feat: Standardization of test execution methods in Windows (#76)

Author: mato533

.github/workflows/ci-smoke.yml

@@ -5,6 +5,10 @@ on:
     # Make this a reusable workflow, no value needed
     # https://docs.github.com/en/actions/using-workflows/reusing-workflows
     inputs:
+      os:
+        description: 'OS for run'
+        type: string
+        default: "['ubuntu-latest', 'windows-latest', 'macos-latest']"
       scenario:
         description: 'Smoke scenario'
         type: string
@@ -19,7 +23,7 @@ jobs:
       fail-fast: false
       matrix:
         node-version: ['20']
-        os: ['ubuntu-latest', 'windows-latest', 'macos-latest']
+        os: ${{ fromJSON(inputs.os) }}
     runs-on: ${{ matrix.os }}
     steps:
       - name: 👷 Checkout

.github/workflows/ci.yml

@@ -74,3 +74,11 @@ jobs:
     uses: ./.github/workflows/ci-smoke.yml
     with:
       scenario: 'env'
+
+  smoke-retro-wdio:
+    name: Smoke - Run test with temporary configuration file when lower version of WDIO
+    needs: [lint, build, e2e]
+    uses: ./.github/workflows/ci-smoke.yml
+    with:
+      os: "['windows-latest']"
+      scenario: 'retro-wdio-win'

e2e/package.json

@@ -19,22 +19,27 @@
     "test:smoke": "run-s test:smoke:*",
     "test:smoke:config": "cross-env VSCODE_WDIO_E2E_SCENARIO=config xvfb-maybe wdio run ./wdioSmoke.conf.ts",
     "test:smoke:timeout": "cross-env VSCODE_WDIO_E2E_SCENARIO=timeout xvfb-maybe wdio run ./wdioSmoke.conf.ts",
+    "test:smoke:retro-wdio-win": "run-s test:smoke:retro-wdio-win:*",
+    "test:smoke:retro-wdio-win:run-latest": "cross-env VSCODE_WDIO_E2E_SCENARIO=mocha VSCODE_WDIO_SMOKE_RETRO_WIN=no xvfb-maybe pnpm run wdio",
+    "test:smoke:retro-wdio-win:prepare": "tsx ./scripts/retro-wdio-win.ts",
+    "test:smoke:retro-wdio-win:run": "cross-env VSCODE_WDIO_E2E_SCENARIO=mocha VSCODE_WDIO_SMOKE_RETRO_WIN=yes xvfb-maybe pnpm run wdio",
+    "test:smoke:retro-wdio-win:cleanup": "git checkout ../samples/e2e/mocha/package.json ../pnpm-lock.yaml && pnpm --filter @vscode-wdio/e2e-mocha install",
     "test:smoke:env": "cross-env VSCODE_WDIO_E2E_SCENARIO=env xvfb-maybe wdio run ./wdioSmoke.conf.ts",
     "wdio": "wdio run ./wdio.conf.ts"
   },
   "devDependencies": {
     "@types/semver": "^7.7.0",
-    "@wdio/cli": "^9.13.0",
-    "@wdio/globals": "^9.12.6",
-    "@wdio/local-runner": "^9.13.0",
-    "@wdio/mocha-framework": "^9.13.0",
-    "@wdio/spec-reporter": "^9.13.0",
-    "@wdio/types": "^9.15.0",
+    "@wdio/cli": "^9.16.2",
+    "@wdio/globals": "^9.16.2",
+    "@wdio/local-runner": "^9.16.2",
+    "@wdio/mocha-framework": "^9.16.2",
+    "@wdio/spec-reporter": "^9.16.2",
+    "@wdio/types": "^9.16.2",
     "chai": "^5.2.0",
     "expect": "^30.0.0",
     "semver": "^7.7.2",
     "wdio-vscode-service": "^6.1.3",
-    "webdriver": "^9.13.0",
-    "webdriverio": "^9.13.0"
+    "webdriver": "^9.16.2",
+    "webdriverio": "^9.16.2"
   }
 }

e2e/scripts/retro-wdio-win.ts

@@ -0,0 +1,17 @@
+import shell from 'shelljs'
+
+import pkg from '../../samples/e2e/mocha/package.json' with { type: 'json' }
+
+const targetPkg = '@vscode-wdio/e2e-mocha'
+const targetWdioVersion = '9.15.0'
+
+const wdioPkgs = Object.keys(pkg.devDependencies).filter((pkg) => pkg.startsWith('@wdio/'))
+
+const targetWdioPkgs = wdioPkgs.map((pkg) => `${pkg}@${targetWdioVersion}`)
+
+const cmd = `pnpm --filter ${targetPkg} install -D ${targetWdioPkgs.join(' ')}`
+
+console.log(`\n>${cmd}\n`)
+
+const result = shell.exec(cmd)
+process.exit(result.code)

e2e/tests/basic.spec.ts

@@ -85,4 +85,21 @@ describe(`VS Code Extension Testing with ${targetFramework}`, function () {
 
         await expect(items).toMatchTreeStructure(expected.runPartially)
     })
+
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    if (process.env.VSCODE_WDIO_SMOKE_RETRO_WIN === 'yes') {
+
+        it('should use temporally configuration file', async function () {
+            await expect(workbench).hasExpectedLog(
+                'Use temporary configuration files to run WDIO: @wdio/utils@9.15.0 < 9.16.0'
+            )
+        })
+
+        // eslint-disable-next-line mocha/no-setup-in-describe
+    } else if (process.env.VSCODE_WDIO_SMOKE_RETRO_WIN === 'no') {
+
+        it('should not use temporally configuration file', async function () {
+            await expect(workbench).hasExpectedLog(/Use cli argument to run WDIO: @wdio\/utils/)
+        })
+    }
 })

e2e/wdio.conf.ts

@@ -20,6 +20,8 @@ const version = isCompatibilityMode ? minimumVersion : 'stable'
 const outputDir = path.join(__dirname, 'logs', [isCompatibilityMode ? 'compatibility' : 'e2e', target].join('-'))
 process.env.VSCODE_WDIO_TRACE_LOG_PATH = outputDir
 
+const loglevel = process.env.VSCODE_WDIO_SMOKE_RETRO_WIN ? 'debug' : 'trace'
+
 function defineSpecs(target: TestTargets) {
     switch (target) {
         case 'cucumber':
@@ -38,7 +40,7 @@ export function createBaseConfig(workspacePath: string, userSettings = {}): Webd
     const resolvedUserSettings = Object.assign(
         {},
         {
-            'webdriverio.logLevel': 'trace',
+            'webdriverio.logLevel': loglevel,
         },
         userSettings
     )

packages/vscode-wdio-reporter/package.json

@@ -16,10 +16,10 @@
   },
   "dependencies": {
     "@vscode-wdio/constants": "workspace:*",
-    "@wdio/reporter": "^9.12.3"
+    "@wdio/reporter": "^9.16.2"
   },
   "devDependencies": {
     "@vscode-wdio/types": "workspace:*",
-    "@wdio/types": "^9.13.0"
+    "@wdio/types": "^9.16.2"
   }
 }

packages/vscode-wdio-types/package.json

@@ -47,7 +47,7 @@
   },
   "devDependencies": {
     "@vscode-wdio/constants": "workspace:*",
-    "@wdio/types": "^9.13.0"
+    "@wdio/types": "^9.16.2"
   },
   "dependencies": {
     "@types/ws": "^8.18.1"

packages/vscode-wdio-worker/package.json

@@ -31,12 +31,13 @@
     "@vscode-wdio/utils": "workspace:*",
     "birpc": "^2.3.0",
     "dotenv": "^16.5.0",
+    "import-meta-resolve": "^4.1.0",
     "recast": "^0.23.11",
     "ws": "^8.18.1"
   },
   "devDependencies": {
     "@types/ws": "^8.18.1",
     "@vscode-wdio/types": "workspace:*",
-    "@wdio/cli": "^9.12.4"
+    "@wdio/cli": "^9.16.2"
   }
 }

packages/vscode-wdio-worker/src/test.ts

@@ -3,7 +3,7 @@ import * as os from 'node:os'
 import { dirname, isAbsolute, join, resolve } from 'node:path'
 
 import { getLauncherInstance } from './cli.js'
-import { getTempConfigCreator, isWindows } from './utils.js'
+import { getTempConfigCreator, isFixedWdio, isWindows } from './utils.js'
 import type { ResultSet } from '@vscode-wdio/types/reporter'
 import type { RunTestOptions, TestResultData } from '@vscode-wdio/types/server'
 import type { ILogger } from '@vscode-wdio/types/utils'
@@ -15,6 +15,13 @@ const VSCODE_REPORTER_PATH = resolve(__dirname, 'reporter.cjs')
 export async function runTest(this: WorkerMetaContext, options: RunTestOptions): Promise<TestResultData> {
     const outputDir = await getOutputDir.call(this)
     let configFile: string | undefined = undefined
+
+    // To avoid this issue, We use temporary configuration files
+    // only on Windows platforms and for versions of @wdio/utils prior to 9.15.0.
+    // https://github.com/webdriverio/webdriverio/issues/14532
+    // This issue was fixed by this PR.
+    // https://github.com/webdriverio/webdriverio/pull/14565
+    const useTempConfigFile = isWindows() && !(await isFixedWdio.call(this, options.configPath))
     try {
         // Prepare launcher options
         const wdioArgs: RunCommandArguments = {
@@ -42,16 +49,14 @@ export async function runTest(this: WorkerMetaContext, options: RunTestOptions):
             await fs.mkdir(logDir, { recursive: true })
         }
 
-        if (isWindows()) {
+        if (useTempConfigFile) {
             const creator = await getTempConfigCreator(this)
             configFile = await creator(options.configPath, outputDir.json!)
             options.configPath = configFile
             wdioArgs.configPath = configFile
-        }
-
-        // The `stdout` must be true because the name of the logger is
-        // the name of the file and the initialization of Write Stream will fail.
-        if (!isWindows()) {
+        } else {
+            // The `stdout` must be true because the name of the logger is
+            // the name of the file and the initialization of Write Stream will fail.
             wdioArgs.reporters = [[VSCODE_REPORTER_PATH, { stdout: true, outputDir: outputDir.json }]]
         }
 
@@ -122,7 +127,7 @@ export async function runTest(this: WorkerMetaContext, options: RunTestOptions):
         if (outputDir.json) {
             await removeResultDir(this.log, outputDir.json)
         }
-        if (isWindows() && configFile) {
+        if (useTempConfigFile && configFile) {
             try {
                 this.log.debug(`Remove temp config file...: ${configFile}`)
                 await fs.rm(configFile, { recursive: true, force: true })