From 0a1455eb15a040191108fc9c9794b6b9b91150ef Mon Sep 17 00:00:00 2001 From: Ben King <9087625+benfdking@users.noreply.github.com> Date: Thu, 26 Jun 2025 14:42:51 +0100 Subject: [PATCH] ci: test extension through code-server and in ci --- .github/workflows/pr.yaml | 37 ++++- .nvmrc | 2 +- vscode/extension/package.json | 7 +- vscode/extension/playwright.config.ts | 2 +- vscode/extension/tests/stop.spec.ts | 50 +++--- vscode/extension/tests/utils_code_server.ts | 165 ++++++++++++++++++++ 6 files changed, 237 insertions(+), 26 deletions(-) create mode 100644 vscode/extension/tests/utils_code_server.ts diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 765aea043f..1d7f6ec8a3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: '20' + node-version: '22' - uses: pnpm/action-setup@v4 with: version: latest @@ -25,3 +25,38 @@ jobs: run: pnpm install - name: Run CI run: pnpm run ci + test-vscode-e2e: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - uses: pnpm/action-setup@v4 + with: + version: latest + - name: Install dependencies + run: pnpm install + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install python dependencies + run: | + python -m venv .venv + source .venv/bin/activate + make install-dev + - name: Fetch VS Code + working-directory: ./vscode/extension + run: pnpm run fetch-vscode + + - name: Install code-server + run: curl -fsSL https://code-server.dev/install.sh | sh + - name: Install Playwright browsers + working-directory: ./vscode/extension + run: pnpm exec playwright install + - name: Run e2e tests + working-directory: ./vscode/extension + run: | + source ../../.venv/bin/activate + pnpm run test:e2e tests/stop.spec.ts diff --git a/.nvmrc b/.nvmrc index 2edeafb09d..209e3ef4b6 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -20 \ No newline at end of file +20 diff --git a/vscode/extension/package.json b/vscode/extension/package.json index cc5722c684..e8b7d5f00b 100644 --- a/vscode/extension/package.json +++ b/vscode/extension/package.json @@ -120,9 +120,10 @@ "lint": "eslint src", "lint:fix": "eslint src --fix", "test:unit": "vitest run", - "test:e2e": "playwright test", - "test:e2e:ui": "playwright test --ui", - "test:e2e:headed": "playwright test --headed", + "code-server": "code-server", + "test:e2e": "pnpm run vscode:package && playwright test", + "test:e2e:ui": "pnpm run vscode:package && playwright test --ui", + "test:e2e:headed": "pnpm run vscode:package && playwright test --headed", "fetch-vscode": "tsx scripts/fetch-vscode.ts", "compile": "pnpm run check-types && node esbuild.js", "check-types": "tsc --noEmit -p ./tsconfig.build.json", diff --git a/vscode/extension/playwright.config.ts b/vscode/extension/playwright.config.ts index 6820f2c8b1..599e5e3738 100644 --- a/vscode/extension/playwright.config.ts +++ b/vscode/extension/playwright.config.ts @@ -11,7 +11,7 @@ export default defineConfig({ use: { // ⭢ we'll launch Electron ourselves – no browser needed browserName: 'chromium', - headless: false, // headed makes screenshots deterministic + headless: true, // headless mode for tests launchOptions: { slowMo: process.env.CI ? 0 : 100, }, diff --git a/vscode/extension/tests/stop.spec.ts b/vscode/extension/tests/stop.spec.ts index 61422991cc..d7b2d08ccd 100644 --- a/vscode/extension/tests/stop.spec.ts +++ b/vscode/extension/tests/stop.spec.ts @@ -1,57 +1,67 @@ import path from 'path' -import { startVSCode, SUSHI_SOURCE_PATH } from './utils' +import { SUSHI_SOURCE_PATH } from './utils' import os from 'os' import { test } from '@playwright/test' import fs from 'fs-extra' +import { startCodeServer, stopCodeServer } from './utils_code_server' -test('Stop server works', async () => { +test('Stop server works', async ({ page }) => { + test.setTimeout(120000) // Increase timeout to 2 minutes + + console.log('Starting test: Stop server works') const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'vscode-test-sushi-')) await fs.copy(SUSHI_SOURCE_PATH, tempDir) + const context = await startCodeServer(tempDir, true) + try { - const { window, close } = await startVSCode(tempDir) + // Navigate to code-server instance + await page.goto(`http://127.0.0.1:${context.codeServerPort}`) + + // Wait for code-server to load + await page.waitForLoadState('networkidle') + await page.waitForSelector('[role="application"]', { timeout: 10000 }) - // Wait for the models folder to be visible - await window.waitForSelector('text=models') + // Wait for the models folder to be visible in the file explorer + await page.waitForSelector('text=models') // Click on the models folder, excluding external_models - await window + await page .getByRole('treeitem', { name: 'models', exact: true }) .locator('a') .click() - // Open the customer_revenue_lifetime model - await window + // Open the customers.sql model + await page .getByRole('treeitem', { name: 'customers.sql', exact: true }) .locator('a') .click() - await window.waitForSelector('text=grain') - await window.waitForSelector('text=Loaded SQLMesh Context') + await page.waitForSelector('text=grain') + await page.waitForSelector('text=Loaded SQLMesh Context') // Stop the server - await window.keyboard.press( + await page.keyboard.press( process.platform === 'darwin' ? 'Meta+Shift+P' : 'Control+Shift+P', ) - await window.keyboard.type('SQLMesh: Stop Server') - await window.keyboard.press('Enter') + await page.keyboard.type('SQLMesh: Stop Server') + await page.keyboard.press('Enter') // Await LSP server stopped message - await window.waitForSelector('text=LSP server stopped') + await page.waitForSelector('text=LSP server stopped') // Render the model - await window.keyboard.press( + await page.keyboard.press( process.platform === 'darwin' ? 'Meta+Shift+P' : 'Control+Shift+P', ) - await window.keyboard.type('Render Model') - await window.keyboard.press('Enter') + await page.keyboard.type('Render Model') + await page.keyboard.press('Enter') // Await error message - await window.waitForSelector( + await page.waitForSelector( 'text="Failed to render model: LSP client not ready."', ) - await close() } finally { - await fs.remove(tempDir) + await stopCodeServer(context) } }) diff --git a/vscode/extension/tests/utils_code_server.ts b/vscode/extension/tests/utils_code_server.ts new file mode 100644 index 0000000000..c38fee615c --- /dev/null +++ b/vscode/extension/tests/utils_code_server.ts @@ -0,0 +1,165 @@ +import { spawn, ChildProcess, execSync } from 'child_process' +import path from 'path' +import fs from 'fs-extra' + +export interface CodeServerContext { + codeServerProcess: ChildProcess + codeServerPort: number + tempDir: string +} + +/** + * @param tempDir - The temporary directory to use for the code-server instance + * @param placeFileWithPythonInterpreter - Whether to place a vscode/settings.json file in the temp directory that points to the python interpreter of the environmen the test is running in. + * @returns The code-server context + */ +export async function startCodeServer( + tempDir: string, + placeFileWithPythonInterpreter: boolean = false, +): Promise { + // Find an available port + const codeServerPort = Math.floor(Math.random() * 10000) + 50000 + + // Create .vscode/settings.json with Python interpreter if requested + if (placeFileWithPythonInterpreter) { + const vscodeDir = path.join(tempDir, '.vscode') + await fs.ensureDir(vscodeDir) + + // Get the current Python interpreter path + const pythonPath = execSync('which python', { + encoding: 'utf-8', + }).trim() + + const settings = { + 'python.defaultInterpreterPath': path.join( + __dirname, + '..', + '..', + '..', + '.venv', + 'bin', + 'python', + ), + } + + await fs.writeJson(path.join(vscodeDir, 'settings.json'), settings, { + spaces: 2, + }) + console.log( + `Created .vscode/settings.json with Python interpreter: ${pythonPath}`, + ) + } + + // Get the extension version from package.json + const extensionDir = path.join(__dirname, '..') + const packageJson = JSON.parse( + fs.readFileSync(path.join(extensionDir, 'package.json'), 'utf-8'), + ) + const version = packageJson.version + const extensionName = packageJson.name || 'sqlmesh' + + // Look for the specific version .vsix file + const vsixFileName = `${extensionName}-${version}.vsix` + const vsixPath = path.join(extensionDir, vsixFileName) + + if (!fs.existsSync(vsixPath)) { + throw new Error( + `Extension file ${vsixFileName} not found. Run "pnpm run vscode:package" first.`, + ) + } + + console.log(`Using extension: ${vsixFileName}`) + + // Install the extension first + const extensionsDir = path.join(tempDir, 'extensions') + console.log('Installing extension...') + execSync( + `pnpm run code-server --user-data-dir "${tempDir}" --extensions-dir "${extensionsDir}" --install-extension "${vsixPath}"`, + { stdio: 'inherit' }, + ) + + // Start code-server instance + const codeServerProcess = spawn( + 'pnpm', + [ + 'run', + 'code-server', + '--bind-addr', + `127.0.0.1:${codeServerPort}`, + '--auth', + 'none', + '--disable-telemetry', + '--disable-update-check', + '--disable-workspace-trust', + '--user-data-dir', + tempDir, + '--extensions-dir', + extensionsDir, + tempDir, + ], + { + stdio: 'pipe', + cwd: path.join(__dirname, '..'), + }, + ) + + // Wait for code-server to be ready + await new Promise((resolve, reject) => { + let output = '' + const timeout = setTimeout(() => { + reject(new Error('Code-server failed to start within timeout')) + }, 30000) + + codeServerProcess.stdout?.on('data', data => { + output += data.toString() + if (output.includes('HTTP server listening on')) { + clearTimeout(timeout) + resolve() + } + }) + + codeServerProcess.stderr?.on('data', data => { + console.error('Code-server stderr:', data.toString()) + }) + + codeServerProcess.on('error', error => { + clearTimeout(timeout) + reject(error) + }) + + codeServerProcess.on('exit', code => { + if (code !== 0) { + clearTimeout(timeout) + reject(new Error(`Code-server exited with code ${code}`)) + } + }) + }) + + return { codeServerProcess, codeServerPort, tempDir } +} + +export async function stopCodeServer( + context: CodeServerContext, +): Promise { + const { codeServerProcess, tempDir } = context + + // Clean up code-server process + codeServerProcess.kill('SIGTERM') + + // Wait for process to exit + await new Promise(resolve => { + codeServerProcess.on('exit', () => { + resolve() + }) + // Force kill after 5 seconds + setTimeout(() => { + if (!codeServerProcess.killed) { + codeServerProcess.kill('SIGKILL') + } + resolve() + }, 5000) + }) + + // Clean up temporary directory + await fs.remove(tempDir) +}