ci: test extension through code-server and in ci

.github/workflows/pr.yaml

@@ -17,11 +17,46 @@
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
-          node-version: '20'
+          node-version: '22'
       - uses: pnpm/action-setup@v4
         with:
           version: latest
       - name: Install dependencies
         run: pnpm install
       - name: Run CI
         run: pnpm run ci
+  test-vscode-e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+      - uses: pnpm/action-setup@v4
+        with:
+          version: latest
+      - name: Install dependencies
+        run: pnpm install
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install python dependencies
+        run: |
+          python -m venv .venv
+          source .venv/bin/activate
+          make install-dev
+      - name: Fetch VS Code
+        working-directory: ./vscode/extension
+        run: pnpm run fetch-vscode
+
+      - name: Install code-server
+        run: curl -fsSL https://code-server.dev/install.sh | sh
+      - name: Install Playwright browsers
+        working-directory: ./vscode/extension
+        run: pnpm exec playwright install
+      - name: Run e2e tests
+        working-directory: ./vscode/extension
+        run: |
+          source ../../.venv/bin/activate
+          pnpm run test:e2e tests/stop.spec.ts

.nvmrc

@@ -1 +1 @@
-20
+20

vscode/extension/package.json

@@ -120,9 +120,10 @@
     "lint": "eslint src",
     "lint:fix": "eslint src --fix",
     "test:unit": "vitest run",
-    "test:e2e": "playwright test",
-    "test:e2e:ui": "playwright test --ui",
-    "test:e2e:headed": "playwright test --headed",
+    "code-server": "code-server",
+    "test:e2e": "pnpm run vscode:package && playwright test",
+    "test:e2e:ui": "pnpm run vscode:package && playwright test --ui",
+    "test:e2e:headed": "pnpm run vscode:package && playwright test --headed",
     "fetch-vscode": "tsx scripts/fetch-vscode.ts",
     "compile": "pnpm run check-types && node esbuild.js",
     "check-types": "tsc --noEmit -p ./tsconfig.build.json",

vscode/extension/playwright.config.ts

@@ -11,7 +11,7 @@ export default defineConfig({
       use: {
         // ⭢ we'll launch Electron ourselves – no browser needed
         browserName: 'chromium',
-        headless: false, // headed makes screenshots deterministic
+        headless: true, // headless mode for tests
         launchOptions: {
           slowMo: process.env.CI ? 0 : 100,
         },

vscode/extension/tests/stop.spec.ts

@@ -1,57 +1,67 @@
 import path from 'path'
-import { startVSCode, SUSHI_SOURCE_PATH } from './utils'
+import { SUSHI_SOURCE_PATH } from './utils'
 import os from 'os'
 import { test } from '@playwright/test'
 import fs from 'fs-extra'
+import { startCodeServer, stopCodeServer } from './utils_code_server'
 
-test('Stop server works', async () => {
+test('Stop server works', async ({ page }) => {
+  test.setTimeout(120000) // Increase timeout to 2 minutes
+
+  console.log('Starting test: Stop server works')
   const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'vscode-test-sushi-'))
   await fs.copy(SUSHI_SOURCE_PATH, tempDir)
 
+  const context = await startCodeServer(tempDir, true)
+
   try {
-    const { window, close } = await startVSCode(tempDir)
+    // Navigate to code-server instance
+    await page.goto(`http://127.0.0.1:${context.codeServerPort}`)
+
+    // Wait for code-server to load
+    await page.waitForLoadState('networkidle')
+    await page.waitForSelector('[role="application"]', { timeout: 10000 })
 
-    //   Wait for the models folder to be visible
-    await window.waitForSelector('text=models')
+    // Wait for the models folder to be visible in the file explorer
+    await page.waitForSelector('text=models')
 
     // Click on the models folder, excluding external_models
-    await window
+    await page
       .getByRole('treeitem', { name: 'models', exact: true })
       .locator('a')
       .click()
 
-    // Open the customer_revenue_lifetime model
-    await window
+    // Open the customers.sql model
+    await page
       .getByRole('treeitem', { name: 'customers.sql', exact: true })
       .locator('a')
       .click()
 
-    await window.waitForSelector('text=grain')
-    await window.waitForSelector('text=Loaded SQLMesh Context')
+    await page.waitForSelector('text=grain')
+    await page.waitForSelector('text=Loaded SQLMesh Context')
 
     // Stop the server
-    await window.keyboard.press(
+    await page.keyboard.press(
       process.platform === 'darwin' ? 'Meta+Shift+P' : 'Control+Shift+P',
     )
-    await window.keyboard.type('SQLMesh: Stop Server')
-    await window.keyboard.press('Enter')
+    await page.keyboard.type('SQLMesh: Stop Server')
+    await page.keyboard.press('Enter')
 
     // Await LSP server stopped message
-    await window.waitForSelector('text=LSP server stopped')
+    await page.waitForSelector('text=LSP server stopped')
 
     // Render the model
-    await window.keyboard.press(
+    await page.keyboard.press(
       process.platform === 'darwin' ? 'Meta+Shift+P' : 'Control+Shift+P',
     )
-    await window.keyboard.type('Render Model')
-    await window.keyboard.press('Enter')
+    await page.keyboard.type('Render Model')
+    await page.keyboard.press('Enter')
 
     // Await error message
-    await window.waitForSelector(
+    await page.waitForSelector(
       'text="Failed to render model: LSP client not ready."',
     )
-    await close()
   } finally {
-    await fs.remove(tempDir)
+    await stopCodeServer(context)
   }
 })

vscode/extension/tests/utils_code_server.ts

@@ -0,0 +1,165 @@
+import { spawn, ChildProcess, execSync } from 'child_process'
+import path from 'path'
+import fs from 'fs-extra'
+
+export interface CodeServerContext {
+  codeServerProcess: ChildProcess
+  codeServerPort: number
+  tempDir: string
+}
+
+/**
+ * @param tempDir - The temporary directory to use for the code-server instance
+ * @param placeFileWithPythonInterpreter - Whether to place a vscode/settings.json file in the temp directory that points to the python interpreter of the environmen the test is running in.
+ * @returns The code-server context
+ */
+export async function startCodeServer(
+  tempDir: string,
+  placeFileWithPythonInterpreter: boolean = false,
+): Promise<CodeServerContext> {
+  // Find an available port
+  const codeServerPort = Math.floor(Math.random() * 10000) + 50000
+
+  // Create .vscode/settings.json with Python interpreter if requested
+  if (placeFileWithPythonInterpreter) {
+    const vscodeDir = path.join(tempDir, '.vscode')
+    await fs.ensureDir(vscodeDir)
+
+    // Get the current Python interpreter path
+    const pythonPath = execSync('which python', {
+      encoding: 'utf-8',
+    }).trim()
+
+    const settings = {
+      'python.defaultInterpreterPath': path.join(
+        __dirname,
+        '..',
+        '..',
+        '..',
+        '.venv',
+        'bin',
+        'python',
+      ),
+    }
+
+    await fs.writeJson(path.join(vscodeDir, 'settings.json'), settings, {
+      spaces: 2,
+    })
+    console.log(
+      `Created .vscode/settings.json with Python interpreter: ${pythonPath}`,
+    )
+  }
+
+  // Get the extension version from package.json
+  const extensionDir = path.join(__dirname, '..')
+  const packageJson = JSON.parse(
+    fs.readFileSync(path.join(extensionDir, 'package.json'), 'utf-8'),
+  )
+  const version = packageJson.version
+  const extensionName = packageJson.name || 'sqlmesh'
+
+  // Look for the specific version .vsix file
+  const vsixFileName = `${extensionName}-${version}.vsix`
+  const vsixPath = path.join(extensionDir, vsixFileName)
+
+  if (!fs.existsSync(vsixPath)) {
+    throw new Error(
+      `Extension file ${vsixFileName} not found. Run "pnpm run vscode:package" first.`,
+    )
+  }
+
+  console.log(`Using extension: ${vsixFileName}`)
+
+  // Install the extension first
+  const extensionsDir = path.join(tempDir, 'extensions')
+  console.log('Installing extension...')
+  execSync(
+    `pnpm run code-server --user-data-dir "${tempDir}" --extensions-dir "${extensionsDir}" --install-extension "${vsixPath}"`,
+    { stdio: 'inherit' },
+  )
+
+  // Start code-server instance
+  const codeServerProcess = spawn(
+    'pnpm',
+    [
+      'run',
+      'code-server',
+      '--bind-addr',
+      `127.0.0.1:${codeServerPort}`,
+      '--auth',
+      'none',
+      '--disable-telemetry',
+      '--disable-update-check',
+      '--disable-workspace-trust',
+      '--user-data-dir',
+      tempDir,
+      '--extensions-dir',
+      extensionsDir,
+      tempDir,
+    ],
+    {
+      stdio: 'pipe',
+      cwd: path.join(__dirname, '..'),
+    },
+  )
+
+  // Wait for code-server to be ready
+  await new Promise<void>((resolve, reject) => {
+    let output = ''
+    const timeout = setTimeout(() => {
+      reject(new Error('Code-server failed to start within timeout'))
+    }, 30000)
+
+    codeServerProcess.stdout?.on('data', data => {
+      output += data.toString()
+      if (output.includes('HTTP server listening on')) {
+        clearTimeout(timeout)
+        resolve()
+      }
+    })
+
+    codeServerProcess.stderr?.on('data', data => {
+      console.error('Code-server stderr:', data.toString())
+    })
+
+    codeServerProcess.on('error', error => {
+      clearTimeout(timeout)
+      reject(error)
+    })
+
+    codeServerProcess.on('exit', code => {
+      if (code !== 0) {
+        clearTimeout(timeout)
+        reject(new Error(`Code-server exited with code ${code}`))
+      }
+    })
+  })
+
+  return { codeServerProcess, codeServerPort, tempDir }
+}
+
+export async function stopCodeServer(
+  context: CodeServerContext,
+): Promise<void> {
+  const { codeServerProcess, tempDir } = context
+
+  // Clean up code-server process
+  codeServerProcess.kill('SIGTERM')
+
+  // Wait for process to exit
+  await new Promise<void>(resolve => {
+    codeServerProcess.on('exit', () => {
+      resolve()
+    })
+    // Force kill after 5 seconds
+    setTimeout(() => {
+      if (!codeServerProcess.killed) {
+        codeServerProcess.kill('SIGKILL')
+      }
+      resolve()
+    }, 5000)
+  })
+
+  // Clean up temporary directory
+  await fs.remove(tempDir)
+}