From ded10ae1e7789804f14a8ce1d32b77ba0860246e Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Sun, 20 Sep 2026 15:38:20 -0400 Subject: [PATCH 1/4] fix(learning): stop treating guidance retrieval as successful reuse --- .../agentic-flow/reasoning-bank/index.ts | 34 +--- .../experience-guidance-outcomes.test.ts | 179 ++++++++++++++++++ 2 files changed, 182 insertions(+), 31 deletions(-) create mode 100644 tests/integration/experience-guidance-outcomes.test.ts diff --git a/src/integrations/agentic-flow/reasoning-bank/index.ts b/src/integrations/agentic-flow/reasoning-bank/index.ts index 7b490ae57..490b72d69 100644 --- a/src/integrations/agentic-flow/reasoning-bank/index.ts +++ b/src/integrations/agentic-flow/reasoning-bank/index.ts @@ -304,20 +304,6 @@ export class EnhancedReasoningBankAdapter { ...guidance.suggestedActions.map(a => `Action: ${a}`), ...result.guidance, ]; - this.stats.tokensSavedEstimate += guidance.estimatedTokenSavings; - this.stats.experiencesApplied++; - - // Record application of each source experience for reuse tracking - for (const src of guidance.sourceExperiences) { - this.experienceReplay.recordApplication( - src.id, - request.task, - true, // success=true at routing time; updated later via recordOutcome - Math.round(guidance.estimatedTokenSavings), - ).catch(err => { - console.warn(`[EnhancedAdapter] Failed to record experience application: ${err}`); - }); - } } } @@ -458,23 +444,8 @@ export class EnhancedReasoningBankAdapter { return null; } - const guidance = await this.experienceReplay.getGuidance(task, domain); - if (guidance) { - this.stats.experiencesApplied++; - - // Record application for reuse tracking (experience_applications table) - for (const src of guidance.sourceExperiences) { - this.experienceReplay.recordApplication( - src.id, - task, - true, - Math.round(guidance.estimatedTokenSavings), - ).catch(err => { - console.warn(`[EnhancedAdapter] Failed to record experience application: ${err}`); - }); - } - } - return guidance; + // Retrieving guidance does not establish that it was applied successfully. + return this.experienceReplay.getGuidance(task, domain); } /** @@ -496,6 +467,7 @@ export class EnhancedReasoningBankAdapter { if (!this.experienceReplay) return; await this.experienceReplay.recordApplication(experienceId, task, success, tokensSaved); + this.stats.experiencesApplied++; this.stats.tokensSavedEstimate += tokensSaved; } diff --git a/tests/integration/experience-guidance-outcomes.test.ts b/tests/integration/experience-guidance-outcomes.test.ts new file mode 100644 index 000000000..459ee4b51 --- /dev/null +++ b/tests/integration/experience-guidance-outcomes.test.ts @@ -0,0 +1,179 @@ +/** Guidance retrieval must not manufacture successful execution evidence. */ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { createServer, type Server } from 'node:http'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { Database } from 'better-sqlite3'; +import type { EnhancedReasoningBankAdapter } from '../../src/integrations/agentic-flow/reasoning-bank/index.js'; + +describe('Experience guidance outcome integrity', () => { + let server: Server; + let projectRoot: string; + let endpoint: string; + let adapter: EnhancedReasoningBankAdapter; + let db: Database; + let adapterModule: typeof import('../../src/integrations/agentic-flow/reasoning-bank/index.js'); + let memoryModule: typeof import('../../src/kernel/unified-memory.js'); + let embeddingsModule: typeof import('../../src/learning/real-embeddings.js'); + let hnswModule: typeof import('../../src/kernel/hnsw-adapter.js'); + let consolidationModule: typeof import('../../src/learning/experience-consolidation.js'); + const task = 'Generate boundary checks for a numeric range'; + const domain = 'test-generation' as const; + + beforeAll(async () => { + projectRoot = mkdtempSync(join(tmpdir(), 'aqe-guidance-outcomes-')); + vi.stubEnv('AQE_PROJECT_ROOT', projectRoot); + vi.stubEnv('AQE_MEMORY_BACKEND', 'memory'); + vi.stubEnv('AQE_EMBEDDER_TOKEN', ''); + + // The only external service is a deterministic loopback embedder. A single + // experience per domain makes retrieval independent of semantic quality. + server = createServer(async (request, response) => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = JSON.parse(Buffer.concat(chunks).toString()) as { input: string | string[] }; + const inputs = Array.isArray(body.input) ? body.input : [body.input]; + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ + data: inputs.map((_, index) => ({ index, embedding: [1, ...Array(383).fill(0)] })), + })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + endpoint = `http://127.0.0.1:${(server.address() as { port: number }).port}`; + vi.stubEnv('AQE_EMBEDDER_ENDPOINT', endpoint); + + // Set isolation before production imports establish singleton defaults. + adapterModule = await import('../../src/integrations/agentic-flow/reasoning-bank/index.js'); + memoryModule = await import('../../src/kernel/unified-memory.js'); + embeddingsModule = await import('../../src/learning/real-embeddings.js'); + hnswModule = await import('../../src/kernel/hnsw-adapter.js'); + consolidationModule = await import('../../src/learning/experience-consolidation.js'); + }); + + beforeEach(async () => { + adapter = new adapterModule.EnhancedReasoningBankAdapter({ + enablePatternEvolution: false, + autoConsolidate: false, + base: { sqlite: { useUnified: true }, embeddings: { endpoint } }, + experienceReplay: { autoPrune: false, embedding: { endpoint } }, + trajectoryTracker: { autoEndTimeoutMs: 1000 }, + }); + await adapter.initialize(); + const memory = memoryModule.getUnifiedMemory(); + expect(memory.getDbPath()).toBe(':memory:'); + db = memory.getDatabase()!; + }); + + afterEach(async () => { + await adapter?.dispose(); + embeddingsModule?.resetInitialization(); + memoryModule?.resetUnifiedMemory(); + hnswModule?.HnswAdapter.closeAll(); + }); + + afterAll(async () => { + server?.closeAllConnections(); + if (server) await new Promise(resolve => server.close(() => resolve())); + vi.unstubAllEnvs(); + if (projectRoot) rmSync(projectRoot, { recursive: true, force: true }); + }); + + async function seedExperience(): Promise { + const trajectory = await adapter.startTaskTrajectory(task, { domain }); + await adapter.recordTaskStep(trajectory, 'Check both range boundaries', { outcome: 'success' }, { quality: 0.6 }); + await adapter.endTaskTrajectory(trajectory, true); + return (db.prepare('SELECT id FROM captured_experiences WHERE domain = ?').get(domain) as { id: string }).id; + } + + function applications() { + return db.prepare('SELECT experience_id, success, tokens_saved FROM experience_applications ORDER BY rowid').all(); + } + + it('returns guidance repeatedly without recording applications or successful reuse', async () => { + const id = await seedExperience(); + for (let i = 0; i < 4; i++) { + const guidance = await adapter.getExperienceGuidance(task, domain); + expect(guidance?.sourceExperiences).toEqual([expect.objectContaining({ id })]); + } + expect(applications()).toEqual([]); + expect(db.prepare('SELECT application_count FROM captured_experiences WHERE id = ?').get(id)).toEqual({ application_count: 0 }); + expect((await adapter.getStats()).adapter.experiencesApplied).toBe(0); + }); + + it('includes routing guidance without recording a completed experience', async () => { + const id = await seedExperience(); + const routed = await adapter.routeTaskWithExperience({ task, domain }); + expect(routed.success).toBe(true); + if (!routed.success) throw routed.error; + expect(routed.value.experienceGuidance?.sourceExperiences).toEqual([expect.objectContaining({ id })]); + expect(routed.value.guidance.some(line => line.startsWith('Strategy:'))).toBe(true); + expect(applications()).toEqual([]); + expect((await adapter.getStats()).adapter).toMatchObject({ tasksRouted: 1, experiencesApplied: 0, tokensSavedEstimate: 0 }); + }); + + it('does not turn failed execution after retrieval into positive reinforcement', async () => { + const id = await seedExperience(); + await adapter.getExperienceGuidance(task, domain); + const trajectory = await adapter.startTaskTrajectory(task, { domain }); + await adapter.recordTaskStep(trajectory, 'Run checks', { outcome: 'failure', error: 'Assertion failed' }, { quality: 0 }); + expect((await adapter.endTaskTrajectory(trajectory, false)).outcome).toBe('failure'); + await adapter.recordExperienceApplication(id, task, false); + expect(applications()).toEqual([{ experience_id: id, success: 0, tokens_saved: 0 }]); + const consolidator = new consolidationModule.ExperienceConsolidator(); + await consolidator.initialize(db); + await consolidator.consolidateDomain(domain); + const quality = (db.prepare('SELECT quality FROM captured_experiences WHERE id = ?').get(id) as { quality: number }).quality; + expect(quality).toBeCloseTo(0.26); + expect((await adapter.getExperienceGuidance(task, domain))?.confidence).toBeCloseTo(0.26); + }); + + it('persists explicit success and failure with matching adapter and replay counts', async () => { + const id = await seedExperience(); + await adapter.recordExperienceApplication(id, 'Successful execution', true, 30); + await adapter.recordExperienceApplication(id, 'Failed execution', false, 5); + expect(applications()).toEqual([ + { experience_id: id, success: 1, tokens_saved: 30 }, + { experience_id: id, success: 0, tokens_saved: 5 }, + ]); + const stats = await adapter.getStats(); + expect(stats.adapter).toMatchObject({ experiencesApplied: 2, tokensSavedEstimate: 35 }); + expect(stats.experienceReplay).toMatchObject({ experiencesApplied: 2, totalTokensSaved: 35 }); + }); + + it('keeps predicted savings from subsequent guidance separate from recorded savings', async () => { + const id = await seedExperience(); + await adapter.recordExperienceApplication(id, task, true, 42); + expect((await adapter.getExperienceGuidance(task, domain))?.estimatedTokenSavings).toBe(42); + const routed = await adapter.routeTaskWithExperience({ task, domain }); + expect(routed.success).toBe(true); + expect(applications()).toHaveLength(1); + expect((await adapter.getStats()).adapter).toMatchObject({ experiencesApplied: 1, tokensSavedEstimate: 42 }); + }); + + it.each([{ success: true, quality: 0.66 }, { success: false, quality: 0.26 }])( + 'reinforces explicitly recorded success=$success through consolidation', + async ({ success, quality }) => { + const id = await seedExperience(); + await adapter.recordExperienceApplication(id, task, success); + const consolidator = new consolidationModule.ExperienceConsolidator(); + await consolidator.initialize(db); + await consolidator.consolidateDomain(domain); + expect((await adapter.getExperienceGuidance(task, domain))?.confidence).toBeCloseTo(quality); + expect(applications()).toHaveLength(1); + }, + ); + + it('leaves application evidence and counters unchanged when no guidance exists', async () => { + expect(await adapter.getExperienceGuidance(task, domain)).toBeNull(); + expect(applications()).toEqual([]); + expect((await adapter.getStats()).adapter).toMatchObject({ experiencesApplied: 0, tokensSavedEstimate: 0 }); + }); + + it('does not count an outcome whose SQLite persistence fails', async () => { + // The schema's real foreign key rejects an unknown source experience. + await expect(adapter.recordExperienceApplication('missing-experience', task, true, 99)).rejects.toThrow(); + expect(applications()).toEqual([]); + expect((await adapter.getStats()).adapter).toMatchObject({ experiencesApplied: 0, tokensSavedEstimate: 0 }); + }); +}); From 0d27284c15c9965194765cc43aa73159249557d9 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Mon, 21 Sep 2026 01:46:07 -0400 Subject: [PATCH 2/4] fix(ci): skip report comments for fork pull requests (cherry picked from commit 0686b7cf5185f6f855c33f629bcdcbaf2c7fb832) --- .github/workflows/mcp-tools-test.yml | 3 +- .github/workflows/optimized-ci.yml | 3 +- .github/workflows/skill-validation.yml | 2 + tests/unit/scripts/fork-pr-comments.test.ts | 49 +++++++++++++++++++++ 4 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 tests/unit/scripts/fork-pr-comments.test.ts diff --git a/.github/workflows/mcp-tools-test.yml b/.github/workflows/mcp-tools-test.yml index a0a409e7d..64c0abced 100644 --- a/.github/workflows/mcp-tools-test.yml +++ b/.github/workflows/mcp-tools-test.yml @@ -229,7 +229,8 @@ jobs: - name: Create summary comment uses: actions/github-script@v9 - if: github.event_name == 'pull_request' + # Fork tokens cannot write PR comments; keep reports and test gates running. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository with: script: | const fs = require('fs'); diff --git a/.github/workflows/optimized-ci.yml b/.github/workflows/optimized-ci.yml index ab346fb95..ffd0d52bd 100644 --- a/.github/workflows/optimized-ci.yml +++ b/.github/workflows/optimized-ci.yml @@ -412,7 +412,8 @@ jobs: path: ci-metrics.md retention-days: 30 - name: Comment on PR - if: github.event_name == 'pull_request' + # Fork tokens cannot write PR comments; keep reports and test gates running. + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository uses: actions/github-script@v9 with: script: | diff --git a/.github/workflows/skill-validation.yml b/.github/workflows/skill-validation.yml index 8b3c9de2e..d3cad5fb2 100644 --- a/.github/workflows/skill-validation.yml +++ b/.github/workflows/skill-validation.yml @@ -451,6 +451,8 @@ jobs: cat report.md - name: Comment on PR + # Fork tokens cannot write PR comments; keep the Tier 3 gate running. + if: github.event.pull_request.head.repo.full_name == github.repository uses: actions/github-script@v9 with: script: | diff --git a/tests/unit/scripts/fork-pr-comments.test.ts b/tests/unit/scripts/fork-pr-comments.test.ts new file mode 100644 index 000000000..f6b620c7e --- /dev/null +++ b/tests/unit/scripts/fork-pr-comments.test.ts @@ -0,0 +1,49 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { runInNewContext } from 'node:vm'; +import { parse } from 'yaml'; +import { describe, expect, it } from 'vitest'; + +type Step = { name?: string; if?: string }; +type Job = { if?: string; steps: Step[] }; +const root = resolve(import.meta.dirname, '../../..'); +const workflows = [ + ['optimized-ci.yml', 'dashboard', 'Comment on PR'], + ['mcp-tools-test.yml', 'mcp-summary', 'Create summary comment'], + ['skill-validation.yml', 'report', 'Comment on PR'], +] as const; + +// These workflow guards use comparisons and boolean operators shared by +// JavaScript and Actions expressions. Exercise the actual YAML conditions. +function allowed(condition: string | undefined, event: string, headRepo?: string): boolean { + if (!condition) return true; + return Boolean(runInNewContext(condition, { + always: () => true, + github: { + event_name: event, + repository: 'upstream/agentic-qe', + event: headRepo ? { pull_request: { head: { repo: { full_name: headRepo } } } } : {}, + }, + })); +} + +describe.each(workflows)('%s PR reporting permissions', (file, jobName, stepName) => { + const workflow = parse(readFileSync(resolve(root, '.github/workflows', file), 'utf8')); + const job: Job = workflow.jobs[jobName]; + const comment = job.steps.find((step) => step.name === stepName)!; + + it('keeps the reporting job available but skips writes for fork PRs', () => { + expect(comment).toBeDefined(); + expect(allowed(job.if, 'pull_request', 'contributor/agentic-qe')).toBe(true); + expect(allowed(comment.if, 'pull_request', 'contributor/agentic-qe')).toBe(false); + }); + + it('retains comments for same-repository PRs', () => { + expect(allowed(job.if, 'pull_request', 'upstream/agentic-qe')).toBe(true); + expect(allowed(comment.if, 'pull_request', 'upstream/agentic-qe')).toBe(true); + }); + + it.each(['push', 'workflow_dispatch'])('does not post a PR comment on %s', (event) => { + expect(allowed(job.if, event) && allowed(comment.if, event)).toBe(false); + }); +}); From 7ba8752702aea185a18444434eb3d5df88e213f8 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Mon, 21 Sep 2026 17:43:48 -0400 Subject: [PATCH 3/4] fix(ci): preserve runner exits and generate valid coverage reports (cherry picked from commit 480c054e3a1f1c74f36a0c189e3401970dbf04fd) --- .github/workflows/mcp-tools-test.yml | 18 ++--- scripts/ci-vitest-run.sh | 50 ++----------- tests/unit/scripts/ci-vitest-run.test.ts | 89 ++++++++++++++++++++++++ vitest.config.ts | 2 +- 4 files changed, 100 insertions(+), 59 deletions(-) create mode 100644 tests/unit/scripts/ci-vitest-run.test.ts diff --git a/.github/workflows/mcp-tools-test.yml b/.github/workflows/mcp-tools-test.yml index 64c0abced..138cd4c5f 100644 --- a/.github/workflows/mcp-tools-test.yml +++ b/.github/workflows/mcp-tools-test.yml @@ -147,21 +147,11 @@ jobs: - run: npm run build - name: Run MCP integration tests - run: | - timeout 480 npm run test:mcp:integration; EXIT=$? - if [ $EXIT -eq 124 ] && [ -f junit.xml ]; then - FAILURES=$(grep -c '/dev/null || echo "0") - if [ "$FAILURES" = "0" ]; then - echo "::warning::Vitest hung after tests passed (exit 124). Treating as success." - exit 0 - fi - fi - exit $EXIT + run: bash scripts/ci-vitest-run.sh tests/integration/mcp/ env: - NODE_OPTIONS: '--max-old-space-size=1024' - # C3: was `continue-on-error: true`, which let real failures pass as - # green. Removed so this job is an actual gate. The exit-124 hang - # tolerance above still absorbs the known vitest-hang flake. + NODE_OPTIONS: '--max-old-space-size=1024 --expose-gc' + CI_VITEST_TIMEOUT: '480' + # Preserve runner errors and timeouts even when a partial report exists. - name: Generate test report uses: dorny/test-reporter@v1 diff --git a/scripts/ci-vitest-run.sh b/scripts/ci-vitest-run.sh index 32260b6ec..7a1ddd706 100755 --- a/scripts/ci-vitest-run.sh +++ b/scripts/ci-vitest-run.sh @@ -1,50 +1,12 @@ #!/usr/bin/env bash -# CI wrapper for vitest that handles process hangs gracefully. -# -# Problem: vitest completes all tests but hangs due to open handles -# (SQLite connections, HNSW models, timers). The `timeout` command -# kills it with exit code 124, which CI treats as failure even though -# all tests passed. -# -# Solution: Capture vitest output via tee. When timeout kills vitest, -# check the captured output for the "Test Files X passed" summary -# line that vitest prints after all tests complete. junit.xml cannot -# be used because vitest writes it only on clean exit, and the killed -# process leaves it as 0 bytes. +# Bound CI test execution without changing Vitest's exit status. +# A passing test summary does not prove coverage/report generation or cleanup +# completed. Runner errors and timeouts must remain failures. # # Usage: scripts/ci-vitest-run.sh [vitest args...] TIMEOUT_SECONDS="${CI_VITEST_TIMEOUT:-480}" -OUTFILE=$(mktemp /tmp/vitest-output.XXXXXX) - -# --foreground: send signal only to the child process, not the process -# group. Without this, timeout kills this wrapper script too. -# Pipe through tee to capture output while still displaying it. -timeout --foreground "$TIMEOUT_SECONDS" npx vitest run "$@" 2>&1 | tee "$OUTFILE" -# PIPESTATUS[0] is timeout's exit code, not tee's -EXIT=${PIPESTATUS[0]} - -if [ "$EXIT" -eq 0 ]; then - rm -f "$OUTFILE" - exit 0 -fi - -# Check captured output for vitest's test summary. -# Vitest prints "Test Files X passed" after all tests complete, -# before the process hangs. If this line exists with no failures, -# tests passed and the exit code is from the timeout kill. -if grep -q "Test Files.*passed" "$OUTFILE" 2>/dev/null; then - if grep -q "Test Files.*failed" "$OUTFILE" 2>/dev/null; then - echo "::error::Some test files failed." - rm -f "$OUTFILE" - exit "$EXIT" - fi - echo "" - echo "::warning::Vitest process hung after all tests passed (exit $EXIT). Treating as success." - rm -f "$OUTFILE" - exit 0 -fi -echo "::error::Vitest was killed before tests completed (exit $EXIT)." -rm -f "$OUTFILE" -exit "$EXIT" +# --foreground sends the timeout signal to the child rather than this wrapper's +# process group. exec preserves the runner's status, including timeout exit 124. +exec timeout --foreground "$TIMEOUT_SECONDS" npx vitest run "$@" diff --git a/tests/unit/scripts/ci-vitest-run.test.ts b/tests/unit/scripts/ci-vitest-run.test.ts new file mode 100644 index 000000000..fa25cad4a --- /dev/null +++ b/tests/unit/scripts/ci-vitest-run.test.ts @@ -0,0 +1,89 @@ +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { afterEach, describe, expect, it } from 'vitest'; + +const wrapper = resolve(import.meta.dirname, '../../../scripts/ci-vitest-run.sh'); +// The wrapper is used by Ubuntu CI. Stock macOS and Windows do not ship all +// of its shell tools; qualify those local runs without letting Linux CI skip. +const timeoutVersion = spawnSync('timeout', ['--version'], { encoding: 'utf8' }); +const shellToolsAvailable = existsSync('/bin/bash') && timeoutVersion.status === 0 + && timeoutVersion.stdout.includes('GNU coreutils'); +if (process.platform === 'linux' && !shellToolsAvailable) { + throw new Error('CI Vitest wrapper tests require /bin/bash and GNU timeout on Linux; this CI prerequisite must not be skipped.'); +} +const skipReason = !shellToolsAvailable ? ' (requires Bash and GNU timeout on this platform)' : ''; +const fixtures: string[] = []; + +afterEach(() => { + for (const fixture of fixtures.splice(0)) { + rmSync(fixture, { recursive: true, force: true }); + } +}); + +function runRunner(exitCode: number, options: { hang?: boolean; summary?: boolean } = {}) { + const fixture = mkdtempSync(join(tmpdir(), 'aqe-ci-verdict-')); + fixtures.push(fixture); + const argsPath = join(fixture, 'args.txt'); + writeFileSync(join(fixture, 'npx'), `#!/bin/sh +printf '%s\n' "$@" > "$AQE_TEST_ARGS" +if [ "$AQE_TEST_SUMMARY" = 'true' ]; then + printf ' Test Files 1 passed (1)\n Tests 18 passed (18)\n' +fi +if [ "$AQE_TEST_HANG" = 'true' ]; then + exec sleep 15 +fi +if [ "$AQE_TEST_EXIT" != '0' ]; then + printf 'Unhandled Error: coverage report generation failed\n' >&2 +fi +exit "$AQE_TEST_EXIT" +`, { mode: 0o755 }); + + const result = spawnSync('/bin/bash', [wrapper, 'tests/fixture with spaces.test.ts', '--coverage'], { + cwd: fixture, + encoding: 'utf8', + timeout: 8000, + env: { + ...process.env, + PATH: `${fixture}:${process.env.PATH}`, + TMPDIR: fixture, + AQE_PROJECT_ROOT: fixture, + AQE_TEST_ARGS: argsPath, + AQE_TEST_EXIT: String(exitCode), + AQE_TEST_SUMMARY: String(options.summary ?? true), + AQE_TEST_HANG: String(options.hang ?? false), + CI_VITEST_TIMEOUT: '5', + }, + }); + return { ...result, args: readFileSync(argsPath, 'utf8').trim().split('\n') }; +} + +describe.skipIf(process.platform !== 'linux' && !shellToolsAvailable)(`CI Vitest runner exit status${skipReason}`, () => { + it('passes a successful runner and forwards arguments without splitting', () => { + const result = runRunner(0); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(0); + expect(result.stdout).toContain('18 passed'); + expect(result.args).toEqual(['vitest', 'run', 'tests/fixture with spaces.test.ts', '--coverage']); + }); + + it.each([1, 2, 124, 137, 143, 255])('preserves exit %i after a passing test summary', (code) => { + const result = runRunner(code); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(code); + expect(result.stdout + result.stderr).toContain('coverage report generation failed'); + expect(result.stdout + result.stderr).not.toContain('Treating as success'); + }); + + it('preserves failures before the test summary', () => { + expect(runRunner(1, { summary: false }).status).toBe(1); + }); + + it('fails an actual timeout even after all tests report passing', () => { + const result = runRunner(0, { hang: true }); + expect(result.error).toBeUndefined(); + expect(result.stdout).toContain('18 passed'); + expect(result.status).toBe(124); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 3fd5fea8e..93e359754 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -38,7 +38,7 @@ export default defineConfig({ ], coverage: { provider: 'v8', - reporter: ['text', 'json', 'html', 'junit'], + reporter: ['text', 'json', 'json-summary', 'html'], include: ['src/**/*.ts'], exclude: ['src/**/*.d.ts', 'src/**/index.ts'], }, From fb52539983191bd8956c271c55154abea57cf795 Mon Sep 17 00:00:00 2001 From: Rudy Celekli Date: Mon, 21 Sep 2026 17:45:26 -0400 Subject: [PATCH 4/4] test(ci): bound local shell capability detection (cherry picked from commit 8152bc578fecaa58142cf4793ae8175e117ee493) --- tests/unit/scripts/ci-vitest-run.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/scripts/ci-vitest-run.test.ts b/tests/unit/scripts/ci-vitest-run.test.ts index fa25cad4a..fe5161af8 100644 --- a/tests/unit/scripts/ci-vitest-run.test.ts +++ b/tests/unit/scripts/ci-vitest-run.test.ts @@ -7,7 +7,7 @@ import { afterEach, describe, expect, it } from 'vitest'; const wrapper = resolve(import.meta.dirname, '../../../scripts/ci-vitest-run.sh'); // The wrapper is used by Ubuntu CI. Stock macOS and Windows do not ship all // of its shell tools; qualify those local runs without letting Linux CI skip. -const timeoutVersion = spawnSync('timeout', ['--version'], { encoding: 'utf8' }); +const timeoutVersion = spawnSync('timeout', ['--version'], { encoding: 'utf8', timeout: 2000 }); const shellToolsAvailable = existsSync('/bin/bash') && timeoutVersion.status === 0 && timeoutVersion.stdout.includes('GNU coreutils'); if (process.platform === 'linux' && !shellToolsAvailable) {