diff --git a/docs/ci-measured-quality-gate.md b/docs/ci-measured-quality-gate.md new file mode 100644 index 000000000..a526b1076 --- /dev/null +++ b/docs/ci-measured-quality-gate.md @@ -0,0 +1,46 @@ +# Measured quality gates in CI + +`aqe ci run --phase quality-gate` evaluates the same seven timestamped evidence +records and shared thresholds as `aqe quality --gate` and MCP +`quality_assess({ runGate: true })`. It does not call a separate domain +`evaluate()` API or derive approval from a static score. + +All seven records must be present, valid, and no more than 24 hours old. Missing, +partial, malformed, future-dated, or stale evidence fails the phase. Failed +checks retain their measured values and thresholds in the reports. CI uses +exit 0 for a passing pipeline and 1 for a failing pipeline; the standalone +`quality` command retains its separate near-threshold exit-2 convention. + +## Enforcement and partial runs + +With enforcement enabled, at least one selected quality gate must run and every +selected gate must pass. Filtering out the gate with `--phase`, disabling it, +or stopping before it executes cannot report a passed gate. + +Use `--no-quality-gate` for an explicitly advisory run, including a partial +pipeline that does not select a gate. A gate that does run still evaluates the +evidence and reports its actual result. A failed advisory gate produces a +pipeline warning and does not stop later phases. Failures in other phases +still fail the pipeline. + +JSON reports expose `qualityGatePassed` as the actual evaluation result, +`qualityGateStatus` as `passed`, `failed`, or `not-run`, and +`qualityGateEnforced` separately. Text and Markdown also distinguish “Not run” +and advisory operation from a passed gate. + +## Artifacts and scope + +`quality-gate.json` contains the aggregate gate result and is reset to not-run +before phase execution. Individual executed gates write +`quality-gate-.json`, so a later passing gate cannot overwrite an +earlier failure. A failed evidence load also writes a failed artifact, replacing +any previous result for that phase number. The aggregate artifact and current +run's referenced phase artifacts are authoritative; unreferenced files from +older runs are historical. + +This uses the canonical evidence contract and fixed shared thresholds. The +legacy CI `quality_gate.thresholds` fields do not configure that evaluator. +It does not change CI YAML parsing, the other CI phase implementations, or +bind independent metric records to a common revision/run. Evidence production +and those contracts remain separate work; running test generation alone does +not supply all seven required measurements. diff --git a/src/cli/commands/ci.ts b/src/cli/commands/ci.ts index 61d2f8f5b..59ceed817 100644 --- a/src/cli/commands/ci.ts +++ b/src/cli/commands/ci.ts @@ -22,6 +22,10 @@ import { } from '../utils/ci-config.js'; import { writeOutput, toJSON } from '../utils/ci-output.js'; import { buildCoverageData } from '../utils/coverage-data.js'; +import { + evaluateQualityEvidence, + loadQualityEvidence, +} from '../../domains/quality-assessment/quality-evidence.js'; // ============================================================================ // Phase Execution @@ -31,7 +35,7 @@ async function executePhase( phase: CIPhase, context: CLIContext, outputDir: string, - outputFormat: string, + phaseIndex: number, ): Promise { const startTime = Date.now(); const artifacts: string[] = []; @@ -172,29 +176,25 @@ async function executePhase( } case 'quality-gate': { - const qualityAPI = await context.kernel!.getDomainAPIAsync!<{ - evaluate(request: Record): Promise<{ success: boolean; value?: unknown; error?: Error }>; - }>('quality-assessment'); - - if (!qualityAPI) { - return makeResult(phase, startTime, 'failed', 1, 'Quality assessment domain not available', artifacts); - } - - const result = await qualityAPI.evaluate({ runGate: true, includeAdvice: true }); - if (result.success && result.value) { - const assessment = result.value as { passed?: boolean; score?: string; grade?: string; checks?: unknown[]; recommendations?: string[]; meetsThreshold?: boolean }; - const passed = assessment.passed ?? assessment.meetsThreshold ?? true; - details = { passed, score: assessment.score || assessment.grade, checks: assessment.checks }; - status = passed ? 'passed' : 'failed'; - summary = `Quality gate: ${passed ? 'PASSED' : 'FAILED'} (score: ${assessment.score || assessment.grade || 'N/A'})`; - - const artifactPath = path.join(outputDir, 'quality-gate.json'); - fs.writeFileSync(artifactPath, toJSON(assessment), 'utf-8'); - artifacts.push(artifactPath); - } else { + // Use the same evidence contract as `aqe quality --gate` and registered + // MCP quality_assess. The domain API has no evaluate() method. + try { + const metrics = await loadQualityEvidence(context.kernel!.memory); + const assessment = evaluateQualityEvidence(metrics); + details = { ...assessment, metrics, evidenceStatus: 'measured' }; + status = assessment.passed ? 'passed' : 'failed'; + summary = `Quality gate: ${assessment.passed ? 'PASSED' : 'FAILED'} (${assessment.checks.filter(check => check.passed).length}/${assessment.checks.length} measured checks passed)`; + } catch (error) { status = 'failed'; - summary = result.error?.message || 'Quality gate evaluation failed'; + summary = `Quality gate: ${error instanceof Error ? error.message : String(error)}`; + details = { passed: false, evidenceStatus: 'unavailable', error: summary, checks: [] }; } + + // Each gate owns its evidence. A later passing gate must not overwrite + // an earlier failure when a pipeline contains multiple gate phases. + const artifactPath = path.join(outputDir, `quality-gate-${phaseIndex + 1}.json`); + fs.writeFileSync(artifactPath, toJSON(details), 'utf-8'); + artifacts.push(artifactPath); break; } @@ -285,7 +285,7 @@ function generateCombinedReport(result: CIRunResult): string { let md = `# AQE CI/CD Report\n\n`; md += `**Status:** ${result.overallStatus === 'passed' ? 'PASSED' : result.overallStatus === 'warning' ? 'WARNING' : 'FAILED'}\n`; md += `**Duration:** ${(result.duration / 1000).toFixed(1)}s\n`; - md += `**Quality Gate:** ${result.qualityGatePassed ? 'Passed' : 'Failed'}\n\n`; + md += `**Quality Gate:** ${describeQualityGate(result)}\n\n`; md += `## Phases\n\n`; md += `| Phase | Type | Status | Duration | Summary |\n`; md += `|-------|------|--------|----------|---------|\n`; @@ -311,6 +311,12 @@ function generateCombinedReport(result: CIRunResult): string { return md; } +function describeQualityGate(result: CIRunResult): string { + const status = result.qualityGateStatus === 'not-run' ? 'Not run' : + result.qualityGatePassed ? 'Passed' : 'Failed'; + return `${status} (${result.qualityGateEnforced ? 'enforced' : 'advisory'})`; +} + // ============================================================================ // Command // ============================================================================ @@ -397,10 +403,16 @@ export function createCICommand( // Create output directory const outputDir = path.resolve(config.output.directory); fs.mkdirSync(outputDir, { recursive: true }); + const gateReportPath = path.join(outputDir, 'quality-gate.json'); + // Invalidate an earlier approval before executing any phase. This also + // covers early stops, disabled gates, and --phase selections without a gate. + fs.writeFileSync(gateReportPath, toJSON({ + passed: false, status: 'not-run', evidenceStatus: 'not-run', + enforced: config.qualityGate.enforced, phases: [], + }), 'utf-8'); // Execute phases const phaseResults: CIPhaseResult[] = []; - let pipelineFailed = false; for (const phase of config.phases) { if (format === 'text') { @@ -408,7 +420,7 @@ export function createCICommand( process.stdout.write(chalk.cyan(spinner)); } - const result = await executePhase(phase, context, outputDir, config.output.format); + const result = await executePhase(phase, context, outputDir, phaseResults.length); phaseResults.push(result); if (format === 'text') { @@ -423,8 +435,7 @@ export function createCICommand( } } - if (result.status === 'failed') { - pipelineFailed = true; + if (result.status === 'failed' && (result.type !== 'quality-gate' || config.qualityGate.enforced)) { if (!phase.continueOnFailure) { if (format === 'text') { console.log(chalk.red(`\n Pipeline stopped: "${phase.name}" failed (continue_on_failure: false)\n`)); @@ -435,13 +446,18 @@ export function createCICommand( } // Determine overall status - const hasFailure = phaseResults.some(r => r.status === 'failed'); - const hasWarning = phaseResults.some(r => r.status === 'warning'); - const qualityGateResult = phaseResults.find(r => r.type === 'quality-gate'); - const qualityGatePassed = !config.qualityGate.enforced || !qualityGateResult || qualityGateResult.status === 'passed'; + const hasFailure = phaseResults.some(r => r.status === 'failed' && r.type !== 'quality-gate'); + const hasWarning = phaseResults.some(r => r.status === 'warning' || + (r.type === 'quality-gate' && r.status === 'failed' && !config.qualityGate.enforced)); + const gateResults = phaseResults.filter(r => r.type === 'quality-gate'); + const expectedGates = config.phases.filter(phase => phase.type === 'quality-gate').length; + const qualityGatePassed = gateResults.length > 0 && gateResults.length === expectedGates && + gateResults.every(r => r.status === 'passed'); + const qualityGateStatus = qualityGatePassed ? 'passed' : + gateResults.some(r => r.status === 'failed') ? 'failed' : 'not-run'; const overallStatus: 'passed' | 'failed' | 'warning' = - hasFailure || !qualityGatePassed ? 'failed' : + hasFailure || (config.qualityGate.enforced && !qualityGatePassed) ? 'failed' : hasWarning ? 'warning' : 'passed'; const completedAt = new Date(); @@ -454,10 +470,21 @@ export function createCICommand( duration, phases: phaseResults, qualityGatePassed, + qualityGateStatus, + qualityGateEnforced: config.qualityGate.enforced, overallStatus, exitCode: overallStatus === 'failed' ? 1 : 0, }; + fs.writeFileSync(gateReportPath, toJSON({ + ...(gateResults.length === 1 ? gateResults[0].details : {}), + passed: qualityGatePassed, + status: qualityGateStatus, + enforced: config.qualityGate.enforced, + ...(qualityGateStatus === 'not-run' ? { evidenceStatus: 'not-run' } : {}), + phases: gateResults, + }), 'utf-8'); + // Write combined report if (config.output.combinedReport) { const reportPath = path.join(outputDir, 'ci-report.md'); @@ -478,7 +505,7 @@ export function createCICommand( const statusColor = overallStatus === 'passed' ? chalk.green : overallStatus === 'failed' ? chalk.red : chalk.yellow; console.log(` ${statusColor(`Pipeline: ${overallStatus.toUpperCase()}`)} (${(duration / 1000).toFixed(1)}s)`); - console.log(` Quality Gate: ${qualityGatePassed ? chalk.green('PASSED') : chalk.red('FAILED')}`); + console.log(` Quality Gate: ${describeQualityGate(runResult)}`); console.log(chalk.gray(` Artifacts: ${outputDir}/`)); console.log(''); } diff --git a/src/cli/utils/ci-config.ts b/src/cli/utils/ci-config.ts index 3c8c54d1a..c87e6cd2c 100644 --- a/src/cli/utils/ci-config.ts +++ b/src/cli/utils/ci-config.ts @@ -87,7 +87,10 @@ export interface CIRunResult { completedAt: Date; duration: number; phases: CIPhaseResult[]; + /** True only when at least one gate ran and every gate passed. */ qualityGatePassed: boolean; + qualityGateStatus: 'passed' | 'failed' | 'not-run'; + qualityGateEnforced: boolean; overallStatus: 'passed' | 'failed' | 'warning'; exitCode: number; } diff --git a/tests/integration/quality-assess-measured-gate.test.ts b/tests/integration/quality-assess-measured-gate.test.ts index 0578add90..2e1fe39d2 100644 --- a/tests/integration/quality-assess-measured-gate.test.ts +++ b/tests/integration/quality-assess-measured-gate.test.ts @@ -9,6 +9,8 @@ import type { MemoryBackend } from '../../src/kernel/interfaces.js'; import type { QualityAssessResult } from '../../src/mcp/handlers/domain-handler-configs.js'; import type { ToolResult } from '../../src/mcp/types.js'; import type { QualityEvidenceValues } from '../../src/domains/quality-assessment/quality-evidence.js'; +import type { CLIContext } from '../../src/cli/handlers/interfaces.js'; +import type { CIRunResult } from '../../src/cli/utils/ci-config.js'; // Routing advice is unrelated to gate enforcement. Keep the real registered // handler, executor, analyzer, kernel memory, and evidence validator in this test. @@ -31,10 +33,28 @@ describe('registered quality_assess measured gate', () => { let memory: MemoryBackend; let evidence: typeof import('../../src/domains/quality-assessment/quality-evidence.js'); let cli: typeof import('../../src/cli/commands/quality.js'); + let ci: typeof import('../../src/cli/commands/ci.js'); let core: typeof import('../../src/mcp/handlers/core-handlers.js'); let resetCache: () => void; const originalCwd = process.cwd(); + async function runCIGate(args: string[] = []) { + const output = join(process.cwd(), 'ci-result.json'); + const cleanup = vi.fn(async (_code: number) => undefined); + const command = ci.createCICommand( + { kernel: core.getFleetState().kernel } as CLIContext, + cleanup as unknown as (code: number) => Promise, + async () => true, + ); + await command.parseAsync([ + 'run', '--phase', 'quality-gate', '--format', 'json', '--output', output, ...args, + ], { from: 'user' }); + return { + report: JSON.parse(await readFile(output, 'utf8')) as CIRunResult, + exitCode: cleanup.mock.calls[0]?.[0], + }; + } + async function callTool(name: string, args: Record): Promise> { const response = await server['handleRequest']({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name, arguments: args }, @@ -82,6 +102,7 @@ describe('registered quality_assess measured gate', () => { core = await import('../../src/mcp/handlers/core-handlers.js'); evidence = await import('../../src/domains/quality-assessment/quality-evidence.js'); cli = await import('../../src/cli/commands/quality.js'); + ci = await import('../../src/cli/commands/ci.js'); ({ resetSessionCache: resetCache } = await import('../../src/optimization/session-cache.js')); server = createMCPProtocolServer(); const fleet = await callTool('fleet_init', { memoryBackend: 'memory', maxAgents: 2 }); @@ -119,6 +140,16 @@ describe('registered quality_assess measured gate', () => { const report = result.data?.savedFiles?.find(file => file.endsWith('_report.md')); expect(report).toBeDefined(); expect(await readFile(report!, 'utf8')).toContain('N/A — measured gate uses individual checks.'); + + const pipeline = await runCIGate(); + expect(pipeline.exitCode).toBe(0); + expect(pipeline.report).toMatchObject({ + overallStatus: 'passed', qualityGatePassed: true, qualityGateStatus: 'passed', + phases: [{ status: 'passed', details: { passed: true, checks: expected.checks } }], + }); + const artifact = JSON.parse(await readFile(pipeline.report.phases[0].artifacts[0], 'utf8')); + expect(artifact).toMatchObject({ passed: true, checks: expected.checks }); + expect(artifact).not.toHaveProperty('score'); }); it.each(failures)('blocks measured %s failure even when static code analysis passes', async (metric, value) => { @@ -131,6 +162,12 @@ describe('registered quality_assess measured gate', () => { expect(result.data).toMatchObject({ passed: false, checks: expected.checks, recommendations: expected.recommendations, riskDecision: { decision: 'block' } }); expect(cli.getMeasuredQualityExitCode(expected)).toBe(1); + const pipeline = await runCIGate(); + expect(pipeline.exitCode).toBe(1); + expect(pipeline.report).toMatchObject({ + overallStatus: 'failed', qualityGatePassed: false, qualityGateStatus: 'failed', + phases: [{ status: 'failed', details: { passed: false, checks: expected.checks } }], + }); }); it.each(['missing', 'partial', 'stale', 'malformed', 'future'] as const)( @@ -151,9 +188,44 @@ describe('registered quality_assess measured gate', () => { expect(result.success).toBe(false); expect(result.error).toBe((expected as Error).message); expect(result.data).toBeUndefined(); + const pipeline = await runCIGate(); + expect(pipeline.exitCode).toBe(1); + expect(pipeline.report.qualityGatePassed).toBe(false); + expect(pipeline.report.phases[0].summary).toContain((expected as Error).message); + const artifact = JSON.parse(await readFile(pipeline.report.phases[0].artifacts[0], 'utf8')); + expect(artifact).toMatchObject({ passed: false, evidenceStatus: 'unavailable' }); }, ); + it('does not report an omitted gate as passed, even when enforcement is explicitly disabled', async () => { + await seed(); + await runCIGate(); + const required = await runCIGate(['--phase', 'no-such-phase']); + expect(required.exitCode).toBe(1); + expect(required.report).toMatchObject({ + phases: [], qualityGatePassed: false, qualityGateStatus: 'not-run', overallStatus: 'failed', + }); + expect(JSON.parse(await readFile(join(process.cwd(), '.aqe-ci-output', 'quality-gate.json'), 'utf8'))) + .toMatchObject({ passed: false, status: 'not-run', evidenceStatus: 'not-run' }); + const advisory = await runCIGate(['--phase', 'no-such-phase', '--no-quality-gate']); + expect(advisory.exitCode).toBe(0); + expect(advisory.report).toMatchObject({ + phases: [], qualityGatePassed: false, qualityGateStatus: 'not-run', qualityGateEnforced: false, + }); + expect(await readFile(join(process.cwd(), '.aqe-ci-output', 'ci-report.md'), 'utf8')) + .toContain('**Quality Gate:** Not run (advisory)'); + }); + + it('retains a failed advisory gate and continues without blocking the pipeline', async () => { + await seed({ ...passing, criticalBugs: 1 }); + const pipeline = await runCIGate(['--no-quality-gate']); + expect(pipeline.exitCode).toBe(0); + expect(pipeline.report).toMatchObject({ + overallStatus: 'warning', qualityGatePassed: false, qualityGateEnforced: false, + qualityGateStatus: 'failed', phases: [{ status: 'failed' }], + }); + }); + it('rechecks changed evidence with identical gate arguments and the session cache enabled', async () => { await seed(); const first = await callTool('quality_assess', { runGate: true }); diff --git a/tests/unit/cli/commands/ci.test.ts b/tests/unit/cli/commands/ci.test.ts new file mode 100644 index 000000000..2a9d0e832 --- /dev/null +++ b/tests/unit/cli/commands/ci.test.ts @@ -0,0 +1,138 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import type { CLIContext } from '../../../../src/cli/handlers/interfaces.js'; +import type { CIConfig, CIPhase, CIRunResult } from '../../../../src/cli/utils/ci-config.js'; +import { getDefaultCIConfig } from '../../../../src/cli/utils/ci-config.js'; +import { createCICommand } from '../../../../src/cli/commands/ci.js'; + +// Test execution/aggregation independently of YAML parsing. The integration +// suite also runs the real command defaults and registered MCP gate together. +vi.mock('../../../../src/cli/utils/ci-config.js', async (importOriginal) => ({ + ...await importOriginal(), + findCIConfigFile: () => null, + getDefaultCIConfig: vi.fn(), +})); + +const passing = { + coverage: 90, testsPassing: 100, criticalBugs: 0, codeSmells: 10, + securityVulnerabilities: 0, technicalDebt: 2, duplications: 3, +}; + +describe('CI gate execution and reporting', () => { + let directory: string; + let config: CIConfig; + let reads: number; + let failFirstGate: boolean; + + function phase(name: string, overrides: Partial = {}): CIPhase { + return { + name, type: 'quality-gate', enabled: true, config: {}, + continueOnFailure: false, timeout: 60, ...overrides, + }; + } + + beforeEach(() => { + directory = mkdtempSync(path.join(tmpdir(), 'aqe-ci-command-')); + reads = 0; + failFirstGate = false; + config = { + version: '1', name: 'fixture', phases: [phase('Gate')], + output: { directory, format: 'json', combinedReport: true }, + qualityGate: { enforced: true, thresholds: {} }, + }; + vi.mocked(getDefaultCIConfig).mockImplementation(() => config); + }); + + afterEach(() => { + rmSync(directory, { recursive: true, force: true }); + vi.restoreAllMocks(); + }); + + async function run(args: string[] = []) { + const context = { + kernel: { + memory: { + get: async (key: string) => { + const metric = key.split(':')[1] as keyof typeof passing; + const failing = failFirstGate && reads++ < 7 && metric === 'coverage'; + return { + schemaVersion: 1, metric, value: failing ? 0 : passing[metric], + source: 'fixture', measuredAt: new Date().toISOString(), + }; + }, + }, + getDomainAPIAsync: async () => undefined, + }, + } as unknown as CLIContext; + const cleanup = vi.fn(async (_code: number) => undefined); + const command = createCICommand(context, cleanup as unknown as (code: number) => Promise, async () => true); + const output = path.join(directory, 'result.json'); + await command.parseAsync(['run', '--format', 'json', '--output', output, ...args], { from: 'user' }); + return { + report: JSON.parse(readFileSync(output, 'utf8')) as CIRunResult, + artifact: JSON.parse(readFileSync(path.join(directory, 'quality-gate.json'), 'utf8')), + exitCode: cleanup.mock.calls[0][0], + }; + } + + it('does not count disabled gates as executed or passed', async () => { + config.phases[0].enabled = false; + const result = await run(); + expect(result.exitCode).toBe(1); + expect(result.report).toMatchObject({ qualityGatePassed: false, qualityGateStatus: 'not-run' }); + expect(result.artifact).toMatchObject({ passed: false, evidenceStatus: 'not-run' }); + }); + + it('keeps each gate artifact and fails the aggregate when a later gate passes', async () => { + failFirstGate = true; + config.phases = [phase('First', { continueOnFailure: true }), phase('Second')]; + const result = await run(); + expect(result.exitCode).toBe(1); + expect(result.report.phases.map(item => item.status)).toEqual(['failed', 'passed']); + expect(result.report.qualityGatePassed).toBe(false); + expect(result.artifact).toMatchObject({ passed: false, status: 'failed' }); + const paths = result.report.phases.map(item => item.artifacts[0]); + expect(new Set(paths).size).toBe(2); + expect(JSON.parse(readFileSync(paths[0], 'utf8')).passed).toBe(false); + expect(JSON.parse(readFileSync(paths[1], 'utf8')).passed).toBe(true); + }); + + it('continues after an advisory gate failure while preserving its failed result', async () => { + failFirstGate = true; + config.phases.push(phase('After gate', { type: 'custom' })); + const result = await run(['--no-quality-gate']); + expect(result.exitCode).toBe(0); + expect(result.report.phases.map(item => item.phase)).toEqual(['Gate', 'After gate']); + expect(result.report).toMatchObject({ + qualityGatePassed: false, qualityGateStatus: 'failed', qualityGateEnforced: false, overallStatus: 'warning', + }); + expect(result.artifact.passed).toBe(false); + }); + + it('does not suppress a non-gate failure when gate enforcement is disabled', async () => { + config.phases.push(phase('Coverage', { type: 'coverage' })); + const result = await run(['--no-quality-gate']); + expect(result.exitCode).toBe(1); + expect(result.report).toMatchObject({ overallStatus: 'failed', qualityGatePassed: true }); + }); + + it('invalidates a prior approval when another phase stops execution before the gate', async () => { + expect((await run()).artifact.passed).toBe(true); + config.phases.unshift(phase('Coverage', { type: 'coverage' })); + const result = await run(); + expect(result.exitCode).toBe(1); + expect(result.report.phases).toHaveLength(1); + expect(result.report.qualityGateStatus).toBe('not-run'); + expect(result.artifact).toMatchObject({ passed: false, evidenceStatus: 'not-run' }); + }); + + it('requires every selected gate to execute, even if an earlier one passed', async () => { + config.phases.push(phase('Coverage', { type: 'coverage' }), phase('Second gate')); + const result = await run(); + expect(result.exitCode).toBe(1); + expect(result.report).toMatchObject({ qualityGatePassed: false, qualityGateStatus: 'not-run' }); + expect(result.artifact.passed).toBe(false); + }); +});