From 3f60781983fdb661eb3699f86852e4d70a5acff5 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 09:26:46 -0700 Subject: [PATCH 01/15] ci: report published package size and new runtime dependencies on each PR, in the build job Measures every published package's npm download and unpacked size in build-storybook, after its build, and compares it with the merge base's measurement that Main now keeps for 14 days. Adds a Published Package Size section to the PR report and the job summary, and fails the job when a package's download grows more than 5% (and at least 1 KB) unless the PR description has a 'Size budget:' line. --- .github/scripts/generate-pr-comment.js | 22 +- .github/scripts/generate-pr-comment.test.mjs | 79 ++++ .github/scripts/lib/package-size.js | 315 +++++++++++++ .github/scripts/lib/package-size.test.mjs | 445 +++++++++++++++++++ .github/scripts/lib/pr-preview.mjs | 32 +- .github/scripts/lib/pr-preview.test.mjs | 27 ++ .github/scripts/package-size.mjs | 104 +++++ .github/workflows/ci.yml | 82 ++++ .github/workflows/deploy.yml | 12 + 9 files changed, 1108 insertions(+), 10 deletions(-) create mode 100644 .github/scripts/lib/package-size.js create mode 100644 .github/scripts/lib/package-size.test.mjs create mode 100644 .github/scripts/package-size.mjs diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.js index addace1fea9cb..13fb8e6a0228d 100644 --- a/.github/scripts/generate-pr-comment.js +++ b/.github/scripts/generate-pr-comment.js @@ -3,13 +3,14 @@ /** * @description Generates and posts PR comment with analysis results - * @input --analysis --base-stats --a11y --visual --storybook-url --sandbox-url --preview-state --source-conclusion --run-url + * @input --analysis --base-stats --package-size --a11y --visual --storybook-url --sandbox-url --preview-state --source-conclusion --run-url * @output Formatted markdown comment body to stdout */ const fs = require('node:fs'); const {buildA11ySection} = require('./lib/a11y-format'); const {buildVisualSection} = require('./lib/visual-format'); +const {renderPackageSizeSection} = require('./lib/package-size'); // Report-file strings render as literal inline text, report numbers as // numbers — the comment's structure comes only from this file's literals. const {inline, num} = require('./lib/report-text'); @@ -22,6 +23,7 @@ const getArg = name => { const analysisFile = getArg('analysis') || 'analysis.json'; const baseStatsFile = getArg('base-stats'); +const packageSizeFile = getArg('package-size'); const a11yFile = getArg('a11y') || 'a11y-report.json'; const visualFile = getArg('visual'); const runUrl = getArg('run-url') || ''; @@ -301,6 +303,22 @@ if (analysis.bundleDelta) { bundleSection += `**Bundle size ${direction}:** ${delta > 0 ? '+' : ''}${delta} bytes\n\n`; } +// Published package sizes from build-storybook's package-size check: the +// download and unpacked size of every published package, base against head, +// and any runtime dependency added or removed. The caller has already matched +// its head. +let packageSizeSection = ''; +if (packageSizeFile) { + try { + packageSizeSection = renderPackageSizeSection( + JSON.parse(fs.readFileSync(packageSizeFile, 'utf8')), + inline, + ); + } catch (e) { + console.error('Warning: Could not read package size report:', e.message); + } +} + // Build storybook link section let storybookSection = ''; if (storybookUrl) { @@ -356,7 +374,7 @@ const body = `## PR Analysis Report ${diffModeCaveat}${previewAvailabilitySection}${storybookSection}${sandboxSection}${componentSection || (sizeSection ? '' : '_No new or modified components detected._\n\n')} -${sizeSection}${budgetSection}${bundleSection} +${sizeSection}${budgetSection}${bundleSection}${packageSizeSection} ${a11ySection} ${visualSection}--- diff --git a/.github/scripts/generate-pr-comment.test.mjs b/.github/scripts/generate-pr-comment.test.mjs index c1b02c57ce48e..a95fdf27d3c03 100644 --- a/.github/scripts/generate-pr-comment.test.mjs +++ b/.github/scripts/generate-pr-comment.test.mjs @@ -306,3 +306,82 @@ describe('generate-pr-comment touch-only budget', () => { expect(out).not.toContain('Touch-only size budget'); }); }); + +describe('published package size', () => { + function withPackageSize(report) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-package-size-')); + try { + const file = path.join(dir, 'package-size.json'); + fs.writeFileSync(file, JSON.stringify(report)); + return runComment({newComponents: [], modifiedComponents: []}, [ + '--package-size', + file, + ]); + } finally { + fs.rmSync(dir, {recursive: true, force: true}); + } + } + + const sizes = (packedBytes, files = 10) => ({ + packedBytes, + unpackedBytes: packedBytes * 4, + files, + }); + + it('renders the section after the bundle summary, with new dependencies', () => { + const out = withPackageSize({ + headSha: 'a'.repeat(40), + budget: {packedPercent: 5, packedMinBytes: 1024}, + packages: [ + { + name: '@astryxdesign/core', + base: sizes(100000, 10), + head: sizes(110000, 12), + overBudget: true, + }, + ], + addedDependencies: [ + { + package: '@astryxdesign/core', + field: 'dependencies', + name: 'left-pad', + range: '^1.3.0', + }, + ], + removedDependencies: [], + justification: null, + verdict: 'over-budget', + }); + expect(out.indexOf('### Bundle Size Summary')).toBeLessThan( + out.indexOf('### Published Package Size'), + ); + expect(out).toContain( + '| @astryxdesign/core | 97.7KB | 107.4KB | +9.8KB (+10.0%) ⚠️ |', + ); + expect(out).toContain('| 10 → 12 |'); + expect(out).toContain('left-pad'); + expect(out).toContain('**Over budget:**'); + }); + + it('renders report strings as literal text', () => { + const out = withPackageSize({ + packages: [ + { + name: '', + base: sizes(1000), + head: sizes(2000), + overBudget: false, + }, + ], + addedDependencies: [], + removedDependencies: [], + verdict: 'pass', + }); + expect(out).not.toContain(' { + const out = runComment({newComponents: [], modifiedComponents: []}); + expect(out).not.toContain('Published Package Size'); + }); +}); diff --git a/.github/scripts/lib/package-size.js b/.github/scripts/lib/package-size.js new file mode 100644 index 0000000000000..1a12ad56cde79 --- /dev/null +++ b/.github/scripts/lib/package-size.js @@ -0,0 +1,315 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file package-size.js + * @input A built workspace (every published package's files on disk) and, to + * compare, two measurements and a pull request description + * @output Each published package's tarball size (gzip, as npm downloads it), + * unpacked size, file count and runtime dependencies; a base-against-head + * comparison with a budget verdict; and that comparison as Markdown + * @position Shared by package-size.mjs (CI's build-storybook job on a pull + * request, and Main's typecheck job, which keeps each main commit's + * measurement for pull requests to compare against) and by + * generate-pr-comment.js, which renders the comparison in the PR report. + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const {execFile} = require('node:child_process'); + +/** + * A package fails the check when its tarball grows by more than this share of + * its base size and by at least MIN_BYTES; the floor keeps a small package + * from failing on a few hundred bytes. + */ +const PACKED_BUDGET_PERCENT = 5; +const PACKED_BUDGET_MIN_BYTES = 1024; + +/** Dependency fields a consumer installs or must provide. */ +const RUNTIME_DEPENDENCY_FIELDS = ['dependencies', 'peerDependencies']; + +/** The workspace directories that can hold a published package. */ +const PACKAGE_GROUPS = ['packages', 'packages/themes']; + +/** Packages npm receives: public, in the scope, not canary-only. */ +function publishedPackages(root) { + const found = []; + for (const group of PACKAGE_GROUPS) { + const groupDir = path.join(root, group); + if (!fs.existsSync(groupDir)) continue; + for (const entry of fs.readdirSync(groupDir, {withFileTypes: true})) { + if (!entry.isDirectory()) continue; + const directory = path.join(groupDir, entry.name); + const manifestPath = path.join(directory, 'package.json'); + if (!fs.existsSync(manifestPath)) continue; + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + if ( + manifest.name?.startsWith('@astryxdesign/') && + manifest.private !== true && + manifest.astryx?.canaryOnly !== true + ) + found.push({directory, manifest}); + } + } + return found.sort((a, b) => a.manifest.name.localeCompare(b.manifest.name)); +} + +function runtimeDependencies(manifest) { + const dependencies = {}; + for (const field of RUNTIME_DEPENDENCY_FIELDS) { + for (const [name, range] of Object.entries(manifest[field] || {})) + dependencies[`${field}:${name}`] = String(range); + } + return dependencies; +} + +/** + * `npm pack --dry-run` lists what the package's `files` field ships and the + * tarball's size without writing it. Lifecycle scripts are skipped: the + * packages are already built, and some packages' prepack builds them again. + */ +function npmPackDryRun(directory) { + return new Promise((resolve, reject) => { + execFile( + 'npm', + ['pack', '--dry-run', '--json', '--ignore-scripts', '--silent'], + {cwd: directory, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024}, + (error, stdout) => { + if (error) return reject(error); + try { + resolve(JSON.parse(stdout)[0]); + } catch (parseError) { + reject(parseError); + } + }, + ); + }); +} + +/** + * Every published package, packed side by side: the largest package's pack + * is most of the time, so the others add little. + */ +async function measurePackages(root, pack = npmPackDryRun) { + const published = publishedPackages(root); + const results = await Promise.all( + published.map(({directory}) => pack(directory)), + ); + const packages = {}; + published.forEach(({manifest}, index) => { + const result = results[index]; + packages[manifest.name] = { + packedBytes: result.size, + unpackedBytes: result.unpackedSize, + files: result.entryCount, + dependencies: runtimeDependencies(manifest), + }; + }); + return {schema: 1, packages}; +} + +function isMeasured(bytes) { + return typeof bytes === 'number' && Number.isFinite(bytes); +} + +/** + * The description's `Size budget:` line, outside HTML comments (a template's + * placeholder is not a justification), or null when it has no text. + */ +function sizeBudgetJustification(body) { + const visible = String(body || '').replace(//g, ''); + const match = visible.match( + /^[ \t>*_-]*Size budget:[ \t]*(?:\*\*|__)?[ \t]*(\S.*)$/im, + ); + return match ? match[1].trim() : null; +} + +function overBudget(baseBytes, headBytes) { + if (!isMeasured(baseBytes) || !isMeasured(headBytes) || baseBytes <= 0) + return false; + const delta = headBytes - baseBytes; + return ( + delta >= PACKED_BUDGET_MIN_BYTES && + (delta / baseBytes) * 100 > PACKED_BUDGET_PERCENT + ); +} + +function splitDependencyKey(key) { + const separator = key.indexOf(':'); + return {field: key.slice(0, separator), name: key.slice(separator + 1)}; +} + +/** + * Head against base, package by package. Without a base measurement every + * row keeps its head figures, nothing is over budget, and the verdict says + * the base was not measured. + */ +function comparePackageSizes({head, base, body, headSha = '', baseSha = ''}) { + const headPackages = head?.packages || {}; + const basePackages = base?.packages || null; + const names = [ + ...new Set([ + ...Object.keys(headPackages), + ...Object.keys(basePackages || {}), + ]), + ].sort(); + const packages = []; + const addedDependencies = []; + const removedDependencies = []; + for (const name of names) { + const now = headPackages[name] || null; + const before = basePackages ? basePackages[name] || null : null; + packages.push({ + name, + base: before && { + packedBytes: before.packedBytes, + unpackedBytes: before.unpackedBytes, + files: before.files, + }, + head: now && { + packedBytes: now.packedBytes, + unpackedBytes: now.unpackedBytes, + files: now.files, + }, + overBudget: overBudget(before?.packedBytes, now?.packedBytes), + }); + if (!basePackages || !now) continue; + const beforeDependencies = before?.dependencies || {}; + const nowDependencies = now.dependencies || {}; + for (const [key, range] of Object.entries(nowDependencies)) { + if (key in beforeDependencies) continue; + addedDependencies.push({ + package: name, + ...splitDependencyKey(key), + range, + }); + } + if (!before) continue; + for (const [key, range] of Object.entries(beforeDependencies)) { + if (key in nowDependencies) continue; + removedDependencies.push({ + package: name, + ...splitDependencyKey(key), + range, + }); + } + } + const justification = sizeBudgetJustification(body); + const over = packages.some(entry => entry.overBudget); + const verdict = !basePackages + ? 'no-base' + : !over + ? 'pass' + : justification + ? 'justified' + : 'over-budget'; + return { + schema: 1, + headSha, + baseSha, + budget: { + packedPercent: PACKED_BUDGET_PERCENT, + packedMinBytes: PACKED_BUDGET_MIN_BYTES, + }, + packages, + addedDependencies, + removedDependencies, + justification, + verdict, + }; +} + +function sizeText(bytes) { + if (!isMeasured(bytes)) return 'N/A'; + if (Math.abs(bytes) < 1024) return `${bytes}B`; + return `${(bytes / 1024).toFixed(1)}KB`; +} + +function deltaText(after, before) { + if (!isMeasured(after) || !isMeasured(before)) return 'N/A'; + const delta = after - before; + const percent = + before > 0 && delta !== 0 + ? ` (${delta > 0 ? '+' : ''}${((delta / before) * 100).toFixed(1)}%)` + : ''; + return `${delta > 0 ? '+' : ''}${sizeText(delta)}${percent}`; +} + +function changed(entry) { + if (!entry.base || !entry.head) return true; + return ( + entry.base.packedBytes !== entry.head.packedBytes || + entry.base.unpackedBytes !== entry.head.unpackedBytes || + entry.base.files !== entry.head.files + ); +} + +/** + * The comparison as Markdown. `literal` renders a report string as inline + * text; the PR comment passes its own escaper, the job summary the identity. + */ +function renderPackageSizeSection(report, literal = text => String(text)) { + if (!report || !Array.isArray(report.packages)) return ''; + const {packedPercent, packedMinBytes} = report.budget || {}; + let out = '### Published Package Size\n\n'; + if (report.verdict === 'no-base') { + out += + "_Main has no kept measurement of this pull request's merge base (it is older than 14 days, or predates the measurement), so only this head's sizes are shown and no budget applies._\n\n"; + } + const rows = + report.verdict === 'no-base' + ? report.packages + : report.packages.filter(changed); + if (rows.length === 0) { + out += `No published package changed size (${report.packages.length} measured).\n\n`; + } else { + out += + '| Package | Download before | Download after | Delta | Unpacked before | Unpacked after | Delta | Files |\n'; + out += + '|---------|-----------------|----------------|-------|-----------------|----------------|-------|-------|\n'; + for (const entry of rows) { + const files = + entry.base && entry.head && entry.base.files !== entry.head.files + ? `${entry.base.files} → ${entry.head.files}` + : String(entry.head?.files ?? entry.base?.files ?? 'N/A'); + out += `| ${literal(entry.name)} | ${sizeText(entry.base?.packedBytes)} | ${sizeText(entry.head?.packedBytes)} | ${deltaText(entry.head?.packedBytes, entry.base?.packedBytes)}${entry.overBudget ? ' ⚠️' : ''} | ${sizeText(entry.base?.unpackedBytes)} | ${sizeText(entry.head?.unpackedBytes)} | ${deltaText(entry.head?.unpackedBytes, entry.base?.unpackedBytes)} | ${files} |\n`; + } + const unchanged = report.packages.length - rows.length; + out += + unchanged === 0 + ? '\n' + : `\n${unchanged} other published ${unchanged === 1 ? 'package is' : 'packages are'} unchanged.\n\n`; + } + if (report.addedDependencies?.length > 0) { + out += '**New runtime dependencies:**\n\n'; + for (const dependency of report.addedDependencies) + out += `- ${literal(dependency.package)}: ${literal(dependency.name)} ${literal(dependency.range)} (${literal(dependency.field)})\n`; + out += '\n'; + } + if (report.removedDependencies?.length > 0) { + out += '**Removed runtime dependencies:**\n\n'; + for (const dependency of report.removedDependencies) + out += `- ${literal(dependency.package)}: ${literal(dependency.name)} (${literal(dependency.field)})\n`; + out += '\n'; + } + if (report.verdict === 'over-budget') { + out += `**Over budget:** a package's download grew more than ${packedPercent}% (and at least ${sizeText(packedMinBytes)}). Shrink it, or add a \`Size budget:\` line to the pull request description saying why the growth is worth it, then re-run the job.\n\n`; + } else if (report.verdict === 'justified') { + out += `**Over budget, justified:** ${literal(report.justification)}\n\n`; + } + out += + 'Download is the gzip tarball npm fetches; unpacked is what lands in `node_modules`.\n\n'; + return out; +} + +module.exports = { + PACKED_BUDGET_PERCENT, + PACKED_BUDGET_MIN_BYTES, + publishedPackages, + runtimeDependencies, + measurePackages, + sizeBudgetJustification, + overBudget, + comparePackageSizes, + renderPackageSizeSection, +}; diff --git a/.github/scripts/lib/package-size.test.mjs b/.github/scripts/lib/package-size.test.mjs new file mode 100644 index 0000000000000..d1a7c5e563ee6 --- /dev/null +++ b/.github/scripts/lib/package-size.test.mjs @@ -0,0 +1,445 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file package-size.test.mjs + * Pins the published-package-size check: which packages count as published, + * what `npm pack` measures, the budget (a tarball growing more than 5% and at + * least 1 KB) and its `Size budget:` justification, the runtime-dependency + * diff, the Markdown, and the CLI's exit codes. + */ + +import {execFileSync, spawnSync} from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {afterEach, describe, expect, it} from 'vitest'; +import sizes from './package-size.js'; + +const { + PACKED_BUDGET_MIN_BYTES, + publishedPackages, + measurePackages, + sizeBudgetJustification, + overBudget, + comparePackageSizes, + renderPackageSizeSection, +} = sizes; + +const CLI = path.join( + path.dirname(fileURLToPath(import.meta.url)), + '..', + 'package-size.mjs', +); + +const temporary = []; +afterEach(() => { + for (const dir of temporary.splice(0)) + fs.rmSync(dir, {recursive: true, force: true}); +}); + +function tempDir() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-package-size-')); + temporary.push(dir); + return dir; +} + +function writePackage(root, relative, manifest, files = {}) { + const dir = path.join(root, relative); + fs.mkdirSync(dir, {recursive: true}); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify(manifest)); + for (const [name, contents] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(dir, name)), {recursive: true}); + fs.writeFileSync(path.join(dir, name), contents); + } + return dir; +} + +function measurement(packages) { + return {schema: 1, packages}; +} + +function pkg(packedBytes, dependencies = {}, extra = {}) { + return { + packedBytes, + unpackedBytes: packedBytes * 4, + files: 10, + dependencies, + ...extra, + }; +} + +describe('publishedPackages', () => { + it('keeps public scoped packages and drops private, canary-only and unscoped ones', () => { + const root = tempDir(); + writePackage(root, 'packages/core', {name: '@astryxdesign/core'}); + writePackage(root, 'packages/themes/neutral', { + name: '@astryxdesign/theme-neutral', + }); + writePackage(root, 'packages/lab', { + name: '@astryxdesign/lab', + private: true, + }); + writePackage(root, 'packages/canary', { + name: '@astryxdesign/canary', + astryx: {canaryOnly: true}, + }); + writePackage(root, 'packages/other', {name: 'unscoped'}); + expect(publishedPackages(root).map(entry => entry.manifest.name)).toEqual([ + '@astryxdesign/core', + '@astryxdesign/theme-neutral', + ]); + }); +}); + +describe('measurePackages', () => { + it('records npm pack sizes and runtime dependencies, not dev dependencies', async () => { + const root = tempDir(); + writePackage(root, 'packages/core', { + name: '@astryxdesign/core', + dependencies: {'@stylexjs/stylex': '^0.19.0'}, + peerDependencies: {react: '>=19'}, + devDependencies: {vitest: '4'}, + }); + const result = await measurePackages(root, async () => ({ + size: 100, + unpackedSize: 400, + entryCount: 3, + })); + expect(result.packages['@astryxdesign/core']).toEqual({ + packedBytes: 100, + unpackedBytes: 400, + files: 3, + dependencies: { + 'dependencies:@stylexjs/stylex': '^0.19.0', + 'peerDependencies:react': '>=19', + }, + }); + }); + + it('measures a real package with npm pack, honouring files and skipping lifecycle scripts', async () => { + const root = tempDir(); + writePackage( + root, + 'packages/core', + { + name: '@astryxdesign/core', + version: '0.0.0', + files: ['dist'], + scripts: {prepack: 'exit 1'}, + }, + { + 'dist/index.js': 'export const a = 1;\n'.repeat(200), + 'src/index.test.ts': 'not shipped', + }, + ); + const {packages} = await measurePackages(root); + const core = packages['@astryxdesign/core']; + // package.json and dist/index.js; the test file is outside `files`. + expect(core.files).toBe(2); + expect(core.unpackedBytes).toBeGreaterThan(4000); + expect(core.packedBytes).toBeGreaterThan(0); + expect(core.packedBytes).toBeLessThan(core.unpackedBytes); + }); +}); + +describe('sizeBudgetJustification', () => { + it.each([ + [ + 'Size budget: the new locale data is the feature', + 'the new locale data is the feature', + ], + ['Intro\n\n- Size budget: worth it\n', 'worth it'], + ['> **Size budget:** bold label', 'bold label'], + ['size budget: lower case label', 'lower case label'], + ])('reads %j', (body, expected) => { + expect(sizeBudgetJustification(body)).toBe(expected); + }); + + it('ignores a template placeholder inside an HTML comment and an empty line', () => { + expect( + sizeBudgetJustification( + '', + ), + ).toBeNull(); + expect( + sizeBudgetJustification( + '', + ), + ).toBeNull(); + expect(sizeBudgetJustification('Size budget: \nnext line')).toBeNull(); + expect(sizeBudgetJustification(null)).toBeNull(); + }); +}); + +describe('overBudget', () => { + it('needs more than 5% and at least the byte floor', () => { + const base = 100 * 1024; + expect(overBudget(base, base + 5 * 1024)).toBe(false); // exactly 5% + expect(overBudget(base, base + 5 * 1024 + 1)).toBe(true); + // 10% of a small package, but under the floor. + expect(overBudget(5000, 5000 + PACKED_BUDGET_MIN_BYTES - 1)).toBe(false); + expect(overBudget(5000, 5000 + PACKED_BUDGET_MIN_BYTES)).toBe(true); + }); + + it('never fires without both measurements', () => { + expect(overBudget(undefined, 999999)).toBe(false); + expect(overBudget(1000, null)).toBe(false); + expect(overBudget(0, 999999)).toBe(false); + }); +}); + +describe('comparePackageSizes', () => { + const base = measurement({ + '@astryxdesign/core': pkg(100_000, { + 'dependencies:@stylexjs/stylex': '^0.19.0', + }), + '@astryxdesign/theme-neutral': pkg(20_000), + }); + + it('passes an unchanged head', () => { + const report = comparePackageSizes({head: base, base, body: ''}); + expect(report.verdict).toBe('pass'); + expect(report.packages.every(entry => !entry.overBudget)).toBe(true); + expect(report.addedDependencies).toEqual([]); + }); + + it('fails a package past the budget, and a Size budget line justifies it', () => { + const head = measurement({ + ...base.packages, + '@astryxdesign/core': pkg( + 110_000, + base.packages['@astryxdesign/core'].dependencies, + ), + }); + const failing = comparePackageSizes({head, base, body: 'No reason given'}); + expect(failing.verdict).toBe('over-budget'); + expect( + failing.packages.find(entry => entry.name === '@astryxdesign/core') + .overBudget, + ).toBe(true); + const justified = comparePackageSizes({ + head, + base, + body: 'Size budget: ships the new locales', + }); + expect(justified.verdict).toBe('justified'); + expect(justified.justification).toBe('ships the new locales'); + }); + + it('lists added and removed runtime dependencies', () => { + const head = measurement({ + '@astryxdesign/core': pkg(100_000, {'dependencies:lodash': '^4.17.21'}), + '@astryxdesign/theme-neutral': pkg(20_000), + }); + const report = comparePackageSizes({head, base}); + expect(report.addedDependencies).toEqual([ + { + package: '@astryxdesign/core', + field: 'dependencies', + name: 'lodash', + range: '^4.17.21', + }, + ]); + expect(report.removedDependencies).toEqual([ + { + package: '@astryxdesign/core', + field: 'dependencies', + name: '@stylexjs/stylex', + range: '^0.19.0', + }, + ]); + }); + + it('reports a newly published package without a budget, its dependencies as new', () => { + const head = measurement({ + ...base.packages, + '@astryxdesign/new': pkg(50_000, {'peerDependencies:react': '>=19'}), + }); + const report = comparePackageSizes({head, base}); + const added = report.packages.find( + entry => entry.name === '@astryxdesign/new', + ); + expect(added.base).toBeNull(); + expect(added.overBudget).toBe(false); + expect(report.verdict).toBe('pass'); + expect(report.addedDependencies).toEqual([ + { + package: '@astryxdesign/new', + field: 'peerDependencies', + name: 'react', + range: '>=19', + }, + ]); + }); + + it('applies no budget and diffs no dependencies without a base', () => { + const head = measurement({ + '@astryxdesign/core': pkg(999_999, {'dependencies:x': '1'}), + }); + const report = comparePackageSizes({head, base: null}); + expect(report.verdict).toBe('no-base'); + expect(report.addedDependencies).toEqual([]); + }); +}); + +describe('renderPackageSizeSection', () => { + const base = measurement({ + '@astryxdesign/core': pkg(100_000), + '@astryxdesign/theme-neutral': pkg(20_000), + '@astryxdesign/theme-stone': pkg(20_000), + }); + + it('shows only changed packages and counts the rest', () => { + const head = measurement({ + ...base.packages, + '@astryxdesign/core': pkg(101_000), + }); + const markdown = renderPackageSizeSection( + comparePackageSizes({head, base}), + ); + expect(markdown).toContain('### Published Package Size'); + expect(markdown).toContain( + '| @astryxdesign/core | 97.7KB | 98.6KB | +1000B (+1.0%) |', + ); + expect(markdown).not.toContain('theme-neutral'); + expect(markdown).toContain('2 other published packages are unchanged.'); + expect(markdown).not.toContain('Over budget'); + }); + + it('says when nothing changed', () => { + const markdown = renderPackageSizeSection( + comparePackageSizes({head: base, base}), + ); + expect(markdown).toContain( + 'No published package changed size (3 measured).', + ); + }); + + it('flags the row and asks for a Size budget line when over budget', () => { + const head = measurement({ + ...base.packages, + '@astryxdesign/core': pkg(110_000), + }); + const markdown = renderPackageSizeSection( + comparePackageSizes({head, base}), + ); + expect(markdown).toMatch( + /\| @astryxdesign\/core \|[^\n]*\(\+10\.0%\) ⚠️ \|/, + ); + expect(markdown).toContain('**Over budget:**'); + expect(markdown).toContain('`Size budget:`'); + }); + + it('shows every head size and no budget without a base', () => { + const markdown = renderPackageSizeSection( + comparePackageSizes({head: base, base: null}), + ); + expect(markdown).toContain('no budget applies'); + expect(markdown).toContain('theme-stone'); + }); + + it('renders report strings through the given escaper', () => { + const head = measurement({ + '@astryxdesign/core': pkg(100_000, {'dependencies:': '1'}), + }); + const markdown = renderPackageSizeSection( + comparePackageSizes({ + head, + base: measurement({'@astryxdesign/core': pkg(100_000)}), + }), + text => `[${text}]`, + ); + expect(markdown).toContain( + '- [@astryxdesign/core]: [] [1] ([dependencies])', + ); + }); +}); + +describe('package-size.mjs', () => { + function runCli(args) { + return spawnSync(process.execPath, [CLI, ...args], {encoding: 'utf8'}); + } + + function compareAndEnforce({head, base, body}) { + const dir = tempDir(); + const file = name => path.join(dir, name); + fs.writeFileSync(file('head.json'), JSON.stringify(head)); + if (base) fs.writeFileSync(file('base.json'), JSON.stringify(base)); + fs.writeFileSync(file('body.txt'), body); + const compare = runCli([ + 'compare', + '--head', + file('head.json'), + '--base', + file('base.json'), + '--body', + file('body.txt'), + '--head-sha', + 'a'.repeat(40), + '--output', + file('report.json'), + '--summary', + file('summary.md'), + ]); + const enforce = runCli(['enforce', '--report', file('report.json')]); + return { + compare, + enforce, + report: JSON.parse(fs.readFileSync(file('report.json'), 'utf8')), + summary: fs.readFileSync(file('summary.md'), 'utf8'), + }; + } + + const base = measurement({'@astryxdesign/core': pkg(100_000)}); + const grown = measurement({'@astryxdesign/core': pkg(110_000)}); + + it('compares without failing, and enforce fails an unjustified overrun', () => { + const {compare, enforce, report, summary} = compareAndEnforce({ + head: grown, + base, + body: '', + }); + expect(compare.status).toBe(0); + expect(report.headSha).toBe('a'.repeat(40)); + expect(summary).toContain('**Over budget:**'); + expect(enforce.status).toBe(1); + expect(enforce.stdout).toContain( + '::error title=Package size over budget::', + ); + }); + + it('enforce passes a justified overrun and a missing base', () => { + expect( + compareAndEnforce({head: grown, base, body: 'Size budget: worth it'}) + .enforce.status, + ).toBe(0); + const noBase = compareAndEnforce({head: grown, base: null, body: ''}); + expect(noBase.report.verdict).toBe('no-base'); + expect(noBase.compare.stdout).toContain('::notice::'); + expect(noBase.enforce.status).toBe(0); + }); + + it('measure writes every published package of a workspace', () => { + const root = tempDir(); + writePackage( + root, + 'packages/core', + {name: '@astryxdesign/core', version: '0.0.0'}, + { + 'index.js': 'export {};\n', + }, + ); + const output = path.join(root, 'out.json'); + execFileSync(process.execPath, [ + CLI, + 'measure', + '--root', + root, + '--output', + output, + ]); + const written = JSON.parse(fs.readFileSync(output, 'utf8')); + expect(Object.keys(written.packages)).toEqual(['@astryxdesign/core']); + }); +}); diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjs index fafbfb4a8c5ac..50894a05a4006 100644 --- a/.github/scripts/lib/pr-preview.mjs +++ b/.github/scripts/lib/pr-preview.mjs @@ -477,18 +477,20 @@ function trustedAnalysis({analysisPath, metadataPath, identity, core}) { * Base measurements come from the same CI run's merge-base build. They are * used only when they name the exact head this comment reports. */ -function trustedBaseStats({baseStatsPath, identity, core}) { +// A size report the generator may read: present and written for the exact +// head being reported. +function trustedHeadReport({reportPath, label, identity, core}) { try { - if (!fs.existsSync(baseStatsPath)) return false; - const stats = readJSON(baseStatsPath); + if (!fs.existsSync(reportPath)) return false; + const report = readJSON(reportPath); if ( - !FULL_SHA.test(String(stats.headSha)) || - stats.headSha !== identity.headSha + !FULL_SHA.test(String(report.headSha)) || + report.headSha !== identity.headSha ) - refuse('base stats head does not match'); + refuse(`${label} head does not match`); return true; } catch (error) { - core.warning(`Ignoring untrusted or stale base stats: ${error.message}`); + core.warning(`Ignoring untrusted or stale ${label}: ${error.message}`); return false; } } @@ -510,6 +512,7 @@ export async function reconcilePrComment({ analysisPath = 'pr-analysis/analysis.json', metadataPath = 'pr-analysis/pr-meta.json', baseStatsPath = 'pr-size-base/size-base.json', + packageSizePath = 'pr-analysis/package-size.json', a11yPath = 'a11y/a11y-report.json', visualPath = 'trusted-visual/verdict.json', createIfMissing = true, @@ -562,9 +565,22 @@ export async function reconcilePrComment({ generator, '--analysis', analysisPath, - ...(trustedBaseStats({baseStatsPath, identity, core}) + ...(trustedHeadReport({ + reportPath: baseStatsPath, + label: 'base stats', + identity, + core, + }) ? ['--base-stats', baseStatsPath] : []), + ...(trustedHeadReport({ + reportPath: packageSizePath, + label: 'package size report', + identity, + core, + }) + ? ['--package-size', packageSizePath] + : []), // No report means the audit did not run (it runs on main and release // checks, or with the ci:heavy-lanes label); the comment says so. ...(fs.existsSync(a11yPath) ? ['--a11y', a11yPath] : []), diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjs index 9e89018876599..c015c2b8bdaf8 100644 --- a/.github/scripts/lib/pr-preview.test.mjs +++ b/.github/scripts/lib/pr-preview.test.mjs @@ -201,6 +201,7 @@ async function reconcile({ fallbackMessage, execute, baseStats, + packageSize, } = {}) { const value = previewAvailable ? {...requested, headRepository: requested.baseRepository} @@ -212,6 +213,12 @@ async function reconcile({ 'size-base.json', ); if (baseStats) fs.writeFileSync(baseStatsPath, JSON.stringify(baseStats)); + const packageSizePath = path.join( + path.dirname(paths.analysis), + 'package-size.json', + ); + if (packageSize) + fs.writeFileSync(packageSizePath, JSON.stringify(packageSize)); const {github, state} = githubFixture({ value: githubFixtureValue, comments, @@ -228,6 +235,7 @@ async function reconcile({ analysisPath: paths.analysis, metadataPath: paths.metadata, baseStatsPath, + packageSizePath, a11yPath: paths.a11y, visualPath: paths.visual, createIfMissing, @@ -964,6 +972,25 @@ describe('PR comment preview reconciliation', () => { }, ); + it.each([ + ['the exact head', HEAD, true], + ['another head', 'b'.repeat(40), false], + ])( + 'passes the package size report only when it names %s', + async (_label, headSha, used) => { + const seen = []; + await reconcile({ + packageSize: {headSha, packages: [], verdict: 'pass'}, + execute: (_node, args) => { + seen.push(args); + return '## PR Analysis Report\n'; + }, + }); + expect(seen).toHaveLength(1); + expect(seen[0].includes('--package-size')).toBe(used); + }, + ); + it('falls back to a safe current message when analysis rendering fails', async () => { const stale = { id: 78, diff --git a/.github/scripts/package-size.mjs b/.github/scripts/package-size.mjs new file mode 100644 index 0000000000000..d740eb8bd0586 --- /dev/null +++ b/.github/scripts/package-size.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Published package size: measure a built workspace, or compare a pull + * request's measurement with its merge base's. + * @input `measure --output [--root ]`: a workspace whose packages + * are built. + * `compare --head [--base ] [--body ] --output + * [--summary ] [--head-sha ] [--base-sha ]`: two + * measurements (a missing or unreadable base compares nothing) and the pull + * request description. + * `enforce --report `: a comparison `compare` wrote. + * @output `measure` writes each published package's sizes and runtime + * dependencies. `compare` writes the comparison and appends it to the job + * summary. `enforce` exits 1 when a package is over budget and the + * description has no `Size budget:` line; it runs after the report's + * artifacts are uploaded, so a failing check still gets its PR report. + * @position CI's build-storybook job measures and compares a pull request's + * head; Main's typecheck job measures each main commit and keeps it for 14 + * days as the base. The PR report renders the comparison. + */ + +import fs from 'node:fs'; +import {createRequire} from 'node:module'; + +const require = createRequire(import.meta.url); +const { + measurePackages, + comparePackageSizes, + renderPackageSizeSection, +} = require('./lib/package-size.js'); + +const [command, ...args] = process.argv.slice(2); +const getArg = name => { + const index = args.indexOf(`--${name}`); + return index !== -1 ? args[index + 1] : null; +}; + +function readJson(file) { + if (!file) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch { + return null; + } +} + +function readText(file) { + if (!file) return ''; + try { + return fs.readFileSync(file, 'utf8'); + } catch { + return ''; + } +} + +if (command === 'measure') { + const output = getArg('output'); + if (!output) throw new Error('measure needs --output '); + const measurement = await measurePackages(getArg('root') || process.cwd()); + fs.writeFileSync(output, `${JSON.stringify(measurement, null, 2)}\n`); + console.log( + `Measured ${Object.keys(measurement.packages).length} published packages.`, + ); +} else if (command === 'compare') { + const output = getArg('output'); + const head = readJson(getArg('head')); + if (!output || !head) + throw new Error('compare needs --head and --output '); + const base = readJson(getArg('base')); + if (getArg('base') && !base) + console.log( + '::notice::No base measurement for the merge base; sizes are shown without a budget.', + ); + const report = comparePackageSizes({ + head, + base, + body: readText(getArg('body')), + headSha: getArg('head-sha') || '', + baseSha: getArg('base-sha') || '', + }); + fs.writeFileSync(output, `${JSON.stringify(report, null, 2)}\n`); + const markdown = renderPackageSizeSection(report); + const summary = getArg('summary'); + if (summary) fs.appendFileSync(summary, markdown); + console.log(markdown); +} else if (command === 'enforce') { + const report = readJson(getArg('report')); + if (!report) throw new Error('enforce needs a readable --report '); + if (report.verdict === 'over-budget') { + console.log( + '::error title=Package size over budget::A published package grew past its budget. Shrink it, or add a "Size budget:" line to the pull request description saying why, then re-run this job.', + ); + process.exitCode = 1; + } else { + console.log(`Package size verdict: ${report.verdict}.`); + } +} else { + console.error( + 'usage: package-size.mjs measure --output | compare --head [--base ] [--body ] --output | enforce --report ', + ); + process.exitCode = 2; +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4503097173f6d..f05d2d75c7ff8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -879,6 +879,8 @@ jobs: runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check' && github.triggering_actor != 'github-actions[bot]') && '4-core-ubuntu' || 'ubuntu-latest' }} timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 360 || 15 }} permissions: + # actions: read finds Main's kept package-size measurement of the merge base. + actions: read contents: read pull-requests: read outputs: @@ -898,6 +900,11 @@ jobs: - name: Decide this job's scope id: scope uses: ./.test-scope/.github/actions/test-scope + with: + # `touches` scopes only the package-size check (a package, the + # lockfile, or the check itself); the build itself follows the + # specialized-lane outputs. + paths: '^(packages/|pnpm-lock\.yaml$|\.github/scripts/package-size\.mjs$|\.github/scripts/lib/package-size\.js$)' - name: Skip heavy work for a specialized lane if: steps.scope.outputs.docsite_only == 'true' || steps.scope.outputs.spec_only == 'true' || steps.scope.outputs.tooling_only == 'true' @@ -1023,6 +1030,74 @@ jobs: --output analysis.json \ "${FORCE_FULL[@]}" + # Published package size: this head's packages are built above, and Main + # keeps each main commit's measurement for 14 days, so the merge base is + # compared without building it. Only a pull request that changes a + # package or the lockfile runs it. + - name: Measure published package size + id: package-size + if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && github.event_name == 'pull_request' && steps.scope.outputs.touches == 'true' + run: node .github/scripts/package-size.mjs measure --output package-size-head.json + + - name: Find the merge base for package size + id: package-size-merge-base + if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size.outcome == 'success' + env: + BASE_REF: ${{ github.base_ref }} + run: echo "sha=$(git merge-base "origin/$BASE_REF" HEAD || true)" >> "$GITHUB_OUTPUT" + + - name: Find main's package size for the merge base + id: package-size-cached + if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size-merge-base.outputs.sha != '' + continue-on-error: true + uses: actions/github-script@v9 + env: + BASE_SHA: ${{ steps.package-size-merge-base.outputs.sha }} + with: + retries: 3 + script: | + const name = `package-size-main-${process.env.BASE_SHA}`; + const {data} = await github.rest.actions.listArtifactsForRepo({...context.repo, name, per_page: 100}); + // Only Main's own run on main for this exact commit: a pull + // request's run can upload an artifact under any name. + const artifact = data.artifacts.find(candidate => + !candidate.expired && + candidate.workflow_run?.head_branch === 'main' && + candidate.workflow_run?.head_sha === process.env.BASE_SHA && + candidate.workflow_run?.head_repository_id === candidate.workflow_run?.repository_id); + core.setOutput('run_id', artifact ? String(artifact.workflow_run.id) : ''); + core.info(artifact ? `Using ${name} from run ${artifact.workflow_run.id}.` : `No ${name}; sizes are shown without a budget.`); + + - name: Download main's package size for the merge base + if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size-cached.outputs.run_id != '' + continue-on-error: true + uses: actions/download-artifact@v8 + with: + name: package-size-main-${{ steps.package-size-merge-base.outputs.sha }} + run-id: ${{ steps.package-size-cached.outputs.run_id }} + github-token: ${{ github.token }} + path: .package-size-base + + # The description is read now, not from the event, so an author who adds + # a `Size budget:` line can re-run this job without pushing. + - name: Check published package size + if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size.outcome == 'success' + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ steps.package-size-merge-base.outputs.sha }} + run: | + gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.body // ""' > pr-body.txt || : > pr-body.txt + node .github/scripts/package-size.mjs compare \ + --head package-size-head.json \ + --base .package-size-base/package-size.json \ + --body pr-body.txt \ + --head-sha "$HEAD_SHA" \ + --base-sha "$BASE_SHA" \ + --output package-size.json \ + --summary "$GITHUB_STEP_SUMMARY" + - name: Write PR comment metadata if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && github.event_name == 'pull_request' run: | @@ -1049,8 +1124,15 @@ jobs: path: | analysis.json pr-meta.json + package-size.json retention-days: 1 + # Last, so an over-budget head still uploads the analysis its PR report + # reads. + - name: Enforce the package size budget + if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size.outcome == 'success' + run: node .github/scripts/package-size.mjs enforce --report package-size.json + # Measures the PR's merge base so the PR comment can show component and # package sizes before and after. It runs beside build-storybook rather than # inside it, so no browser lane waits for the second build; the head sizes diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5f93ce343e37b..a65af1395c56c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -142,6 +142,18 @@ jobs: packages/themes/*/dist/ retention-days: 14 + # A pull request compares its published package sizes with its merge + # base's from this measurement (ci.yml build-storybook). + - name: Measure published package size + run: node .github/scripts/package-size.mjs measure --output package-size.json + + - name: Keep this commit's package size for PR size reports + uses: actions/upload-artifact@v7 + with: + name: package-size-main-${{ needs.main-commit.outputs.sha }} + path: package-size.json + retention-days: 14 + - name: Verify package exports run: node scripts/verify-exports.mjs From 947e1eddcd33c013d6d819cd32071437ff89402d Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 09:28:56 -0700 Subject: [PATCH 02/15] ci: write package sizes past 1 MB in MB --- .github/scripts/lib/package-size.js | 3 ++- .github/scripts/lib/package-size.test.mjs | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/scripts/lib/package-size.js b/.github/scripts/lib/package-size.js index 1a12ad56cde79..579484f5eac66 100644 --- a/.github/scripts/lib/package-size.js +++ b/.github/scripts/lib/package-size.js @@ -222,7 +222,8 @@ function comparePackageSizes({head, base, body, headSha = '', baseSha = ''}) { function sizeText(bytes) { if (!isMeasured(bytes)) return 'N/A'; if (Math.abs(bytes) < 1024) return `${bytes}B`; - return `${(bytes / 1024).toFixed(1)}KB`; + if (Math.abs(bytes) < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`; + return `${(bytes / (1024 * 1024)).toFixed(2)}MB`; } function deltaText(after, before) { diff --git a/.github/scripts/lib/package-size.test.mjs b/.github/scripts/lib/package-size.test.mjs index d1a7c5e563ee6..bccc72dfe73fe 100644 --- a/.github/scripts/lib/package-size.test.mjs +++ b/.github/scripts/lib/package-size.test.mjs @@ -307,6 +307,19 @@ describe('renderPackageSizeSection', () => { expect(markdown).not.toContain('Over budget'); }); + it('writes a package past 1 MB in MB', () => { + const big = measurement({'@astryxdesign/core': pkg(9_000_000)}); + const markdown = renderPackageSizeSection( + comparePackageSizes({ + head: measurement({'@astryxdesign/core': pkg(9_100_000)}), + base: big, + }), + ); + expect(markdown).toContain( + '| @astryxdesign/core | 8.58MB | 8.68MB | +97.7KB (+1.1%) |', + ); + }); + it('says when nothing changed', () => { const markdown = renderPackageSizeSection( comparePackageSizes({head: base, base}), From 4c9cfea8ea42d20975efc6137de2075f8fb67cf8 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 09:56:31 -0700 Subject: [PATCH 03/15] ci: per-suite ci:run-visual, ci:run-a11y and ci:run-rtl labels replace ci:heavy-lanes Each label opts a pull request into one heavy suite. A run-labels gate job reads the event's labels and, for each suite label, the newest labeler's role (write, maintain or admin; fail closed), from a decision script in the merge commit's sparse checkout; each suite's owners need it and run only on their own suite's output. A label still takes effect on the next push or reopen, ci:run stays the fork gate, and the release check is unchanged. The pull-request comment names the label for a suite that did not run. AST-030, CONTRIBUTING and the visual and RTL docs name the new labels. --- .github/scripts/ci-heavy-lanes.test.mjs | 80 ++++-- .github/scripts/ci-pr-gate.test.mjs | 2 +- .github/scripts/lib/a11y-format.js | 5 +- .github/scripts/lib/a11y-format.test.mjs | 29 ++- .github/scripts/lib/pr-preview.mjs | 2 +- .../scripts/release-check-workflow.test.mjs | 3 + .github/scripts/run-labels.cjs | 131 ++++++++++ .github/scripts/run-labels.test.mjs | 239 ++++++++++++++++++ .github/scripts/visual-gate/README.md | 2 +- .../visual-gate/workflow-concurrency.test.mjs | 2 +- .../visual-maintenance-workflow.test.mjs | 9 +- .github/workflows/ci.yml | 79 ++++-- CONTRIBUTING.md | 18 +- apps/storybook/rtl-audit/README.md | 4 +- docs/architecture/rtl-audit-evidence.md | 2 +- docs/specs/AST-030/spec.md | 24 +- 16 files changed, 561 insertions(+), 70 deletions(-) create mode 100644 .github/scripts/run-labels.cjs create mode 100644 .github/scripts/run-labels.test.mjs diff --git a/.github/scripts/ci-heavy-lanes.test.mjs b/.github/scripts/ci-heavy-lanes.test.mjs index da1d133992bbd..cf737baa690d7 100644 --- a/.github/scripts/ci-heavy-lanes.test.mjs +++ b/.github/scripts/ci-heavy-lanes.test.mjs @@ -21,34 +21,82 @@ const ci = read('ci.yml'); const heavy = read('main-heavy-lanes.yml'); const main = read('deploy.yml'); -const LABEL = 'ci:heavy-lanes'; -const HEAVY = [ - 'pr-a11y-guards', - 'pr-a11y-specs', - 'pr-a11y', - 'visual-scout', - 'pr-visual', - 'pr-rtl-shard', - 'pr-rtl-merge', - 'pr-rtl', -]; -const PR_CLAUSE = `(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, '${LABEL}'))`; +const SUITE = { + 'pr-a11y-guards': 'a11y', + 'pr-a11y-specs': 'a11y', + 'pr-a11y': 'a11y', + 'visual-scout': 'visual', + 'pr-visual': 'visual', + 'pr-rtl-shard': 'rtl', + 'pr-rtl-merge': 'rtl', + 'pr-rtl': 'rtl', +}; +const HEAVY = Object.keys(SUITE); +const prClause = suite => + `(github.event_name == 'pull_request' && needs.run-labels.outputs.${suite} == 'true')`; describe('visual, accessibility and RTL owners', () => { // release-check-workflow.test.mjs evaluates them for release-check and - // visual-maintenance-workflow.test.mjs for labelled and unlabelled pull + // visual-maintenance-workflow.test.mjs for opted-in and other pull // requests; this pins the shape on every owner. it.each(HEAVY)( - '%s runs on a pull request only with the ci:heavy-lanes label', + '%s runs on a pull request only when run-labels opts in its own suite', name => { const condition = ungated(ci.jobs[name].if); - expect(condition).toContain(PR_CLAUSE); - expect(condition.replaceAll(PR_CLAUSE, '')).not.toContain( + expect(condition).toContain(prClause(SUITE[name])); + expect(condition.replaceAll(prClause(SUITE[name]), '')).not.toContain( "'pull_request'", ); + // No other suite's decision, and no label read directly. + for (const other of ['visual', 'a11y', 'rtl']) { + if (other !== SUITE[name]) + expect(condition).not.toContain(`outputs.${other}`); + } + expect(condition).not.toMatch(/labels\.\*\.name, 'ci:(heavy|run-)/); + expect([ci.jobs[name].needs].flat()).toContain('run-labels'); }, ); + it('decides the suites in one gate job from the trusted decision script', () => { + const gate = ci.jobs['run-labels']; + expect(gate['runs-on']).toBe('ubuntu-slim'); + expect(gate.permissions).toEqual({ + contents: 'read', + 'pull-requests': 'read', + }); + expect(gate.outputs).toEqual({ + visual: '${{ steps.decide.outputs.visual }}', + a11y: '${{ steps.decide.outputs.a11y }}', + rtl: '${{ steps.decide.outputs.rtl }}', + }); + expect(gate.steps[0].with).toMatchObject({ + 'sparse-checkout': '.github/scripts/run-labels.cjs', + path: '.run-labels', + 'persist-credentials': false, + }); + expect(gate.steps[0].with.ref).toBeUndefined(); + expect(gate.steps[1].with.script).toContain( + '.run-labels/.github/scripts/run-labels.cjs', + ); + // It runs for the release check too, so its owners' needs succeed there. + expect(ungated(gate.if)).toBe( + "github.event_name != 'workflow_dispatch' || inputs.operation == 'release-check'", + ); + }); + + it('names no catch-all heavy label anywhere in the workflows', () => { + for (const name of fs.readdirSync( + new URL('../workflows/', import.meta.url), + )) + expect( + fs.readFileSync( + new URL(`../workflows/${name}`, import.meta.url), + 'utf8', + ), + name, + ).not.toContain('ci:heavy-lanes'); + }); + it('leaves the required joins without a heavy owner, so they still report on every pull request', () => { for (const join of ['test', 'build', 'docsite-test']) { const needs = [ci.jobs[join].needs].flat(); diff --git a/.github/scripts/ci-pr-gate.test.mjs b/.github/scripts/ci-pr-gate.test.mjs index f170e27f62281..f22da8cfff428 100644 --- a/.github/scripts/ci-pr-gate.test.mjs +++ b/.github/scripts/ci-pr-gate.test.mjs @@ -228,7 +228,7 @@ describe('pull-request CI gate (spec:AST-030 FR17)', () => { expect(runs({})).toBe(true); expect(runs({fork: true})).toBe(false); expect(runs({fork: true, labels: ['ci:run']})).toBe(true); - expect(runs({fork: true, labels: ['ci:heavy-lanes']})).toBe(false); + expect(runs({fork: true, labels: ['ci:run-visual']})).toBe(false); expect(runs({draft: true})).toBe(false); expect(runs({draft: true, fork: true, labels: ['ci:run']})).toBe(false); }); diff --git a/.github/scripts/lib/a11y-format.js b/.github/scripts/lib/a11y-format.js index bd2b189c52573..1f2f82aaa861c 100644 --- a/.github/scripts/lib/a11y-format.js +++ b/.github/scripts/lib/a11y-format.js @@ -6,6 +6,7 @@ */ const {inline, num, safeUrl} = require('./report-text'); +const {notRunSentence} = require('../run-labels.cjs'); // Impact emoji badges const impactEmoji = { @@ -45,9 +46,9 @@ function formatWcagTags(tags) { function buildA11ySection(a11yReport, storiesAuditedByComponent) { let a11ySection = '### Accessibility Audit\n\n'; // No report: the audit did not run (pull requests run it only with the - // `ci:heavy-lanes` label, spec:AST-030 FR12). Never claim a clean result. + // `ci:run-a11y` label, spec:AST-030 FR12). Never claim a clean result. if (!a11yReport) { - return a11ySection + '**Status:** The accessibility audit did not run on this pull request. It runs after merges to `main` and before each release; pull requests that touch components run it with the `ci:heavy-lanes` label.\n\n'; + return `${a11ySection}**Status:** ${notRunSentence('a11y')} It runs after merges to \`main\` and before each release.\n\n`; } const totalViolations = Object.values(a11yReport.components || {}) .reduce((sum, comp) => sum + (comp.violations?.length || 0), 0); diff --git a/.github/scripts/lib/a11y-format.test.mjs b/.github/scripts/lib/a11y-format.test.mjs index a2ee2983fdc23..63b9180f80c43 100644 --- a/.github/scripts/lib/a11y-format.test.mjs +++ b/.github/scripts/lib/a11y-format.test.mjs @@ -3,19 +3,28 @@ import {describe, expect, it} from 'vitest'; import {createRequire} from 'node:module'; -const {buildA11ySection, formatWcagTags} = createRequire(import.meta.url)('./a11y-format.js'); +const {buildA11ySection, formatWcagTags} = createRequire(import.meta.url)( + './a11y-format.js', +); -const report = (violations) => ({ +const report = violations => ({ components: {Button: {storiesAudited: 4, violations}}, }); describe('buildA11ySection', () => { it('says so plainly when there is nothing to report', () => { - expect(buildA11ySection({components: {}})).toContain('No accessibility violations detected'); + expect(buildA11ySection({components: {}})).toContain( + 'No accessibility violations detected', + ); }); it('says the audit did not run when there is no report', () => { - expect(buildA11ySection(null)).toContain('did not run on this pull request'); + expect(buildA11ySection(null)).toContain( + 'did not run on this pull request', + ); + expect(buildA11ySection(null)).toContain( + 'Accessibility did not run on this pull request; add `ci:run-a11y` to run it.', + ); }); it('lists a violation with its rule, story count, and help link', () => { @@ -31,10 +40,14 @@ describe('buildA11ySection', () => { }, ]), ); - expect(section).toContain('**serious**: Elements must have sufficient color contrast'); + expect(section).toContain( + '**serious**: Elements must have sufficient color contrast', + ); expect(section).toContain('Rule: `color-contrast`'); expect(section).toContain('Affects 2/4 stories'); - expect(section).toContain('[Learn more](https://dequeuniversity.com/rules/axe/4.10/color-contrast)'); + expect(section).toContain( + '[Learn more](https://dequeuniversity.com/rules/axe/4.10/color-contrast)', + ); expect(section).toContain('WCAG: 1.4.3 (Level AA)'); }); @@ -49,7 +62,9 @@ describe('buildA11ySection', () => { }, ]), ); - expect(section).toContain('line one line two with <em>markup</em> and \\[brackets\\]'); + expect(section).toContain( + 'line one line two with <em>markup</em> and \\[brackets\\]', + ); // Rule ids keep their own alphabet only, so the code span stays a code span. expect(section).toContain('Rule: `ruleoddname`'); // A help value that is not a plain https URL renders no link at all. diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjs index fafbfb4a8c5ac..6424e2899a3dc 100644 --- a/.github/scripts/lib/pr-preview.mjs +++ b/.github/scripts/lib/pr-preview.mjs @@ -566,7 +566,7 @@ export async function reconcilePrComment({ ? ['--base-stats', baseStatsPath] : []), // No report means the audit did not run (it runs on main and release - // checks, or with the ci:heavy-lanes label); the comment says so. + // checks, or with the ci:run-a11y label); the comment says so. ...(fs.existsSync(a11yPath) ? ['--a11y', a11yPath] : []), ...(fs.existsSync(visualPath) ? ['--visual', visualPath] : []), ...(storybook ? ['--storybook-url', `${previewOrigin}/storybook/`] : []), diff --git a/.github/scripts/release-check-workflow.test.mjs b/.github/scripts/release-check-workflow.test.mjs index 02a3d28e22f94..a7d17e1b2c818 100644 --- a/.github/scripts/release-check-workflow.test.mjs +++ b/.github/scripts/release-check-workflow.test.mjs @@ -25,6 +25,9 @@ const packageJson = JSON.parse( const releaseJobs = [ 'release-authority', 'test-registry-e2e', + // Decides the pull-request opt-ins; on a release check it opts nothing in + // and succeeds, so the owners that need it run. + 'run-labels', 'build-storybook', 'typecheck', 'visual-scout', diff --git a/.github/scripts/run-labels.cjs b/.github/scripts/run-labels.cjs new file mode 100644 index 0000000000000..8ecc256c8dacf --- /dev/null +++ b/.github/scripts/run-labels.cjs @@ -0,0 +1,131 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Decides which heavy suites a pull request opted into with a + * `ci:run-` label (spec:AST-030 FR12, DEC-7). + * + * A suite runs on a pull request only while the pull request carries its + * label, and only when the newest application of that label came from someone + * with write, maintain or admin access. Agent lanes push with write access, + * so their labels count. The decision reads the payload of the event that + * started the run, so applying or removing a label starts nothing; it takes + * effect on the next push or reopen (FR17). Outside a pull request no label + * applies, and the release check runs every suite on its own terms. Any + * lookup that fails denies the suite (fail closed). + */ + +'use strict'; + +/** Suites a pull request can opt into, by output name and label. */ +const SUITE_LABELS = Object.freeze({ + visual: 'ci:run-visual', + a11y: 'ci:run-a11y', + rtl: 'ci:run-rtl', +}); + +const WRITE_ROLES = new Set(['admin', 'maintain', 'write']); + +/** Display names for the pull-request comment's not-run sentence. */ +const SUITE_NAMES = Object.freeze({ + visual: 'Visual', + a11y: 'Accessibility', + rtl: 'RTL', + perf: 'Perf', +}); + +/** + * The comment's sentence for a suite a pull request did not run, with the + * label that runs it. + */ +function notRunSentence(suite) { + return `${SUITE_NAMES[suite]} did not run on this pull request; add \`ci:run-${suite}\` to run it.`; +} + +/** + * The newest actor that applied each label, from the pull request's timeline. + * Only the newest application counts: a later one by someone without write + * access does not inherit an earlier maintainer's. + */ +async function newestLabelers(github, {owner, repo, number}) { + const data = await github.graphql( + `query($owner: String!, $repo: String!, $number: Int!) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $number) { + timelineItems(last: 100, itemTypes: [LABELED_EVENT]) { + nodes { ... on LabeledEvent { actor { login } label { name } } } + } + } + } + }`, + {owner, repo, number}, + ); + const labelers = new Map(); + for (const event of data.repository.pullRequest.timelineItems.nodes) { + const name = event?.label?.name; + if (name != null) labelers.set(name, event.actor?.login ?? null); + } + return labelers; +} + +/** + * Decides every suite and sets one `true`/`false` output per suite. + * @returns {Promise>} + */ +async function decideRunLabels({github, context, core}) { + const decisions = Object.fromEntries( + Object.keys(SUITE_LABELS).map(suite => [suite, false]), + ); + const pr = context.payload.pull_request; + if (context.eventName === 'pull_request' && pr != null) { + const carried = new Set((pr.labels ?? []).map(label => label.name)); + const wanted = Object.entries(SUITE_LABELS).filter(([, label]) => + carried.has(label), + ); + if (wanted.length > 0) { + const {owner, repo} = context.repo; + let labelers = new Map(); + const roles = new Map(); + try { + labelers = await newestLabelers(github, { + owner, + repo, + number: pr.number, + }); + for (const [, label] of wanted) { + const login = labelers.get(label); + if (login != null && !roles.has(login)) { + const {data} = + await github.rest.repos.getCollaboratorPermissionLevel({ + owner, + repo, + username: login, + }); + roles.set(login, data.role_name); + } + } + } catch (error) { + core.warning( + `Could not check who applied the ci:run labels: ${error.message}`, + ); + } + for (const [suite, label] of wanted) { + const login = labelers.get(label) ?? null; + const role = login == null ? null : (roles.get(login) ?? null); + decisions[suite] = WRITE_ROLES.has(role); + if (!decisions[suite]) { + core.notice( + login == null + ? `No ${label} label event could be read; the ${suite} suite does not run.` + : `${label} was applied by ${login} (${role ?? 'unknown role'}); only write, maintain or admin access runs the ${suite} suite.`, + ); + } + } + } + } + for (const [suite, runs] of Object.entries(decisions)) { + core.setOutput(suite, String(runs)); + } + return decisions; +} + +module.exports = {SUITE_LABELS, decideRunLabels, notRunSentence}; diff --git a/.github/scripts/run-labels.test.mjs b/.github/scripts/run-labels.test.mjs new file mode 100644 index 0000000000000..64fd4ab894c54 --- /dev/null +++ b/.github/scripts/run-labels.test.mjs @@ -0,0 +1,239 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Per-suite opt-in labels for the heavy owners. + * @input run-labels.cjs with a mocked GitHub API, and ci.yml's owner conditions + * @output Each `ci:run-` label, applied with write access, runs only its + * own suite; a missing, removed or unauthorized label runs none + * @position Node contracts for AST-030 FR12 and FR17 + */ + +import fs from 'node:fs'; +import {createRequire} from 'node:module'; +import {describe, expect, it} from 'vitest'; +import yaml from 'yaml'; +import {ungated} from './ci-pr-gate.mjs'; + +const require = createRequire(import.meta.url); +const { + SUITE_LABELS, + decideRunLabels, + notRunSentence, +} = require('./run-labels.cjs'); + +const ci = yaml.parse( + fs.readFileSync(new URL('../workflows/ci.yml', import.meta.url), 'utf8'), +); + +/** Runs the decision against a mocked pull request and GitHub API. */ +async function decide({ + eventName = 'pull_request', + labels = [], + events = labels.map(name => ({actor: {login: 'maintainer'}, label: {name}})), + roles = {maintainer: 'maintain'}, + failLookup = false, +} = {}) { + const outputs = {}; + const notices = []; + const lookups = []; + const core = { + setOutput: (name, value) => { + outputs[name] = value; + }, + notice: message => notices.push(message), + warning: message => notices.push(message), + }; + const github = { + graphql: async (_query, variables) => { + if (failLookup) throw new Error('rate limited'); + expect(variables).toEqual({owner: 'meta', repo: 'astryx', number: 7}); + return {repository: {pullRequest: {timelineItems: {nodes: events}}}}; + }, + rest: { + repos: { + getCollaboratorPermissionLevel: async ({username}) => { + lookups.push(username); + return {data: {role_name: roles[username] ?? 'read'}}; + }, + }, + }, + }; + const context = { + eventName, + repo: {owner: 'meta', repo: 'astryx'}, + payload: { + pull_request: + eventName === 'pull_request' + ? {number: 7, labels: labels.map(name => ({name}))} + : undefined, + }, + }; + await decideRunLabels({github, context, core}); + return {outputs, notices, lookups}; +} + +describe('run-labels decision', () => { + it('words the not-run sentence the comment shows for each suite', () => { + expect(notRunSentence('visual')).toBe( + 'Visual did not run on this pull request; add `ci:run-visual` to run it.', + ); + expect(notRunSentence('a11y')).toBe( + 'Accessibility did not run on this pull request; add `ci:run-a11y` to run it.', + ); + expect(notRunSentence('rtl')).toBe( + 'RTL did not run on this pull request; add `ci:run-rtl` to run it.', + ); + expect(notRunSentence('perf')).toBe( + 'Perf did not run on this pull request; add `ci:run-perf` to run it.', + ); + }); + + it('names one label per suite', () => { + expect(SUITE_LABELS).toEqual({ + visual: 'ci:run-visual', + a11y: 'ci:run-a11y', + rtl: 'ci:run-rtl', + }); + }); + + it.each(Object.entries(SUITE_LABELS))( + 'opts in only %s for %s', + async (suite, label) => { + const {outputs} = await decide({labels: [label]}); + for (const other of Object.keys(SUITE_LABELS)) + expect(outputs[other], other).toBe(String(other === suite)); + }, + ); + + it('runs a suite when its label came from write, maintain or admin access', async () => { + for (const role of ['write', 'maintain', 'admin']) { + const {outputs} = await decide({ + labels: ['ci:run-a11y'], + roles: {maintainer: role}, + }); + expect(outputs.a11y, role).toBe('true'); + } + }); + + it('does not run a suite labelled with triage or read access, and says why', async () => { + for (const role of ['triage', 'read']) { + const {outputs, notices} = await decide({ + labels: ['ci:run-rtl'], + roles: {maintainer: role}, + }); + expect(outputs.rtl, role).toBe('false'); + expect(notices.join('\n')).toContain('maintainer'); + } + }); + + it("judges each label's newest application, not an earlier one", async () => { + const {outputs} = await decide({ + labels: ['ci:run-visual', 'ci:run-rtl'], + events: [ + {actor: {login: 'maintainer'}, label: {name: 'ci:run-visual'}}, + {actor: {login: 'maintainer'}, label: {name: 'ci:run-rtl'}}, + {actor: {login: 'triager'}, label: {name: 'ci:run-visual'}}, + ], + roles: {maintainer: 'maintain', triager: 'triage'}, + }); + expect(outputs).toEqual({visual: 'false', a11y: 'false', rtl: 'true'}); + }); + + it('runs nothing for a removed label, even though its event is in the timeline', async () => { + const {outputs} = await decide({ + labels: [], + events: [{actor: {login: 'maintainer'}, label: {name: 'ci:run-visual'}}], + }); + expect(outputs).toEqual({visual: 'false', a11y: 'false', rtl: 'false'}); + }); + + it('fails closed when the lookup fails or finds no label event', async () => { + expect( + (await decide({labels: ['ci:run-visual'], failLookup: true})).outputs, + ).toEqual({visual: 'false', a11y: 'false', rtl: 'false'}); + expect( + (await decide({labels: ['ci:run-visual'], events: []})).outputs.visual, + ).toBe('false'); + }); + + it('looks each labeler up once and makes no call without a suite label', async () => { + const {lookups} = await decide({ + labels: ['ci:run-visual', 'ci:run-a11y', 'ci:run-rtl'], + }); + expect(lookups).toEqual(['maintainer']); + const none = await decide({labels: ['ci:run', 'bug']}); + expect(none.lookups).toEqual([]); + expect(none.outputs).toEqual({ + visual: 'false', + a11y: 'false', + rtl: 'false', + }); + }); + + it('opts nothing in outside a pull request', async () => { + for (const eventName of ['push', 'workflow_dispatch', 'merge_group']) { + const {outputs} = await decide({eventName}); + expect(outputs, eventName).toEqual({ + visual: 'false', + a11y: 'false', + rtl: 'false', + }); + } + }); +}); + +// Replay: each label's decision through every heavy owner's condition on a +// pull request whose other needs succeeded. +describe('each label runs only its own suite', () => { + const OWNERS = { + visual: ['visual-scout', 'pr-visual'], + a11y: ['pr-a11y-guards', 'pr-a11y-specs', 'pr-a11y'], + rtl: ['pr-rtl-shard', 'pr-rtl-merge', 'pr-rtl'], + }; + + function runsOnPullRequest(job, decision) { + const expression = ungated(job.if) + .replace(/^\$\{\{\s*|\s*\}\}$/g, '') + .replace(/needs\.([a-z][a-z0-9-]*)/g, 'needs["$1"]'); + const needs = Object.fromEntries( + [job.needs].flat().map(name => [ + name, + { + result: 'success', + outputs: name === 'run-labels' ? decision : {}, + }, + ]), + ); + return Function( + 'github', + 'inputs', + 'needs', + 'always', + 'cancelled', + `return Boolean(${expression});`, + )( + {event_name: 'pull_request', event: {pull_request: {labels: []}}}, + {}, + needs, + () => true, + () => false, + ); + } + + it.each([ + [[]], + [['ci:run-visual']], + [['ci:run-a11y']], + [['ci:run-rtl']], + [['ci:run-visual', 'ci:run-rtl']], + ])('with %j', async labels => { + const {outputs} = await decide({labels}); + const ran = Object.entries(OWNERS) + .flatMap(([, owners]) => owners) + .filter(owner => runsOnPullRequest(ci.jobs[owner], outputs)); + const expected = Object.entries(OWNERS) + .filter(([suite]) => labels.includes(SUITE_LABELS[suite])) + .flatMap(([, owners]) => owners); + expect(ran.sort()).toEqual(expected.sort()); + }); +}); diff --git a/.github/scripts/visual-gate/README.md b/.github/scripts/visual-gate/README.md index 69630706566b4..a70b86b278d6b 100644 --- a/.github/scripts/visual-gate/README.md +++ b/.github/scripts/visual-gate/README.md @@ -4,7 +4,7 @@ Every visual-regression test uses the shared Storybook framework and `.github/workflows/ci.yml` → `pr-visual` (**Stable visual regression**). It runs in `release-check`, which `main-heavy-lanes.yml` dispatches after merges to `main` (a burst is checked once, at its newest commit) and the release gate dispatches before a cut; a pull request runs it only -with the `ci:heavy-lanes` label (AST-030 FR12, DEC-7). +with the `ci:run-visual` label (AST-030 FR12, DEC-7). [AST-030 FR12](../../../docs/specs/AST-030/spec.md) owns this requirement. Add coverage to that owner, not to a new component, package, theme, scheduled, post-CI, or post-merge workflow. diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs index 352f875843d76..8cb4c7978d294 100644 --- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs +++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs @@ -89,7 +89,7 @@ describe('PR report and deployment workflow contracts', () => { value.indexOf(' pr-rtl-shard:'), ); expect(visual).toContain( - '[build-storybook, maintenance-request, visual-scout]', + '[build-storybook, maintenance-request, visual-scout, run-labels]', ); expect(visual).toContain( 'name: storybook-${{ needs.build-storybook.outputs.short_hash }}', diff --git a/.github/scripts/visual-maintenance-workflow.test.mjs b/.github/scripts/visual-maintenance-workflow.test.mjs index 344258cce840a..10cae5b6bb1de 100644 --- a/.github/scripts/visual-maintenance-workflow.test.mjs +++ b/.github/scripts/visual-maintenance-workflow.test.mjs @@ -122,11 +122,16 @@ describe('CI baseline maintenance routing', () => { it('runs on an opted-in pull request when the maintenance guard is skipped', () => { const needs = dependencies(visual); needs['maintenance-request'].result = 'skipped'; - const optedIn = {event: 'pull_request', needs, labels: ['ci:heavy-lanes']}; + // run-labels decided the pull request opted into visual (ci:run-visual). + needs['run-labels'].outputs = {visual: 'true', a11y: 'false', rtl: 'false'}; + const optedIn = {event: 'pull_request', needs}; // The job decides its stable visual scope in its own steps. expect(runs(visual, optedIn)).toBe(true); - // Without the opt-in label a pull request update runs no visual owner. + // Without the opt-in a pull request update runs no visual owner, and + // another suite's label does not opt it in. + needs['run-labels'].outputs = {visual: 'false', a11y: 'true', rtl: 'true'}; expect(runs(visual, {event: 'pull_request', needs})).toBe(false); + needs['run-labels'].outputs = {visual: 'true', a11y: 'false', rtl: 'false'}; needs['build-storybook'].result = 'failure'; expect(runs(visual, optedIn)).toBe(false); expect(runs(visual, {event: 'merge_group', needs})).toBe(false); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e9d912a1b290..b438e1a9343ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,9 +57,11 @@ permissions: {} # Visual, accessibility and RTL owners (pr-a11y*, visual-scout, pr-visual, # pr-rtl*) do not run on pull request updates: they run for release-check, # which the Main heavy lanes workflow (main-heavy-lanes.yml) dispatches on every -# push to main and the release gate dispatches before a cut, and on a pull request labelled `ci:heavy-lanes` (add the label, -# then push or close and reopen). A main failure is fixed forward -# (spec:AST-030 FR12, DEC-7). +# push to main and the release gate dispatches before a cut. A pull request +# opts into one suite with its label, `ci:run-visual`, `ci:run-a11y` or +# `ci:run-rtl`, applied by someone with write access (add the label, then push +# or close and reopen); run-labels decides. A main failure is fixed forward +# (spec:AST-030 FR12, FR17, DEC-7). # # Runners: every job uses GitHub's standard hosted runners, except the release # path's long jobs, which take a paid larger-runner pool so pull request load @@ -874,6 +876,45 @@ jobs: # test). Uploads the storybook preview and analysis artifacts for downstream # jobs. Sandbox previews come from the exact-head Vercel deployment, so CI # builds no Sandbox of its own. + # Which heavy suites this pull request opted into: one output per suite, + # `true` while the pull request carries that suite's `ci:run-` label + # and its newest application came from someone with write, maintain or + # admin access (spec:AST-030 FR12). It reads the event that started the run, + # so a label takes effect on the next push or reopen (FR17). Outside a pull + # request every output is `false`; the release check runs every suite on its + # own terms. The decision script comes from the merge commit's trusted + # sparse checkout, and a lookup that fails denies the suite. + run-labels: + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (github.event_name != 'workflow_dispatch' || inputs.operation == 'release-check') }} + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + visual: ${{ steps.decide.outputs.visual }} + a11y: ${{ steps.decide.outputs.a11y }} + rtl: ${{ steps.decide.outputs.rtl }} + steps: + - name: Check out the decision script + uses: actions/checkout@v7 + with: + sparse-checkout: .github/scripts/run-labels.cjs + sparse-checkout-cone-mode: false + path: .run-labels + persist-credentials: false + + - name: Decide which suites run + id: decide + uses: actions/github-script@v9 + with: + retries: 3 + script: | + const {decideRunLabels} = require( + `${process.env.GITHUB_WORKSPACE}/.run-labels/.github/scripts/run-labels.cjs`, + ); + await decideRunLabels({github, context, core}); + build-storybook: if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && ((github.event_name != 'workflow_dispatch' || inputs.operation == 'release-check') && always() && !cancelled()) }} # Every browser lane waits for this job, and the package builds and the @@ -1403,8 +1444,8 @@ jobs: pr-a11y-guards: name: pr-a11y (audit and guards) - needs: [build-storybook] - if: ((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:heavy-lanes')) || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check')) + needs: [build-storybook, run-labels] + if: ((github.event_name == 'pull_request' && needs.run-labels.outputs.a11y == 'true') || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check')) runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check' && github.triggering_actor != 'github-actions[bot]') && '2-core-ubuntu-arm' || 'ubuntu-24.04-arm' }} timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 360 || 15 }} permissions: @@ -1534,8 +1575,8 @@ jobs: # inside the budget without weakening the stable pr-a11y join. pr-a11y-specs: name: pr-a11y (browser specs) - needs: [build-storybook] - if: ((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:heavy-lanes')) || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check')) + needs: [build-storybook, run-labels] + if: ((github.event_name == 'pull_request' && needs.run-labels.outputs.a11y == 'true') || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check')) runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check' && github.triggering_actor != 'github-actions[bot]') && '2-core-ubuntu-arm' || 'ubuntu-24.04-arm' }} timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && 360 || 15 }} permissions: @@ -1624,8 +1665,8 @@ jobs: # Stable required context joining the scoped audit/guards and browser specs. pr-a11y: - needs: [pr-a11y-guards, pr-a11y-specs] - if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (always() && ((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:heavy-lanes')) || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check'))) }} + needs: [pr-a11y-guards, pr-a11y-specs, run-labels] + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (always() && ((github.event_name == 'pull_request' && needs.run-labels.outputs.a11y == 'true') || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check'))) }} runs-on: ubuntu-slim timeout-minutes: 5 permissions: @@ -2059,11 +2100,11 @@ jobs: # joins them only when every shard succeeded, and otherwise scouts inline. visual-scout: name: Stable visual scout (${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [build-storybook] + needs: [build-storybook, run-labels] if: >- (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (!cancelled() && needs.build-storybook.result == 'success' && - ((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:heavy-lanes')) || + ((github.event_name == 'pull_request' && needs.run-labels.outputs.visual == 'true') || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check'))) # Five legs: at three, each leg's scout took ~4.5 minutes and the visual # regression waited for the slowest one. @@ -2174,10 +2215,10 @@ jobs: # Audit and interaction fixtures stay with their existing browser checks. pr-visual: name: Stable visual regression - needs: [build-storybook, maintenance-request, visual-scout] + needs: [build-storybook, maintenance-request, visual-scout, run-labels] if: >- (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (!cancelled() && - (((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:heavy-lanes')) && + (((github.event_name == 'pull_request' && needs.run-labels.outputs.visual == 'true') && needs.build-storybook.result == 'success') || (github.event_name == 'workflow_dispatch' && ((inputs.operation == 'release-check' && @@ -2492,9 +2533,9 @@ jobs: # To promote findings too, remove the audit step's continue-on-error. pr-rtl-shard: name: pr-rtl (${{ matrix.package }}${{ matrix.parts > 1 && format(' {0}/{1}', matrix.part, matrix.parts) || '' }}) - needs: [build-storybook] + needs: [build-storybook, run-labels] if: >- - ((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:heavy-lanes')) || + ((github.event_name == 'pull_request' && needs.run-labels.outputs.rtl == 'true') || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check')) # Core is the largest package, so its audit is split three ways # (`--shard`) and joined by pr-rtl-merge; every other package runs whole. @@ -2694,10 +2735,10 @@ jobs: # part fails here. pr-rtl-merge: name: pr-rtl (core) - needs: [build-storybook, pr-rtl-shard] + needs: [build-storybook, pr-rtl-shard, run-labels] if: >- (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (always() && !cancelled() && - ((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:heavy-lanes')) || + ((github.event_name == 'pull_request' && needs.run-labels.outputs.rtl == 'true') || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check'))) runs-on: ${{ (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check' && github.triggering_actor != 'github-actions[bot]') && '2-core-ubuntu-arm' || 'ubuntu-24.04-arm' }} timeout-minutes: 15 @@ -2802,8 +2843,8 @@ jobs: # Stable required context joining the five package shards above. pr-rtl: - needs: [pr-rtl-shard, pr-rtl-merge] - if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (always() && !cancelled() && ((github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:heavy-lanes')) || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check'))) }} + needs: [pr-rtl-shard, pr-rtl-merge, run-labels] + if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft && (github.event.pull_request.head.repo.full_name == github.repository || contains(github.event.pull_request.labels.*.name, 'ci:run'))) && (always() && !cancelled() && ((github.event_name == 'pull_request' && needs.run-labels.outputs.rtl == 'true') || (github.event_name == 'workflow_dispatch' && inputs.operation == 'release-check'))) }} runs-on: ubuntu-slim timeout-minutes: 5 permissions: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d160091f3967d..9cf556d6af9a1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -386,7 +386,7 @@ Two repo-side rules worth restating here: one is a review reject. - CI is the enforcement layer, not a replacement for the checklist: the `pr-a11y` job in `ci.yml` runs an axe audit after merges to `main` and - before each release (and on a pull request labelled `ci:heavy-lanes`), at + before each release (and on a pull request labelled `ci:run-a11y`), at full scope across the component surface, and the `useAnnounce` lint rule rejects hand-wired `aria-live` regions. axe only catches static, DOM-level issues — keyboard behavior, focus management, and @@ -533,7 +533,7 @@ any other value: ``` These routes apply when a pull request runs the accessibility owner, which it -does only with the `ci:heavy-lanes` label; `main` and release checks run every +does only with the `ci:run-a11y` label; `main` and release checks run every spec. `pnpm test:a11y-contract:pr origin/main --dry-run` prints what a pull request from your branch would run (component tokens need CI's `analysis.json`; without it the runner says so and leaves those specs out). A spec writes its receipts under @@ -548,7 +548,7 @@ component set, or has no audit spec has an entry in ### Accessibility audits Merges to `main` (checked at the newest commit of a burst), every release -check, and a PR labelled `ci:heavy-lanes` +check, and a PR labelled `ci:run-a11y` that touches components run an axe-core audit (the `pr-a11y` CI job); on a PR it covers the Storybook stories of the changed components. The job **fails** when it finds a violation that is not listed in the checked-in baseline, @@ -924,10 +924,14 @@ Storybook story for anything you added or altered. push, or when the maintainer closes and reopens the pull request; CI then runs on each push while the label stays. Until then the required checks fail with "Waiting for a maintainer's ci:run label". -- **`ci:heavy-lanes` opts a pull request into the visual, accessibility and - RTL suites**, which otherwise run on `main` after it merges. A maintainer - applies it when a change needs that evidence before merging; like `ci:run`, - it takes effect on the next push or a close and reopen. +- **`ci:run-visual`, `ci:run-a11y` and `ci:run-rtl` each opt a pull request + into one heavy suite** (visual regression, accessibility, RTL), which + otherwise runs on `main` after the pull request merges. Someone with write + access applies the label for the evidence a change needs before merging; a + label applied without write access runs nothing. Like `ci:run`, a label + takes effect on the next push or a close and reopen, and removing it stops + that suite from the next run. The pull-request comment names the label for + each suite that did not run. - **Pull requests from a branch in this repository run CI without the label**, since pushing a branch here needs write access. Maintainers who work from a fork either push their branches here or apply `ci:run` themselves. diff --git a/apps/storybook/rtl-audit/README.md b/apps/storybook/rtl-audit/README.md index b237e75025dde..31084cfd8592d 100644 --- a/apps/storybook/rtl-audit/README.md +++ b/apps/storybook/rtl-audit/README.md @@ -19,7 +19,7 @@ independent auto passes: **D1 (icon-mirror)**, **D5 (positional-mirror)**, and **D6 (directional-decoration)**. > **Scope in CI.** `pr-rtl` runs after merges to `main` and in release checks at -> full scope, and on a PR only with the `ci:heavy-lanes` label (spec:AST-030 +> full scope, and on a PR only with the `ci:run-rtl` label (spec:AST-030 > FR12). On a labelled PR it passes `--filter` with the components the PR > touched (from `analysis.json`), matching `pr-a11y`. Eligibility is classified > by the component policy and package registry loaded from the trusted base ref; @@ -36,7 +36,7 @@ independent auto passes: **D1 (icon-mirror)**, **D5 (positional-mirror)**, and > policy requires full scope, every manifest must be runnable with an empty > filter and all five package reports must arrive. A planned scan ending in ERROR > is incomplete. Missing or malformed analysis/index/manifest input, a noncanonical owner, stale output, a missing or unexpected report, and -> an all-skipped required matrix all fail closed. An RTL-harness-only PR labelled `ci:heavy-lanes` runs the fixed +> an all-skipped required matrix all fail closed. An RTL-harness-only PR labelled `ci:run-rtl` runs the fixed > `Chart,ChartLegend` routing smoke scope so package discovery and curated aliases > cannot skip their own check; without the label, `main`'s `release-check` covers > the harness after merge (spec:AST-030 FR12). The diff --git a/docs/architecture/rtl-audit-evidence.md b/docs/architecture/rtl-audit-evidence.md index 17ea4b7491d50..70124d51c800e 100644 --- a/docs/architecture/rtl-audit-evidence.md +++ b/docs/architecture/rtl-audit-evidence.md @@ -152,7 +152,7 @@ Lint and rendered audit are complementary: - **INV7 — PR scope and full-roster scope answer different questions.** The RTL audit runs after merges to `main` (a burst of pushes is checked once, at its newest commit) and before each release, at full scope; a pull request runs it - only with the `ci:heavy-lanes` label (spec:AST-030 FR12, DEC-7). When a pull + only with the `ci:run-rtl` label (spec:AST-030 FR12, DEC-7). When a pull request runs it, it checks components affected by the change; it may run per package, but the PR result is the join of every package scope and cannot pass with one scope missing. The full-scope run checks the live roster and shared diff --git a/docs/specs/AST-030/spec.md b/docs/specs/AST-030/spec.md index af737b39e54b3..9fdc9bf939497 100644 --- a/docs/specs/AST-030/spec.md +++ b/docs/specs/AST-030/spec.md @@ -191,10 +191,13 @@ they are separately named. visual plan and unfiltered accessibility/RTL component rosters. Changed-file, focused, smoke, and no-op paths MUST NOT narrow release checks. On a pull request, the visual (`visual-scout`, `pr-visual`), accessibility (`pr-a11y` - with its guards and specs) and RTL (`pr-rtl` with its legs) owners MUST run - only when the pull request carries the `ci:heavy-lanes` label at the event; a - pull request update without it runs none of them, and no required context - depends on them. An opted-in PR's a11y MUST remain scoped to changed components + with its guards and specs) and RTL (`pr-rtl` with its legs) owners MUST each + run only when the pull request carries that suite's label at the event: + `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl`, in each case last applied by + someone with write, maintain or admin access, and a lookup that cannot + confirm that runs nothing. A label runs only its own suite; a pull request + update without one runs none of them, and no required context depends on + them. An opted-in PR's a11y MUST remain scoped to changed components plus fast interaction guards; full a11y sweeps, whole-repository spec-test contracts, and their evidence uploads MUST run only for `release-check` in this workflow. Missing, @@ -232,7 +235,7 @@ they are separately named. reads sources for it (the TypeScript compiler, a bundler) counts as reading sources; and every test that loads a changed workspace package from its build, by the package's name or a path into its `dist/`. When a pull request runs - them (FR12: only with the `ci:heavy-lanes` label), the visual and RTL owners + them (FR12: only with their `ci:run-visual` or `ci:run-rtl` label), the visual and RTL owners audit one affected set of component folders: the changed components and their reverse dependents, the folders of the components, in any package, whose built bundle, `import()` targets included, reaches a changed module. @@ -241,8 +244,8 @@ they are separately named. audits every component each of those folders implements. A Changeset renders nothing and widens no audit. A changed module that no component bundle holds adds none, and without a readable build the audits run in full. PR accessibility, when it runs, stays scoped to changed components (FR12). Shared infrastructure runs the full suite of - the owners the pull request runs (visual, accessibility and RTL only with the - `ci:heavy-lanes` label, FR12): root + the owners the pull request runs (visual, accessibility and RTL only with + their `ci:run-*` labels, FR12): root dependency manifests and lockfile, root configuration, a package's manifest, build or test configuration and test setup, `internal/` (the test harness included), `theme-build` paths (tokens and themes), Storybook configuration, @@ -301,7 +304,8 @@ they are separately named. suites. A re-run, which is the pre-merge restart, is exempt from both limits, and when the limits cannot be looked up the suite runs. Merge-queue entries, `main`, release checks and the visual, accessibility and RTL owners a - maintainer opts a pull request into with `ci:heavy-lanes` are unaffected. + maintainer opts a pull request into with their `ci:run-*` labels are + unaffected. - **FR17 — A pull request runs CI when it is ready and a maintainer can vouch for it.** A draft pull request runs no CI, and marking it ready for review starts its pull-request run, which runs what the change reaches (FR13, @@ -472,7 +476,7 @@ retroactive status as release evidence. Run the visual, accessibility and RTL owners on every push to `main`, through the same exact-commit `release-check` the release gate uses, and before a cut; -a pull request runs them only when it opts in with the `ci:heavy-lanes` label. Main-push checks are throttled: one runs at a time, a +a pull request runs each only when it opts in with that suite's `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` label. Main-push checks are throttled: one runs at a time, a burst of pushes collapses to its newest commit, and a running check is never cancelled, so every check that starts finishes. Pull request updates keep the fast checks and every required context; the @@ -500,7 +504,7 @@ read sources through the file system run whole, because no import reaches what they read. Shared infrastructure, uncertainty, `main`, merge-group entries and release checks run everything, so no landing or release gate narrows; a pull request runs the visual, accessibility and RTL owners only with the -`ci:heavy-lanes` label (DEC-7). +suite's `ci:run-*` label (DEC-7). Rejected: a full run triggered by approval, a label or a draft-to-ready change, which tests the pull request's head rather than the merged result. From 207c6fa7cabf74c34ef13d1ac36df57662729ef8 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 10:07:02 -0700 Subject: [PATCH 04/15] ci: the PR report leads with one status table: this head's preview, size, visual, accessibility, RTL, perf when it ran, and the CI jobs; touched components link their docs and stories --- .github/scripts/generate-pr-comment.js | 133 +++--- .github/scripts/generate-pr-comment.test.mjs | 138 ++++--- .github/scripts/lib/pr-preview.mjs | 138 ++++++- .github/scripts/lib/pr-preview.test.mjs | 143 ++++++- .github/scripts/lib/pr-status.js | 379 ++++++++++++++++++ .github/scripts/lib/pr-status.test.mjs | 301 ++++++++++++++ .../visual-gate/workflow-concurrency.test.mjs | 12 +- .github/workflows/pr-comment.yml | 3 + 8 files changed, 1123 insertions(+), 124 deletions(-) create mode 100644 .github/scripts/lib/pr-status.js create mode 100644 .github/scripts/lib/pr-status.test.mjs diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.js index 13fb8e6a0228d..a97567eb1bc1e 100644 --- a/.github/scripts/generate-pr-comment.js +++ b/.github/scripts/generate-pr-comment.js @@ -3,14 +3,22 @@ /** * @description Generates and posts PR comment with analysis results - * @input --analysis --base-stats --package-size --a11y --visual --storybook-url --sandbox-url --preview-state --source-conclusion --run-url - * @output Formatted markdown comment body to stdout + * @input --analysis --base-stats --package-size --a11y --visual --preview --jobs --touched --source-conclusion --run-url + * @output Formatted markdown comment body to stdout: one status table first + * (this head's preview, size, visual, accessibility, RTL, perf when it ran, + * and the CI jobs), then the touched components with their preview links + * and results, then the detail sections, collapsed */ const fs = require('node:fs'); const {buildA11ySection} = require('./lib/a11y-format'); const {buildVisualSection} = require('./lib/visual-format'); const {renderPackageSizeSection} = require('./lib/package-size'); +const { + buildStatusTable, + buildJobTable, + buildTouchedComponents, +} = require('./lib/pr-status'); // Report-file strings render as literal inline text, report numbers as // numbers — the comment's structure comes only from this file's literals. const {inline, num} = require('./lib/report-text'); @@ -28,11 +36,30 @@ const a11yFile = getArg('a11y') || 'a11y-report.json'; const visualFile = getArg('visual'); const runUrl = getArg('run-url') || ''; const prNumber = getArg('pr-number') || ''; -const storybookUrl = getArg('storybook-url') || ''; -const sandboxUrl = getArg('sandbox-url') || ''; -const previewState = getArg('preview-state') || ''; const sourceConclusion = getArg('source-conclusion') || ''; +// Optional JSON inputs the comment job writes: this head's preview +// ({state, url, sha, aliasOrigin}), the source run's jobs, and the +// touched-components report. A missing or unreadable file leaves its part out. +function readOptionalJson(name) { + const file = getArg(name); + if (!file) return null; + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (e) { + console.error(`Warning: Could not read --${name}:`, e.message); + return null; + } +} +const preview = readOptionalJson('preview'); +const jobs = readOptionalJson('jobs'); +const touched = readOptionalJson('touched'); +// Links go to this head's own deployment when it is ready, else to the +// branch alias (the latest ready deployment). +const previewOrigin = + preview?.state === 'ready' ? preview.url : preview?.aliasOrigin || ''; +const storybookUrl = previewOrigin ? `${previewOrigin}/storybook/` : ''; + // Read analysis results let analysis = { newComponents: [], @@ -308,57 +335,66 @@ if (analysis.bundleDelta) { // and any runtime dependency added or removed. The caller has already matched // its head. let packageSizeSection = ''; +let packageSize = null; if (packageSizeFile) { try { - packageSizeSection = renderPackageSizeSection( - JSON.parse(fs.readFileSync(packageSizeFile, 'utf8')), - inline, - ); + packageSize = JSON.parse(fs.readFileSync(packageSizeFile, 'utf8')); + packageSizeSection = renderPackageSizeSection(packageSize, inline); } catch (e) { console.error('Warning: Could not read package size report:', e.message); } } -// Build storybook link section -let storybookSection = ''; -if (storybookUrl) { - storybookSection = `### 📚 Storybook Preview - -**${extLink('View Storybook for this PR', storybookUrl)}** - -`; -} - -// Build sandbox link section -let sandboxSection = ''; -if (sandboxUrl) { - sandboxSection = `### 🧪 Sandbox Preview +// Footer links: the full CI run. +const footerLinksStr = runUrl + ? ` | ${extLink('View full report', runUrl)}` + : ''; -**${extLink('View Sandbox for this PR', sandboxUrl)}** - -`; -} +// A run that did not succeed may have published nothing for this head. +const conclusionNote = + sourceConclusion && sourceConclusion !== 'success' + ? `> **Current run:** CI concluded ${inline(sourceConclusion)}; parts of this report may be missing.\n\n` + : ''; -// Both previews come from the branch's Vercel alias, which serves the -// branch's latest deployment; until a push's deployment is ready it still -// shows the previous head. Vercel shortens a long alias with a hash, and then there are -// no links. -let previewAvailabilitySection = ''; -if (previewState) { - if (!storybookUrl || !sandboxUrl) - previewAvailabilitySection = `> **Preview availability:** this branch's preview address cannot be known in advance; open the preview from Vercel's comment on this pull request.${sourceConclusion && sourceConclusion !== 'success' ? ` CI concluded ${sourceConclusion}.` : ''}\n\n`; - else - previewAvailabilitySection = - "> **Preview:** the links below open this branch's latest Vercel deployment, which can trail a new push until that push's deployment is ready.\n\n"; -} +const statusTable = buildStatusTable({ + preview, + analysis, + budgetFlags, + packageSize, + a11yReport, + visualVerdict, + jobs, +}); +const touchedSection = buildTouchedComponents({ + analysis, + origin: previewOrigin, + touched, + a11yReport, + visualVerdict, +}); -// Build footer with links -let footerLinks = []; -if (storybookUrl) footerLinks.push(extLink('Storybook', storybookUrl)); -if (sandboxUrl) footerLinks.push(extLink('Sandbox', sandboxUrl)); -if (runUrl) footerLinks.push(extLink('View full report', runUrl)); -const footerLinksStr = - footerLinks.length > 0 ? ` | ${footerLinks.join(' | ')}` : ''; +// Detail sections, collapsed: the table above carries their verdicts. +const details = (summary, content) => + content + ? `
\n${summary}\n\n${content}\n
\n\n` + : ''; +const sizeDetails = details( + 'Size details', + `${sizeSection}${budgetSection}${bundleSection}${packageSizeSection}`, +); +const componentDetails = details('Component details', componentSection); +// Visual changes are a question for the reviewer: their frames stay open. +const visualDetails = visualSection + ? visualVerdict?.changes?.length > 0 + ? visualSection + : details('Visual regression', visualSection) + : ''; +// The accessibility detail only matters when the audit found something. +const a11yViolations = Object.values(a11yReport?.components || {}).some( + component => (component.violations?.length || 0) > 0, +); +const a11yDetails = a11yViolations ? a11ySection : ''; +const jobTable = buildJobTable(jobs); // A one-line caveat shown when the analysis fell back to the approximate // two-dot diff. The file list may include base-only churn and, worse, may @@ -373,10 +409,7 @@ const diffModeCaveat = const body = `## PR Analysis Report -${diffModeCaveat}${previewAvailabilitySection}${storybookSection}${sandboxSection}${componentSection || (sizeSection ? '' : '_No new or modified components detected._\n\n')} -${sizeSection}${budgetSection}${bundleSection}${packageSizeSection} -${a11ySection} -${visualSection}--- +${conclusionNote}${diffModeCaveat}${statusTable}${touchedSection}${sizeDetails}${componentDetails}${visualDetails}${a11yDetails}${jobTable}--- Generated by PR Enrichment workflow${footerLinksStr} `; diff --git a/.github/scripts/generate-pr-comment.test.mjs b/.github/scripts/generate-pr-comment.test.mjs index a95fdf27d3c03..178a96faf658a 100644 --- a/.github/scripts/generate-pr-comment.test.mjs +++ b/.github/scripts/generate-pr-comment.test.mjs @@ -66,63 +66,103 @@ describe('generate-pr-comment diffMode caveat', () => { }); }); -describe('generate-pr-comment preview availability', () => { +describe('generate-pr-comment status table', () => { + function withInputs(inputs, analysis = baseAnalysis, extra = []) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-status-')); + try { + const args = []; + for (const [name, value] of Object.entries(inputs)) { + const file = path.join(dir, `${name}.json`); + fs.writeFileSync(file, JSON.stringify(value)); + args.push(`--${name}`, file); + } + return runComment(analysis, [...args, ...extra]); + } finally { + fs.rmSync(dir, {recursive: true, force: true}); + } + } + const HEAD = 'a'.repeat(40); + const READY = 'https://astryx-abc123-fbopensource.vercel.app'; + const ALIAS = 'https://astryx-git-my-branch-fbopensource.vercel.app'; + + it('leads with the status table, the preview first', () => { + const out = withInputs({preview: {state: 'ready', sha: HEAD, url: READY}}); + const table = out.indexOf('| Check | Result |'); + expect(table).toBeGreaterThan(out.indexOf('')); + expect(out.slice(table)).toMatch( + /^\| Check \| Result \|\n\|-------\|--------\|\n\| Preview \|/, + ); + expect(out).toContain( + `| Preview | [Storybook](${READY}/storybook/) · [Sandbox](${READY}/sandbox/) · [Docs](${READY}/) · built from \`aaaaaaa\` |`, + ); + expect(out).not.toMatch(/CodeBunny/i); + }); + it.each([ - ['none', [], false, false], [ - 'storybook', - ['--storybook-url', 'https://example.test/pr/1/'], - true, - false, + 'building', + {state: 'building', sha: HEAD, aliasOrigin: ALIAS}, + `⏳ Preview building for \`aaaaaaa\`… Until it's ready, [Storybook](${ALIAS}/storybook/)`, ], + ['failed', {state: 'failed', sha: HEAD}, 'failed to build'], [ - 'sandbox', - ['--sandbox-url', 'https://example.test/pr/1/sandbox/'], - false, - true, - ], - [ - 'both', - [ - '--storybook-url', - 'https://example.test/pr/1/', - '--sandbox-url', - 'https://example.test/pr/1/sandbox/', - ], - true, - true, + 'fork', + {state: 'fork', sha: HEAD}, + 'No preview for pull requests from forks.', ], - ])( - 'renders independent %s availability', - (state, urls, storybook, sandbox) => { - const stdout = runComment(baseAnalysis, [ - ...urls, - '--preview-state', - state, - '--source-conclusion', - 'success', - ]); + ['unknown', {state: 'unknown', sha: HEAD}, 'cannot be known in advance'], + ])('says what a %s preview is, honestly', (_label, preview, text) => { + const out = withInputs({preview}); + expect(out).toContain(text); + if (preview.state !== 'building') expect(out).not.toContain('/storybook/'); + }); - expect(stdout.includes('View Storybook for this PR')).toBe(storybook); - expect(stdout.includes('View Sandbox for this PR')).toBe(sandbox); - expect(stdout.includes('> **Preview availability:**')).toBe( - !storybook || !sandbox, - ); - expect(stdout).toContain(''); - }, - ); + it('shows no preview row on a draft, which links no preview', () => { + const out = withInputs({}); + expect(out).toContain('| Check | Result |'); + expect(out).not.toContain('| Preview |'); + }); - it('explains that failed CI published no previews', () => { - const stdout = runComment(baseAnalysis, [ - '--preview-state', - 'none', - '--source-conclusion', - 'failure', - ]); + it('says a heavy suite did not run, with the label that runs it', () => { + const out = withInputs({ + jobs: [ + {name: 'pr-visual', conclusion: 'skipped', steps: []}, + {name: 'pr-rtl', conclusion: 'skipped', steps: []}, + ], + }); + expect(out).toContain( + '| Visual | ⏭ Visual did not run on this pull request; add `ci:run-visual` to run it. |', + ); + expect(out).toContain( + '| Accessibility | ⏭ Accessibility did not run on this pull request; add `ci:run-a11y` to run it. |', + ); + expect(out).toContain( + '| RTL | ⏭ RTL did not run on this pull request; add `ci:run-rtl` to run it. |', + ); + expect(out).not.toContain('| Perf |'); + }); + + it('notes a run that did not succeed', () => { + const out = runComment(baseAnalysis, ['--source-conclusion', 'failure']); + expect(out).toContain('CI concluded failure'); + }); - expect(stdout).toContain('CI concluded failure'); - expect(stdout).not.toContain('View Storybook for this PR'); - expect(stdout).not.toContain('View Sandbox for this PR'); + it('collapses the size tables', () => { + const out = runComment({ + ...baseAnalysis, + componentSizes: { + 'core/Card': { + package: '@astryxdesign/core', + importGzipBytes: 2000, + cssGzipBytes: 100, + }, + }, + }); + expect(out).toContain('Size details'); + expect(out.indexOf('Size details')).toBeLessThan( + out.indexOf('### Component Size'), + ); + expect(out).toMatch(/\| Size \| ✅ 1 component\(s\) within budget/); }); }); diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjs index 50894a05a4006..23540aaddcb63 100644 --- a/.github/scripts/lib/pr-preview.mjs +++ b/.github/scripts/lib/pr-preview.mjs @@ -2,6 +2,7 @@ import {execFileSync} from 'node:child_process'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import process from 'node:process'; import {fileURLToPath} from 'node:url'; @@ -49,6 +50,99 @@ function refuse(message) { throw new Error(`PR preview refused: ${message}`); } +const VERCEL_DEPLOYMENT_URL = /^https:\/\/[a-z0-9-]+\.vercel\.app\/?$/; + +/** + * This head's Vercel preview, from the deployment Vercel reports to GitHub for + * the exact commit: `ready` (that deployment's own address, so every link + * shows this head), `failed`, or `building` until Vercel reports it (the + * branch alias then serves the previous ready deployment). Previews are not + * built for forks. Two reads in the comment job; no workflow runs on a + * deployment event. + */ +export async function headPreview({ + github, + owner, + repo, + identity, + aliasOrigin, +}) { + const sha = identity.headSha; + if (identity.headRepository !== identity.baseRepository) + return {state: 'fork', sha}; + try { + const {data: deployments} = await github.rest.repos.listDeployments({ + owner, + repo, + sha, + per_page: 20, + }); + const deployment = deployments + .filter( + candidate => + candidate.creator?.login === 'vercel[bot]' && + String(candidate.environment).toLowerCase() === 'preview', + ) + .sort((a, b) => b.id - a.id)[0]; + if (deployment) { + const {data: statuses} = await github.rest.repos.listDeploymentStatuses({ + owner, + repo, + deployment_id: deployment.id, + per_page: 1, + }); + const latest = statuses[0]; + if ( + latest?.state === 'success' && + VERCEL_DEPLOYMENT_URL.test(String(latest.environment_url ?? '')) + ) + return { + state: 'ready', + sha, + url: latest.environment_url.replace(/\/$/, ''), + }; + if (latest?.state === 'failure' || latest?.state === 'error') + return {state: 'failed', sha}; + } + } catch { + // An unreadable deployment list leaves the preview as still building. + } + return aliasOrigin + ? {state: 'building', sha, aliasOrigin} + : {state: 'unknown', sha}; +} + +/** + * The source run's jobs (latest attempt) with each step's conclusion, so the + * report can say which jobs ran and which skipped their work by path. Null + * when the list cannot be read. + */ +export async function sourceRunJobs({github, owner, repo, runId}) { + try { + const jobs = await github.paginate( + github.rest.actions.listJobsForWorkflowRun, + { + owner, + repo, + run_id: runId, + filter: 'latest', + per_page: 100, + }, + ); + return jobs.map(job => ({ + name: job.name, + status: job.status, + conclusion: job.conclusion, + steps: (job.steps || []).map(step => ({ + name: step.name, + conclusion: step.conclusion, + })), + })); + } catch { + return null; + } +} + /** * The pull request moved on from the source run: it closed, merged, or got a * newer head. That is ordinary churn, not a forged identity. Nothing is @@ -415,13 +509,6 @@ export function validateAnalysisMetadata(metadata, identity) { return metadata; } -function previewState(storybook, sandbox) { - if (storybook && sandbox) return 'both'; - if (storybook) return 'storybook'; - if (sandbox) return 'sandbox'; - return 'none'; -} - // The same preview wording as the generator's report: the branch alias // serves the branch's latest deployment, which can trail a new push. A draft // or an update-only reconcile (no alias attempted) says nothing about previews. @@ -513,6 +600,7 @@ export async function reconcilePrComment({ metadataPath = 'pr-analysis/pr-meta.json', baseStatsPath = 'pr-size-base/size-base.json', packageSizePath = 'pr-analysis/package-size.json', + touchedPath = 'pr-analysis/touched-components.json', a11yPath = 'a11y/a11y-report.json', visualPath = 'trusted-visual/verdict.json', createIfMissing = true, @@ -545,8 +633,6 @@ export async function reconcilePrComment({ // preview and says nothing about one. const aliasAttempted = createIfMissing && !identity.draft; const previewOrigin = aliasAttempted ? branchPreviewOrigin(identity) : null; - const storybook = previewOrigin !== null; - const sandbox = storybook; const comments = await allComments(github, owner, repo, identity.prNumber); const botComments = comments.filter(comment => comment.user?.type === 'Bot'); const botComment = @@ -561,6 +647,27 @@ export async function reconcilePrComment({ let body; if (analysisReady) { + // The comment job's own reads, handed to the generator as files. + const inputs = fs.mkdtempSync(path.join(os.tmpdir(), 'pr-report-')); + const preview = aliasAttempted + ? await headPreview({ + github, + owner, + repo, + identity, + aliasOrigin: previewOrigin, + }) + : null; + const jobs = await sourceRunJobs({ + github, + owner, + repo, + runId: identity.sourceRunId, + }); + const previewPath = path.join(inputs, 'preview.json'); + const jobsPath = path.join(inputs, 'jobs.json'); + if (preview) fs.writeFileSync(previewPath, JSON.stringify(preview)); + if (jobs) fs.writeFileSync(jobsPath, JSON.stringify(jobs)); const args = [ generator, '--analysis', @@ -585,11 +692,16 @@ export async function reconcilePrComment({ // checks, or with the ci:heavy-lanes label); the comment says so. ...(fs.existsSync(a11yPath) ? ['--a11y', a11yPath] : []), ...(fs.existsSync(visualPath) ? ['--visual', visualPath] : []), - ...(storybook ? ['--storybook-url', `${previewOrigin}/storybook/`] : []), - ...(sandbox ? ['--sandbox-url', `${previewOrigin}/sandbox/`] : []), - ...(aliasAttempted - ? ['--preview-state', previewState(storybook, sandbox)] + ...(trustedHeadReport({ + reportPath: touchedPath, + label: 'touched-components report', + identity, + core, + }) + ? ['--touched', touchedPath] : []), + ...(preview ? ['--preview', previewPath] : []), + ...(jobs ? ['--jobs', jobsPath] : []), '--source-conclusion', identity.sourceConclusion, '--run-url', diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjs index c015c2b8bdaf8..720473aedd88e 100644 --- a/.github/scripts/lib/pr-preview.test.mjs +++ b/.github/scripts/lib/pr-preview.test.mjs @@ -815,16 +815,141 @@ describe('PR comment preview reconciliation', () => { ['alias known', true], ])('shows both or neither branch links: %s', async (_label, ready) => { const {result} = await reconcile({previewAvailable: ready}); - expect(result.body.includes('View Storybook for this PR')).toBe(ready); - expect(result.body.includes('View Sandbox for this PR')).toBe(ready); - expect(result.body.includes('> **Preview availability:**')).toBe(!ready); - if (ready) { - expect(result.body).toContain(`${VERCEL_ORIGIN}/storybook/`); - expect(result.body).toContain(`${VERCEL_ORIGIN}/sandbox/`); - } + expect( + result.body.includes(`[Storybook](${VERCEL_ORIGIN}/storybook/)`), + ).toBe(ready); + expect(result.body.includes(`[Sandbox](${VERCEL_ORIGIN}/sandbox/)`)).toBe( + ready, + ); + // identity() opens from a fork, which Vercel builds no preview for. + expect( + result.body.includes('No preview for pull requests from forks.'), + ).toBe(!ready); expect(result.body).not.toContain('facebook.github.io/astryx/pr/'); }); + describe("this head's preview and the run's jobs", () => { + const deployment = (overrides = {}) => ({ + id: 7, + environment: 'Preview', + creator: {login: 'vercel[bot]'}, + ...overrides, + }); + async function withGithub({deployments = [], statuses = [], jobs = []}) { + const value = identity({headRepository: 'meta/astryx'}); + const harness = githubFixture({value}); + harness.github.rest.repos = { + listDeployments: vi.fn(async () => ({data: deployments})), + listDeploymentStatuses: vi.fn(async () => ({data: statuses})), + }; + harness.github.rest.actions.listJobsForWorkflowRun = vi.fn(async () => ({ + data: jobs, + })); + const paths = fixture(value); + const seen = []; + const result = await reconcilePrComment({ + github: harness.github, + core: {info: vi.fn(), warning: vi.fn()}, + context: { + repo: {owner: 'meta', repo: 'astryx'}, + serverUrl: 'https://github.com', + }, + expectedIdentity: value, + analysisPath: paths.analysis, + metadataPath: paths.metadata, + a11yPath: paths.a11y, + visualPath: paths.visual, + execute: (node, args, options) => { + seen.push(args); + return execFileSync(node, args, options); + }, + }); + return {result, harness, args: seen[0]}; + } + + it("links Vercel's ready deployment of this exact head", async () => { + const {result, harness} = await withGithub({ + deployments: [deployment()], + statuses: [ + { + state: 'success', + environment_url: 'https://astryx-abc123-fbopensource.vercel.app', + }, + ], + }); + expect(harness.github.rest.repos.listDeployments).toHaveBeenCalledWith( + expect.objectContaining({sha: HEAD}), + ); + expect(result.body).toContain( + '[Storybook](https://astryx-abc123-fbopensource.vercel.app/storybook/)', + ); + expect(result.body).toContain('built from `aaaaaaa`'); + }); + + it("ignores a deployment Vercel did not create, and an address that is not Vercel's", async () => { + for (const setup of [ + { + deployments: [deployment({creator: {login: 'someone'}})], + statuses: [ + { + state: 'success', + environment_url: 'https://astryx-abc123-fbopensource.vercel.app', + }, + ], + }, + { + deployments: [deployment()], + statuses: [ + {state: 'success', environment_url: 'https://evil.example/'}, + ], + }, + ]) { + const {result} = await withGithub(setup); + expect(result.body).not.toContain('astryx-abc123'); + expect(result.body).not.toContain('evil.example'); + expect(result.body).toContain('Preview building'); + } + }); + + it('says a failed preview failed', async () => { + const {result} = await withGithub({ + deployments: [deployment()], + statuses: [{state: 'failure'}], + }); + expect(result.body).toContain('failed to build'); + }); + + it("passes the run's jobs, so the report says what ran and what path scoping skipped", async () => { + const {result, args} = await withGithub({ + jobs: [ + { + name: 'test-node (1/4)', + status: 'completed', + conclusion: 'success', + steps: [ + {name: "Decide this job's scope", conclusion: 'success'}, + {name: 'Run tests', conclusion: 'success'}, + ], + }, + { + name: 'docsite-test', + status: 'completed', + conclusion: 'success', + steps: [ + {name: "Decide this job's scope", conclusion: 'success'}, + {name: 'Build docsite', conclusion: 'skipped'}, + ], + }, + ], + }); + expect(args).toContain('--jobs'); + expect(result.body).toContain('| test-node | ✅ passed | |'); + expect(result.body).toContain( + '| docsite-test | ⏭ skipped | nothing in its paths changed |', + ); + }); + }); + describe('the fallback report, when there is no trusted analysis', () => { const failRender = () => { throw new Error('malformed analysis'); @@ -1095,10 +1220,10 @@ describe('PR comment preview reconciliation', () => { expect(harness.state.updated).toHaveLength(1); expect(harness.state.comments).toHaveLength(1); expect(harness.state.comments[0].body).toContain( - 'View Storybook for this PR', + `[Storybook](${VERCEL_ORIGIN}/storybook/)`, ); expect(harness.state.comments[0].body).toContain( - 'View Sandbox for this PR', + `[Sandbox](${VERCEL_ORIGIN}/sandbox/)`, ); expect(harness.state.listedIssues).toEqual([5697, 5697]); }); diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js new file mode 100644 index 0000000000000..3a5f1acf70e6c --- /dev/null +++ b/.github/scripts/lib/pr-status.js @@ -0,0 +1,379 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file pr-status.js + * @input The source CI run's job list (names, conclusions, steps), this + * head's Vercel preview state, the analysis, the size reports, the + * accessibility report, the visual verdict and, when present, the + * touched-components report + * @output The PR report's top: one status table (preview first, then size, + * visual, accessibility, RTL, perf when it ran, and CI jobs), the touched + * components with their preview links and results, and the collapsed CI job + * table + * @position Used by generate-pr-comment.js. Everything here reads data the CI + * run and the comment job already have; nothing starts a job. + */ + +const {inline, num} = require('./report-text'); + +/** The heavy suites a pull request runs only on a maintainer's label. */ +const SUITES = [ + {id: 'visual', label: 'Visual', jobs: ['visual-scout', 'pr-visual']}, + {id: 'a11y', label: 'Accessibility', jobs: ['pr-a11y']}, + {id: 'rtl', label: 'RTL', jobs: ['pr-rtl', 'pr-rtl-shard', 'pr-rtl-merge']}, + {id: 'perf', label: 'Perf', jobs: ['perf']}, +]; + +/** The sentence a heavy suite shows when it did not run (one wording everywhere). */ +function notRunSentence(suite) { + return `${suite.label} did not run on this pull request; add \`ci:run-${suite.id}\` to run it.`; +} + +const SCOPE_STEP = "Decide this job's scope"; +const BOOKKEEPING_STEP = /^(Set up job|Complete job|Post )/; + +/** A matrix leg's job name without its `(1/3)` suffix. */ +function baseJobName(name) { + return String(name ?? '').replace(/\s*\([^)]*\)\s*$/, ''); +} + +/** + * What one job did: `ran` (with its conclusion), `scoped-out` (it started, + * decided nothing in its scope changed, and skipped its work), or `not-run` + * (its run condition was false). + */ +function classifyJob(job) { + if (job.conclusion === 'skipped') return {state: 'not-run'}; + const steps = job.steps || []; + const skipStep = steps.find( + step => /^Skip /.test(step.name) && step.conclusion === 'success', + ); + if (skipStep && job.conclusion === 'success') + return { + state: 'scoped-out', + reason: 'specialized change (spec, docsite or tooling only)', + }; + const scopeIndex = steps.findIndex(step => step.name === SCOPE_STEP); + if (scopeIndex !== -1 && job.conclusion === 'success') { + const work = steps + .slice(scopeIndex + 1) + .filter(step => !BOOKKEEPING_STEP.test(step.name)); + if (work.length > 0 && work.every(step => step.conclusion === 'skipped')) + return {state: 'scoped-out', reason: 'nothing in its paths changed'}; + } + return {state: 'ran', conclusion: job.conclusion || job.status || 'unknown'}; +} + +/** Jobs grouped by name (matrix legs together), each with one state. */ +function summarizeJobs(jobs) { + const groups = new Map(); + for (const job of jobs || []) { + const name = baseJobName(job.name); + if (!groups.has(name)) groups.set(name, []); + groups.get(name).push(classifyJob(job)); + } + const summary = []; + for (const [name, legs] of groups) { + const ran = legs.filter(leg => leg.state === 'ran'); + if (ran.length > 0) { + const failed = ran.find(leg => leg.conclusion !== 'success'); + summary.push({ + name, + state: 'ran', + conclusion: failed ? failed.conclusion : 'success', + }); + } else if (legs.some(leg => leg.state === 'scoped-out')) { + summary.push({ + name, + state: 'scoped-out', + reason: legs.find(leg => leg.reason).reason, + }); + } else { + summary.push({name, state: 'not-run'}); + } + } + return summary.sort((a, b) => a.name.localeCompare(b.name)); +} + +function suiteOf(jobName) { + return SUITES.find(suite => suite.jobs.includes(jobName)) ?? null; +} + +const CONCLUSION_TEXT = { + success: '✅ passed', + failure: '❌ failed', + cancelled: '⏹ cancelled', + timed_out: '❌ timed out', +}; + +function conclusionText(conclusion) { + return CONCLUSION_TEXT[conclusion] ?? inline(conclusion); +} + +/** One suite's state from its jobs: ran (and how), scoped out, or not run. */ +function suiteJobState(suite, jobSummary) { + const jobs = jobSummary.filter(job => suite.jobs.includes(job.name)); + if (jobs.length === 0) return null; + const ran = jobs.filter(job => job.state === 'ran'); + if (ran.length > 0) { + const failed = ran.find(job => job.conclusion !== 'success'); + return {state: 'ran', conclusion: failed ? failed.conclusion : 'success'}; + } + if (jobs.some(job => job.state === 'scoped-out')) + return {state: 'scoped-out'}; + return {state: 'not-run'}; +} + +function previewCell(preview) { + if (!preview || preview.state === 'none') return null; + if (preview.state === 'fork') + return 'No preview for pull requests from forks.'; + if (preview.state === 'unknown') + return "This branch's preview address cannot be known in advance; open it from Vercel's comment."; + const links = origin => + `[Storybook](${origin}/storybook/) · [Sandbox](${origin}/sandbox/) · [Docs](${origin}/)`; + if (preview.state === 'ready') + return `${links(preview.url)} · built from \`${inline(String(preview.sha).slice(0, 7))}\``; + if (preview.state === 'failed') + return `❌ The preview for \`${inline(String(preview.sha).slice(0, 7))}\` failed to build; see Vercel's comment.`; + // Building, or Vercel has not reported this head yet: the branch alias + // serves the latest ready deployment, which can be an earlier push. + return `⏳ Preview building for \`${inline(String(preview.sha).slice(0, 7))}\`… ${preview.aliasOrigin ? `Until it's ready, ${links(preview.aliasOrigin)} open the branch's previous preview.` : ''}`.trim(); +} + +function visualCell(verdict, jobState, suite) { + if (verdict) { + if (verdict.status === 'skipped') + return `⏭ skipped: ${inline(verdict.reason)}`; + if (verdict.status === 'failed') + return `❌ ${num(verdict.counts?.failed)} shot(s) could not be captured`; + const changed = verdict.changes?.length ?? 0; + const added = verdict.added?.length ?? 0; + const removed = verdict.removed?.length ?? 0; + if (changed > 0) + return `🔍 ${changed} of ${num(verdict.counts?.total)} shots changed: review below`; + if (added > 0 || removed > 0) + return `🔍 ${added} added · ${removed} removed`; + return `✅ no change across ${num(verdict.counts?.total)} shots`; + } + return jobCell(jobState, suite); +} + +function a11yCell(report, jobState, suite) { + if (report) { + const violations = Object.values(report.components || {}).reduce( + (sum, component) => sum + (component.violations?.length || 0), + 0, + ); + return violations === 0 + ? '✅ no violations' + : `❌ ${violations} violation(s): details below`; + } + return jobCell(jobState, suite); +} + +function jobCell(jobState, suite) { + if (!jobState || jobState.state === 'not-run') + return `⏭ ${notRunSentence(suite)}`; + if (jobState.state === 'scoped-out') + return '⏭ skipped: nothing it covers changed'; + return conclusionText(jobState.conclusion); +} + +function sizeCell({analysis, budgetFlags, packageSize}) { + const parts = []; + const head = analysis.componentSizes || {}; + const base = analysis.baseComponentSizes || {}; + const deltas = Object.keys(head) + .map(key => ({ + key, + delta: + typeof head[key]?.importGzipBytes === 'number' && + typeof base[key]?.importGzipBytes === 'number' + ? head[key].importGzipBytes - base[key].importGzipBytes + : null, + })) + .filter(entry => entry.delta !== null); + if (Object.keys(head).length > 0) { + const largest = deltas.sort((a, b) => b.delta - a.delta)[0]; + const largestText = largest + ? `, largest ${largest.delta > 0 ? '+' : ''}${bytes(largest.delta)} (${inline(largest.key)})` + : ''; + parts.push( + budgetFlags.length > 0 + ? `⚠️ ${budgetFlags.length} component(s) over the touch-only budget${largestText}` + : `✅ ${Object.keys(head).length} component(s) within budget${largestText}`, + ); + } + if (packageSize) { + if (packageSize.verdict === 'over-budget') + parts.push('❌ a published package is over budget'); + else if (packageSize.verdict === 'justified') + parts.push('⚠️ a published package is over budget (justified)'); + else if (packageSize.verdict === 'no-base') + parts.push('packages measured (no base to compare)'); + else parts.push('packages within budget'); + const added = packageSize.addedDependencies || []; + if (added.length > 0) + parts.push( + `new dependencies: ${added.map(dep => inline(dep.name)).join(', ')}`, + ); + } + return parts.length > 0 ? parts.join(' · ') : 'no published package changed'; +} + +function bytes(value) { + const magnitude = Math.abs(value); + if (magnitude < 1024) return `${value}B`; + return `${(value / 1024).toFixed(1)}KB`; +} + +function ciCell(jobSummary) { + if (!jobSummary || jobSummary.length === 0) return null; + const ran = jobSummary.filter(job => job.state === 'ran'); + const failed = ran.filter(job => job.conclusion !== 'success'); + const scoped = jobSummary.filter(job => job.state === 'scoped-out'); + const parts = [`${ran.length} ran`]; + if (failed.length > 0) + parts.push(`❌ failed: ${failed.map(job => inline(job.name)).join(', ')}`); + if (scoped.length > 0) parts.push(`${scoped.length} skipped by path`); + return `${parts.join(' · ')} (table below)`; +} + +/** The status table every report leads with. */ +function buildStatusTable({ + preview, + analysis, + budgetFlags = [], + packageSize = null, + a11yReport = null, + visualVerdict = null, + jobs = null, +}) { + const jobSummary = jobs ? summarizeJobs(jobs) : []; + const rows = []; + const previewText = previewCell(preview); + if (previewText) rows.push(['Preview', previewText]); + rows.push(['Size', sizeCell({analysis, budgetFlags, packageSize})]); + const [visual, a11y, rtl, perf] = SUITES; + rows.push([ + visual.label, + visualCell(visualVerdict, suiteJobState(visual, jobSummary), visual), + ]); + rows.push([ + a11y.label, + a11yCell(a11yReport, suiteJobState(a11y, jobSummary), a11y), + ]); + rows.push([rtl.label, jobCell(suiteJobState(rtl, jobSummary), rtl)]); + const perfState = suiteJobState(perf, jobSummary); + if (perfState?.state === 'ran') + rows.push([perf.label, jobCell(perfState, perf)]); + const ci = ciCell(jobSummary); + if (ci) rows.push(['CI jobs', ci]); + return `| Check | Result |\n|-------|--------|\n${rows + .map(([check, result]) => `| ${check} | ${result} |`) + .join('\n')}\n\n`; +} + +/** Every job of the run that applies to a pull request, with what it did. */ +function buildJobTable(jobs) { + const summary = summarizeJobs(jobs || []); + const shown = summary.filter( + job => job.state !== 'not-run' || suiteOf(job.name) !== null, + ); + if (shown.length === 0) return ''; + const hidden = summary.length - shown.length; + const rows = shown.map(job => { + if (job.state === 'ran') + return `| ${inline(job.name)} | ${conclusionText(job.conclusion)} | |`; + if (job.state === 'scoped-out') + return `| ${inline(job.name)} | ⏭ skipped | ${inline(job.reason)} |`; + return `| ${inline(job.name)} | ⏭ not run | ${notRunSentence(suiteOf(job.name))} |`; + }); + return `
\nCI jobs: ${shown.filter(job => job.state === 'ran').length} ran, ${shown.filter(job => job.state === 'scoped-out').length} skipped by path\n\n| Job | Result | Why |\n|-----|--------|-----|\n${rows.join('\n')}\n${hidden > 0 ? `\n${hidden} release-only job(s) do not apply to pull requests.\n` : ''}\n
\n\n`; +} + +/** A Storybook docs link from a story title (`Core/Button` → `core-button`). */ +function storybookLink(origin, storyTitle) { + const title = Array.isArray(storyTitle) ? storyTitle[0] : storyTitle; + if (!origin || !title) return null; + const id = String(title) + .toLowerCase() + .replace(/\//g, '-') + .replace(/[^a-z0-9_-]/g, ''); + return `${origin}/storybook/?path=/docs/${id}--docs`; +} + +const RESULT_TEXT = { + pass: '✅', + fail: '❌', + changed: '🔍 changed', + 'not-run': '⏭', +}; + +function resultText(value) { + if (value == null) return '—'; + return RESULT_TEXT[value] ?? inline(value); +} + +/** + * The components this pull request touches: a docs page and Storybook stories + * in this head's preview, and each suite's result for it. Results come from + * the touched-components report when the run wrote one; otherwise from the + * accessibility report and visual verdict, and are blank for a suite with no + * per-component result. + */ +function buildTouchedComponents({ + analysis, + origin, + touched = null, + a11yReport = null, + visualVerdict = null, +}) { + const names = [ + ...(analysis.newComponents || []), + ...(analysis.modifiedComponents || []), + ]; + const fromReport = new Map( + (touched?.components || []).map(component => [component.name, component]), + ); + for (const name of fromReport.keys()) + if (!names.includes(name)) names.push(name); + if (names.length === 0) return ''; + const visualChanges = visualVerdict?.changes || []; + const rows = names.map(name => { + const stats = analysis.componentStats?.[name] || {}; + const reported = fromReport.get(name)?.results || {}; + const docs = origin + ? `[Docs](${origin}/components/${encodeURIComponent(name)})` + : '—'; + const stories = storybookLink(origin, stats.storyTitle); + const a11y = + reported.a11y ?? + (a11yReport + ? (a11yReport.components?.[name]?.violations?.length ?? 0) > 0 + ? 'fail' + : 'pass' + : null); + const visual = + reported.visual ?? + (visualVerdict && visualVerdict.status !== 'skipped' + ? visualChanges.some(change => change.component === name) + ? 'changed' + : 'pass' + : null); + return `| ${inline(name)} | ${docs} | ${stories ? `[Stories](${stories})` : '—'} | ${resultText(visual)} | ${resultText(a11y)} | ${resultText(reported.rtl)} |`; + }); + return `**Touched components**\n\n| Component | Docs | Storybook | Visual | A11y | RTL |\n|-----------|------|-----------|--------|------|-----|\n${rows.join('\n')}\n\n`; +} + +module.exports = { + SUITES, + notRunSentence, + classifyJob, + summarizeJobs, + buildStatusTable, + buildJobTable, + buildTouchedComponents, + storybookLink, +}; diff --git a/.github/scripts/lib/pr-status.test.mjs b/.github/scripts/lib/pr-status.test.mjs new file mode 100644 index 0000000000000..186b640e22415 --- /dev/null +++ b/.github/scripts/lib/pr-status.test.mjs @@ -0,0 +1,301 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file pr-status.test.mjs + * Pins the PR report's top: the status table (preview first, no review-bot + * row, a heavy suite that did not run names its label), how a job is read as + * ran / skipped by path / not run, the touched components' preview links and + * results, and the collapsed CI job table. + */ + +import {describe, expect, it} from 'vitest'; +import status from './pr-status.js'; + +const { + notRunSentence, + classifyJob, + summarizeJobs, + buildStatusTable, + buildJobTable, + buildTouchedComponents, + SUITES, +} = status; + +const SCOPE = "Decide this job's scope"; +const job = (name, conclusion, steps = []) => ({ + name, + status: 'completed', + conclusion, + steps: steps.map(([stepName, stepConclusion]) => ({ + name: stepName, + conclusion: stepConclusion, + })), +}); + +describe('classifyJob', () => { + it('reads a job whose condition was false as not run', () => { + expect(classifyJob(job('pr-visual', 'skipped'))).toEqual({ + state: 'not-run', + }); + }); + + it('reads a job that decided its scope and skipped every work step as skipped by path', () => { + const scoped = job('docsite-test', 'success', [ + ['Set up job', 'success'], + [SCOPE, 'success'], + ['Checkout', 'skipped'], + ['Build', 'skipped'], + ['Post Checkout', 'success'], + ['Complete job', 'success'], + ]); + expect(classifyJob(scoped)).toEqual({ + state: 'scoped-out', + reason: 'nothing in its paths changed', + }); + }); + + it('reads a job that ran any work step as ran', () => { + const ran = job('test-node', 'success', [ + [SCOPE, 'success'], + ['Checkout', 'success'], + ['Test', 'skipped'], + ]); + expect(classifyJob(ran)).toEqual({state: 'ran', conclusion: 'success'}); + expect(classifyJob(job('lint', 'failure', [['Lint', 'failure']]))).toEqual({ + state: 'ran', + conclusion: 'failure', + }); + }); + + it('reads a specialized-lane skip as skipped by path', () => { + const skipped = job('build-storybook', 'success', [ + [SCOPE, 'success'], + ['Skip heavy work for a specialized lane', 'success'], + ['Checkout', 'skipped'], + ]); + expect(classifyJob(skipped).state).toBe('scoped-out'); + }); +}); + +describe('summarizeJobs', () => { + it('folds matrix legs into one row; any failed leg fails it', () => { + const summary = summarizeJobs([ + job('test-node (1/2)', 'success', [['Test', 'success']]), + job('test-node (2/2)', 'failure', [['Test', 'failure']]), + ]); + expect(summary).toEqual([ + {name: 'test-node', state: 'ran', conclusion: 'failure'}, + ]); + }); +}); + +describe('notRunSentence', () => { + it.each([ + [ + 'visual', + 'Visual did not run on this pull request; add `ci:run-visual` to run it.', + ], + [ + 'a11y', + 'Accessibility did not run on this pull request; add `ci:run-a11y` to run it.', + ], + [ + 'rtl', + 'RTL did not run on this pull request; add `ci:run-rtl` to run it.', + ], + [ + 'perf', + 'Perf did not run on this pull request; add `ci:run-perf` to run it.', + ], + ])('names the %s label', (id, sentence) => { + expect(notRunSentence(SUITES.find(suite => suite.id === id))).toBe( + sentence, + ); + }); +}); + +describe('buildStatusTable', () => { + const analysis = {newComponents: [], modifiedComponents: []}; + + it('puts the preview first and has no review-bot row', () => { + const table = buildStatusTable({ + preview: { + state: 'ready', + sha: 'a'.repeat(40), + url: 'https://x.vercel.app', + }, + analysis, + }); + const rows = table + .trim() + .split('\n') + .slice(2) + .map(row => row.split(' | ')[0]); + expect(rows).toEqual([ + '| Preview', + '| Size', + '| Visual', + '| Accessibility', + '| RTL', + ]); + expect(table).not.toMatch(/CodeBunny|\| Review/i); + }); + + it('shows perf only when it ran', () => { + const ran = buildStatusTable({ + analysis, + jobs: [job('perf', 'success', [['Measure', 'success']])], + }); + expect(ran).toContain('| Perf | ✅ passed |'); + expect( + buildStatusTable({analysis, jobs: [job('perf', 'skipped')]}), + ).not.toContain('| Perf |'); + }); + + it('reads the visual verdict and the accessibility report before the jobs', () => { + const table = buildStatusTable({ + analysis, + visualVerdict: {status: 'ok', changes: [{}, {}], counts: {total: 10}}, + a11yReport: {components: {Card: {violations: [{id: 'x'}]}}}, + jobs: [job('pr-visual', 'success', [['Run', 'success']])], + }); + expect(table).toContain( + '| Visual | 🔍 2 of 10 shots changed: review below |', + ); + expect(table).toContain( + '| Accessibility | ❌ 1 violation(s): details below |', + ); + }); + + it('summarizes size: components against the touch budget, packages and new dependencies', () => { + const table = buildStatusTable({ + analysis: { + ...analysis, + componentSizes: { + 'core/Card': {importGzipBytes: 5000}, + 'core/Chip': {importGzipBytes: 900}, + }, + baseComponentSizes: { + 'core/Card': {importGzipBytes: 2000}, + 'core/Chip': {importGzipBytes: 900}, + }, + }, + budgetFlags: [{comp: 'core/Card'}], + packageSize: { + verdict: 'pass', + addedDependencies: [{name: 'left-pad'}], + }, + }); + expect(table).toContain( + '| Size | ⚠️ 1 component(s) over the touch-only budget, largest +2.9KB (core/Card) · packages within budget · new dependencies: left-pad |', + ); + }); + + it('counts the CI jobs and names a failed one', () => { + const table = buildStatusTable({ + analysis, + jobs: [ + job('lint', 'failure', [['Lint', 'failure']]), + job('test-ui', 'success', [['Test', 'success']]), + job('docsite-test', 'success', [ + [SCOPE, 'success'], + ['Build', 'skipped'], + ]), + job('release-check', 'skipped'), + ], + }); + expect(table).toContain( + '| CI jobs | 2 ran · ❌ failed: lint · 1 skipped by path (table below) |', + ); + }); +}); + +describe('buildJobTable', () => { + it('lists what each job did and why, collapsed, and leaves out release-only jobs', () => { + const table = buildJobTable([ + job('test-ui', 'success', [['Test', 'success']]), + job('docsite-test', 'success', [ + [SCOPE, 'success'], + ['Build', 'skipped'], + ]), + job('pr-rtl', 'skipped'), + job('release-check', 'skipped'), + ]); + expect(table).toMatch( + /^
\nCI jobs: 1 ran, 1 skipped by path<\/summary>/, + ); + expect(table).toContain('| test-ui | ✅ passed | |'); + expect(table).toContain( + '| docsite-test | ⏭ skipped | nothing in its paths changed |', + ); + expect(table).toContain( + '| pr-rtl | ⏭ not run | RTL did not run on this pull request; add `ci:run-rtl` to run it. |', + ); + expect(table).not.toContain('release-check'); + expect(table).toContain( + '1 release-only job(s) do not apply to pull requests.', + ); + }); +}); + +describe('buildTouchedComponents', () => { + const origin = 'https://astryx-abc123-fbopensource.vercel.app'; + const analysis = { + newComponents: [], + modifiedComponents: ['Button'], + componentStats: { + Button: {package: '@astryxdesign/core', storyTitle: 'Core/Button'}, + }, + }; + + it("links each touched component's docs page and stories in this head's preview", () => { + const table = buildTouchedComponents({analysis, origin}); + expect(table).toContain( + `| Button | [Docs](${origin}/components/Button) | [Stories](${origin}/storybook/?path=/docs/core-button--docs) | — | — | — |`, + ); + }); + + it('takes per-component results from the touched-components report first', () => { + const table = buildTouchedComponents({ + analysis, + origin, + touched: { + components: [ + { + name: 'Button', + results: {visual: 'changed', a11y: 'pass', rtl: 'fail'}, + }, + ], + }, + a11yReport: {components: {Button: {violations: [{id: 'x'}]}}}, + }); + expect(table).toContain('| 🔍 changed | ✅ | ❌ |'); + }); + + it('falls back to the accessibility report and visual verdict', () => { + const table = buildTouchedComponents({ + analysis, + origin, + a11yReport: {components: {Button: {violations: []}}}, + visualVerdict: {status: 'ok', changes: [{component: 'Button'}]}, + }); + expect(table).toContain('| 🔍 changed | ✅ | — |'); + }); + + it('renders nothing when no component was touched', () => { + expect( + buildTouchedComponents({ + analysis: {newComponents: [], modifiedComponents: []}, + origin, + }), + ).toBe(''); + }); + + it('renders component names as literal text', () => { + const table = buildTouchedComponents({ + analysis: {modifiedComponents: [''], componentStats: {}}, + origin, + }); + expect(table).not.toContain(''); + }); +}); diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs index 352f875843d76..1441fbad4493c 100644 --- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs +++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs @@ -64,7 +64,9 @@ describe('PR report and deployment workflow contracts', () => { expect(value.jobs[job].concurrency['cancel-in-progress']).toBe(false); } expect(value.jobs.comment.needs).toBe('resolve'); - expect(value.jobs.comment.permissions.deployments).toBeUndefined(); + // The comment job reads this head's preview state once; no job runs on a + // deployment event, and none waits for one. + expect(value.jobs.comment.permissions.deployments).toBe('read'); expect(workflow('pr-comment.yml')).toContain( 'validateAnalysisMetadata(metadata, {', ); @@ -137,8 +139,12 @@ describe('PR report and deployment workflow contracts', () => { 'utf8', ); expect(reconciler).toContain('branchPreviewOrigin(identity)'); - expect(reconciler).toContain('`${previewOrigin}/storybook/`'); - expect(reconciler).toContain('`${previewOrigin}/sandbox/`'); + const status = fs.readFileSync( + path.join(ROOT, '.github/scripts/lib/pr-status.js'), + 'utf8', + ); + expect(status).toContain('[Storybook](${origin}/storybook/)'); + expect(status).toContain('[Sandbox](${origin}/sandbox/)'); }); it('publishes only the static compatibility landing to GitHub Pages', () => { diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml index 75c491e4ece3a..d16bcf1534527 100644 --- a/.github/workflows/pr-comment.yml +++ b/.github/workflows/pr-comment.yml @@ -199,7 +199,10 @@ jobs: runs-on: ubuntu-24.04-arm permissions: pull-requests: write + # actions: the artifacts and the source run's job list; deployments: + # this head's Vercel preview state. actions: read + deployments: read # The artifact downloads are optional (a run may not produce them), so # they are not failures of this job. outputs: From 104373c72e691405ae0bbd511f15fe78afe16bbe Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 10:08:22 -0700 Subject: [PATCH 05/15] ci: the report's not-run sentence comes from run-labels.cjs --- .github/scripts/lib/pr-status.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js index 3a5f1acf70e6c..cd7645988bc09 100644 --- a/.github/scripts/lib/pr-status.js +++ b/.github/scripts/lib/pr-status.js @@ -16,7 +16,9 @@ const {inline, num} = require('./report-text'); -/** The heavy suites a pull request runs only on a maintainer's label. */ +const runLabels = require('../run-labels.cjs'); + +/** The heavy suites a pull request runs only with its `ci:run-` label. */ const SUITES = [ {id: 'visual', label: 'Visual', jobs: ['visual-scout', 'pr-visual']}, {id: 'a11y', label: 'Accessibility', jobs: ['pr-a11y']}, @@ -24,9 +26,9 @@ const SUITES = [ {id: 'perf', label: 'Perf', jobs: ['perf']}, ]; -/** The sentence a heavy suite shows when it did not run (one wording everywhere). */ +/** The sentence a heavy suite shows when it did not run (run-labels.cjs owns it). */ function notRunSentence(suite) { - return `${suite.label} did not run on this pull request; add \`ci:run-${suite.id}\` to run it.`; + return runLabels.notRunSentence(suite.id); } const SCOPE_STEP = "Decide this job's scope"; From b1dc73e5a1bfd250651c3ddbbe20d0a836a8af41 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 10:14:25 -0700 Subject: [PATCH 06/15] ci: the package budget is advisory; the merge-base lookups share one helper; the check also follows root build inputs - A package over its budget is flagged (job summary, PR report, a workflow warning) and fails nothing, like the component touch-only flags; the 'Size budget:' line and the enforce step are gone. - The package-size merge-base step deepens once when the 50-commit fetch holds no merge base, as size-base does. - Report counts render through num(). - tsconfig.json and scripts/ are in the check's paths scope. - size-base and the package-size lookup share lib/main-artifact.cjs. --- .github/scripts/ci-tooling-routing.test.mjs | 18 ++-- .github/scripts/generate-pr-comment.test.mjs | 3 +- .github/scripts/lib/main-artifact.cjs | 35 +++++++ .github/scripts/lib/main-artifact.test.mjs | 71 +++++++++++++ .github/scripts/lib/package-size.js | 51 +++------ .github/scripts/lib/package-size.test.mjs | 105 +++++-------------- .github/scripts/package-size.mjs | 35 ++----- .github/workflows/ci.yml | 59 ++++------- 8 files changed, 191 insertions(+), 186 deletions(-) create mode 100644 .github/scripts/lib/main-artifact.cjs create mode 100644 .github/scripts/lib/main-artifact.test.mjs diff --git a/.github/scripts/ci-tooling-routing.test.mjs b/.github/scripts/ci-tooling-routing.test.mjs index 95673a78e0a46..8f6611543a3cf 100644 --- a/.github/scripts/ci-tooling-routing.test.mjs +++ b/.github/scripts/ci-tooling-routing.test.mjs @@ -261,15 +261,17 @@ describe('Node-tooling CI routing', () => { const install = step(job, "Install the merge base's dependencies"); expect(install.run).toBe('pnpm install --frozen-lockfile'); expect(install.if).not.toContain('steps.download'); - // Only Main's own run on main for the merge-base commit counts. + // Only Main's own run on main for the merge-base commit counts; both + // merge-base lookups share one helper (main-artifact.test.mjs). const find = step(job, "Find main's build of the merge base").with.script; - expect(find).toContain("candidate.workflow_run?.head_branch === 'main'"); - expect(find).toContain( - 'candidate.workflow_run?.head_sha === process.env.BASE_SHA', - ); - expect(find).toContain( - 'candidate.workflow_run?.head_repository_id === candidate.workflow_run?.repository_id', - ); + expect(find).toContain('.github/scripts/lib/main-artifact.cjs'); + expect(find).toContain('sha: process.env.BASE_SHA'); + const packageFind = step( + ci.jobs['build-storybook'], + "Find main's package size for the merge base", + ).with.script; + expect(packageFind).toContain('.github/scripts/lib/main-artifact.cjs'); + expect(packageFind).toContain('sha: process.env.BASE_SHA'); }); it('builds no Sandbox and builds Storybook once per pull-request run', () => { diff --git a/.github/scripts/generate-pr-comment.test.mjs b/.github/scripts/generate-pr-comment.test.mjs index a95fdf27d3c03..c96e933971354 100644 --- a/.github/scripts/generate-pr-comment.test.mjs +++ b/.github/scripts/generate-pr-comment.test.mjs @@ -349,7 +349,6 @@ describe('published package size', () => { }, ], removedDependencies: [], - justification: null, verdict: 'over-budget', }); expect(out.indexOf('### Bundle Size Summary')).toBeLessThan( @@ -360,7 +359,7 @@ describe('published package size', () => { ); expect(out).toContain('| 10 → 12 |'); expect(out).toContain('left-pad'); - expect(out).toContain('**Over budget:**'); + expect(out).toContain('**⚠️ Over the package budget:**'); }); it('renders report strings as literal text', () => { diff --git a/.github/scripts/lib/main-artifact.cjs b/.github/scripts/lib/main-artifact.cjs new file mode 100644 index 0000000000000..4bbd3b294a9df --- /dev/null +++ b/.github/scripts/lib/main-artifact.cjs @@ -0,0 +1,35 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file main-artifact.cjs + * @input An Octokit client, the repository, an artifact name, and the commit + * it must belong to + * @output The id of the workflow run on `main` that uploaded that artifact for + * that exact commit, or null + * @position Shared by the CI jobs that read what Main keeps for a merge base + * (its package builds, its package-size measurement), so each trusts the + * same thing: only Main's own run on `main` for this exact commit, in this + * repository. A pull request's run can upload an artifact under any name. + */ + +'use strict'; + +async function mainArtifactRunId({github, owner, repo, name, sha}) { + const {data} = await github.rest.actions.listArtifactsForRepo({ + owner, + repo, + name, + per_page: 100, + }); + const artifact = data.artifacts.find( + candidate => + !candidate.expired && + candidate.workflow_run?.head_branch === 'main' && + candidate.workflow_run?.head_sha === sha && + candidate.workflow_run?.head_repository_id === + candidate.workflow_run?.repository_id, + ); + return artifact ? String(artifact.workflow_run.id) : null; +} + +module.exports = {mainArtifactRunId}; diff --git a/.github/scripts/lib/main-artifact.test.mjs b/.github/scripts/lib/main-artifact.test.mjs new file mode 100644 index 0000000000000..160d1accb7950 --- /dev/null +++ b/.github/scripts/lib/main-artifact.test.mjs @@ -0,0 +1,71 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file main-artifact.test.mjs + * Pins which kept artifact a merge-base lookup trusts: only one Main's own run + * on `main` uploaded for that exact commit in this repository, unexpired. + */ + +import {describe, expect, it, vi} from 'vitest'; +import lookup from './main-artifact.cjs'; + +const {mainArtifactRunId} = lookup; +const SHA = 'a'.repeat(40); + +function github(artifacts) { + return { + rest: { + actions: { + listArtifactsForRepo: vi.fn(async () => ({data: {artifacts}})), + }, + }, + }; +} + +const artifact = (overrides = {}, run = {}) => ({ + expired: false, + ...overrides, + workflow_run: { + id: 42, + head_branch: 'main', + head_sha: SHA, + head_repository_id: 1, + repository_id: 1, + ...run, + }, +}); + +describe('mainArtifactRunId', () => { + it("returns Main's run for the exact commit", async () => { + const client = github([artifact()]); + await expect( + mainArtifactRunId({ + github: client, + owner: 'meta', + repo: 'astryx', + name: 'x', + sha: SHA, + }), + ).resolves.toBe('42'); + expect(client.rest.actions.listArtifactsForRepo).toHaveBeenCalledWith( + expect.objectContaining({name: 'x', owner: 'meta', repo: 'astryx'}), + ); + }); + + it.each([ + ['an expired artifact', artifact({expired: true})], + ['a pull request branch', artifact({}, {head_branch: 'feature'})], + ['another commit', artifact({}, {head_sha: 'b'.repeat(40)})], + ['a fork', artifact({}, {head_repository_id: 2})], + ])('ignores %s', async (_label, candidate) => { + await expect( + mainArtifactRunId({ + github: github([candidate]), + owner: 'meta', + repo: 'astryx', + name: 'x', + sha: SHA, + }), + ).resolves.toBeNull(); + }); +}); diff --git a/.github/scripts/lib/package-size.js b/.github/scripts/lib/package-size.js index 579484f5eac66..31dee287c39a2 100644 --- a/.github/scripts/lib/package-size.js +++ b/.github/scripts/lib/package-size.js @@ -3,10 +3,10 @@ /** * @file package-size.js * @input A built workspace (every published package's files on disk) and, to - * compare, two measurements and a pull request description + * compare, two measurements * @output Each published package's tarball size (gzip, as npm downloads it), * unpacked size, file count and runtime dependencies; a base-against-head - * comparison with a budget verdict; and that comparison as Markdown + * comparison with an advisory budget flag; and that comparison as Markdown * @position Shared by package-size.mjs (CI's build-storybook job on a pull * request, and Main's typecheck job, which keeps each main commit's * measurement for pull requests to compare against) and by @@ -16,11 +16,13 @@ const fs = require('node:fs'); const path = require('node:path'); const {execFile} = require('node:child_process'); +const {num} = require('./report-text'); /** - * A package fails the check when its tarball grows by more than this share of - * its base size and by at least MIN_BYTES; the floor keeps a small package - * from failing on a few hundred bytes. + * A package is flagged when its tarball grows by more than this share of its + * base size and by at least MIN_BYTES; the floor keeps a small package from + * being flagged for a few hundred bytes. The flag is advisory: it asks the + * author, as the component touch-only budget does, and fails nothing. */ const PACKED_BUDGET_PERCENT = 5; const PACKED_BUDGET_MIN_BYTES = 1024; @@ -112,18 +114,6 @@ function isMeasured(bytes) { return typeof bytes === 'number' && Number.isFinite(bytes); } -/** - * The description's `Size budget:` line, outside HTML comments (a template's - * placeholder is not a justification), or null when it has no text. - */ -function sizeBudgetJustification(body) { - const visible = String(body || '').replace(//g, ''); - const match = visible.match( - /^[ \t>*_-]*Size budget:[ \t]*(?:\*\*|__)?[ \t]*(\S.*)$/im, - ); - return match ? match[1].trim() : null; -} - function overBudget(baseBytes, headBytes) { if (!isMeasured(baseBytes) || !isMeasured(headBytes) || baseBytes <= 0) return false; @@ -144,7 +134,7 @@ function splitDependencyKey(key) { * row keeps its head figures, nothing is over budget, and the verdict says * the base was not measured. */ -function comparePackageSizes({head, base, body, headSha = '', baseSha = ''}) { +function comparePackageSizes({head, base, headSha = '', baseSha = ''}) { const headPackages = head?.packages || {}; const basePackages = base?.packages || null; const names = [ @@ -194,15 +184,8 @@ function comparePackageSizes({head, base, body, headSha = '', baseSha = ''}) { }); } } - const justification = sizeBudgetJustification(body); const over = packages.some(entry => entry.overBudget); - const verdict = !basePackages - ? 'no-base' - : !over - ? 'pass' - : justification - ? 'justified' - : 'over-budget'; + const verdict = !basePackages ? 'no-base' : over ? 'over-budget' : 'pass'; return { schema: 1, headSha, @@ -214,7 +197,6 @@ function comparePackageSizes({head, base, body, headSha = '', baseSha = ''}) { packages, addedDependencies, removedDependencies, - justification, verdict, }; } @@ -236,6 +218,10 @@ function deltaText(after, before) { return `${delta > 0 ? '+' : ''}${sizeText(delta)}${percent}`; } +function fileCount(value) { + return isMeasured(value) ? String(num(value)) : 'N/A'; +} + function changed(entry) { if (!entry.base || !entry.head) return true; return ( @@ -262,7 +248,7 @@ function renderPackageSizeSection(report, literal = text => String(text)) { ? report.packages : report.packages.filter(changed); if (rows.length === 0) { - out += `No published package changed size (${report.packages.length} measured).\n\n`; + out += `No published package changed size (${num(report.packages.length)} measured).\n\n`; } else { out += '| Package | Download before | Download after | Delta | Unpacked before | Unpacked after | Delta | Files |\n'; @@ -271,8 +257,8 @@ function renderPackageSizeSection(report, literal = text => String(text)) { for (const entry of rows) { const files = entry.base && entry.head && entry.base.files !== entry.head.files - ? `${entry.base.files} → ${entry.head.files}` - : String(entry.head?.files ?? entry.base?.files ?? 'N/A'); + ? `${fileCount(entry.base.files)} → ${fileCount(entry.head.files)}` + : fileCount(entry.head?.files ?? entry.base?.files); out += `| ${literal(entry.name)} | ${sizeText(entry.base?.packedBytes)} | ${sizeText(entry.head?.packedBytes)} | ${deltaText(entry.head?.packedBytes, entry.base?.packedBytes)}${entry.overBudget ? ' ⚠️' : ''} | ${sizeText(entry.base?.unpackedBytes)} | ${sizeText(entry.head?.unpackedBytes)} | ${deltaText(entry.head?.unpackedBytes, entry.base?.unpackedBytes)} | ${files} |\n`; } const unchanged = report.packages.length - rows.length; @@ -294,9 +280,7 @@ function renderPackageSizeSection(report, literal = text => String(text)) { out += '\n'; } if (report.verdict === 'over-budget') { - out += `**Over budget:** a package's download grew more than ${packedPercent}% (and at least ${sizeText(packedMinBytes)}). Shrink it, or add a \`Size budget:\` line to the pull request description saying why the growth is worth it, then re-run the job.\n\n`; - } else if (report.verdict === 'justified') { - out += `**Over budget, justified:** ${literal(report.justification)}\n\n`; + out += `**⚠️ Over the package budget:** a package's download grew more than ${num(packedPercent)}% (and at least ${sizeText(num(packedMinBytes))}). If the growth is not intended, shrink it; if it is, say why in the pull request description.\n\n`; } out += 'Download is the gzip tarball npm fetches; unpacked is what lands in `node_modules`.\n\n'; @@ -309,7 +293,6 @@ module.exports = { publishedPackages, runtimeDependencies, measurePackages, - sizeBudgetJustification, overBudget, comparePackageSizes, renderPackageSizeSection, diff --git a/.github/scripts/lib/package-size.test.mjs b/.github/scripts/lib/package-size.test.mjs index bccc72dfe73fe..fdef429bf5203 100644 --- a/.github/scripts/lib/package-size.test.mjs +++ b/.github/scripts/lib/package-size.test.mjs @@ -3,9 +3,9 @@ /** * @file package-size.test.mjs * Pins the published-package-size check: which packages count as published, - * what `npm pack` measures, the budget (a tarball growing more than 5% and at - * least 1 KB) and its `Size budget:` justification, the runtime-dependency - * diff, the Markdown, and the CLI's exit codes. + * what `npm pack` measures, the advisory budget flag (a tarball growing more + * than 5% and at least 1 KB), the runtime-dependency diff, the Markdown, and + * the CLI, which flags and never fails. */ import {execFileSync, spawnSync} from 'node:child_process'; @@ -20,7 +20,6 @@ const { PACKED_BUDGET_MIN_BYTES, publishedPackages, measurePackages, - sizeBudgetJustification, overBudget, comparePackageSizes, renderPackageSizeSection, @@ -143,35 +142,6 @@ describe('measurePackages', () => { }); }); -describe('sizeBudgetJustification', () => { - it.each([ - [ - 'Size budget: the new locale data is the feature', - 'the new locale data is the feature', - ], - ['Intro\n\n- Size budget: worth it\n', 'worth it'], - ['> **Size budget:** bold label', 'bold label'], - ['size budget: lower case label', 'lower case label'], - ])('reads %j', (body, expected) => { - expect(sizeBudgetJustification(body)).toBe(expected); - }); - - it('ignores a template placeholder inside an HTML comment and an empty line', () => { - expect( - sizeBudgetJustification( - '', - ), - ).toBeNull(); - expect( - sizeBudgetJustification( - '', - ), - ).toBeNull(); - expect(sizeBudgetJustification('Size budget: \nnext line')).toBeNull(); - expect(sizeBudgetJustification(null)).toBeNull(); - }); -}); - describe('overBudget', () => { it('needs more than 5% and at least the byte floor', () => { const base = 100 * 1024; @@ -198,13 +168,13 @@ describe('comparePackageSizes', () => { }); it('passes an unchanged head', () => { - const report = comparePackageSizes({head: base, base, body: ''}); + const report = comparePackageSizes({head: base, base}); expect(report.verdict).toBe('pass'); expect(report.packages.every(entry => !entry.overBudget)).toBe(true); expect(report.addedDependencies).toEqual([]); }); - it('fails a package past the budget, and a Size budget line justifies it', () => { + it('flags a package past the budget', () => { const head = measurement({ ...base.packages, '@astryxdesign/core': pkg( @@ -212,19 +182,12 @@ describe('comparePackageSizes', () => { base.packages['@astryxdesign/core'].dependencies, ), }); - const failing = comparePackageSizes({head, base, body: 'No reason given'}); - expect(failing.verdict).toBe('over-budget'); + const flagged = comparePackageSizes({head, base}); + expect(flagged.verdict).toBe('over-budget'); expect( - failing.packages.find(entry => entry.name === '@astryxdesign/core') + flagged.packages.find(entry => entry.name === '@astryxdesign/core') .overBudget, ).toBe(true); - const justified = comparePackageSizes({ - head, - base, - body: 'Size budget: ships the new locales', - }); - expect(justified.verdict).toBe('justified'); - expect(justified.justification).toBe('ships the new locales'); }); it('lists added and removed runtime dependencies', () => { @@ -304,7 +267,7 @@ describe('renderPackageSizeSection', () => { ); expect(markdown).not.toContain('theme-neutral'); expect(markdown).toContain('2 other published packages are unchanged.'); - expect(markdown).not.toContain('Over budget'); + expect(markdown).not.toContain('Over the package budget'); }); it('writes a package past 1 MB in MB', () => { @@ -329,7 +292,7 @@ describe('renderPackageSizeSection', () => { ); }); - it('flags the row and asks for a Size budget line when over budget', () => { + it('flags the row and asks the author when over budget', () => { const head = measurement({ ...base.packages, '@astryxdesign/core': pkg(110_000), @@ -340,8 +303,8 @@ describe('renderPackageSizeSection', () => { expect(markdown).toMatch( /\| @astryxdesign\/core \|[^\n]*\(\+10\.0%\) ⚠️ \|/, ); - expect(markdown).toContain('**Over budget:**'); - expect(markdown).toContain('`Size budget:`'); + expect(markdown).toContain('**⚠️ Over the package budget:**'); + expect(markdown).toContain('say why in the pull request description'); }); it('shows every head size and no budget without a base', () => { @@ -374,20 +337,17 @@ describe('package-size.mjs', () => { return spawnSync(process.execPath, [CLI, ...args], {encoding: 'utf8'}); } - function compareAndEnforce({head, base, body}) { + function compare({head, base}) { const dir = tempDir(); const file = name => path.join(dir, name); fs.writeFileSync(file('head.json'), JSON.stringify(head)); if (base) fs.writeFileSync(file('base.json'), JSON.stringify(base)); - fs.writeFileSync(file('body.txt'), body); - const compare = runCli([ + const result = runCli([ 'compare', '--head', file('head.json'), '--base', file('base.json'), - '--body', - file('body.txt'), '--head-sha', 'a'.repeat(40), '--output', @@ -395,10 +355,8 @@ describe('package-size.mjs', () => { '--summary', file('summary.md'), ]); - const enforce = runCli(['enforce', '--report', file('report.json')]); return { - compare, - enforce, + result, report: JSON.parse(fs.readFileSync(file('report.json'), 'utf8')), summary: fs.readFileSync(file('summary.md'), 'utf8'), }; @@ -407,30 +365,23 @@ describe('package-size.mjs', () => { const base = measurement({'@astryxdesign/core': pkg(100_000)}); const grown = measurement({'@astryxdesign/core': pkg(110_000)}); - it('compares without failing, and enforce fails an unjustified overrun', () => { - const {compare, enforce, report, summary} = compareAndEnforce({ - head: grown, - base, - body: '', - }); - expect(compare.status).toBe(0); + it('flags an overrun with a warning and never fails', () => { + const {result, report, summary} = compare({head: grown, base}); + expect(result.status).toBe(0); expect(report.headSha).toBe('a'.repeat(40)); - expect(summary).toContain('**Over budget:**'); - expect(enforce.status).toBe(1); - expect(enforce.stdout).toContain( - '::error title=Package size over budget::', + expect(summary).toContain('**⚠️ Over the package budget:**'); + expect(result.stdout).toContain( + '::warning title=Package size over budget::', ); + expect(result.stdout).not.toContain('::error'); }); - it('enforce passes a justified overrun and a missing base', () => { - expect( - compareAndEnforce({head: grown, base, body: 'Size budget: worth it'}) - .enforce.status, - ).toBe(0); - const noBase = compareAndEnforce({head: grown, base: null, body: ''}); - expect(noBase.report.verdict).toBe('no-base'); - expect(noBase.compare.stdout).toContain('::notice::'); - expect(noBase.enforce.status).toBe(0); + it('compares without a base, with a notice and no flag', () => { + const {result, report} = compare({head: grown, base: null}); + expect(result.status).toBe(0); + expect(report.verdict).toBe('no-base'); + expect(result.stdout).toContain('::notice::'); + expect(result.stdout).not.toContain('::warning'); }); it('measure writes every published package of a workspace', () => { diff --git a/.github/scripts/package-size.mjs b/.github/scripts/package-size.mjs index d740eb8bd0586..045e55968e879 100644 --- a/.github/scripts/package-size.mjs +++ b/.github/scripts/package-size.mjs @@ -6,16 +6,12 @@ * request's measurement with its merge base's. * @input `measure --output [--root ]`: a workspace whose packages * are built. - * `compare --head [--base ] [--body ] --output + * `compare --head [--base ] --output * [--summary ] [--head-sha ] [--base-sha ]`: two - * measurements (a missing or unreadable base compares nothing) and the pull - * request description. - * `enforce --report `: a comparison `compare` wrote. + * measurements (a missing or unreadable base compares nothing). * @output `measure` writes each published package's sizes and runtime - * dependencies. `compare` writes the comparison and appends it to the job - * summary. `enforce` exits 1 when a package is over budget and the - * description has no `Size budget:` line; it runs after the report's - * artifacts are uploaded, so a failing check still gets its PR report. + * dependencies. `compare` writes the comparison, appends it to the job + * summary, and flags (never fails) a package over its budget. * @position CI's build-storybook job measures and compares a pull request's * head; Main's typecheck job measures each main commit and keeps it for 14 * days as the base. The PR report renders the comparison. @@ -46,15 +42,6 @@ function readJson(file) { } } -function readText(file) { - if (!file) return ''; - try { - return fs.readFileSync(file, 'utf8'); - } catch { - return ''; - } -} - if (command === 'measure') { const output = getArg('output'); if (!output) throw new Error('measure needs --output '); @@ -76,7 +63,6 @@ if (command === 'measure') { const report = comparePackageSizes({ head, base, - body: readText(getArg('body')), headSha: getArg('head-sha') || '', baseSha: getArg('base-sha') || '', }); @@ -85,20 +71,13 @@ if (command === 'measure') { const summary = getArg('summary'); if (summary) fs.appendFileSync(summary, markdown); console.log(markdown); -} else if (command === 'enforce') { - const report = readJson(getArg('report')); - if (!report) throw new Error('enforce needs a readable --report '); - if (report.verdict === 'over-budget') { + if (report.verdict === 'over-budget') console.log( - '::error title=Package size over budget::A published package grew past its budget. Shrink it, or add a "Size budget:" line to the pull request description saying why, then re-run this job.', + '::warning title=Package size over budget::A published package grew past its advisory budget. If the growth is not intended, shrink it; if it is, say why in the pull request description.', ); - process.exitCode = 1; - } else { - console.log(`Package size verdict: ${report.verdict}.`); - } } else { console.error( - 'usage: package-size.mjs measure --output | compare --head [--base ] [--body ] --output | enforce --report ', + 'usage: package-size.mjs measure --output | compare --head [--base ] --output ', ); process.exitCode = 2; } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f05d2d75c7ff8..f52a24e6cd095 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -902,9 +902,9 @@ jobs: uses: ./.test-scope/.github/actions/test-scope with: # `touches` scopes only the package-size check (a package, the - # lockfile, or the check itself); the build itself follows the - # specialized-lane outputs. - paths: '^(packages/|pnpm-lock\.yaml$|\.github/scripts/package-size\.mjs$|\.github/scripts/lib/package-size\.js$)' + # lockfile, the root build inputs, or the check itself); the build + # itself follows the specialized-lane outputs. + paths: '^(packages/|pnpm-lock\.yaml$|tsconfig\.json$|scripts/|\.github/scripts/package-size\.mjs$|\.github/scripts/lib/package-size\.js$)' - name: Skip heavy work for a specialized lane if: steps.scope.outputs.docsite_only == 'true' || steps.scope.outputs.spec_only == 'true' || steps.scope.outputs.tooling_only == 'true' @@ -1039,12 +1039,19 @@ jobs: if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && github.event_name == 'pull_request' && steps.scope.outputs.touches == 'true' run: node .github/scripts/package-size.mjs measure --output package-size-head.json + # A branch far behind main may have no merge base within the fetched + # history; deepen once, as size-base does, and compare nothing without one. - name: Find the merge base for package size id: package-size-merge-base if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size.outcome == 'success' env: BASE_REF: ${{ github.base_ref }} - run: echo "sha=$(git merge-base "origin/$BASE_REF" HEAD || true)" >> "$GITHUB_OUTPUT" + run: | + if ! BASE_SHA=$(git merge-base "origin/$BASE_REF" HEAD); then + git fetch --deepen=200 origin "$BASE_REF:refs/remotes/origin/$BASE_REF" || true + BASE_SHA=$(git merge-base "origin/$BASE_REF" HEAD || true) + fi + echo "sha=$BASE_SHA" >> "$GITHUB_OUTPUT" - name: Find main's package size for the merge base id: package-size-cached @@ -1056,17 +1063,11 @@ jobs: with: retries: 3 script: | + const {mainArtifactRunId} = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/lib/main-artifact.cjs`); const name = `package-size-main-${process.env.BASE_SHA}`; - const {data} = await github.rest.actions.listArtifactsForRepo({...context.repo, name, per_page: 100}); - // Only Main's own run on main for this exact commit: a pull - // request's run can upload an artifact under any name. - const artifact = data.artifacts.find(candidate => - !candidate.expired && - candidate.workflow_run?.head_branch === 'main' && - candidate.workflow_run?.head_sha === process.env.BASE_SHA && - candidate.workflow_run?.head_repository_id === candidate.workflow_run?.repository_id); - core.setOutput('run_id', artifact ? String(artifact.workflow_run.id) : ''); - core.info(artifact ? `Using ${name} from run ${artifact.workflow_run.id}.` : `No ${name}; sizes are shown without a budget.`); + const runId = await mainArtifactRunId({github, ...context.repo, name, sha: process.env.BASE_SHA}); + core.setOutput('run_id', runId ?? ''); + core.info(runId ? `Using ${name} from run ${runId}.` : `No ${name}; sizes are shown without a budget.`); - name: Download main's package size for the merge base if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size-cached.outputs.run_id != '' @@ -1078,21 +1079,17 @@ jobs: github-token: ${{ github.token }} path: .package-size-base - # The description is read now, not from the event, so an author who adds - # a `Size budget:` line can re-run this job without pushing. - - name: Check published package size + # Advisory: a package over its budget is flagged in the job summary and + # the PR report, and fails nothing. + - name: Compare published package size if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size.outcome == 'success' env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} BASE_SHA: ${{ steps.package-size-merge-base.outputs.sha }} run: | - gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER" --jq '.body // ""' > pr-body.txt || : > pr-body.txt node .github/scripts/package-size.mjs compare \ --head package-size-head.json \ --base .package-size-base/package-size.json \ - --body pr-body.txt \ --head-sha "$HEAD_SHA" \ --base-sha "$BASE_SHA" \ --output package-size.json \ @@ -1127,12 +1124,6 @@ jobs: package-size.json retention-days: 1 - # Last, so an over-budget head still uploads the analysis its PR report - # reads. - - name: Enforce the package size budget - if: steps.scope.outputs.docsite_only != 'true' && steps.scope.outputs.spec_only != 'true' && steps.scope.outputs.tooling_only != 'true' && steps.package-size.outcome == 'success' - run: node .github/scripts/package-size.mjs enforce --report package-size.json - # Measures the PR's merge base so the PR comment can show component and # package sizes before and after. It runs beside build-storybook rather than # inside it, so no browser lane waits for the second build; the head sizes @@ -1215,17 +1206,11 @@ jobs: with: retries: 3 script: | + const {mainArtifactRunId} = require(`${process.env.GITHUB_WORKSPACE}/.github/scripts/lib/main-artifact.cjs`); const name = `dists-main-${process.env.BASE_SHA}`; - const {data} = await github.rest.actions.listArtifactsForRepo({...context.repo, name, per_page: 100}); - // Only Main's own run on main for this exact commit: a pull - // request's run can upload an artifact under any name. - const artifact = data.artifacts.find(candidate => - !candidate.expired && - candidate.workflow_run?.head_branch === 'main' && - candidate.workflow_run?.head_sha === process.env.BASE_SHA && - candidate.workflow_run?.head_repository_id === candidate.workflow_run?.repository_id); - core.setOutput('run_id', artifact ? String(artifact.workflow_run.id) : ''); - core.info(artifact ? `Using ${name} from run ${artifact.workflow_run.id}.` : `No ${name}; building the merge base.`); + const runId = await mainArtifactRunId({github, ...context.repo, name, sha: process.env.BASE_SHA}); + core.setOutput('run_id', runId ?? ''); + core.info(runId ? `Using ${name} from run ${runId}.` : `No ${name}; building the merge base.`); - name: Download main's build of the merge base id: download From a4d7aac8d3f80048b6b522e64033978dac61ffb8 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 10:22:49 -0700 Subject: [PATCH 07/15] Merge the advisory package budget into the status table --- .github/scripts/lib/pr-status.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js index cd7645988bc09..43c4f3b221157 100644 --- a/.github/scripts/lib/pr-status.js +++ b/.github/scripts/lib/pr-status.js @@ -209,9 +209,7 @@ function sizeCell({analysis, budgetFlags, packageSize}) { } if (packageSize) { if (packageSize.verdict === 'over-budget') - parts.push('❌ a published package is over budget'); - else if (packageSize.verdict === 'justified') - parts.push('⚠️ a published package is over budget (justified)'); + parts.push('⚠️ a published package grew past its budget'); else if (packageSize.verdict === 'no-base') parts.push('packages measured (no base to compare)'); else parts.push('packages within budget'); From 918fa698584d301686d1156c7fe1c31df2ac0468 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 10:27:42 -0700 Subject: [PATCH 08/15] ci: the status table reads suite jobs by display name, and a job's skip reason is its own skip step --- .github/scripts/lib/pr-status.js | 27 ++++++++++++++++++-------- .github/scripts/lib/pr-status.test.mjs | 26 ++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js index 43c4f3b221157..d1b5534aef025 100644 --- a/.github/scripts/lib/pr-status.js +++ b/.github/scripts/lib/pr-status.js @@ -18,12 +18,23 @@ const {inline, num} = require('./report-text'); const runLabels = require('../run-labels.cjs'); -/** The heavy suites a pull request runs only with its `ci:run-` label. */ +/** + * The heavy suites a pull request runs only with its `ci:run-` label, + * and the job display names (matrix suffix removed) that belong to each. + */ const SUITES = [ - {id: 'visual', label: 'Visual', jobs: ['visual-scout', 'pr-visual']}, - {id: 'a11y', label: 'Accessibility', jobs: ['pr-a11y']}, - {id: 'rtl', label: 'RTL', jobs: ['pr-rtl', 'pr-rtl-shard', 'pr-rtl-merge']}, - {id: 'perf', label: 'Perf', jobs: ['perf']}, + { + id: 'visual', + label: 'Visual', + jobs: /^(Stable visual|visual-scout|pr-visual)/i, + }, + { + id: 'a11y', + label: 'Accessibility', + jobs: /^(pr-a11y|Accessibility spec-test)/i, + }, + {id: 'rtl', label: 'RTL', jobs: /^pr-rtl/i}, + {id: 'perf', label: 'Perf', jobs: /^(perf|pr-perf)/i}, ]; /** The sentence a heavy suite shows when it did not run (run-labels.cjs owns it). */ @@ -53,7 +64,7 @@ function classifyJob(job) { if (skipStep && job.conclusion === 'success') return { state: 'scoped-out', - reason: 'specialized change (spec, docsite or tooling only)', + reason: skipStep.name, }; const scopeIndex = steps.findIndex(step => step.name === SCOPE_STEP); if (scopeIndex !== -1 && job.conclusion === 'success') { @@ -98,7 +109,7 @@ function summarizeJobs(jobs) { } function suiteOf(jobName) { - return SUITES.find(suite => suite.jobs.includes(jobName)) ?? null; + return SUITES.find(suite => suite.jobs.test(jobName)) ?? null; } const CONCLUSION_TEXT = { @@ -114,7 +125,7 @@ function conclusionText(conclusion) { /** One suite's state from its jobs: ran (and how), scoped out, or not run. */ function suiteJobState(suite, jobSummary) { - const jobs = jobSummary.filter(job => suite.jobs.includes(job.name)); + const jobs = jobSummary.filter(job => suite.jobs.test(job.name)); if (jobs.length === 0) return null; const ran = jobs.filter(job => job.state === 'ran'); if (ran.length > 0) { diff --git a/.github/scripts/lib/pr-status.test.mjs b/.github/scripts/lib/pr-status.test.mjs index 186b640e22415..90815660ae0fe 100644 --- a/.github/scripts/lib/pr-status.test.mjs +++ b/.github/scripts/lib/pr-status.test.mjs @@ -73,7 +73,31 @@ describe('classifyJob', () => { ['Skip heavy work for a specialized lane', 'success'], ['Checkout', 'skipped'], ]); - expect(classifyJob(skipped).state).toBe('scoped-out'); + expect(classifyJob(skipped)).toEqual({ + state: 'scoped-out', + reason: 'Skip heavy work for a specialized lane', + }); + }); +}); + +describe('suite jobs by display name', () => { + it.each([ + ['Stable visual regression', 'Visual'], + [ + 'Stable visual scout (${{ matrix.shard }}/${{ strategy.job-total }})', + 'Visual', + ], + ['pr-a11y (browser specs)', 'Accessibility'], + ['Accessibility spec-test contracts', 'Accessibility'], + ['pr-rtl (core)', 'RTL'], + ])('reads a skipped %s as the %s suite not running', (name, label) => { + const table = buildStatusTable({ + analysis: {newComponents: [], modifiedComponents: []}, + jobs: [job(name, 'skipped')], + }); + expect(table).toContain( + `| ${label} | ⏭ ${label} did not run on this pull request;`, + ); }); }); From 197273fd230652725b2aa51a5de7a49aaaf14cbe Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 10:33:39 -0700 Subject: [PATCH 09/15] ci: fold a skipped matrix's unexpanded job name into its job --- .github/scripts/lib/pr-status.js | 8 ++++++-- .github/scripts/lib/pr-status.test.mjs | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js index d1b5534aef025..fcdb892756a12 100644 --- a/.github/scripts/lib/pr-status.js +++ b/.github/scripts/lib/pr-status.js @@ -45,9 +45,13 @@ function notRunSentence(suite) { const SCOPE_STEP = "Decide this job's scope"; const BOOKKEEPING_STEP = /^(Set up job|Complete job|Post )/; -/** A matrix leg's job name without its `(1/3)` suffix. */ +/** + * A job's name without its parenthesized suffix: a matrix leg's `(1/3)`, a + * variant's `(browser specs)`, or a skipped matrix's unexpanded expression, + * which can hold parentheses of its own. + */ function baseJobName(name) { - return String(name ?? '').replace(/\s*\([^)]*\)\s*$/, ''); + return String(name ?? '').replace(/\s*\(.*\)\s*$/, ''); } /** diff --git a/.github/scripts/lib/pr-status.test.mjs b/.github/scripts/lib/pr-status.test.mjs index 90815660ae0fe..fa661b33a7296 100644 --- a/.github/scripts/lib/pr-status.test.mjs +++ b/.github/scripts/lib/pr-status.test.mjs @@ -80,6 +80,19 @@ describe('classifyJob', () => { }); }); +describe('job names', () => { + it("folds a skipped matrix's unexpanded name, parentheses and all", () => { + const summary = summarizeJobs([ + job('pr-rtl', 'skipped'), + job( + "pr-rtl (${{ matrix.package }}${{ (((matrix.parts > 1) && format(' {0}/{1}', matrix.part, matrix.parts)) || '') }})", + 'skipped', + ), + ]); + expect(summary).toEqual([{name: 'pr-rtl', state: 'not-run'}]); + }); +}); + describe('suite jobs by display name', () => { it.each([ ['Stable visual regression', 'Visual'], From 2b2736fb9dfa40c82ff29c32ec0b8cd212012ff9 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 11:04:58 -0700 Subject: [PATCH 10/15] =?UTF-8?q?ci:=20the=20PR=20report=20follows=20the?= =?UTF-8?q?=20agreed=20mockup:=20preview=20first,=20one=20modified-compone?= =?UTF-8?q?nts=20table=20(name=20links=20docs,=20Storybook,=20size,=20LoC,?= =?UTF-8?q?=20Visual=20=C2=B7=20A11y=20=C2=B7=20RTL=20marks),=20collapsed?= =?UTF-8?q?=20visual=20thumbnails,=20package=20size=20lines,=20one=20not-r?= =?UTF-8?q?un=20hint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changed shots' frames are published to the PR's assets branch by publish-frames.mjs so the report can show before/after/diff thumbnails; the touched-components report (wide-change flag included) is read from its own artifact. --- .github/scripts/generate-pr-comment.js | 408 ++--------- .github/scripts/generate-pr-comment.test.mjs | 410 +++-------- .github/scripts/lib/pr-preview.mjs | 10 +- .github/scripts/lib/pr-preview.test.mjs | 80 +- .github/scripts/lib/pr-status.js | 692 ++++++++++-------- .github/scripts/lib/pr-status.test.mjs | 580 ++++++++------- .../scripts/visual-gate/publish-frames.mjs | 163 +++++ .../visual-gate/publish-frames.test.mjs | 134 ++++ .github/workflows/pr-comment.yml | 39 +- 9 files changed, 1300 insertions(+), 1216 deletions(-) create mode 100644 .github/scripts/visual-gate/publish-frames.mjs create mode 100644 .github/scripts/visual-gate/publish-frames.test.mjs diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.js index a97567eb1bc1e..cc8539def9755 100644 --- a/.github/scripts/generate-pr-comment.js +++ b/.github/scripts/generate-pr-comment.js @@ -2,23 +2,20 @@ // Copyright (c) Meta Platforms, Inc. and affiliates. /** - * @description Generates and posts PR comment with analysis results - * @input --analysis --base-stats --package-size --a11y --visual --preview --jobs --touched --source-conclusion --run-url - * @output Formatted markdown comment body to stdout: one status table first - * (this head's preview, size, visual, accessibility, RTL, perf when it ran, - * and the CI jobs), then the touched components with their preview links - * and results, then the detail sections, collapsed + * @description Generates the PR comment from the run's analysis results + * @input --analysis --base-stats --package-size --a11y --visual --visual-image-url --preview --jobs --touched --source-conclusion --run-url + * @output Formatted markdown comment body to stdout, compact by design: this + * head's preview first, the modified components (preview links, size, lines + * of code, a mark per browser suite that ran; three rows visible, the rest + * collapsed), visual changes as thumbnails, one line for package size and + * one for suites that did not run. Accessibility violations add one + * collapsed section. GitHub shows + * the CI checks and CodeBunny posts its own review; neither is repeated. */ const fs = require('node:fs'); const {buildA11ySection} = require('./lib/a11y-format'); -const {buildVisualSection} = require('./lib/visual-format'); -const {renderPackageSizeSection} = require('./lib/package-size'); -const { - buildStatusTable, - buildJobTable, - buildTouchedComponents, -} = require('./lib/pr-status'); +const {buildCompactReport} = require('./lib/pr-status'); // Report-file strings render as literal inline text, report numbers as // numbers — the comment's structure comes only from this file's literals. const {inline, num} = require('./lib/report-text'); @@ -31,18 +28,12 @@ const getArg = name => { const analysisFile = getArg('analysis') || 'analysis.json'; const baseStatsFile = getArg('base-stats'); -const packageSizeFile = getArg('package-size'); const a11yFile = getArg('a11y') || 'a11y-report.json'; -const visualFile = getArg('visual'); const runUrl = getArg('run-url') || ''; -const prNumber = getArg('pr-number') || ''; const sourceConclusion = getArg('source-conclusion') || ''; -// Optional JSON inputs the comment job writes: this head's preview -// ({state, url, sha, aliasOrigin}), the source run's jobs, and the -// touched-components report. A missing or unreadable file leaves its part out. -function readOptionalJson(name) { - const file = getArg(name); +// Optional JSON inputs. A missing or unreadable file leaves its part out. +function readOptionalJson(name, file = getArg(name)) { if (!file) return null; try { return JSON.parse(fs.readFileSync(file, 'utf8')); @@ -51,351 +42,74 @@ function readOptionalJson(name) { return null; } } -const preview = readOptionalJson('preview'); -const jobs = readOptionalJson('jobs'); -const touched = readOptionalJson('touched'); -// Links go to this head's own deployment when it is ready, else to the -// branch alias (the latest ready deployment). -const previewOrigin = - preview?.state === 'ready' ? preview.url : preview?.aliasOrigin || ''; -const storybookUrl = previewOrigin ? `${previewOrigin}/storybook/` : ''; -// Read analysis results -let analysis = { - newComponents: [], - modifiedComponents: [], - componentStats: {}, - totalBundle: {}, -}; -let a11yReport = null; // stays null when the audit did not run +let analysis = {newComponents: [], modifiedComponents: [], componentStats: {}}; try { analysis = JSON.parse(fs.readFileSync(analysisFile, 'utf8')); } catch (e) { console.error('Warning: Could not read analysis file:', e.message); } -// Base measurements from a built checkout of the merge base, written by -// `analyze-pr.js --base-only`. The caller has already matched their head. -if (baseStatsFile) { - try { - const base = JSON.parse(fs.readFileSync(baseStatsFile, 'utf8')); - analysis.baseComponentStats = base.baseComponentStats || undefined; - analysis.baseBundlePackages = base.baseBundlePackages || undefined; - analysis.baseComponentSizes = base.baseComponentSizes || undefined; - } catch (e) { - console.error('Warning: Could not read base stats:', e.message); - } +// Base measurements of the merge base, when the caller has one for this +// exact head (it has matched the head). +const baseStats = readOptionalJson('base-stats', baseStatsFile); +if (baseStats) { + analysis.baseComponentStats = baseStats.baseComponentStats || undefined; + analysis.baseBundlePackages = baseStats.baseBundlePackages || undefined; + analysis.baseComponentSizes = baseStats.baseComponentSizes || undefined; } +// This head's preview ({state, url, sha, aliasOrigin}), the source run's +// jobs, the touched-components report, the package-size report, and the +// browser suites' results. The accessibility report is absent when the +// audit did not run; the visual verdict when the visual job did not. +const preview = readOptionalJson('preview'); +const jobs = readOptionalJson('jobs'); +const touched = readOptionalJson('touched'); +const packageSize = readOptionalJson('package-size'); +const visualVerdict = readOptionalJson('visual'); +let a11yReport = null; try { a11yReport = JSON.parse(fs.readFileSync(a11yFile, 'utf8')); } catch (e) { console.error('Warning: Could not read a11y report:', e.message); } -// Build a Storybook deep link URL for a component -// Story titles like "Core/XDSButton" become path "/docs/core-xdsbutton--docs" -function getStorybookLink(storybookBaseUrl, storyTitle) { - if (!storybookBaseUrl || !storyTitle) return null; - // storyTitle may be a single string or an array when multiple story files match - const title = Array.isArray(storyTitle) ? storyTitle[0] : storyTitle; - if (!title) return null; - // Storybook converts "Core/XDSButton" to "core-xdsbutton" as the story ID - // prefix. Story ids only ever contain [a-z0-9_-]; the title came from a - // report file, so drop anything outside that alphabet before it lands in - // the href. - const storyPath = title - .toLowerCase() - .replace(/\//g, '-') - .replace(/[^a-z0-9_-]/g, ''); - return `${storybookBaseUrl}?path=/docs/${storyPath}--docs`; -} - -// Build an external link that opens in a new tab function extLink(text, url) { return `${text}`; } -// Difference of two measured byte counts, or null when either side was not -// measured. A missing size is never treated as 0. -function bytesDelta(after, before) { - return typeof after === 'number' && - Number.isFinite(after) && - typeof before === 'number' && - Number.isFinite(before) - ? after - before - : null; -} - -// A measured byte count as text, or N/A. Only numbers reach the output. -function sizeText(bytes) { - if (typeof bytes !== 'number' || !Number.isFinite(bytes)) return 'N/A'; - if (Math.abs(bytes) < 1024) return `${bytes}B`; - return `${(bytes / 1024).toFixed(1)}KB`; -} - -function signedSizeText(delta) { - if (delta === null) return 'N/A'; - return delta > 0 ? `+${sizeText(delta)}` : sizeText(delta); -} - -// Touch-only budget per component (architecture:component-style-authoring -// INV6, spec:AST-080 FR8): 2 KB gzip JavaScript, 1 KB gzip CSS. The report -// cannot tell whether growth runs only on a coarse pointer, so it flags any -// growth over budget and asks the author. -const JS_BUDGET_BYTES = 2 * 1024; -const CSS_BUDGET_BYTES = 1024; -const budgetFlags = []; - -// Build component stats section -let componentSection = ''; -if (analysis.newComponents && analysis.newComponents.length > 0) { - componentSection += `### New Components\n\n`; - for (const comp of analysis.newComponents) { - const stats = analysis.componentStats[comp] || {}; - const sbLink = getStorybookLink(storybookUrl, stats.storyTitle); - const sbBadge = sbLink ? ` · ${extLink('View in Storybook', sbLink)}` : ''; - const pkgBadge = stats.package - ? ` (${inline(stats.package)})` - : ''; - componentSection += `
\n${inline(comp)}${pkgBadge}${sbBadge}\n\n`; - componentSection += `| Metric | Value |\n|--------|-------|\n`; - componentSection += `| Entry File (ESM) | ${inline(stats.esmSize) || 'N/A'} |\n`; - if (stats.cjsSize) - componentSection += `| Entry File (CJS) | ${inline(stats.cjsSize)} |\n`; - componentSection += `| Lines of Code | ${inline(stats.linesOfCode) || 'N/A'} |\n`; - componentSection += `| Source Files | ${inline(stats.fileCount) || 'N/A'} |\n`; - componentSection += `| Complexity | ${inline(stats.complexityRating) || 'N/A'} (${num(stats.complexity)}) |\n`; - componentSection += `| Exports | ${stats.exports?.map(inline).join(', ') || 'N/A'} |\n`; - componentSection += `| Props Count | ${inline(stats.propsCount) || 'N/A'} |\n`; - componentSection += `| Has Tests | ${stats.hasTests ? 'Yes' : 'No'} |\n`; - componentSection += `| Has Stories | ${stats.hasStories ? 'Yes' : 'No'} |\n`; - componentSection += `\n
\n\n`; - } -} - -if (analysis.modifiedComponents && analysis.modifiedComponents.length > 0) { - componentSection += `### Modified Components\n\n`; - for (const comp of analysis.modifiedComponents) { - const stats = analysis.componentStats[comp] || {}; - const baseStats = analysis.baseComponentStats?.[comp] || {}; - const sbLink = getStorybookLink(storybookUrl, stats.storyTitle); - const sbBadge = sbLink ? ` · ${extLink('View in Storybook', sbLink)}` : ''; - - // Check if this modified directory has new exports - const newExports = analysis.newExports || []; - const compExports = stats.exports || []; - const newInComp = compExports.filter(e => newExports.includes(e)); - const newBadge = - newInComp.length > 0 ? ` · 🆕 ${newInComp.map(inline).join(', ')}` : ''; - const pkgBadge = stats.package - ? ` (${inline(stats.package)})` - : ''; - - componentSection += `
\n${inline(comp)}${pkgBadge}${sbBadge}${newBadge}\n\n`; - componentSection += `| Metric | Before | After | Delta |\n|--------|--------|-------|-------|\n`; - - const esmDelta = bytesDelta(stats.esmBytes, baseStats.esmBytes); - const esmDeltaStr = - esmDelta !== null - ? esmDelta > 0 - ? `+${esmDelta}B` - : `${esmDelta}B` - : 'N/A'; - - componentSection += `| Entry File (ESM) | ${inline(baseStats.esmSize) || 'N/A'} | ${inline(stats.esmSize) || 'N/A'} | ${esmDeltaStr} |\n`; - componentSection += `| Lines of Code | ${inline(baseStats.linesOfCode) || 'N/A'} | ${inline(stats.linesOfCode) || 'N/A'} | - |\n`; - componentSection += `| Complexity | ${inline(baseStats.complexityRating) || 'N/A'} | ${inline(stats.complexityRating) || 'N/A'} (${num(stats.complexity)}) | - |\n`; - componentSection += `\n
\n\n`; - } -} - -// Import and CSS size of every public component whose implementation changed, -// family folders included, before and after. -let sizeSection = ''; -const componentSizes = analysis.componentSizes || {}; -const baseComponentSizes = analysis.baseComponentSizes || {}; -const sizeKeys = [ - ...new Set([ - ...Object.keys(componentSizes), - ...Object.keys(baseComponentSizes), - ]), -].sort(); -if (sizeKeys.length > 0) { - sizeSection += '### Component Size\n\n'; - sizeSection += - '| Component | Import before | Import after | Delta | CSS before | CSS after | Delta |\n'; - sizeSection += - '|-----------|---------------|--------------|-------|------------|-----------|-------|\n'; - for (const key of sizeKeys) { - const head = componentSizes[key] || {}; - const base = baseComponentSizes[key] || {}; - const importDelta = bytesDelta(head.importGzipBytes, base.importGzipBytes); - const cssDelta = bytesDelta(head.cssGzipBytes, base.cssGzipBytes); - const jsOver = importDelta !== null && importDelta > JS_BUDGET_BYTES; - const cssOver = cssDelta !== null && cssDelta > CSS_BUDGET_BYTES; - if (jsOver || cssOver) - budgetFlags.push({comp: key, importDelta, cssDelta, jsOver, cssOver}); - sizeSection += `| ${inline(key)} | ${sizeText(base.importGzipBytes)} | ${sizeText(head.importGzipBytes)} | ${signedSizeText(importDelta)}${jsOver ? ' ⚠️' : ''} | ${sizeText(base.cssGzipBytes)} | ${sizeText(head.cssGzipBytes)} | ${signedSizeText(cssDelta)}${cssOver ? ' ⚠️' : ''} |\n`; - } - sizeSection += - '\nSizes are gzip. Import counts the JavaScript the import loads eagerly; CSS counts the package stylesheet rules referenced by the modules it can load from its own package, lazily loaded ones included.\n\n'; -} - -// Components whose growth passes the touch-only budget. -let budgetSection = ''; -if (budgetFlags.length > 0) { - budgetSection += '### ⚠️ Touch-only size budget\n\n'; - budgetSection += - 'These components grew past the touch-only budget (2 KB gzip JavaScript or 1 KB gzip CSS per component). If the growth runs only on a coarse pointer, load its JavaScript lazily so the import does not carry it, or justify the cost in the pull request description; CSS stays in the package stylesheet, so justify it.\n\n'; - budgetSection += - '| Component | JavaScript | CSS |\n|-----------|------------|-----|\n'; - for (const flag of budgetFlags) { - budgetSection += `| ${inline(flag.comp)} | ${signedSizeText(flag.importDelta)}${flag.jsOver ? ' ⚠️' : ''} | ${signedSizeText(flag.cssDelta)}${flag.cssOver ? ' ⚠️' : ''} |\n`; - } - budgetSection += '\n'; -} - -// Build accessibility section using shared module -const a11ySection = buildA11ySection(a11yReport); +const details = (summary, content) => + content + ? `
\n${summary}\n\n${content}\n
\n\n` + : ''; -// Visual regression is optional: the pr-visual job is skipped when no -// components changed, and absent entirely on older runs. -let visualVerdict = null; -if (visualFile) { - try { - visualVerdict = JSON.parse(fs.readFileSync(visualFile, 'utf8')); - } catch { - visualVerdict = null; - } -} -const visualSection = buildVisualSection( - visualVerdict, - getArg('visual-report-url'), - getArg('visual-image-url'), +// The audit's findings, collapsed: the component table carries each verdict. +const violations = Object.values(a11yReport?.components || {}).reduce( + (sum, component) => sum + (component.violations?.length || 0), + 0, ); +const a11yDetails = + violations > 0 + ? details( + `Accessibility: ${num(violations)} violation(s)`, + buildA11ySection(a11yReport), + ) + : ''; -// Build bundle size section — one row per package the PR actually touched. -let bundleSection = '### Bundle Size Summary\n\n'; -// Prefer the multi-package list; fall back to the legacy single-core shape so -// an older analysis.json still renders. -const bundlePackages = - analysis.bundlePackages && analysis.bundlePackages.length > 0 - ? analysis.bundlePackages - : analysis.totalBundle - ? [{package: '@astryxdesign/core', ...analysis.totalBundle}] - : []; - -const baseBundles = analysis.baseBundlePackages; -const signedBytes = delta => - delta === null ? 'N/A' : delta > 0 ? `+${delta}B` : `${delta}B`; -if (bundlePackages.length > 0 && Array.isArray(baseBundles)) { - bundleSection += `Gzip figures measure each package's entry file, not the code an import pulls in.\n\n`; - bundleSection += `| Package | Entry (ESM) | Entry gzip before | Entry gzip after | Delta |\n`; - bundleSection += `|---------|-------------|-------------------|------------------|-------|\n`; - for (const b of bundlePackages) { - const base = baseBundles.find(entry => entry.package === b.package) || {}; - const delta = bytesDelta(b.gzipBytes, base.gzipBytes); - bundleSection += `| ${inline(b.package)} | ${inline(b.esmSize) || 'N/A'} | ${inline(base.gzipSize) || 'N/A'} | ${inline(b.gzipSize) || 'N/A'} | ${signedBytes(delta)} |\n`; - } - bundleSection += `\n`; - const sheets = bundlePackages.filter( - b => typeof b.stylesheetGzipBytes === 'number', - ); - if (sheets.length > 0) { - bundleSection += `| Package stylesheet | Gzip before | Gzip after | Delta |\n`; - bundleSection += `|--------------------|-------------|------------|-------|\n`; - for (const b of sheets) { - const base = baseBundles.find(entry => entry.package === b.package) || {}; - const delta = bytesDelta(b.stylesheetGzipBytes, base.stylesheetGzipBytes); - bundleSection += `| ${inline(b.package)} | ${sizeText(base.stylesheetGzipBytes)} | ${sizeText(b.stylesheetGzipBytes)} | ${signedSizeText(delta)} |\n`; - } - bundleSection += `\n`; - } -} else if (bundlePackages.length > 0) { - bundleSection += `| Package | Size (ESM) | Size (CJS) | Gzipped |\n`; - bundleSection += `|---------|------------|------------|----------|\n`; - for (const b of bundlePackages) { - bundleSection += `| ${inline(b.package)} | ${inline(b.esmSize) || 'N/A'} | ${inline(b.cjsSize) || 'N/A'} | ${inline(b.gzipSize) || 'N/A'} |\n`; - } - bundleSection += `\n`; -} else { - bundleSection += '_No component packages changed._\n\n'; -} - -if (analysis.bundleDelta) { - const delta = num(analysis.bundleDelta); - const direction = - delta > 0 ? 'increased' : delta < 0 ? 'decreased' : 'unchanged'; - bundleSection += `**Bundle size ${direction}:** ${delta > 0 ? '+' : ''}${delta} bytes\n\n`; -} - -// Published package sizes from build-storybook's package-size check: the -// download and unpacked size of every published package, base against head, -// and any runtime dependency added or removed. The caller has already matched -// its head. -let packageSizeSection = ''; -let packageSize = null; -if (packageSizeFile) { - try { - packageSize = JSON.parse(fs.readFileSync(packageSizeFile, 'utf8')); - packageSizeSection = renderPackageSizeSection(packageSize, inline); - } catch (e) { - console.error('Warning: Could not read package size report:', e.message); - } -} - -// Footer links: the full CI run. -const footerLinksStr = runUrl - ? ` | ${extLink('View full report', runUrl)}` - : ''; +// Shots the visual job could not capture: one line; changed shots get their +// own thumbnail table in the report. +const visualFailed = + visualVerdict?.status === 'failed' + ? `> **Visual:** ${num(visualVerdict.counts?.failed)} shot(s) could not be captured.\n\n` + : ''; -// A run that did not succeed may have published nothing for this head. +// A run that did not succeed may have published only part of this report. const conclusionNote = sourceConclusion && sourceConclusion !== 'success' ? `> **Current run:** CI concluded ${inline(sourceConclusion)}; parts of this report may be missing.\n\n` : ''; -const statusTable = buildStatusTable({ - preview, - analysis, - budgetFlags, - packageSize, - a11yReport, - visualVerdict, - jobs, -}); -const touchedSection = buildTouchedComponents({ - analysis, - origin: previewOrigin, - touched, - a11yReport, - visualVerdict, -}); - -// Detail sections, collapsed: the table above carries their verdicts. -const details = (summary, content) => - content - ? `
\n${summary}\n\n${content}\n
\n\n` - : ''; -const sizeDetails = details( - 'Size details', - `${sizeSection}${budgetSection}${bundleSection}${packageSizeSection}`, -); -const componentDetails = details('Component details', componentSection); -// Visual changes are a question for the reviewer: their frames stay open. -const visualDetails = visualSection - ? visualVerdict?.changes?.length > 0 - ? visualSection - : details('Visual regression', visualSection) - : ''; -// The accessibility detail only matters when the audit found something. -const a11yViolations = Object.values(a11yReport?.components || {}).some( - component => (component.violations?.length || 0) > 0, -); -const a11yDetails = a11yViolations ? a11ySection : ''; -const jobTable = buildJobTable(jobs); - // A one-line caveat shown when the analysis fell back to the approximate // two-dot diff. The file list may include base-only churn and, worse, may // miss a component the PR actually changed (if that change also exists on @@ -405,14 +119,26 @@ const diffModeCaveat = ? '> ⚠️ **Approximate analysis** - the diff could not be resolved to a merge base, so a two-dot fallback was used. The component list may include base-only changes or miss a component whose change also landed on base.\n\n' : ''; -// Build the full comment +const report = buildCompactReport({ + analysis, + preview, + jobs, + touched, + a11yReport, + visualVerdict, + packageSize, + visualImageUrl: getArg('visual-image-url'), +}); +const footerLinksStr = runUrl + ? ` | ${extLink('View full report', runUrl)}` + : ''; + const body = `## PR Analysis Report -${conclusionNote}${diffModeCaveat}${statusTable}${touchedSection}${sizeDetails}${componentDetails}${visualDetails}${a11yDetails}${jobTable}--- +${report || '_No component or package changed._\n\n'}${conclusionNote}${diffModeCaveat}${visualFailed}${a11yDetails}--- Generated by PR Enrichment workflow${footerLinksStr} `; -// Output to stdout console.log(body); diff --git a/.github/scripts/generate-pr-comment.test.mjs b/.github/scripts/generate-pr-comment.test.mjs index fe3828d727883..b47529285ffd2 100644 --- a/.github/scripts/generate-pr-comment.test.mjs +++ b/.github/scripts/generate-pr-comment.test.mjs @@ -66,9 +66,9 @@ describe('generate-pr-comment diffMode caveat', () => { }); }); -describe('generate-pr-comment status table', () => { +describe('generate-pr-comment compact report', () => { function withInputs(inputs, analysis = baseAnalysis, extra = []) { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-status-')); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-report-')); try { const args = []; for (const [name, value] of Object.entries(inputs)) { @@ -83,344 +83,170 @@ describe('generate-pr-comment status table', () => { } const HEAD = 'a'.repeat(40); const READY = 'https://astryx-abc123-fbopensource.vercel.app'; - const ALIAS = 'https://astryx-git-my-branch-fbopensource.vercel.app'; - - it('leads with the status table, the preview first', () => { - const out = withInputs({preview: {state: 'ready', sha: HEAD, url: READY}}); - const table = out.indexOf('| Check | Result |'); - expect(table).toBeGreaterThan(out.indexOf('')); - expect(out.slice(table)).toMatch( - /^\| Check \| Result \|\n\|-------\|--------\|\n\| Preview \|/, - ); - expect(out).toContain( - `| Preview | [Storybook](${READY}/storybook/) · [Sandbox](${READY}/sandbox/) · [Docs](${READY}/) · built from \`aaaaaaa\` |`, - ); - expect(out).not.toMatch(/CodeBunny/i); + const size = (importGzipBytes, cssGzipBytes = 1000) => ({ + package: '@astryxdesign/core', + importGzipBytes, + cssGzipBytes, }); - - it.each([ - [ - 'building', - {state: 'building', sha: HEAD, aliasOrigin: ALIAS}, - `⏳ Preview building for \`aaaaaaa\`… Until it's ready, [Storybook](${ALIAS}/storybook/)`, - ], - ['failed', {state: 'failed', sha: HEAD}, 'failed to build'], - [ - 'fork', - {state: 'fork', sha: HEAD}, - 'No preview for pull requests from forks.', - ], - ['unknown', {state: 'unknown', sha: HEAD}, 'cannot be known in advance'], - ])('says what a %s preview is, honestly', (_label, preview, text) => { - const out = withInputs({preview}); - expect(out).toContain(text); - if (preview.state !== 'building') expect(out).not.toContain('/storybook/'); + const sized = names => ({ + ...baseAnalysis, + modifiedComponents: [], + componentSizes: Object.fromEntries( + names.map(([key, after]) => [key, size(after)]), + ), + baseComponentSizes: Object.fromEntries( + names.map(([key, , before]) => [key, size(before)]), + ), }); - it('shows no preview row on a draft, which links no preview', () => { - const out = withInputs({}); - expect(out).toContain('| Check | Result |'); - expect(out).not.toContain('| Preview |'); + it("opens with this head's preview, before anything else", () => { + const out = withInputs({preview: {state: 'ready', sha: HEAD, url: READY}}); + const body = out.slice(out.indexOf('')); + expect(body.split('\n')[2]).toBe('### 🔎 Preview · `aaaaaaa`'); + expect(body.split('\n')[4]).toBe( + `**[Storybook](${READY}/storybook/) · [Sandbox](${READY}/sandbox/)** · built from this PR's head`, + ); }); - it('says a heavy suite did not run, with the label that runs it', () => { - const out = withInputs({ - jobs: [ - {name: 'pr-visual', conclusion: 'skipped', steps: []}, - {name: 'pr-rtl', conclusion: 'skipped', steps: []}, - ], - }); - expect(out).toContain( - '| Visual | ⏭ Visual did not run on this pull request; add `ci:run-visual` to run it. |', + it('shows three components and collapses the rest, the biggest growth first', () => { + const out = withInputs( + {preview: {state: 'ready', sha: HEAD, url: READY}}, + sized([ + ['core/A', 5000, 5000], + ['core/B', 9000, 8000], + ['core/C', 7000, 6900], + ['core/D', 4000, 4100], + ['core/E', 3000, 2990], + ]), ); - expect(out).toContain( - '| Accessibility | ⏭ Accessibility did not run on this pull request; add `ci:run-a11y` to run it. |', - ); - expect(out).toContain( - '| RTL | ⏭ RTL did not run on this pull request; add `ci:run-rtl` to run it. |', + const visible = out.slice(0, out.indexOf('
')); + expect(visible).toContain('### Modified components'); + expect(visible.match(/^\| \[[A-E]\]/gm)).toHaveLength(3); + expect(visible.indexOf('[B]')).toBeLessThan(visible.indexOf('[C]')); + expect(visible.indexOf('[C]')).toBeLessThan(visible.indexOf('[E]')); + expect(out).toContain('2 more components'); + expect(out.slice(out.indexOf('
'))).toContain( + `| [D](${READY}/components/D) |`, ); - expect(out).not.toContain('| Perf |'); - }); - - it('notes a run that did not succeed', () => { - const out = runComment(baseAnalysis, ['--source-conclusion', 'failure']); - expect(out).toContain('CI concluded failure'); + expect(out).toContain(`| [B](${READY}/components/B) |`); }); - it('collapses the size tables', () => { + it('flags growth past the touch-only budget, family folders included', () => { const out = runComment({ ...baseAnalysis, - componentSizes: { - 'core/Card': { - package: '@astryxdesign/core', - importGzipBytes: 2000, - cssGzipBytes: 100, - }, - }, + componentSizes: {'core/Chat': size(9000, 1000)}, + baseComponentSizes: {'core/Chat': size(6000, 1000)}, }); - expect(out).toContain('Size details'); - expect(out.indexOf('Size details')).toBeLessThan( - out.indexOf('### Component Size'), + expect(out).toContain( + '| Chat | 🔴 8.8 kB (**+2.9 kB**) · 1.0 kB CSS (0) |', ); - expect(out).toMatch(/\| Size \| ✅ 1 component\(s\) within budget/); }); -}); - -describe('generate-pr-comment base sizes', () => { - const sized = { - ...baseAnalysis, - componentStats: { - Card: {package: '@astryxdesign/core', esmSize: '160B', esmBytes: 160}, - }, - bundlePackages: [ - { - package: '@astryxdesign/core', - esmSize: '900B', - gzipSize: '300B', - gzipBytes: 300, - }, - ], - }; - it('fills Before and Delta from base stats', () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-base-stats-')); - try { - const baseFile = path.join(dir, 'size-base.json'); - fs.writeFileSync( - baseFile, - JSON.stringify({ - baseComponentStats: { - Card: { - package: '@astryxdesign/core', - esmSize: '100B', - esmBytes: 100, - }, - }, - baseBundlePackages: [ - {package: '@astryxdesign/core', gzipSize: '280B', gzipBytes: 280}, - ], - }), - ); - const out = runComment(sized, ['--base-stats', baseFile]); - expect(out).toContain('| Entry File (ESM) | 100B | 160B | +60B |'); - expect(out).toContain( - '| @astryxdesign/core | 900B | 280B | 300B | +20B |', - ); - } finally { - fs.rmSync(dir, {recursive: true, force: true}); - } + it('never reports a delta for an unmeasured size', () => { + const out = runComment({ + ...baseAnalysis, + componentSizes: {'core/Card': size(null), 'core/New': size(3000)}, + baseComponentSizes: {'core/Card': size(2000)}, + }); + expect(out).toContain('| Card | 1.0 kB CSS (0) |'); + expect(out).toContain('| New | 2.9 kB (new) · 1.0 kB CSS (new) |'); }); - it('never reports a delta for an unmeasured size', () => { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-base-null-')); + it('fills the base from base stats', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-base-')); try { - const baseFile = path.join(dir, 'size-base.json'); + const baseFile = path.join(dir, 'base.json'); fs.writeFileSync( baseFile, - JSON.stringify({ - baseComponentStats: { - Card: { - package: '@astryxdesign/charts', - esmSize: null, - esmBytes: null, - }, - }, - baseBundlePackages: [ - {package: '@astryxdesign/core', gzipSize: null, gzipBytes: null}, - ], - }), + JSON.stringify({baseComponentSizes: {'core/Card': size(2000)}}), ); const out = runComment( - { - ...sized, - componentStats: { - Card: { - package: '@astryxdesign/charts', - esmSize: null, - esmBytes: null, - }, - }, - bundlePackages: [ - { - package: '@astryxdesign/core', - esmSize: '900B', - gzipSize: '300B', - gzipBytes: 300, - }, - ], - }, + {...baseAnalysis, componentSizes: {'core/Card': size(1500)}}, ['--base-stats', baseFile], ); - expect(out).toContain('| Entry File (ESM) | N/A | N/A | N/A |'); - expect(out).toContain('| @astryxdesign/core | 900B | N/A | 300B | N/A |'); - expect(out).not.toMatch(/\| [+-]?0B \|/); + expect(out).toContain('| Card | 1.5 kB (−500 B) · 1.0 kB CSS (0) |'); } finally { fs.rmSync(dir, {recursive: true, force: true}); } }); - it('keeps N/A without base stats', () => { - const out = runComment(sized); - expect(out).toContain('| Entry File (ESM) | N/A | 160B | N/A |'); - expect(out).toContain('| Package | Size (ESM) | Size (CJS) | Gzipped |'); - }); -}); - -describe('generate-pr-comment touch-only budget', () => { - function withBase(head, base) { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-budget-')); - try { - const baseFile = path.join(dir, 'size-base.json'); - fs.writeFileSync( - baseFile, - JSON.stringify({ - baseComponentSizes: {'core/BottomSheet': base}, - baseBundlePackages: [], - }), - ); - return runComment( - { - ...baseAnalysis, - modifiedComponents: [], - componentSizes: {'core/BottomSheet': head}, - bundlePackages: [], - }, - ['--base-stats', baseFile], - ); - } finally { - fs.rmSync(dir, {recursive: true, force: true}); - } - } - - it('flags JavaScript growth over 2 KB and CSS growth over 1 KB, family folders included', () => { - const out = withBase( - { - package: '@astryxdesign/core', - importGzipBytes: 10000, - cssGzipBytes: 3000, - }, - { - package: '@astryxdesign/core', - importGzipBytes: 7000, - cssGzipBytes: 1900, - }, - ); - expect(out).toContain('### ⚠️ Touch-only size budget'); + it('says in one line which suites did not run and how to run them', () => { + const out = withInputs({ + jobs: [ + {name: 'Stable visual regression', conclusion: 'skipped', steps: []}, + {name: 'pr-a11y (browser specs)', conclusion: 'skipped', steps: []}, + {name: 'pr-rtl', conclusion: 'skipped', steps: []}, + ], + }); expect(out).toContain( - '| core/BottomSheet | 6.8KB | 9.8KB | +2.9KB ⚠️ | 1.9KB | 2.9KB | +1.1KB ⚠️ |', + 'Not run on this PR: add `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` to run a full suite.', ); + expect(out).not.toContain('| Check |'); + expect(out).not.toMatch(/CodeBunny/i); }); - it('does not flag growth within budget', () => { - const out = withBase( - { - package: '@astryxdesign/core', - importGzipBytes: 9000, - cssGzipBytes: 2900, - }, - { - package: '@astryxdesign/core', - importGzipBytes: 7000, - cssGzipBytes: 1900, + it('puts package size on one line, with new dependencies', () => { + const entry = (name, before, after) => ({ + name, + base: {packedBytes: before}, + head: {packedBytes: after}, + overBudget: false, + }); + const out = withInputs({ + 'package-size': { + headSha: HEAD, + verdict: 'pass', + packages: [ + entry('@astryxdesign/core', 100000, 101229), + entry('@astryxdesign/build', 5000, 5000), + ], + addedDependencies: [ + { + package: '@astryxdesign/core', + field: 'dependencies', + name: 'left-pad', + range: '1', + }, + ], }, - ); - expect(out).not.toContain('Touch-only size budget'); + }); expect(out).toContain( - '| core/BottomSheet | 6.8KB | 8.8KB | +2.0KB | 1.9KB | 2.8KB | +1000B |', + '**Package size:** @astryxdesign/core 98.9 kB packed (**+1.2 kB**); new dependency: left-pad.', ); }); - it('does not flag a component that is new at head', () => { - const out = withBase( - { - package: '@astryxdesign/core', - importGzipBytes: 90000, - cssGzipBytes: 9000, - }, - { - package: '@astryxdesign/core', - importGzipBytes: null, - cssGzipBytes: null, - }, - ); - expect(out).not.toContain('Touch-only size budget'); - }); -}); - -describe('published package size', () => { - function withPackageSize(report) { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-package-size-')); + it('collapses accessibility violations', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-a11y-')); try { - const file = path.join(dir, 'package-size.json'); - fs.writeFileSync(file, JSON.stringify(report)); - return runComment({newComponents: [], modifiedComponents: []}, [ - '--package-size', + const file = path.join(dir, 'a11y.json'); + fs.writeFileSync( file, - ]); + JSON.stringify({ + components: { + Card: { + violations: [{id: 'label', impact: 'serious', description: 'x'}], + }, + }, + }), + ); + const out = runComment(baseAnalysis, ['--a11y', file]); + expect(out).toContain('Accessibility: 1 violation(s)'); + expect(out).toContain('| Card | — | — | — · ✗ · — |'); } finally { fs.rmSync(dir, {recursive: true, force: true}); } - } - - const sizes = (packedBytes, files = 10) => ({ - packedBytes, - unpackedBytes: packedBytes * 4, - files, }); - it('renders the section after the bundle summary, with new dependencies', () => { - const out = withPackageSize({ - headSha: 'a'.repeat(40), - budget: {packedPercent: 5, packedMinBytes: 1024}, - packages: [ - { - name: '@astryxdesign/core', - base: sizes(100000, 10), - head: sizes(110000, 12), - overBudget: true, - }, - ], - addedDependencies: [ - { - package: '@astryxdesign/core', - field: 'dependencies', - name: 'left-pad', - range: '^1.3.0', - }, - ], - removedDependencies: [], - verdict: 'over-budget', - }); - expect(out.indexOf('### Bundle Size Summary')).toBeLessThan( - out.indexOf('### Published Package Size'), - ); - expect(out).toContain( - '| @astryxdesign/core | 97.7KB | 107.4KB | +9.8KB (+10.0%) ⚠️ |', - ); - expect(out).toContain('| 10 → 12 |'); - expect(out).toContain('left-pad'); - expect(out).toContain('**⚠️ Over the package budget:**'); + it('notes a run that did not succeed', () => { + const out = runComment(baseAnalysis, ['--source-conclusion', 'failure']); + expect(out).toContain('CI concluded failure'); }); it('renders report strings as literal text', () => { - const out = withPackageSize({ - packages: [ - { - name: '', - base: sizes(1000), - head: sizes(2000), - overBudget: false, - }, - ], - addedDependencies: [], - removedDependencies: [], - verdict: 'pass', + const out = runComment({ + ...baseAnalysis, + componentSizes: {'core/': size(1000)}, }); - expect(out).not.toContain(' { - const out = runComment({newComponents: [], modifiedComponents: []}); - expect(out).not.toContain('Published Package Size'); + expect(out).not.toContain(''); }); }); diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjs index 9cf6af5606445..a62d7f23ebbf7 100644 --- a/.github/scripts/lib/pr-preview.mjs +++ b/.github/scripts/lib/pr-preview.mjs @@ -600,9 +600,10 @@ export async function reconcilePrComment({ metadataPath = 'pr-analysis/pr-meta.json', baseStatsPath = 'pr-size-base/size-base.json', packageSizePath = 'pr-analysis/package-size.json', - touchedPath = 'pr-analysis/touched-components.json', + touchedPath = 'touched-components/touched-components.json', a11yPath = 'a11y/a11y-report.json', visualPath = 'trusted-visual/verdict.json', + visualImageUrl = null, createIfMissing = true, fallbackMessage, generator = GENERATOR, @@ -692,6 +693,13 @@ export async function reconcilePrComment({ // checks, or with the ci:run-a11y label); the comment says so. ...(fs.existsSync(a11yPath) ? ['--a11y', a11yPath] : []), ...(fs.existsSync(visualPath) ? ['--visual', visualPath] : []), + // Frames publish-frames.mjs put on this PR's assets branch, and only + // from raw.githubusercontent.com. + ...(/^https:\/\/raw\.githubusercontent\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/assets\/pr-\d+\/visual\/[0-9a-f]{40}\/$/.test( + String(visualImageUrl ?? ''), + ) + ? ['--visual-image-url', visualImageUrl] + : []), ...(trustedHeadReport({ reportPath: touchedPath, label: 'touched-components report', diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjs index 720473aedd88e..3ad50af4750aa 100644 --- a/.github/scripts/lib/pr-preview.test.mjs +++ b/.github/scripts/lib/pr-preview.test.mjs @@ -202,6 +202,8 @@ async function reconcile({ execute, baseStats, packageSize, + touched, + visualImageUrl, } = {}) { const value = previewAvailable ? {...requested, headRepository: requested.baseRepository} @@ -219,6 +221,11 @@ async function reconcile({ ); if (packageSize) fs.writeFileSync(packageSizePath, JSON.stringify(packageSize)); + const touchedPath = path.join( + path.dirname(paths.analysis), + 'touched-components.json', + ); + if (touched) fs.writeFileSync(touchedPath, JSON.stringify(touched)); const {github, state} = githubFixture({ value: githubFixtureValue, comments, @@ -236,6 +243,8 @@ async function reconcile({ metadataPath: paths.metadata, baseStatsPath, packageSizePath, + touchedPath, + visualImageUrl, a11yPath: paths.a11y, visualPath: paths.visual, createIfMissing, @@ -883,7 +892,8 @@ describe('PR comment preview reconciliation', () => { expect(result.body).toContain( '[Storybook](https://astryx-abc123-fbopensource.vercel.app/storybook/)', ); - expect(result.body).toContain('built from `aaaaaaa`'); + expect(result.body).toContain('### 🔎 Preview · `aaaaaaa`'); + expect(result.body).toContain("built from this PR's head"); }); it("ignores a deployment Vercel did not create, and an address that is not Vercel's", async () => { @@ -907,7 +917,7 @@ describe('PR comment preview reconciliation', () => { const {result} = await withGithub(setup); expect(result.body).not.toContain('astryx-abc123'); expect(result.body).not.toContain('evil.example'); - expect(result.body).toContain('Preview building'); + expect(result.body).toContain('⏳ Building…'); } }); @@ -919,33 +929,26 @@ describe('PR comment preview reconciliation', () => { expect(result.body).toContain('failed to build'); }); - it("passes the run's jobs, so the report says what ran and what path scoping skipped", async () => { + it("passes the run's jobs, so the report says which suites did not run", async () => { const {result, args} = await withGithub({ jobs: [ { name: 'test-node (1/4)', status: 'completed', conclusion: 'success', - steps: [ - {name: "Decide this job's scope", conclusion: 'success'}, - {name: 'Run tests', conclusion: 'success'}, - ], + steps: [], }, { - name: 'docsite-test', + name: 'pr-rtl (core)', status: 'completed', - conclusion: 'success', - steps: [ - {name: "Decide this job's scope", conclusion: 'success'}, - {name: 'Build docsite', conclusion: 'skipped'}, - ], + conclusion: 'skipped', + steps: [], }, ], }); expect(args).toContain('--jobs'); - expect(result.body).toContain('| test-node | ✅ passed | |'); expect(result.body).toContain( - '| docsite-test | ⏭ skipped | nothing in its paths changed |', + 'Not run on this PR: add `ci:run-rtl` to run it.', ); }); }); @@ -999,7 +1002,7 @@ describe('PR comment preview reconciliation', () => { it('says the accessibility audit did not run when there is no report', async () => { const {result} = await reconcile({a11y: false}); - expect(result.body).toContain('did not run on this pull request'); + expect(result.body).toMatch(/Not run on this PR: add [^\n]*`ci:run-a11y`/); expect(result.body).not.toContain('No accessibility violations detected'); }); @@ -1007,7 +1010,7 @@ describe('PR comment preview reconciliation', () => { const value = identity({sourceConclusion: 'failure'}); const {result} = await reconcile({value}); - expect(result.body).toContain('Modified Components'); + expect(result.body).toContain('Modified components'); expect(result.body).toContain('Card'); expect(result.body).toContain('CI concluded failure'); expect(result.body).not.toContain('View Storybook for this PR'); @@ -1116,6 +1119,49 @@ describe('PR comment preview reconciliation', () => { }, ); + it.each([ + ['the exact head', HEAD, true], + ['another head', 'b'.repeat(40), false], + ])( + 'passes the touched-components report only when it names %s', + async (_label, headSha, used) => { + const seen = []; + await reconcile({ + touched: {headSha, components: []}, + execute: (_node, args) => { + seen.push(args); + return '## PR Analysis Report\n'; + }, + }); + expect(seen[0].includes('--touched')).toBe(used); + }, + ); + + it.each([ + [ + `https://raw.githubusercontent.com/meta/astryx/assets/pr-5697/visual/${HEAD}/`, + true, + ], + ['https://evil.example/visual/', false], + [ + `https://raw.githubusercontent.com/meta/astryx/main/visual/${HEAD}/`, + false, + ], + ])( + 'passes visual frames only from a PR assets branch: %s', + async (url, used) => { + const seen = []; + await reconcile({ + visualImageUrl: url, + execute: (_node, args) => { + seen.push(args); + return '## PR Analysis Report\n'; + }, + }); + expect(seen[0].includes('--visual-image-url')).toBe(used); + }, + ); + it('falls back to a safe current message when analysis rendering fails', async () => { const stale = { id: 78, diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js index fcdb892756a12..2e5d792843331 100644 --- a/.github/scripts/lib/pr-status.js +++ b/.github/scripts/lib/pr-status.js @@ -2,48 +2,55 @@ /** * @file pr-status.js - * @input The source CI run's job list (names, conclusions, steps), this - * head's Vercel preview state, the analysis, the size reports, the - * accessibility report, the visual verdict and, when present, the - * touched-components report - * @output The PR report's top: one status table (preview first, then size, - * visual, accessibility, RTL, perf when it ran, and CI jobs), the touched - * components with their preview links and results, and the collapsed CI job - * table + * @input The analysis (changed components, their import sizes and source + * stats at head and at the merge base), this head's Vercel preview state, + * the package-size report, the accessibility report, the visual verdict, + * the source CI run's jobs and, when present, the touched-components report + * @output The PR report's compact body: this head's preview first; one table + * of modified components (preview links, JS and CSS size and lines of code + * against main, and a fixed-position mark per browser suite when any ran), + * three rows visible and the rest collapsed; the visual deltas as + * before/after/diff thumbnails when shots changed; one line for package + * size; one line for suites that did not run * @position Used by generate-pr-comment.js. Everything here reads data the CI - * run and the comment job already have; nothing starts a job. + * run and the comment job already have; nothing starts a job. GitHub shows + * the CI checks and CodeBunny posts its own review, so neither is repeated. */ const {inline, num} = require('./report-text'); -const runLabels = require('../run-labels.cjs'); - /** * The heavy suites a pull request runs only with its `ci:run-` label, - * and the job display names (matrix suffix removed) that belong to each. + * and the job display names (any parenthesized suffix removed) of each. */ const SUITES = [ { id: 'visual', label: 'Visual', + short: 'Visual', jobs: /^(Stable visual|visual-scout|pr-visual)/i, }, { id: 'a11y', label: 'Accessibility', + short: 'A11y', jobs: /^(pr-a11y|Accessibility spec-test)/i, }, - {id: 'rtl', label: 'RTL', jobs: /^pr-rtl/i}, - {id: 'perf', label: 'Perf', jobs: /^(perf|pr-perf)/i}, + {id: 'rtl', label: 'RTL', short: 'RTL', jobs: /^pr-rtl/i}, + {id: 'perf', label: 'Perf', short: 'Perf', jobs: /^(perf|pr-perf)/i}, ]; -/** The sentence a heavy suite shows when it did not run (run-labels.cjs owns it). */ -function notRunSentence(suite) { - return runLabels.notRunSentence(suite.id); -} +/** Rows shown before the rest of a table is collapsed. */ +const VISIBLE_ROWS = 3; +/** + * Changed shots shown with frames, largest change first; publish-frames.mjs + * publishes the same shots in the same order. + */ +const VISUAL_FRAME_LIMIT = 12; -const SCOPE_STEP = "Decide this job's scope"; -const BOOKKEEPING_STEP = /^(Set up job|Complete job|Post )/; +/** Touch-only budget per component (architecture:component-style-authoring INV6). */ +const JS_BUDGET_BYTES = 2 * 1024; +const CSS_BUDGET_BYTES = 1024; /** * A job's name without its parenthesized suffix: a matrix leg's `(1/3)`, a @@ -55,257 +62,99 @@ function baseJobName(name) { } /** - * What one job did: `ran` (with its conclusion), `scoped-out` (it started, - * decided nothing in its scope changed, and skipped its work), or `not-run` - * (its run condition was false). + * Whether a suite ran on this run: `ran`, or `not-run` when every one of its + * jobs was skipped by its run condition (the label was absent). A suite with + * no jobs in the run is unknown (null). */ -function classifyJob(job) { - if (job.conclusion === 'skipped') return {state: 'not-run'}; - const steps = job.steps || []; - const skipStep = steps.find( - step => /^Skip /.test(step.name) && step.conclusion === 'success', +function suiteRunState(suite, jobs) { + const own = (jobs || []).filter(job => + suite.jobs.test(baseJobName(job.name)), ); - if (skipStep && job.conclusion === 'success') - return { - state: 'scoped-out', - reason: skipStep.name, - }; - const scopeIndex = steps.findIndex(step => step.name === SCOPE_STEP); - if (scopeIndex !== -1 && job.conclusion === 'success') { - const work = steps - .slice(scopeIndex + 1) - .filter(step => !BOOKKEEPING_STEP.test(step.name)); - if (work.length > 0 && work.every(step => step.conclusion === 'skipped')) - return {state: 'scoped-out', reason: 'nothing in its paths changed'}; - } - return {state: 'ran', conclusion: job.conclusion || job.status || 'unknown'}; + if (own.length === 0) return null; + return own.every(job => job.conclusion === 'skipped') ? 'not-run' : 'ran'; } -/** Jobs grouped by name (matrix legs together), each with one state. */ -function summarizeJobs(jobs) { - const groups = new Map(); - for (const job of jobs || []) { - const name = baseJobName(job.name); - if (!groups.has(name)) groups.set(name, []); - groups.get(name).push(classifyJob(job)); - } - const summary = []; - for (const [name, legs] of groups) { - const ran = legs.filter(leg => leg.state === 'ran'); - if (ran.length > 0) { - const failed = ran.find(leg => leg.conclusion !== 'success'); - summary.push({ - name, - state: 'ran', - conclusion: failed ? failed.conclusion : 'success', - }); - } else if (legs.some(leg => leg.state === 'scoped-out')) { - summary.push({ - name, - state: 'scoped-out', - reason: legs.find(leg => leg.reason).reason, - }); - } else { - summary.push({name, state: 'not-run'}); - } - } - return summary.sort((a, b) => a.name.localeCompare(b.name)); +function isMeasured(value) { + return typeof value === 'number' && Number.isFinite(value); } -function suiteOf(jobName) { - return SUITES.find(suite => suite.jobs.test(jobName)) ?? null; +function kb(bytes) { + return `${(bytes / 1024).toFixed(1)} kB`; } -const CONCLUSION_TEXT = { - success: '✅ passed', - failure: '❌ failed', - cancelled: '⏹ cancelled', - timed_out: '❌ timed out', -}; - -function conclusionText(conclusion) { - return CONCLUSION_TEXT[conclusion] ?? inline(conclusion); -} - -/** One suite's state from its jobs: ran (and how), scoped out, or not run. */ -function suiteJobState(suite, jobSummary) { - const jobs = jobSummary.filter(job => suite.jobs.test(job.name)); - if (jobs.length === 0) return null; - const ran = jobs.filter(job => job.state === 'ran'); - if (ran.length > 0) { - const failed = ran.find(job => job.conclusion !== 'success'); - return {state: 'ran', conclusion: failed ? failed.conclusion : 'success'}; - } - if (jobs.some(job => job.state === 'scoped-out')) - return {state: 'scoped-out'}; - return {state: 'not-run'}; +function signedSize(value) { + const sign = value > 0 ? '+' : '−'; + const magnitude = Math.abs(value); + return magnitude < 1024 ? `${sign}${magnitude} B` : `${sign}${kb(magnitude)}`; } -function previewCell(preview) { - if (!preview || preview.state === 'none') return null; - if (preview.state === 'fork') - return 'No preview for pull requests from forks.'; - if (preview.state === 'unknown') - return "This branch's preview address cannot be known in advance; open it from Vercel's comment."; - const links = origin => - `[Storybook](${origin}/storybook/) · [Sandbox](${origin}/sandbox/) · [Docs](${origin}/)`; - if (preview.state === 'ready') - return `${links(preview.url)} · built from \`${inline(String(preview.sha).slice(0, 7))}\``; - if (preview.state === 'failed') - return `❌ The preview for \`${inline(String(preview.sha).slice(0, 7))}\` failed to build; see Vercel's comment.`; - // Building, or Vercel has not reported this head yet: the branch alias - // serves the latest ready deployment, which can be an earlier push. - return `⏳ Preview building for \`${inline(String(preview.sha).slice(0, 7))}\`… ${preview.aliasOrigin ? `Until it's ready, ${links(preview.aliasOrigin)} open the branch's previous preview.` : ''}`.trim(); +/** A change against main; growth in bold, so it stands out. */ +function delta(value, format) { + if (value === null) return 'new'; + if (value === 0) return '0'; + const text = format(value); + return value > 0 ? `**${text}**` : text; } -function visualCell(verdict, jobState, suite) { - if (verdict) { - if (verdict.status === 'skipped') - return `⏭ skipped: ${inline(verdict.reason)}`; - if (verdict.status === 'failed') - return `❌ ${num(verdict.counts?.failed)} shot(s) could not be captured`; - const changed = verdict.changes?.length ?? 0; - const added = verdict.added?.length ?? 0; - const removed = verdict.removed?.length ?? 0; - if (changed > 0) - return `🔍 ${changed} of ${num(verdict.counts?.total)} shots changed: review below`; - if (added > 0 || removed > 0) - return `🔍 ${added} added · ${removed} removed`; - return `✅ no change across ${num(verdict.counts?.total)} shots`; - } - return jobCell(jobState, suite); +function change(after, before) { + return isMeasured(after) && isMeasured(before) ? after - before : null; } -function a11yCell(report, jobState, suite) { - if (report) { - const violations = Object.values(report.components || {}).reduce( - (sum, component) => sum + (component.violations?.length || 0), - 0, - ); - return violations === 0 - ? '✅ no violations' - : `❌ ${violations} violation(s): details below`; - } - return jobCell(jobState, suite); +/** JS and CSS growth, and whether either passes the touch-only budget. */ +function sizeChange(head = {}, base = {}) { + const js = change(head.importGzipBytes, base.importGzipBytes); + const css = change(head.cssGzipBytes, base.cssGzipBytes); + return { + js, + css, + total: js === null && css === null ? null : (js ?? 0) + (css ?? 0), + overBudget: + (js !== null && js > JS_BUDGET_BYTES) || + (css !== null && css > CSS_BUDGET_BYTES), + }; } -function jobCell(jobState, suite) { - if (!jobState || jobState.state === 'not-run') - return `⏭ ${notRunSentence(suite)}`; - if (jobState.state === 'scoped-out') - return '⏭ skipped: nothing it covers changed'; - return conclusionText(jobState.conclusion); -} - -function sizeCell({analysis, budgetFlags, packageSize}) { - const parts = []; - const head = analysis.componentSizes || {}; - const base = analysis.baseComponentSizes || {}; - const deltas = Object.keys(head) - .map(key => ({ - key, - delta: - typeof head[key]?.importGzipBytes === 'number' && - typeof base[key]?.importGzipBytes === 'number' - ? head[key].importGzipBytes - base[key].importGzipBytes - : null, - })) - .filter(entry => entry.delta !== null); - if (Object.keys(head).length > 0) { - const largest = deltas.sort((a, b) => b.delta - a.delta)[0]; - const largestText = largest - ? `, largest ${largest.delta > 0 ? '+' : ''}${bytes(largest.delta)} (${inline(largest.key)})` - : ''; - parts.push( - budgetFlags.length > 0 - ? `⚠️ ${budgetFlags.length} component(s) over the touch-only budget${largestText}` - : `✅ ${Object.keys(head).length} component(s) within budget${largestText}`, - ); - } - if (packageSize) { - if (packageSize.verdict === 'over-budget') - parts.push('⚠️ a published package grew past its budget'); - else if (packageSize.verdict === 'no-base') - parts.push('packages measured (no base to compare)'); - else parts.push('packages within budget'); - const added = packageSize.addedDependencies || []; - if (added.length > 0) - parts.push( - `new dependencies: ${added.map(dep => inline(dep.name)).join(', ')}`, - ); - } - return parts.length > 0 ? parts.join(' · ') : 'no published package changed'; -} - -function bytes(value) { - const magnitude = Math.abs(value); - if (magnitude < 1024) return `${value}B`; - return `${(value / 1024).toFixed(1)}KB`; +/** + * JS and CSS import cost (gzip), each with its change against main, and a 🔴 + * when the growth passes the touch-only budget (advisory). + */ +function sizeCell(head = {}, base = {}) { + const js = head.importGzipBytes; + const css = head.cssGzipBytes; + if (!isMeasured(js) && !isMeasured(css)) return '—'; + const growth = sizeChange(head, base); + const part = (after, changed, label) => + isMeasured(after) + ? `${kb(after)}${label} (${delta(changed, signedSize)})` + : null; + return `${growth.overBudget ? '🔴 ' : ''}${[ + part(js, growth.js, ''), + part(css, growth.css, ' CSS'), + ] + .filter(Boolean) + .join(' · ')}`; } -function ciCell(jobSummary) { - if (!jobSummary || jobSummary.length === 0) return null; - const ran = jobSummary.filter(job => job.state === 'ran'); - const failed = ran.filter(job => job.conclusion !== 'success'); - const scoped = jobSummary.filter(job => job.state === 'scoped-out'); - const parts = [`${ran.length} ran`]; - if (failed.length > 0) - parts.push(`❌ failed: ${failed.map(job => inline(job.name)).join(', ')}`); - if (scoped.length > 0) parts.push(`${scoped.length} skipped by path`); - return `${parts.join(' · ')} (table below)`; +/** Source lines of code with their change against main. */ +function locCell(head, base) { + if (!isMeasured(head)) return '—'; + const count = value => Math.abs(value).toLocaleString('en-US'); + return `${count(num(head))} (${delta( + change(head, base), + value => `${value > 0 ? '+' : '−'}${count(value)}`, + )})`; } -/** The status table every report leads with. */ -function buildStatusTable({ - preview, - analysis, - budgetFlags = [], - packageSize = null, - a11yReport = null, - visualVerdict = null, - jobs = null, -}) { - const jobSummary = jobs ? summarizeJobs(jobs) : []; - const rows = []; - const previewText = previewCell(preview); - if (previewText) rows.push(['Preview', previewText]); - rows.push(['Size', sizeCell({analysis, budgetFlags, packageSize})]); - const [visual, a11y, rtl, perf] = SUITES; - rows.push([ - visual.label, - visualCell(visualVerdict, suiteJobState(visual, jobSummary), visual), - ]); - rows.push([ - a11y.label, - a11yCell(a11yReport, suiteJobState(a11y, jobSummary), a11y), - ]); - rows.push([rtl.label, jobCell(suiteJobState(rtl, jobSummary), rtl)]); - const perfState = suiteJobState(perf, jobSummary); - if (perfState?.state === 'ran') - rows.push([perf.label, jobCell(perfState, perf)]); - const ci = ciCell(jobSummary); - if (ci) rows.push(['CI jobs', ci]); - return `| Check | Result |\n|-------|--------|\n${rows - .map(([check, result]) => `| ${check} | ${result} |`) - .join('\n')}\n\n`; -} +/** + * One mark per browser suite, always in the order Visual · A11y · RTL: ✓ + * passed, ✗ failed or changed (its shots are in the visual section), — did + * not run for this component. + */ +const CHECK_SUITES = ['visual', 'a11y', 'rtl']; +const MARK = {pass: '✓', fail: '✗', changed: '✗'}; -/** Every job of the run that applies to a pull request, with what it did. */ -function buildJobTable(jobs) { - const summary = summarizeJobs(jobs || []); - const shown = summary.filter( - job => job.state !== 'not-run' || suiteOf(job.name) !== null, - ); - if (shown.length === 0) return ''; - const hidden = summary.length - shown.length; - const rows = shown.map(job => { - if (job.state === 'ran') - return `| ${inline(job.name)} | ${conclusionText(job.conclusion)} | |`; - if (job.state === 'scoped-out') - return `| ${inline(job.name)} | ⏭ skipped | ${inline(job.reason)} |`; - return `| ${inline(job.name)} | ⏭ not run | ${notRunSentence(suiteOf(job.name))} |`; - }); - return `
\nCI jobs: ${shown.filter(job => job.state === 'ran').length} ran, ${shown.filter(job => job.state === 'scoped-out').length} skipped by path\n\n| Job | Result | Why |\n|-----|--------|-----|\n${rows.join('\n')}\n${hidden > 0 ? `\n${hidden} release-only job(s) do not apply to pull requests.\n` : ''}\n
\n\n`; +function checksCell(results) { + return CHECK_SUITES.map(id => MARK[results[id]] ?? '—').join(' · '); } /** A Storybook docs link from a story title (`Core/Button` → `core-button`). */ @@ -319,76 +168,319 @@ function storybookLink(origin, storyTitle) { return `${origin}/storybook/?path=/docs/${id}--docs`; } -const RESULT_TEXT = { - pass: '✅', - fail: '❌', - changed: '🔍 changed', - 'not-run': '⏭', -}; - -function resultText(value) { - if (value == null) return '—'; - return RESULT_TEXT[value] ?? inline(value); -} - /** - * The components this pull request touches: a docs page and Storybook stories - * in this head's preview, and each suite's result for it. Results come from - * the touched-components report when the run wrote one; otherwise from the - * accessibility report and visual verdict, and are blank for a suite with no - * per-component result. + * One row per modified component: every component this change sized (its + * folder changed or its bundle reaches a changed module) and every new or + * modified component folder. Results come from the touched-components report + * when the run wrote one, otherwise from the accessibility report and the + * visual verdict. */ -function buildTouchedComponents({ - analysis, - origin, - touched = null, - a11yReport = null, - visualVerdict = null, -}) { - const names = [ - ...(analysis.newComponents || []), - ...(analysis.modifiedComponents || []), - ]; - const fromReport = new Map( +function componentRows({analysis, touched, a11yReport, visualVerdict}) { + const head = analysis.componentSizes || {}; + const base = analysis.baseComponentSizes || {}; + const stats = analysis.componentStats || {}; + const baseStats = analysis.baseComponentStats || {}; + const byName = new Map( (touched?.components || []).map(component => [component.name, component]), ); - for (const name of fromReport.keys()) - if (!names.includes(name)) names.push(name); - if (names.length === 0) return ''; + const keys = new Set(Object.keys(head)); + for (const name of [ + ...(analysis.newComponents || []), + ...(analysis.modifiedComponents || []), + ]) { + const pkg = String(stats[name]?.package || '@astryxdesign/core').replace( + /^@astryxdesign\//, + '', + ); + keys.add(`${pkg}/${name}`); + } + for (const component of touched?.components || []) { + if (component.key) keys.add(component.key); + } const visualChanges = visualVerdict?.changes || []; - const rows = names.map(name => { - const stats = analysis.componentStats?.[name] || {}; - const reported = fromReport.get(name)?.results || {}; - const docs = origin - ? `[Docs](${origin}/components/${encodeURIComponent(name)})` - : '—'; - const stories = storybookLink(origin, stats.storyTitle); + return [...keys].map(key => { + const name = key.slice(key.indexOf('/') + 1); + const reported = byName.get(name)?.results || {}; + // The audit covers the touched components only; a component it did not + // audit has no accessibility result. + const audited = a11yReport?.components?.[name]; const a11y = reported.a11y ?? - (a11yReport - ? (a11yReport.components?.[name]?.violations?.length ?? 0) > 0 + (audited + ? (audited.violations?.length ?? 0) > 0 ? 'fail' : 'pass' : null); const visual = reported.visual ?? (visualVerdict && visualVerdict.status !== 'skipped' - ? visualChanges.some(change => change.component === name) + ? visualChanges.some(entry => entry.component === name) ? 'changed' : 'pass' : null); - return `| ${inline(name)} | ${docs} | ${stories ? `[Stories](${stories})` : '—'} | ${resultText(visual)} | ${resultText(a11y)} | ${resultText(reported.rtl)} |`; + return { + key, + name, + storyTitle: stats[name]?.storyTitle, + head: head[key], + base: base[key], + growth: sizeChange(head[key], base[key]), + loc: [stats[name]?.linesOfCode, baseStats[name]?.linesOfCode], + results: {visual, a11y, rtl: reported.rtl ?? null}, + }; }); - return `**Touched components**\n\n| Component | Docs | Storybook | Visual | A11y | RTL |\n|-----------|------|-----------|--------|------|-----|\n${rows.join('\n')}\n\n`; +} + +/** Failed or changed results first, then budget overruns, then the biggest growth. */ +function importance(row) { + const attention = Object.values(row.results).some( + value => value === 'fail' || value === 'changed', + ); + return [ + attention ? 0 : 1, + row.growth.overBudget ? 0 : 1, + -(row.growth.total ?? -Infinity), + ]; +} + +function compareRows(a, b) { + const [x, y] = [importance(a), importance(b)]; + for (let i = 0; i < x.length; i++) if (x[i] !== y[i]) return x[i] - y[i]; + return a.key.localeCompare(b.key); +} + +function buildComponentTable({ + analysis, + origin, + touched, + a11yReport, + visualVerdict, +}) { + const rows = componentRows({ + analysis, + touched, + a11yReport, + visualVerdict, + }).sort(compareRows); + if (rows.length === 0) return ''; + // The checks column appears only when a browser suite has a result here, + // or when the change is too wide for the touched-components check, which + // then ran none of them (every mark —). + const wide = touched?.wideChange === true; + const checks = + wide || rows.some(row => CHECK_SUITES.some(id => row.results[id] != null)); + const columns = [ + 'Component', + 'Size', + 'LoC', + ...(checks ? ['Visual · A11y · RTL'] : []), + ]; + const header = `| ${columns.join(' | ')} |\n|${columns.map(() => '---').join('|')}|\n`; + // The name links its docs page in this head's preview; Storybook follows. + const line = row => { + // Core components go by name; another package's carry its prefix. + const label = inline(row.key.startsWith('core/') ? row.name : row.key); + const name = origin + ? `[${label}](${origin}/components/${encodeURIComponent(row.name)})` + : label; + const stories = storybookLink(origin, row.storyTitle); + const cells = [ + stories ? `${name} · [Storybook](${stories})` : name, + sizeCell(row.head, row.base), + locCell(...row.loc), + ...(checks ? [checksCell(wide ? {} : row.results)] : []), + ]; + return `| ${cells.join(' | ')} |`; + }; + const visible = rows.slice(0, VISIBLE_ROWS); + const rest = rows.slice(VISIBLE_ROWS); + let out = `### Modified components\n\n${header}${visible.map(line).join('\n')}\n\n`; + if (rest.length > 0) + out += `
\n${rest.length} more component${rest.length === 1 ? '' : 's'}\n\n${header}${rest.map(line).join('\n')}\n\n
\n\n`; + return out; +} + +/** + * The changed shots as before/after/diff thumbnails, largest change first: + * three rows, then the rest collapsed. Frames come from `imageBase` + * (`/.png`); without it the rows still name + * the changed shots. + */ +function buildVisualDeltas({verdict, imageBase}) { + if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed') + return ''; + const all = [ + ...[...(verdict.changes || [])] + .sort((a, b) => Number(b.diffPixels ?? 0) - Number(a.diffPixels ?? 0)) + .map(change => ({...change, kinds: ['before', 'after', 'diff']})), + ...(verdict.added || []).map(key => ({key, name: key, kinds: ['after']})), + ...(verdict.removed || []).map(key => ({ + key, + name: key, + kinds: ['before'], + })), + ]; + if (all.length === 0) return ''; + const rows = all.slice(0, VISUAL_FRAME_LIMIT); + const unshown = all.length - rows.length; + const base = imageBase ? `${String(imageBase).replace(/\/+$/, '')}/` : null; + const frame = (row, kind) => { + if (!row.kinds.includes(kind)) return '—'; + if (!base) return 'frame unavailable'; + const key = encodeURIComponent(String(row.key ?? '')); + return `${kind} frame`; + }; + const label = row => { + const story = [row.component || row.title, row.name] + .filter(Boolean) + .map(inline) + .join(' · '); + const variant = [row.theme, row.mode] + .filter(Boolean) + .map(inline) + .join(', '); + return variant ? `${story} (${variant})` : story; + }; + const header = + '| Component · story | Before | After | Diff |\n|---|---|---|---|\n'; + const line = row => + `| ${label(row)} | ${frame(row, 'before')} | ${frame(row, 'after')} | ${frame(row, 'diff')} |`; + // A shot key is `---__-`. + const componentOf = row => + String( + row.component ?? + String(row.key) + .split('--')[0] + .replace(/^[a-z]+-/, ''), + ) + .toLowerCase() + .replace(/[^a-z0-9]/g, ''); + const components = new Set(all.map(componentOf)); + return `
\nVisual changes: ${num(all.length)} snapshot${all.length === 1 ? '' : 's'} in ${components.size} component${components.size === 1 ? '' : 's'}\n\nA change is a question for the reviewer, not a failure.\n\n${header}${rows.map(line).join('\n')}\n\n${unshown > 0 ? `${num(unshown)} more are in the run's visual report.\n\n` : ''}
\n\n`; +} + +function previewLine(preview) { + if (!preview || preview.state === 'none') return ''; + const sha = `\`${inline(String(preview.sha ?? '').slice(0, 7))}\``; + const links = origin => + `**[Storybook](${origin}/storybook/) · [Sandbox](${origin}/sandbox/)**`; + let text; + if (preview.state === 'fork') + text = 'No preview for pull requests from forks.'; + else if (preview.state === 'unknown') + text = + "This branch's preview address cannot be known in advance; open it from Vercel's comment."; + else if (preview.state === 'ready') + text = `${links(preview.url)} · built from this PR's head`; + else if (preview.state === 'failed') + text = "❌ The preview failed to build; see Vercel's comment."; + else + text = `⏳ Building…${preview.aliasOrigin ? ` Until it's ready, ${links(preview.aliasOrigin)} open the branch's previous preview.` : ''}`; + return `### 🔎 Preview · ${sha}\n\n${text}\n\n`; +} + +/** + * Package size: each published package whose download changed, one line each + * (the first three, then the rest collapsed), with any new runtime dependency. + */ +function packagesLine(report) { + if (!report || !Array.isArray(report.packages)) return ''; + if (report.verdict === 'no-base') + return '**Package size:** measured; main has no measurement of the merge base to compare yet.\n\n'; + const added = report.addedDependencies || []; + const changed = report.packages.filter( + entry => + (isMeasured(entry.base?.packedBytes) && + isMeasured(entry.head?.packedBytes) && + entry.base.packedBytes !== entry.head.packedBytes) || + added.some(dep => dep.package === entry.name), + ); + if (changed.length === 0) + return '**Package size:** no published package changed; no new dependencies.\n\n'; + const packed = bytes => + bytes >= 1024 * 1024 + ? `${(bytes / (1024 * 1024)).toFixed(2)} MB` + : kb(bytes); + const describe = entry => { + const value = change(entry.head?.packedBytes, entry.base?.packedBytes); + const deps = added.filter(dep => dep.package === entry.name); + const size = isMeasured(entry.head?.packedBytes) + ? `${packed(entry.head.packedBytes)} packed (${delta(value, signedSize)})${entry.overBudget ? ' 🔴' : ''}` + : 'new package'; + return `${inline(entry.name)} ${size}; ${ + deps.length > 0 + ? `new dependenc${deps.length === 1 ? 'y' : 'ies'}: ${deps.map(dep => inline(dep.name)).join(', ')}` + : 'no new dependencies' + }`; + }; + if (changed.length === 1) + return `**Package size:** ${describe(changed[0])}.\n\n`; + const lines = changed.map(entry => `- ${describe(entry)}`); + const rest = lines.slice(VISIBLE_ROWS); + return `**Package size:**\n\n${lines.slice(0, VISIBLE_ROWS).join('\n')}\n\n${ + rest.length > 0 + ? `
\n${rest.length} more package${rest.length === 1 ? '' : 's'}\n\n${rest.join('\n')}\n\n
\n\n` + : '' + }`; +} + +/** One line for the browser suites that did not run, with the labels that run them. */ +function notRunLine({jobs, a11yReport, visualVerdict, touched = null}) { + if (touched?.wideChange === true) + return `Wide change (${num(touched.componentCount)} components): add \`ci:run-visual\` / \`ci:run-a11y\` / \`ci:run-rtl\` to run the full suite.\n\n`; + const notRun = SUITES.filter(suite => { + const state = suiteRunState(suite, jobs); + if (state !== null) return state === 'not-run'; + if (jobs) return false; + if (suite.id === 'visual') return !visualVerdict; + if (suite.id === 'a11y') return !a11yReport; + return false; + }); + if (notRun.length === 0) return ''; + const labels = notRun.map(suite => `\`ci:run-${suite.id}\``); + const list = + labels.length === 1 + ? labels[0] + : `${labels.slice(0, -1).join(', ')} or ${labels.at(-1)}`; + return `Not run on this PR: add ${list} to run ${labels.length === 1 ? 'it' : 'a full suite'}.\n\n`; +} + +/** The compact report body, between the heading and the footer. */ +function buildCompactReport({ + analysis, + preview = null, + jobs = null, + touched = null, + a11yReport = null, + visualVerdict = null, + packageSize = null, + visualImageUrl = null, +}) { + const origin = + preview?.state === 'ready' ? preview.url : preview?.aliasOrigin || ''; + return `${previewLine(preview)}${buildComponentTable({ + analysis, + origin, + touched, + a11yReport, + visualVerdict, + })}${buildVisualDeltas({verdict: visualVerdict, imageBase: visualImageUrl})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict, touched})}`; } module.exports = { SUITES, - notRunSentence, - classifyJob, - summarizeJobs, - buildStatusTable, - buildJobTable, - buildTouchedComponents, + VISUAL_FRAME_LIMIT, + baseJobName, + suiteRunState, + sizeCell, + locCell, + componentRows, + buildComponentTable, + buildVisualDeltas, + checksCell, + previewLine, + packagesLine, + notRunLine, + buildCompactReport, storybookLink, }; diff --git a/.github/scripts/lib/pr-status.test.mjs b/.github/scripts/lib/pr-status.test.mjs index fa661b33a7296..5fcf097a948d4 100644 --- a/.github/scripts/lib/pr-status.test.mjs +++ b/.github/scripts/lib/pr-status.test.mjs @@ -2,337 +2,389 @@ /** * @file pr-status.test.mjs - * Pins the PR report's top: the status table (preview first, no review-bot - * row, a heavy suite that did not run names its label), how a job is read as - * ran / skipped by path / not run, the touched components' preview links and - * results, and the collapsed CI job table. + * Pins the compact PR report: the preview line's states; one table of + * modified components (the name linking its docs page, a Storybook link, JS + * and CSS size and lines of code against main with growth in bold, and the + * fixed-position Visual · A11y · RTL marks only when a suite ran), three rows + * visible and the rest collapsed, most important first; the visual deltas + * with frames for the published shots only; the package line; and the one + * not-run or wide-change line. */ import {describe, expect, it} from 'vitest'; import status from './pr-status.js'; const { - notRunSentence, - classifyJob, - summarizeJobs, - buildStatusTable, - buildJobTable, - buildTouchedComponents, + baseJobName, + suiteRunState, + sizeCell, + locCell, + checksCell, + buildComponentTable, + buildVisualDeltas, + previewLine, + packagesLine, + notRunLine, SUITES, + VISUAL_FRAME_LIMIT, } = status; -const SCOPE = "Decide this job's scope"; -const job = (name, conclusion, steps = []) => ({ - name, - status: 'completed', - conclusion, - steps: steps.map(([stepName, stepConclusion]) => ({ - name: stepName, - conclusion: stepConclusion, - })), +const HEAD = 'a'.repeat(40); +const ORIGIN = 'https://astryx-abc123-fbopensource.vercel.app'; +const size = (importGzipBytes, cssGzipBytes = 1024) => ({ + importGzipBytes, + cssGzipBytes, }); -describe('classifyJob', () => { - it('reads a job whose condition was false as not run', () => { - expect(classifyJob(job('pr-visual', 'skipped'))).toEqual({ - state: 'not-run', - }); +describe('previewLine', () => { + it.each([ + [ + 'ready', + {state: 'ready', sha: HEAD, url: ORIGIN}, + `### 🔎 Preview · \`aaaaaaa\`\n\n**[Storybook](${ORIGIN}/storybook/) · [Sandbox](${ORIGIN}/sandbox/)** · built from this PR's head`, + ], + [ + 'building', + {state: 'building', sha: HEAD, aliasOrigin: ORIGIN}, + `⏳ Building… Until it's ready, **[Storybook](${ORIGIN}/storybook/)`, + ], + ['failed', {state: 'failed', sha: HEAD}, 'failed to build'], + [ + 'fork', + {state: 'fork', sha: HEAD}, + 'No preview for pull requests from forks.', + ], + ['unknown', {state: 'unknown', sha: HEAD}, 'cannot be known in advance'], + ])('says what a %s preview is', (_label, preview, text) => { + expect(previewLine(preview)).toContain(text); }); - it('reads a job that decided its scope and skipped every work step as skipped by path', () => { - const scoped = job('docsite-test', 'success', [ - ['Set up job', 'success'], - [SCOPE, 'success'], - ['Checkout', 'skipped'], - ['Build', 'skipped'], - ['Post Checkout', 'success'], - ['Complete job', 'success'], - ]); - expect(classifyJob(scoped)).toEqual({ - state: 'scoped-out', - reason: 'nothing in its paths changed', - }); + it('says nothing on a draft', () => { + expect(previewLine(null)).toBe(''); }); +}); - it('reads a job that ran any work step as ran', () => { - const ran = job('test-node', 'success', [ - [SCOPE, 'success'], - ['Checkout', 'success'], - ['Test', 'skipped'], - ]); - expect(classifyJob(ran)).toEqual({state: 'ran', conclusion: 'success'}); - expect(classifyJob(job('lint', 'failure', [['Lint', 'failure']]))).toEqual({ - state: 'ran', - conclusion: 'failure', - }); +describe('sizeCell and locCell', () => { + it('shows JS and CSS with their change, growth in bold', () => { + expect(sizeCell(size(105_370, 25_600), size(105_062, 25_700))).toBe( + '102.9 kB (**+308 B**) · 25.0 kB CSS (−100 B)', + ); + expect(sizeCell(size(1024, 1024), size(1024, 1024))).toBe( + '1.0 kB (0) · 1.0 kB CSS (0)', + ); }); - it('reads a specialized-lane skip as skipped by path', () => { - const skipped = job('build-storybook', 'success', [ - [SCOPE, 'success'], - ['Skip heavy work for a specialized lane', 'success'], - ['Checkout', 'skipped'], - ]); - expect(classifyJob(skipped)).toEqual({ - state: 'scoped-out', - reason: 'Skip heavy work for a specialized lane', - }); + it('marks growth past the touch-only budget, JavaScript or CSS', () => { + expect(sizeCell(size(5000), size(2000))).toMatch(/^🔴 /); + expect(sizeCell(size(1000, 3000), size(1000, 1000))).toMatch(/^🔴 /); + expect(sizeCell(size(4000), size(2000))).not.toMatch(/🔴/); }); -}); -describe('job names', () => { - it("folds a skipped matrix's unexpanded name, parentheses and all", () => { - const summary = summarizeJobs([ - job('pr-rtl', 'skipped'), - job( - "pr-rtl (${{ matrix.package }}${{ (((matrix.parts > 1) && format(' {0}/{1}', matrix.part, matrix.parts)) || '') }})", - 'skipped', - ), - ]); - expect(summary).toEqual([{name: 'pr-rtl', state: 'not-run'}]); + it('shows a new component as new, and an unmeasured one as a dash', () => { + expect(sizeCell(size(1024), {})).toBe('1.0 kB (new) · 1.0 kB CSS (new)'); + expect(sizeCell({}, size(1))).toBe('—'); + }); + + it('counts lines of code against main', () => { + expect(locCell(124, 120)).toBe('124 (**+4**)'); + expect(locCell(120, 124)).toBe('120 (−4)'); + expect(locCell(120, undefined)).toBe('120 (new)'); + expect(locCell(undefined, 1)).toBe('—'); }); }); -describe('suite jobs by display name', () => { - it.each([ - ['Stable visual regression', 'Visual'], - [ - 'Stable visual scout (${{ matrix.shard }}/${{ strategy.job-total }})', - 'Visual', - ], - ['pr-a11y (browser specs)', 'Accessibility'], - ['Accessibility spec-test contracts', 'Accessibility'], - ['pr-rtl (core)', 'RTL'], - ])('reads a skipped %s as the %s suite not running', (name, label) => { - const table = buildStatusTable({ - analysis: {newComponents: [], modifiedComponents: []}, - jobs: [job(name, 'skipped')], - }); - expect(table).toContain( - `| ${label} | ⏭ ${label} did not run on this pull request;`, +describe('checksCell', () => { + it('keeps Visual · A11y · RTL in fixed positions', () => { + expect(checksCell({visual: 'pass', a11y: null, rtl: 'fail'})).toBe( + '✓ · — · ✗', ); + expect(checksCell({visual: 'changed'})).toBe('✗ · — · —'); }); }); -describe('summarizeJobs', () => { - it('folds matrix legs into one row; any failed leg fails it', () => { - const summary = summarizeJobs([ - job('test-node (1/2)', 'success', [['Test', 'success']]), - job('test-node (2/2)', 'failure', [['Test', 'failure']]), - ]); - expect(summary).toEqual([ - {name: 'test-node', state: 'ran', conclusion: 'failure'}, - ]); +describe('buildComponentTable', () => { + const analysis = rows => ({ + componentSizes: Object.fromEntries( + rows.map(([key, after]) => [key, size(after)]), + ), + baseComponentSizes: Object.fromEntries( + rows.map(([key, , before]) => [key, size(before)]), + ), + componentStats: {B: {storyTitle: 'Core/B', linesOfCode: 50}}, + baseComponentStats: {B: {linesOfCode: 40}}, }); -}); -describe('notRunSentence', () => { - it.each([ - [ - 'visual', - 'Visual did not run on this pull request; add `ci:run-visual` to run it.', - ], - [ - 'a11y', - 'Accessibility did not run on this pull request; add `ci:run-a11y` to run it.', - ], - [ - 'rtl', - 'RTL did not run on this pull request; add `ci:run-rtl` to run it.', - ], - [ - 'perf', - 'Perf did not run on this pull request; add `ci:run-perf` to run it.', - ], - ])('names the %s label', (id, sentence) => { - expect(notRunSentence(SUITES.find(suite => suite.id === id))).toBe( - sentence, + it('shows three rows, the most important first, and collapses the rest', () => { + const table = buildComponentTable({ + analysis: analysis([ + ['core/A', 1000, 1000], + ['core/B', 2000, 1500], + ['core/C', 1200, 1100], + ['core/D', 900, 1000], + ]), + origin: ORIGIN, + a11yReport: {components: {D: {violations: [{id: 'x'}]}}}, + }); + const [visible, rest] = table.split('
'); + // D's failed accessibility result comes first, then the biggest growth. + expect(visible.match(/^\| \[[A-D]\]/gm)).toEqual([ + '| [D]', + '| [B]', + '| [C]', + ]); + expect(visible).toContain('### Modified components'); + expect(visible).toContain( + '| Component | Size | LoC | Visual · A11y · RTL |', ); + expect(rest).toContain('1 more component'); + expect(rest).toContain('| [A]'); + expect(table).toContain( + `| [B](${ORIGIN}/components/B) · [Storybook](${ORIGIN}/storybook/?path=/docs/core-b--docs) | 2.0 kB (**+500 B**) · 1.0 kB CSS (0) | 50 (**+10**) | — · — · — |`, + ); + expect(table).toContain('| — · ✗ · — |'); }); -}); -describe('buildStatusTable', () => { - const analysis = {newComponents: [], modifiedComponents: []}; + it('has no checks column when no suite ran, and no disclosure for three rows', () => { + const table = buildComponentTable({ + analysis: analysis([['core/A', 1000, 1000]]), + origin: '', + }); + expect(table).toContain('| Component | Size | LoC |\n'); + expect(table).toContain('| A |'); + expect(table).not.toContain('
'); + }); - it('puts the preview first and has no review-bot row', () => { - const table = buildStatusTable({ - preview: { - state: 'ready', - sha: 'a'.repeat(40), - url: 'https://x.vercel.app', + it('takes results from the touched-components report first', () => { + const table = buildComponentTable({ + analysis: analysis([['core/A', 1000, 1000]]), + origin: ORIGIN, + touched: { + components: [{name: 'A', results: {visual: 'changed', rtl: 'fail'}}], }, - analysis, + a11yReport: {components: {A: {violations: []}}}, }); - const rows = table - .trim() - .split('\n') - .slice(2) - .map(row => row.split(' | ')[0]); - expect(rows).toEqual([ - '| Preview', - '| Size', - '| Visual', - '| Accessibility', - '| RTL', - ]); - expect(table).not.toMatch(/CodeBunny|\| Review/i); + expect(table).toMatch(/\| ✗ · ✓ · ✗ \|$/m); }); - it('shows perf only when it ran', () => { - const ran = buildStatusTable({ - analysis, - jobs: [job('perf', 'success', [['Measure', 'success']])], + it('shows every mark as not run for a wide change', () => { + const table = buildComponentTable({ + analysis: analysis([['core/A', 1000, 1000]]), + origin: ORIGIN, + touched: {wideChange: true, componentCount: 60, components: []}, }); - expect(ran).toContain('| Perf | ✅ passed |'); - expect( - buildStatusTable({analysis, jobs: [job('perf', 'skipped')]}), - ).not.toContain('| Perf |'); + expect(table).toMatch(/\| — · — · — \|$/m); }); - it('reads the visual verdict and the accessibility report before the jobs', () => { - const table = buildStatusTable({ - analysis, - visualVerdict: {status: 'ok', changes: [{}, {}], counts: {total: 10}}, - a11yReport: {components: {Card: {violations: [{id: 'x'}]}}}, - jobs: [job('pr-visual', 'success', [['Run', 'success']])], + it('lists a modified component folder with no size', () => { + const table = buildComponentTable({ + analysis: { + modifiedComponents: ['Card'], + componentStats: {Card: {package: '@astryxdesign/lab', linesOfCode: 9}}, + }, + origin: '', }); - expect(table).toContain( - '| Visual | 🔍 2 of 10 shots changed: review below |', + expect(table).toContain('| lab/Card | — | 9 (new) |'); + }); + + it('renders nothing when no component changed', () => { + expect(buildComponentTable({analysis: {}, origin: ORIGIN})).toBe(''); + }); +}); + +describe('buildVisualDeltas', () => { + const change = (key, diffPixels, component = 'Card') => ({ + key, + component, + name: key, + theme: 'neutral', + mode: 'light', + diffPixels, + }); + + it('collapses the changed shots under their count, largest change first, with frames', () => { + const verdict = { + status: 'changed', + counts: {total: 20}, + changes: [change('a', 1), change('b', 50), change('c', 9, 'Chip')], + added: ['core-dialog--default__neutral-light'], + }; + const section = buildVisualDeltas({ + verdict, + imageBase: 'https://raw.example/frames', + }); + expect(section).toMatch( + /^
\nVisual changes: 4 snapshots in 3 components<\/summary>/, ); - expect(table).toContain( - '| Accessibility | ❌ 1 violation(s): details below |', + expect(section.match(/^\| \S+ · [a-c] /gm)).toEqual([ + '| Card · b ', + '| Chip · c ', + '| Card · a ', + ]); + expect(section).toContain( + 'before frame', + ); + // An added shot has an after frame only. + expect(section).toContain( + '| — | after frame | — |', ); }); - it('summarizes size: components against the touch budget, packages and new dependencies', () => { - const table = buildStatusTable({ - analysis: { - ...analysis, - componentSizes: { - 'core/Card': {importGzipBytes: 5000}, - 'core/Chip': {importGzipBytes: 900}, - }, - baseComponentSizes: { - 'core/Card': {importGzipBytes: 2000}, - 'core/Chip': {importGzipBytes: 900}, + it('frames only the shots publish-frames.mjs publishes, and counts the rest', () => { + const changes = Array.from({length: VISUAL_FRAME_LIMIT + 3}, (_, i) => + change(`s${i}`, 100 - i), + ); + const section = buildVisualDeltas({ + verdict: {status: 'changed', counts: {total: 99}, changes}, + imageBase: 'https://raw.example/f/', + }); + expect(section.match(/ { + expect(buildVisualDeltas({verdict: {status: 'ok', changes: []}})).toBe(''); + expect(buildVisualDeltas({verdict: null})).toBe(''); + expect( + buildVisualDeltas({ + verdict: { + status: 'changed', + changes: [change('a', 1)], + counts: {total: 1}, }, - }, - budgetFlags: [{comp: 'core/Card'}], - packageSize: { + }), + ).toContain('| frame unavailable |'); + }); +}); + +describe('packagesLine', () => { + const entry = (name, before, after, overBudget = false) => ({ + name, + base: {packedBytes: before}, + head: {packedBytes: after}, + overBudget, + }); + + it('puts one changed package on one line', () => { + expect( + packagesLine({ verdict: 'pass', - addedDependencies: [{name: 'left-pad'}], - }, - }); - expect(table).toContain( - '| Size | ⚠️ 1 component(s) over the touch-only budget, largest +2.9KB (core/Card) · packages within budget · new dependencies: left-pad |', + packages: [ + entry('@astryxdesign/core', 3_570_000, 3_574_096), + entry('@astryxdesign/build', 5000, 5000), + ], + addedDependencies: [], + }), + ).toBe( + '**Package size:** @astryxdesign/core 3.41 MB packed (**+4.0 kB**); no new dependencies.\n\n', ); }); - it('counts the CI jobs and names a failed one', () => { - const table = buildStatusTable({ - analysis, - jobs: [ - job('lint', 'failure', [['Lint', 'failure']]), - job('test-ui', 'success', [['Test', 'success']]), - job('docsite-test', 'success', [ - [SCOPE, 'success'], - ['Build', 'skipped'], - ]), - job('release-check', 'skipped'), + it('lists several changed packages, three then the rest collapsed, unchanged ones left out', () => { + const line = packagesLine({ + verdict: 'over-budget', + packages: [ + entry('@astryxdesign/core', 100000, 120000, true), + entry('@astryxdesign/richtext', 5000, 4000), + entry('@astryxdesign/cli', 9000, 9100), + entry('@astryxdesign/theme-neutral', 9000, 9000), + entry('@astryxdesign/build', 9000, 9200), + ], + addedDependencies: [ + { + package: '@astryxdesign/cli', + field: 'dependencies', + name: 'left-pad', + range: '1', + }, ], }); - expect(table).toContain( - '| CI jobs | 2 ran · ❌ failed: lint · 1 skipped by path (table below) |', + expect(line).toContain( + '- @astryxdesign/core 117.2 kB packed (**+19.5 kB**) 🔴; no new dependencies', ); - }); -}); - -describe('buildJobTable', () => { - it('lists what each job did and why, collapsed, and leaves out release-only jobs', () => { - const table = buildJobTable([ - job('test-ui', 'success', [['Test', 'success']]), - job('docsite-test', 'success', [ - [SCOPE, 'success'], - ['Build', 'skipped'], - ]), - job('pr-rtl', 'skipped'), - job('release-check', 'skipped'), - ]); - expect(table).toMatch( - /^
\nCI jobs: 1 ran, 1 skipped by path<\/summary>/, + expect(line).toContain( + '- @astryxdesign/richtext 3.9 kB packed (−1000 B); no new dependencies', ); - expect(table).toContain('| test-ui | ✅ passed | |'); - expect(table).toContain( - '| docsite-test | ⏭ skipped | nothing in its paths changed |', + expect(line).toContain( + '- @astryxdesign/cli 8.9 kB packed (**+100 B**); new dependency: left-pad', ); - expect(table).toContain( - '| pr-rtl | ⏭ not run | RTL did not run on this pull request; add `ci:run-rtl` to run it. |', + expect(line).toContain('1 more package'); + expect(line).not.toContain('theme-neutral'); + }); + + it('says when nothing changed, and when main has no base yet', () => { + expect( + packagesLine({ + verdict: 'pass', + packages: [entry('@astryxdesign/core', 1, 1)], + }), + ).toBe( + '**Package size:** no published package changed; no new dependencies.\n\n', ); - expect(table).not.toContain('release-check'); - expect(table).toContain( - '1 release-only job(s) do not apply to pull requests.', + expect(packagesLine({verdict: 'no-base', packages: []})).toContain( + 'no measurement of the merge base', ); + expect(packagesLine(null)).toBe(''); }); }); -describe('buildTouchedComponents', () => { - const origin = 'https://astryx-abc123-fbopensource.vercel.app'; - const analysis = { - newComponents: [], - modifiedComponents: ['Button'], - componentStats: { - Button: {package: '@astryxdesign/core', storyTitle: 'Core/Button'}, - }, - }; - - it("links each touched component's docs page and stories in this head's preview", () => { - const table = buildTouchedComponents({analysis, origin}); - expect(table).toContain( - `| Button | [Docs](${origin}/components/Button) | [Stories](${origin}/storybook/?path=/docs/core-button--docs) | — | — | — |`, - ); - }); +describe('notRunLine', () => { + const skipped = name => ({name, conclusion: 'skipped', steps: []}); - it('takes per-component results from the touched-components report first', () => { - const table = buildTouchedComponents({ - analysis, - origin, - touched: { - components: [ - { - name: 'Button', - results: {visual: 'changed', a11y: 'pass', rtl: 'fail'}, - }, + it('names the label for one suite', () => { + expect( + notRunLine({ + jobs: [ + skipped('pr-rtl'), + {name: 'Stable visual regression', conclusion: 'success'}, + {name: 'pr-a11y', conclusion: 'success'}, ], - }, - a11yReport: {components: {Button: {violations: [{id: 'x'}]}}}, - }); - expect(table).toContain('| 🔍 changed | ✅ | ❌ |'); + }), + ).toBe('Not run on this PR: add `ci:run-rtl` to run it.\n\n'); }); - it('falls back to the accessibility report and visual verdict', () => { - const table = buildTouchedComponents({ - analysis, - origin, - a11yReport: {components: {Button: {violations: []}}}, - visualVerdict: {status: 'ok', changes: [{component: 'Button'}]}, - }); - expect(table).toContain('| 🔍 changed | ✅ | — |'); + it('names several suites in one line', () => { + expect( + notRunLine({ + jobs: [ + skipped( + 'Stable visual scout (${{ matrix.shard }}/${{ strategy.job-total }})', + ), + skipped('Accessibility spec-test contracts'), + skipped('pr-rtl (core)'), + ], + }), + ).toBe( + 'Not run on this PR: add `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` to run a full suite.\n\n', + ); }); - it('renders nothing when no component was touched', () => { + it('says a wide change runs the full suite by label', () => { expect( - buildTouchedComponents({ - analysis: {newComponents: [], modifiedComponents: []}, - origin, - }), - ).toBe(''); + notRunLine({jobs: [], touched: {wideChange: true, componentCount: 60}}), + ).toBe( + 'Wide change (60 components): add `ci:run-visual` / `ci:run-a11y` / `ci:run-rtl` to run the full suite.\n\n', + ); }); - it('renders component names as literal text', () => { - const table = buildTouchedComponents({ - analysis: {modifiedComponents: [''], componentStats: {}}, - origin, - }); - expect(table).not.toContain(''); + it('says nothing when every suite ran, and falls back to the reports without a job list', () => { + expect(notRunLine({jobs: []})).toBe(''); + expect(notRunLine({jobs: null, a11yReport: {components: {}}})).toBe( + 'Not run on this PR: add `ci:run-visual` to run it.\n\n', + ); + }); +}); + +describe('job names', () => { + it("folds a skipped matrix's unexpanded name, parentheses and all", () => { + expect( + baseJobName( + "pr-rtl (${{ matrix.package }}${{ (((matrix.parts > 1) && format(' {0}/{1}', matrix.part, matrix.parts)) || '') }})", + ), + ).toBe('pr-rtl'); + const rtl = SUITES.find(suite => suite.id === 'rtl'); + expect( + suiteRunState(rtl, [{name: 'pr-rtl (core)', conclusion: 'success'}]), + ).toBe('ran'); + expect(suiteRunState(rtl, [])).toBeNull(); }); }); diff --git a/.github/scripts/visual-gate/publish-frames.mjs b/.github/scripts/visual-gate/publish-frames.mjs new file mode 100644 index 0000000000000..12bf136f0a58c --- /dev/null +++ b/.github/scripts/visual-gate/publish-frames.mjs @@ -0,0 +1,163 @@ +#!/usr/bin/env node +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Publishes the changed visual shots' frames for the PR report. + * @input `--input ` (publish-pr-report.mjs's output: verdict.json and + * re-encoded `before|after|diff/.png`), `--pr `, + * `--head-sha `, `--repository `, `--limit `, and a + * token in GITHUB_TOKEN. + * @output Commits up to `limit` changed shots' frames, largest change first, + * to the pull request's `assets/pr-` branch (the path PR screenshots + * already use) under `visual//`, and writes `image_base` (their + * raw URL prefix) to GITHUB_OUTPUT. Writes nothing when no shot changed. + * @position The PR Comment workflow's comment job, after the visual report is + * prepared and before the report renders. It writes only that branch, and + * only frames the trusted publisher re-encoded. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import {fileURLToPath} from 'node:url'; + +const KEY = /^(?!\.+$)[A-Za-z0-9._-]{1,240}$/; +const KINDS = ['before', 'after', 'diff']; + +/** The frames to publish: changed shots by pixels changed, then added and removed. */ +export function framesToPublish(verdict, limit) { + if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed') + return []; + const changes = [...(verdict.changes || [])].sort( + (a, b) => Number(b.diffPixels ?? 0) - Number(a.diffPixels ?? 0), + ); + const shots = [ + ...changes.map(change => ({key: change.key, kinds: KINDS})), + ...(verdict.added || []).map(key => ({key, kinds: ['after']})), + ...(verdict.removed || []).map(key => ({key, kinds: ['before']})), + ].filter(shot => KEY.test(String(shot.key ?? ''))); + return shots.slice(0, limit); +} + +async function api(request, method, route, body) { + const response = await request(method, route, body); + if (response.status === 404) return null; + if (response.status >= 300) + throw new Error(`${method} ${route} failed: ${response.status}`); + return response.json(); +} + +/** + * One commit on `assets/pr-` holding the frames, on top of the branch when + * it exists. Returns the raw URL prefix of the published frames, or null. + */ +export async function publishFrames({ + input, + pr, + headSha, + repository, + limit, + request, +}) { + const verdict = JSON.parse( + fs.readFileSync(path.join(input, 'verdict.json'), 'utf8'), + ); + const files = []; + for (const shot of framesToPublish(verdict, limit)) { + for (const kind of shot.kinds) { + const file = path.join(input, kind, `${shot.key}.png`); + if (fs.existsSync(file)) + files.push({path: `visual/${headSha}/${kind}/${shot.key}.png`, file}); + } + } + if (files.length === 0) return null; + const branch = `assets/pr-${pr}`; + const ref = await api( + request, + 'GET', + `/repos/${repository}/git/ref/heads/${branch}`, + ); + const parent = ref?.object?.sha ?? null; + const parentCommit = parent + ? await api(request, 'GET', `/repos/${repository}/git/commits/${parent}`) + : null; + const tree = []; + for (const entry of files) { + const blob = await api(request, 'POST', `/repos/${repository}/git/blobs`, { + content: fs.readFileSync(entry.file).toString('base64'), + encoding: 'base64', + }); + tree.push({path: entry.path, mode: '100644', type: 'blob', sha: blob.sha}); + } + const newTree = await api(request, 'POST', `/repos/${repository}/git/trees`, { + ...(parentCommit ? {base_tree: parentCommit.tree.sha} : {}), + tree, + }); + const commit = await api( + request, + 'POST', + `/repos/${repository}/git/commits`, + { + message: `Visual frames for #${pr} at ${headSha.slice(0, 7)}`, + tree: newTree.sha, + parents: parent ? [parent] : [], + }, + ); + if (parent) + await api( + request, + 'PATCH', + `/repos/${repository}/git/refs/heads/${branch}`, + { + sha: commit.sha, + }, + ); + else + await api(request, 'POST', `/repos/${repository}/git/refs`, { + ref: `refs/heads/${branch}`, + sha: commit.sha, + }); + return `https://raw.githubusercontent.com/${repository}/${branch}/visual/${headSha}/`; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const flag = name => { + const index = process.argv.indexOf(`--${name}`); + return index === -1 ? null : process.argv[index + 1]; + }; + const pr = Number(flag('pr')); + const headSha = String(flag('head-sha') ?? ''); + const repository = String(flag('repository') ?? ''); + if (!Number.isSafeInteger(pr) || pr <= 0) + throw new Error('--pr must be a PR number'); + if (!/^[0-9a-f]{40}$/.test(headSha)) + throw new Error('--head-sha must be a full SHA'); + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) + throw new Error('--repository must be owner/repo'); + const token = process.env.GITHUB_TOKEN; + const request = (method, route, body) => + fetch(`https://api.github.com${route}`, { + method, + headers: { + authorization: `Bearer ${token}`, + accept: 'application/vnd.github+json', + 'content-type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }); + const imageBase = await publishFrames({ + input: path.resolve(flag('input')), + pr, + headSha, + repository, + limit: Number(flag('limit') ?? 12), + request, + }); + if (imageBase && process.env.GITHUB_OUTPUT) + fs.appendFileSync(process.env.GITHUB_OUTPUT, `image_base=${imageBase}\n`); + console.log( + imageBase + ? `Published frames under ${imageBase}` + : 'No changed shots to publish.', + ); +} diff --git a/.github/scripts/visual-gate/publish-frames.test.mjs b/.github/scripts/visual-gate/publish-frames.test.mjs new file mode 100644 index 0000000000000..d3f1935b25e50 --- /dev/null +++ b/.github/scripts/visual-gate/publish-frames.test.mjs @@ -0,0 +1,134 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file publish-frames.test.mjs + * Pins which visual frames the PR report publishes (the largest changes + * first, then added and removed shots, at most `limit`, safe keys only) and + * how: one commit on `assets/pr-`, on top of the branch when it exists. + */ + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import {afterEach, describe, expect, it} from 'vitest'; +import {framesToPublish, publishFrames} from './publish-frames.mjs'; + +const HEAD = 'a'.repeat(40); +const roots = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, {recursive: true, force: true}); +}); + +function input(verdict) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'frames-')); + roots.push(root); + fs.writeFileSync(path.join(root, 'verdict.json'), JSON.stringify(verdict)); + for (const kind of ['before', 'after', 'diff']) { + fs.mkdirSync(path.join(root, kind)); + for (const key of ['a', 'b', 'c', 'added']) + fs.writeFileSync(path.join(root, kind, `${key}.png`), `${kind}-${key}`); + } + return root; +} + +function fakeGitHub({branchExists}) { + const calls = []; + const request = async (method, route, body) => { + calls.push({method, route, body}); + const json = data => ({status: 200, json: async () => data}); + if (method === 'GET' && route.includes('/git/ref/')) + return branchExists + ? json({object: {sha: 'parent'}}) + : {status: 404, json: async () => ({})}; + if (method === 'GET' && route.includes('/git/commits/')) + return json({tree: {sha: 'parent-tree'}}); + if (route.endsWith('/git/blobs')) + return json({sha: `blob-${calls.length}`}); + if (route.endsWith('/git/trees')) return json({sha: 'tree'}); + if (route.endsWith('/git/commits')) return json({sha: 'commit'}); + return json({}); + }; + return {calls, request}; +} + +describe('framesToPublish', () => { + it('takes the largest changes first, then added and removed shots, up to the limit', () => { + const verdict = { + changes: [ + {key: 'a', diffPixels: 1}, + {key: 'b', diffPixels: 9}, + {key: '../escape', diffPixels: 99}, + ], + added: ['added'], + removed: ['gone'], + }; + expect(framesToPublish(verdict, 3).map(shot => shot.key)).toEqual([ + 'b', + 'a', + 'added', + ]); + expect(framesToPublish({...verdict, status: 'skipped'}, 3)).toEqual([]); + }); +}); + +describe('publishFrames', () => { + it("commits the frames on top of the PR's existing assets branch", async () => { + const {calls, request} = fakeGitHub({branchExists: true}); + const base = await publishFrames({ + input: input({changes: [{key: 'b', diffPixels: 2}]}), + pr: 9171, + headSha: HEAD, + repository: 'meta/astryx', + limit: 12, + request, + }); + expect(base).toBe( + `https://raw.githubusercontent.com/meta/astryx/assets/pr-9171/visual/${HEAD}/`, + ); + const tree = calls.find(call => call.route.endsWith('/git/trees')).body; + expect(tree.base_tree).toBe('parent-tree'); + expect(tree.tree.map(entry => entry.path)).toEqual([ + `visual/${HEAD}/before/b.png`, + `visual/${HEAD}/after/b.png`, + `visual/${HEAD}/diff/b.png`, + ]); + expect( + calls.find(call => call.route.endsWith('/git/commits')).body.parents, + ).toEqual(['parent']); + expect(calls.at(-1)).toMatchObject({ + method: 'PATCH', + route: '/repos/meta/astryx/git/refs/heads/assets/pr-9171', + body: {sha: 'commit'}, + }); + }); + + it('creates the branch when it does not exist, and writes nothing without changes', async () => { + const created = fakeGitHub({branchExists: false}); + await publishFrames({ + input: input({added: ['added']}), + pr: 5, + headSha: HEAD, + repository: 'meta/astryx', + limit: 12, + request: created.request, + }); + expect(created.calls.at(-1)).toMatchObject({ + method: 'POST', + route: '/repos/meta/astryx/git/refs', + body: {ref: 'refs/heads/assets/pr-5'}, + }); + const none = fakeGitHub({branchExists: true}); + await expect( + publishFrames({ + input: input({changes: []}), + pr: 5, + headSha: HEAD, + repository: 'meta/astryx', + limit: 12, + request: none.request, + }), + ).resolves.toBeNull(); + expect(none.calls).toEqual([]); + }); +}); diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml index d16bcf1534527..a506c93133ce5 100644 --- a/.github/workflows/pr-comment.yml +++ b/.github/workflows/pr-comment.yml @@ -200,8 +200,11 @@ jobs: permissions: pull-requests: write # actions: the artifacts and the source run's job list; deployments: - # this head's Vercel preview state. + # this head's Vercel preview state; contents: the changed visual frames + # on the pull request's assets branch (publish-frames.mjs writes only + # that branch). actions: read + contents: write deployments: read # The artifact downloads are optional (a run may not produce them), so # they are not failures of this job. @@ -212,6 +215,7 @@ jobs: steps.identity.outcome == 'failure' || steps.cross-check.outcome == 'failure' || steps.visual.outcome == 'failure' || + steps.frames.outcome == 'failure' || steps.publish.outcome == 'failure' }} steps: - name: Checkout trusted default-branch code @@ -304,6 +308,18 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} continue-on-error: true + # The touched-components check's per-component results (or its wide- + # change flag); absent when the check did not run. + - name: Download the touched-components report from the CI run + if: steps.identity.outputs.valid == 'true' + uses: actions/download-artifact@v8 + with: + name: touched-components + path: touched-components + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true + - name: Download a11y artifact from the CI run if: steps.identity.outputs.valid == 'true' uses: actions/download-artifact@v8 @@ -369,6 +385,25 @@ jobs: --run-id "$RUN_ID" \ --run-attempt "$RUN_ATTEMPT" + # The report shows changed shots as before/after/diff thumbnails. Their + # re-encoded frames go to the pull request's assets branch, the path PR + # screenshots already use; nothing is written when no shot changed. + - name: Publish changed visual frames + id: frames + continue-on-error: true + if: steps.identity.outputs.valid == 'true' && steps.visual.outcome == 'success' + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ steps.identity.outputs.pr_number }} + HEAD_SHA: ${{ steps.identity.outputs.head_sha }} + run: | + node .github/scripts/visual-gate/publish-frames.mjs \ + --input trusted-visual \ + --pr "$PR_NUMBER" \ + --head-sha "$HEAD_SHA" \ + --repository "$GITHUB_REPOSITORY" \ + --limit 12 + - name: Generate and post PR comment id: publish continue-on-error: true @@ -385,6 +420,7 @@ jobs: RUN_ID: ${{ steps.identity.outputs.run_id }} RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }} SOURCE_CONCLUSION: ${{ steps.identity.outputs.source_conclusion }} + VISUAL_IMAGE_URL: ${{ steps.frames.outputs.image_base }} with: retries: 3 script: | @@ -396,6 +432,7 @@ jobs: github, core, context, + visualImageUrl: process.env.VISUAL_IMAGE_URL || null, expectedIdentity: { prNumber: Number(process.env.PR_NUMBER), headSha: process.env.HEAD_SHA, From 6c19dca215857f83d02354b0b2bd6bd434f0a7b6 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 11:14:51 -0700 Subject: [PATCH 11/15] ci: the frames step publishes only validated PNG/WebP frames to assets/pr-, and the daily pruner removes that branch once its PR closes --- .../scripts/visual-gate/publish-frames.mjs | 42 ++++++++-- .../visual-gate/publish-frames.test.mjs | 77 ++++++++++++++++++- scripts/prune-branches.sh | 9 ++- scripts/prune-branches.test.mjs | 77 +++++++++++++++++++ 4 files changed, 196 insertions(+), 9 deletions(-) create mode 100644 scripts/prune-branches.test.mjs diff --git a/.github/scripts/visual-gate/publish-frames.mjs b/.github/scripts/visual-gate/publish-frames.mjs index 12bf136f0a58c..3d5219cfcf6c5 100644 --- a/.github/scripts/visual-gate/publish-frames.mjs +++ b/.github/scripts/visual-gate/publish-frames.mjs @@ -7,7 +7,8 @@ * re-encoded `before|after|diff/.png`), `--pr `, * `--head-sha `, `--repository `, `--limit `, and a * token in GITHUB_TOKEN. - * @output Commits up to `limit` changed shots' frames, largest change first, + * @output Commits up to `limit` (at most 12) changed shots' frames, PNG or + * WebP of at most 500 KB each by their own bytes, largest change first, * to the pull request's `assets/pr-` branch (the path PR screenshots * already use) under `visual//`, and writes `image_base` (their * raw URL prefix) to GITHUB_OUTPUT. Writes nothing when no shot changed. @@ -23,6 +24,31 @@ import {fileURLToPath} from 'node:url'; const KEY = /^(?!\.+$)[A-Za-z0-9._-]{1,240}$/; const KINDS = ['before', 'after', 'diff']; +/** Shots published per head, whatever `--limit` asks for. */ +export const MAX_SHOTS = 12; +export const MAX_FRAME_BYTES = 500 * 1024; + +/** A PNG or WebP no larger than MAX_FRAME_BYTES, by its own bytes. */ +export function isPublishableImage(bytes) { + if ( + !Buffer.isBuffer(bytes) || + bytes.length === 0 || + bytes.length > MAX_FRAME_BYTES + ) + return false; + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + if (bytes.subarray(0, 8).equals(png)) return true; + return ( + bytes.subarray(0, 4).toString('latin1') === 'RIFF' && + bytes.subarray(8, 12).toString('latin1') === 'WEBP' + ); +} + +/** The only ref this script writes: PR #'s assets branch. */ +export function assetsBranch(pr) { + if (!Number.isSafeInteger(pr) || pr <= 0) throw new Error('not a PR number'); + return `assets/pr-${pr}`; +} /** The frames to publish: changed shots by pixels changed, then added and removed. */ export function framesToPublish(verdict, limit) { @@ -36,7 +62,7 @@ export function framesToPublish(verdict, limit) { ...(verdict.added || []).map(key => ({key, kinds: ['after']})), ...(verdict.removed || []).map(key => ({key, kinds: ['before']})), ].filter(shot => KEY.test(String(shot.key ?? ''))); - return shots.slice(0, limit); + return shots.slice(0, Math.min(Number(limit) || 0, MAX_SHOTS)); } async function api(request, method, route, body) { @@ -66,12 +92,16 @@ export async function publishFrames({ for (const shot of framesToPublish(verdict, limit)) { for (const kind of shot.kinds) { const file = path.join(input, kind, `${shot.key}.png`); - if (fs.existsSync(file)) - files.push({path: `visual/${headSha}/${kind}/${shot.key}.png`, file}); + if (!fs.existsSync(file)) continue; + const bytes = fs.readFileSync(file); + if (!isPublishableImage(bytes)) continue; + files.push({path: `visual/${headSha}/${kind}/${shot.key}.png`, bytes}); } } if (files.length === 0) return null; - const branch = `assets/pr-${pr}`; + if (!/^[0-9a-f]{40}$/.test(String(headSha))) + throw new Error('not a head SHA'); + const branch = assetsBranch(pr); const ref = await api( request, 'GET', @@ -84,7 +114,7 @@ export async function publishFrames({ const tree = []; for (const entry of files) { const blob = await api(request, 'POST', `/repos/${repository}/git/blobs`, { - content: fs.readFileSync(entry.file).toString('base64'), + content: entry.bytes.toString('base64'), encoding: 'base64', }); tree.push({path: entry.path, mode: '100644', type: 'blob', sha: blob.sha}); diff --git a/.github/scripts/visual-gate/publish-frames.test.mjs b/.github/scripts/visual-gate/publish-frames.test.mjs index d3f1935b25e50..0cc3239b078d3 100644 --- a/.github/scripts/visual-gate/publish-frames.test.mjs +++ b/.github/scripts/visual-gate/publish-frames.test.mjs @@ -11,7 +11,19 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import {afterEach, describe, expect, it} from 'vitest'; -import {framesToPublish, publishFrames} from './publish-frames.mjs'; +import { + MAX_FRAME_BYTES, + MAX_SHOTS, + assetsBranch, + framesToPublish, + isPublishableImage, + publishFrames, +} from './publish-frames.mjs'; + +const PNG = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from('rest'), +]); const HEAD = 'a'.repeat(40); const roots = []; @@ -27,7 +39,7 @@ function input(verdict) { for (const kind of ['before', 'after', 'diff']) { fs.mkdirSync(path.join(root, kind)); for (const key of ['a', 'b', 'c', 'added']) - fs.writeFileSync(path.join(root, kind, `${key}.png`), `${kind}-${key}`); + fs.writeFileSync(path.join(root, kind, `${key}.png`), PNG); } return root; } @@ -72,6 +84,40 @@ describe('framesToPublish', () => { }); }); +describe('image and ref limits', () => { + it('accepts only PNG or WebP bytes no larger than 500 KB', () => { + expect(isPublishableImage(PNG)).toBe(true); + const webp = Buffer.concat([ + Buffer.from('RIFF'), + Buffer.alloc(4), + Buffer.from('WEBPVP8 '), + ]); + expect(isPublishableImage(webp)).toBe(true); + expect(isPublishableImage(Buffer.from(''))).toBe( + false, + ); + expect(isPublishableImage(Buffer.from('GIF89a'))).toBe(false); + expect( + isPublishableImage(Buffer.concat([PNG, Buffer.alloc(MAX_FRAME_BYTES)])), + ).toBe(false); + }); + + it('publishes at most 12 shots, whatever the limit asks', () => { + const changes = Array.from({length: 30}, (_, i) => ({ + key: `s${i}`, + diffPixels: i, + })); + expect(framesToPublish({changes}, 100)).toHaveLength(MAX_SHOTS); + expect(MAX_SHOTS).toBe(12); + }); + + it('writes only assets/pr-', () => { + expect(assetsBranch(42)).toBe('assets/pr-42'); + for (const bad of [0, -1, 1.5, NaN, '42/../main']) + expect(() => assetsBranch(bad)).toThrow(); + }); +}); + describe('publishFrames', () => { it("commits the frames on top of the PR's existing assets branch", async () => { const {calls, request} = fakeGitHub({branchExists: true}); @@ -103,6 +149,33 @@ describe('publishFrames', () => { }); }); + it('skips a frame that is not an image, and touches no other ref', async () => { + const root = input({changes: [{key: 'a', diffPixels: 1}]}); + fs.writeFileSync(path.join(root, 'diff', 'a.png'), ''); + const {calls, request} = fakeGitHub({branchExists: true}); + await publishFrames({ + input: root, + pr: 7, + headSha: HEAD, + repository: 'meta/astryx', + limit: 12, + request, + }); + const tree = calls.find(call => call.route.endsWith('/git/trees')).body; + expect(tree.tree.map(entry => entry.path)).toEqual([ + `visual/${HEAD}/before/a.png`, + `visual/${HEAD}/after/a.png`, + ]); + const refs = calls.filter(call => call.route.includes('/git/ref')); + expect( + refs.every( + call => + call.route.includes('assets/pr-7') || + call.route.endsWith('/git/refs'), + ), + ).toBe(true); + }); + it('creates the branch when it does not exist, and writes nothing without changes', async () => { const created = fakeGitHub({branchExists: false}); await publishFrames({ diff --git a/scripts/prune-branches.sh b/scripts/prune-branches.sh index f5ad24b161c67..e21b26cfe2359 100755 --- a/scripts/prune-branches.sh +++ b/scripts/prune-branches.sh @@ -2,6 +2,7 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # scripts/prune-branches.sh — Delete remote branches with merged/closed PRs +# (a PR's `assets/pr-` branch goes with PR #) # # Usage: # ./scripts/prune-branches.sh # dry run (default) @@ -90,7 +91,13 @@ stale_cutoff="$3" repo="$4" safe_name=$(echo "$branch" | tr '/' '_') -state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error") +# A PR's assets branch (`assets/pr-`: screenshots and the PR report's +# visual frames) belongs to PR #, not to a PR whose head it is. +if [[ "$branch" =~ ^assets/pr-([0-9]+)$ ]]; then + state=$(gh pr view "${BASH_REMATCH[1]}" --repo "$repo" --json state --jq '.state' 2>/dev/null || echo "error") +else + state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error") +fi if [ "$state" = "MERGED" ] || [ "$state" = "CLOSED" ]; then echo "$branch" > "$tmp_dir/delete/$safe_name" diff --git a/scripts/prune-branches.test.mjs b/scripts/prune-branches.test.mjs new file mode 100644 index 0000000000000..7408ea135d160 --- /dev/null +++ b/scripts/prune-branches.test.mjs @@ -0,0 +1,77 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file prune-branches.test.mjs + * Pins that the daily branch pruner deletes a PR's `assets/pr-` branch + * once PR # is merged or closed, and keeps it while the PR is open. + */ + +import {execFileSync} from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {afterEach, describe, expect, it} from 'vitest'; + +const SCRIPT = path.join( + path.dirname(fileURLToPath(import.meta.url)), + 'prune-branches.sh', +); +const roots = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, {recursive: true, force: true}); +}); + +function setup(prStates) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'prune-')); + roots.push(root); + const git = (cwd, ...args) => + execFileSync('git', args, {cwd, encoding: 'utf8'}); + const origin = path.join(root, 'origin.git'); + const work = path.join(root, 'work'); + git(root, 'init', '-q', '--bare', origin); + git(root, 'init', '-q', '-b', 'main', work); + git(work, 'config', 'user.email', 'test@test.co'); + git(work, 'config', 'user.name', 'Test'); + git(work, 'commit', '-q', '--allow-empty', '-m', 'base'); + git(work, 'remote', 'add', 'origin', origin); + git( + work, + 'push', + '-q', + 'origin', + 'main', + 'main:assets/pr-1', + 'main:assets/pr-2', + ); + // A stand-in for gh: `pr view ` answers from prStates; `pr list` finds no PR. + const bin = path.join(root, 'bin'); + fs.mkdirSync(bin); + fs.writeFileSync( + path.join(bin, 'gh'), + `#!/usr/bin/env bash\nif [ "$1 $2" = "pr view" ]; then\n case "$3" in\n${Object.entries( + prStates, + ) + .map(([n, state]) => ` ${n}) echo ${state} ;;`) + .join('\n')}\n esac\nelse\n echo none\nfi\n`, + {mode: 0o755}, + ); + return {work, bin}; +} + +describe('prune-branches.sh', () => { + it("lists a merged or closed PR's assets branch for deletion and keeps an open PR's", () => { + const {work, bin} = setup({1: 'MERGED', 2: 'OPEN'}); + const out = execFileSync('bash', [SCRIPT], { + cwd: work, + encoding: 'utf8', + env: {...process.env, PATH: `${bin}:${process.env.PATH}`}, + }).replace(/\u001b\[[0-9;]*m/g, ''); + const willDelete = out.slice( + out.indexOf('Merged/closed PR (will delete):'), + ); + expect(willDelete).toContain('assets/pr-1'); + expect(willDelete).not.toContain('assets/pr-2'); + }); +}); From 219ccc531df87a4f8192f6cd8521dcb5ed47ec3c Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 11:17:59 -0700 Subject: [PATCH 12/15] ci: the pruner deletes only assets branches the PR report created (marked .pr-report-frames); hand-pushed assets branches keep the existing rule --- .../scripts/visual-gate/publish-frames.mjs | 24 +++++++++++++++- .../visual-gate/publish-frames.test.mjs | 7 +++++ scripts/prune-branches.sh | 10 ++++--- scripts/prune-branches.test.mjs | 28 ++++++++++--------- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/.github/scripts/visual-gate/publish-frames.mjs b/.github/scripts/visual-gate/publish-frames.mjs index 3d5219cfcf6c5..89871033d9012 100644 --- a/.github/scripts/visual-gate/publish-frames.mjs +++ b/.github/scripts/visual-gate/publish-frames.mjs @@ -10,7 +10,8 @@ * @output Commits up to `limit` (at most 12) changed shots' frames, PNG or * WebP of at most 500 KB each by their own bytes, largest change first, * to the pull request's `assets/pr-` branch (the path PR screenshots - * already use) under `visual//`, and writes `image_base` (their + * already use) under `visual//`, marking a branch it creates with + * MARKER so only such branches are pruned, and writes `image_base` (their * raw URL prefix) to GITHUB_OUTPUT. Writes nothing when no shot changed. * @position The PR Comment workflow's comment job, after the visual report is * prepared and before the report renders. It writes only that branch, and @@ -44,6 +45,13 @@ export function isPublishableImage(bytes) { ); } +/** + * The file that marks an assets branch as created by this script. The daily + * branch pruner deletes only marked branches, once their PR closes; a branch + * someone pushed by hand never gets the marker. + */ +export const MARKER = '.pr-report-frames'; + /** The only ref this script writes: PR #'s assets branch. */ export function assetsBranch(pr) { if (!Number.isSafeInteger(pr) || pr <= 0) throw new Error('not a PR number'); @@ -112,6 +120,20 @@ export async function publishFrames({ ? await api(request, 'GET', `/repos/${repository}/git/commits/${parent}`) : null; const tree = []; + if (!parent) { + const marker = await api( + request, + 'POST', + `/repos/${repository}/git/blobs`, + { + content: Buffer.from( + 'Created by the PR report; pruned when the PR closes.\n', + ).toString('base64'), + encoding: 'base64', + }, + ); + tree.push({path: MARKER, mode: '100644', type: 'blob', sha: marker.sha}); + } for (const entry of files) { const blob = await api(request, 'POST', `/repos/${repository}/git/blobs`, { content: entry.bytes.toString('base64'), diff --git a/.github/scripts/visual-gate/publish-frames.test.mjs b/.github/scripts/visual-gate/publish-frames.test.mjs index 0cc3239b078d3..4534108fdac43 100644 --- a/.github/scripts/visual-gate/publish-frames.test.mjs +++ b/.github/scripts/visual-gate/publish-frames.test.mjs @@ -14,6 +14,7 @@ import {afterEach, describe, expect, it} from 'vitest'; import { MAX_FRAME_BYTES, MAX_SHOTS, + MARKER, assetsBranch, framesToPublish, isPublishableImage, @@ -134,6 +135,7 @@ describe('publishFrames', () => { ); const tree = calls.find(call => call.route.endsWith('/git/trees')).body; expect(tree.base_tree).toBe('parent-tree'); + expect(tree.tree.map(entry => entry.path)).not.toContain(MARKER); expect(tree.tree.map(entry => entry.path)).toEqual([ `visual/${HEAD}/before/b.png`, `visual/${HEAD}/after/b.png`, @@ -186,6 +188,11 @@ describe('publishFrames', () => { limit: 12, request: created.request, }); + // Only a branch this script creates carries the pruning marker. + const createdTree = created.calls.find(call => + call.route.endsWith('/git/trees'), + ).body; + expect(createdTree.tree.map(entry => entry.path)).toContain(MARKER); expect(created.calls.at(-1)).toMatchObject({ method: 'POST', route: '/repos/meta/astryx/git/refs', diff --git a/scripts/prune-branches.sh b/scripts/prune-branches.sh index e21b26cfe2359..b20ebdc594b31 100755 --- a/scripts/prune-branches.sh +++ b/scripts/prune-branches.sh @@ -2,7 +2,7 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # scripts/prune-branches.sh — Delete remote branches with merged/closed PRs -# (a PR's `assets/pr-` branch goes with PR #) +# (an `assets/pr-` branch the PR report created goes with PR #) # # Usage: # ./scripts/prune-branches.sh # dry run (default) @@ -91,10 +91,12 @@ stale_cutoff="$3" repo="$4" safe_name=$(echo "$branch" | tr '/' '_') -# A PR's assets branch (`assets/pr-`: screenshots and the PR report's -# visual frames) belongs to PR #, not to a PR whose head it is. -if [[ "$branch" =~ ^assets/pr-([0-9]+)$ ]]; then +# An `assets/pr-` branch the PR report created (it carries the +# `.pr-report-frames` marker) goes with PR #. Every other branch, including +# assets branches pushed by hand, keeps the rule below. +if [[ "$branch" =~ ^assets/pr-([0-9]+)$ ]] && git cat-file -e "origin/$branch:.pr-report-frames" 2>/dev/null; then state=$(gh pr view "${BASH_REMATCH[1]}" --repo "$repo" --json state --jq '.state' 2>/dev/null || echo "error") + [ "$state" = "error" ] && state="OPEN" else state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error") fi diff --git a/scripts/prune-branches.test.mjs b/scripts/prune-branches.test.mjs index 7408ea135d160..d621e20a95127 100644 --- a/scripts/prune-branches.test.mjs +++ b/scripts/prune-branches.test.mjs @@ -2,8 +2,10 @@ /** * @file prune-branches.test.mjs - * Pins that the daily branch pruner deletes a PR's `assets/pr-` branch - * once PR # is merged or closed, and keeps it while the PR is open. + * Pins that the daily branch pruner deletes an `assets/pr-` branch the PR + * report created (it carries `.pr-report-frames`) once PR # is merged or + * closed, keeps it while the PR is open, and does not delete an assets branch + * without the marker, such as screenshots pushed by hand, for its PR closing. */ import {execFileSync} from 'node:child_process'; @@ -36,15 +38,12 @@ function setup(prStates) { git(work, 'config', 'user.name', 'Test'); git(work, 'commit', '-q', '--allow-empty', '-m', 'base'); git(work, 'remote', 'add', 'origin', origin); - git( - work, - 'push', - '-q', - 'origin', - 'main', - 'main:assets/pr-1', - 'main:assets/pr-2', - ); + // assets/pr-3 is pushed by hand: no marker. + git(work, 'push', '-q', 'origin', 'main', 'main:assets/pr-3'); + fs.writeFileSync(path.join(work, '.pr-report-frames'), 'marker\n'); + git(work, 'add', '.pr-report-frames'); + git(work, 'commit', '-q', '-m', 'frames'); + git(work, 'push', '-q', 'origin', 'HEAD:assets/pr-1', 'HEAD:assets/pr-2'); // A stand-in for gh: `pr view ` answers from prStates; `pr list` finds no PR. const bin = path.join(root, 'bin'); fs.mkdirSync(bin); @@ -61,8 +60,8 @@ function setup(prStates) { } describe('prune-branches.sh', () => { - it("lists a merged or closed PR's assets branch for deletion and keeps an open PR's", () => { - const {work, bin} = setup({1: 'MERGED', 2: 'OPEN'}); + it("deletes a closed PR's marked assets branch, keeps an open PR's, and never an unmarked one", () => { + const {work, bin} = setup({1: 'MERGED', 2: 'OPEN', 3: 'CLOSED'}); const out = execFileSync('bash', [SCRIPT], { cwd: work, encoding: 'utf8', @@ -73,5 +72,8 @@ describe('prune-branches.sh', () => { ); expect(willDelete).toContain('assets/pr-1'); expect(willDelete).not.toContain('assets/pr-2'); + // assets/pr-3 has no marker: its PR is closed, yet it is not deleted. + expect(out).not.toMatch(/^\s+assets\/pr-3$/m); + expect(out).toContain('0 branches (no PR'); }); }); From 72718975f7d5ea5ee119911dbcc9c3f8561b0534 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 12:46:02 -0700 Subject: [PATCH 13/15] ci: the PR report lists changed snapshots without thumbnails (frames move to their own PR); a failed visual run never shows a pass; an unmeasured base says so; shared sizeText; shorter not-run line --- .github/scripts/generate-pr-comment.js | 3 +- .github/scripts/generate-pr-comment.test.mjs | 11 +- .github/scripts/lib/package-size.test.mjs | 4 +- .github/scripts/lib/pr-preview.mjs | 8 - .github/scripts/lib/pr-preview.test.mjs | 27 --- .github/scripts/lib/pr-status.js | 84 +++---- .github/scripts/lib/pr-status.test.mjs | 68 ++++-- .github/scripts/lib/report-text.js | 6 +- .../scripts/visual-gate/publish-frames.mjs | 215 ------------------ .../visual-gate/publish-frames.test.mjs | 214 ----------------- .../visual-gate/workflow-concurrency.test.mjs | 2 + .github/workflows/pr-comment.yml | 27 +-- scripts/prune-branches.sh | 11 +- scripts/prune-branches.test.mjs | 79 ------- 14 files changed, 104 insertions(+), 655 deletions(-) delete mode 100644 .github/scripts/visual-gate/publish-frames.mjs delete mode 100644 .github/scripts/visual-gate/publish-frames.test.mjs delete mode 100644 scripts/prune-branches.test.mjs diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.js index cc8539def9755..17df462c0a988 100644 --- a/.github/scripts/generate-pr-comment.js +++ b/.github/scripts/generate-pr-comment.js @@ -3,7 +3,7 @@ /** * @description Generates the PR comment from the run's analysis results - * @input --analysis --base-stats --package-size --a11y --visual --visual-image-url --preview --jobs --touched --source-conclusion --run-url + * @input --analysis --base-stats --package-size --a11y --visual --preview --jobs --touched --source-conclusion --run-url * @output Formatted markdown comment body to stdout, compact by design: this * head's preview first, the modified components (preview links, size, lines * of code, a mark per browser suite that ran; three rows visible, the rest @@ -127,7 +127,6 @@ const report = buildCompactReport({ a11yReport, visualVerdict, packageSize, - visualImageUrl: getArg('visual-image-url'), }); const footerLinksStr = runUrl ? ` | ${extLink('View full report', runUrl)}` diff --git a/.github/scripts/generate-pr-comment.test.mjs b/.github/scripts/generate-pr-comment.test.mjs index 0dcb5b1a6984b..5b18d64df5de6 100644 --- a/.github/scripts/generate-pr-comment.test.mjs +++ b/.github/scripts/generate-pr-comment.test.mjs @@ -138,7 +138,7 @@ describe('generate-pr-comment compact report', () => { baseComponentSizes: {'core/Chat': size(6000, 1000)}, }); expect(out).toContain( - '| Chat | 🔴 8.8 kB (**+2.9 kB**) · 1.0 kB CSS (0) |', + '| Chat | 🔴 8.8 kB (**+2.9 kB**) · 1000 B CSS (0) |', ); }); @@ -148,8 +148,8 @@ describe('generate-pr-comment compact report', () => { componentSizes: {'core/Card': size(null), 'core/New': size(3000)}, baseComponentSizes: {'core/Card': size(2000)}, }); - expect(out).toContain('| Card | 1.0 kB CSS (0) |'); - expect(out).toContain('| New | 2.9 kB (new) · 1.0 kB CSS (new) |'); + expect(out).toContain('| Card | 1000 B CSS (0) |'); + expect(out).toContain('| New | 2.9 kB (new) · 1000 B CSS (new) |'); }); it('fills the base from base stats', () => { @@ -164,7 +164,7 @@ describe('generate-pr-comment compact report', () => { {...baseAnalysis, componentSizes: {'core/Card': size(1500)}}, ['--base-stats', baseFile], ); - expect(out).toContain('| Card | 1.5 kB (−500 B) · 1.0 kB CSS (0) |'); + expect(out).toContain('| Card | 1.5 kB (−500 B) · 1000 B CSS (0) |'); } finally { fs.rmSync(dir, {recursive: true, force: true}); } @@ -179,7 +179,7 @@ describe('generate-pr-comment compact report', () => { ], }); expect(out).toContain( - 'Not run on this PR: add `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` to run a full suite.', + 'Not run on this PR: add `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` to run them.', ); expect(out).not.toContain('| Check |'); expect(out).not.toMatch(/CodeBunny/i); @@ -250,4 +250,3 @@ describe('generate-pr-comment compact report', () => { expect(out).not.toContain(''); }); }); - diff --git a/.github/scripts/lib/package-size.test.mjs b/.github/scripts/lib/package-size.test.mjs index 6c4235ed25fa2..bef6930a1d2a6 100644 --- a/.github/scripts/lib/package-size.test.mjs +++ b/.github/scripts/lib/package-size.test.mjs @@ -263,7 +263,7 @@ describe('renderPackageSizeSection', () => { ); expect(markdown).toContain('### Published Package Size'); expect(markdown).toContain( - '| @astryxdesign/core | 97.7KB | 98.6KB | +1000B (+1.0%) |', + '| @astryxdesign/core | 97.7 kB | 98.6 kB | +1000 B (+1.0%) |', ); expect(markdown).not.toContain('theme-neutral'); expect(markdown).toContain('2 other published packages are unchanged.'); @@ -279,7 +279,7 @@ describe('renderPackageSizeSection', () => { }), ); expect(markdown).toContain( - '| @astryxdesign/core | 8.58MB | 8.68MB | +97.7KB (+1.1%) |', + '| @astryxdesign/core | 8.58 MB | 8.68 MB | +97.7 kB (+1.1%) |', ); }); diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjs index a62d7f23ebbf7..44914fe836eed 100644 --- a/.github/scripts/lib/pr-preview.mjs +++ b/.github/scripts/lib/pr-preview.mjs @@ -603,7 +603,6 @@ export async function reconcilePrComment({ touchedPath = 'touched-components/touched-components.json', a11yPath = 'a11y/a11y-report.json', visualPath = 'trusted-visual/verdict.json', - visualImageUrl = null, createIfMissing = true, fallbackMessage, generator = GENERATOR, @@ -693,13 +692,6 @@ export async function reconcilePrComment({ // checks, or with the ci:run-a11y label); the comment says so. ...(fs.existsSync(a11yPath) ? ['--a11y', a11yPath] : []), ...(fs.existsSync(visualPath) ? ['--visual', visualPath] : []), - // Frames publish-frames.mjs put on this PR's assets branch, and only - // from raw.githubusercontent.com. - ...(/^https:\/\/raw\.githubusercontent\.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+\/assets\/pr-\d+\/visual\/[0-9a-f]{40}\/$/.test( - String(visualImageUrl ?? ''), - ) - ? ['--visual-image-url', visualImageUrl] - : []), ...(trustedHeadReport({ reportPath: touchedPath, label: 'touched-components report', diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjs index 3ad50af4750aa..98b81806a30cc 100644 --- a/.github/scripts/lib/pr-preview.test.mjs +++ b/.github/scripts/lib/pr-preview.test.mjs @@ -203,7 +203,6 @@ async function reconcile({ baseStats, packageSize, touched, - visualImageUrl, } = {}) { const value = previewAvailable ? {...requested, headRepository: requested.baseRepository} @@ -244,7 +243,6 @@ async function reconcile({ baseStatsPath, packageSizePath, touchedPath, - visualImageUrl, a11yPath: paths.a11y, visualPath: paths.visual, createIfMissing, @@ -1137,31 +1135,6 @@ describe('PR comment preview reconciliation', () => { }, ); - it.each([ - [ - `https://raw.githubusercontent.com/meta/astryx/assets/pr-5697/visual/${HEAD}/`, - true, - ], - ['https://evil.example/visual/', false], - [ - `https://raw.githubusercontent.com/meta/astryx/main/visual/${HEAD}/`, - false, - ], - ])( - 'passes visual frames only from a PR assets branch: %s', - async (url, used) => { - const seen = []; - await reconcile({ - visualImageUrl: url, - execute: (_node, args) => { - seen.push(args); - return '## PR Analysis Report\n'; - }, - }); - expect(seen[0].includes('--visual-image-url')).toBe(used); - }, - ); - it('falls back to a safe current message when analysis rendering fails', async () => { const stale = { id: 78, diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js index 2e5d792843331..c79ccc60f6752 100644 --- a/.github/scripts/lib/pr-status.js +++ b/.github/scripts/lib/pr-status.js @@ -17,7 +17,7 @@ * the CI checks and CodeBunny posts its own review, so neither is repeated. */ -const {inline, num} = require('./report-text'); +const {inline, num, sizeText} = require('./report-text'); /** * The heavy suites a pull request runs only with its `ci:run-` label, @@ -78,19 +78,17 @@ function isMeasured(value) { return typeof value === 'number' && Number.isFinite(value); } -function kb(bytes) { - return `${(bytes / 1024).toFixed(1)} kB`; -} - function signedSize(value) { - const sign = value > 0 ? '+' : '−'; - const magnitude = Math.abs(value); - return magnitude < 1024 ? `${sign}${magnitude} B` : `${sign}${kb(magnitude)}`; + return `${value > 0 ? '+' : '−'}${sizeText(Math.abs(value))}`; } -/** A change against main; growth in bold, so it stands out. */ -function delta(value, format) { - if (value === null) return 'new'; +/** + * A change against main; growth in bold, so it stands out. Without a number, + * the component is new when main was measured, and unmeasured when it was + * not (no record of the merge base). + */ +function delta(value, format, baseMeasured = true) { + if (value === null) return baseMeasured ? 'new' : 'not measured'; if (value === 0) return '0'; const text = format(value); return value > 0 ? `**${text}**` : text; @@ -118,14 +116,14 @@ function sizeChange(head = {}, base = {}) { * JS and CSS import cost (gzip), each with its change against main, and a 🔴 * when the growth passes the touch-only budget (advisory). */ -function sizeCell(head = {}, base = {}) { +function sizeCell(head = {}, base = {}, baseMeasured = true) { const js = head.importGzipBytes; const css = head.cssGzipBytes; if (!isMeasured(js) && !isMeasured(css)) return '—'; const growth = sizeChange(head, base); const part = (after, changed, label) => isMeasured(after) - ? `${kb(after)}${label} (${delta(changed, signedSize)})` + ? `${sizeText(after)}${label} (${delta(changed, signedSize, baseMeasured)})` : null; return `${growth.overBudget ? '🔴 ' : ''}${[ part(js, growth.js, ''), @@ -136,12 +134,13 @@ function sizeCell(head = {}, base = {}) { } /** Source lines of code with their change against main. */ -function locCell(head, base) { +function locCell(head, base, baseMeasured = true) { if (!isMeasured(head)) return '—'; const count = value => Math.abs(value).toLocaleString('en-US'); return `${count(num(head))} (${delta( change(head, base), value => `${value > 0 ? '+' : '−'}${count(value)}`, + baseMeasured, )})`; } @@ -198,6 +197,11 @@ function componentRows({analysis, touched, a11yReport, visualVerdict}) { if (component.key) keys.add(component.key); } const visualChanges = visualVerdict?.changes || []; + // Main's sizes of the merge base were read (a base record or build). + const baseMeasured = + Boolean(analysis.baseSha) || + Object.keys(base).length > 0 || + Object.keys(baseStats).length > 0; return [...keys].map(key => { const name = key.slice(key.indexOf('/') + 1); const reported = byName.get(name)?.results || {}; @@ -214,9 +218,12 @@ function componentRows({analysis, touched, a11yReport, visualVerdict}) { const visual = reported.visual ?? (visualVerdict && visualVerdict.status !== 'skipped' - ? visualChanges.some(entry => entry.component === name) - ? 'changed' - : 'pass' + ? // A run that failed compared nothing: never a pass. + visualVerdict.status === 'failed' + ? 'fail' + : visualChanges.some(entry => entry.component === name) + ? 'changed' + : 'pass' : null); return { key, @@ -227,6 +234,7 @@ function componentRows({analysis, touched, a11yReport, visualVerdict}) { growth: sizeChange(head[key], base[key]), loc: [stats[name]?.linesOfCode, baseStats[name]?.linesOfCode], results: {visual, a11y, rtl: reported.rtl ?? null}, + baseMeasured, }; }); } @@ -286,8 +294,8 @@ function buildComponentTable({ const stories = storybookLink(origin, row.storyTitle); const cells = [ stories ? `${name} · [Storybook](${stories})` : name, - sizeCell(row.head, row.base), - locCell(...row.loc), + sizeCell(row.head, row.base, row.baseMeasured), + locCell(...row.loc, row.baseMeasured), ...(checks ? [checksCell(wide ? {} : row.results)] : []), ]; return `| ${cells.join(' | ')} |`; @@ -301,12 +309,11 @@ function buildComponentTable({ } /** - * The changed shots as before/after/diff thumbnails, largest change first: - * three rows, then the rest collapsed. Frames come from `imageBase` - * (`/.png`); without it the rows still name - * the changed shots. + * The changed shots, largest change first: up to VISUAL_FRAME_LIMIT rows + * naming each shot and its change, inside one disclosure whose summary + * carries the counts. The run's visual report holds the frames. */ -function buildVisualDeltas({verdict, imageBase}) { +function buildVisualDeltas({verdict}) { if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed') return ''; const all = [ @@ -323,13 +330,12 @@ function buildVisualDeltas({verdict, imageBase}) { if (all.length === 0) return ''; const rows = all.slice(0, VISUAL_FRAME_LIMIT); const unshown = all.length - rows.length; - const base = imageBase ? `${String(imageBase).replace(/\/+$/, '')}/` : null; - const frame = (row, kind) => { - if (!row.kinds.includes(kind)) return '—'; - if (!base) return 'frame unavailable'; - const key = encodeURIComponent(String(row.key ?? '')); - return `${kind} frame`; - }; + const changeText = row => + row.kinds.length === 3 + ? `changed (${num(row.diffPixels ?? 0).toLocaleString('en-US')} px)` + : row.kinds[0] === 'after' + ? 'added' + : 'removed'; const label = row => { const story = [row.component || row.title, row.name] .filter(Boolean) @@ -341,10 +347,8 @@ function buildVisualDeltas({verdict, imageBase}) { .join(', '); return variant ? `${story} (${variant})` : story; }; - const header = - '| Component · story | Before | After | Diff |\n|---|---|---|---|\n'; - const line = row => - `| ${label(row)} | ${frame(row, 'before')} | ${frame(row, 'after')} | ${frame(row, 'diff')} |`; + const header = '| Component · story | Change |\n|---|---|\n'; + const line = row => `| ${label(row)} | ${changeText(row)} |`; // A shot key is `---__-`. const componentOf = row => String( @@ -397,10 +401,7 @@ function packagesLine(report) { ); if (changed.length === 0) return '**Package size:** no published package changed; no new dependencies.\n\n'; - const packed = bytes => - bytes >= 1024 * 1024 - ? `${(bytes / (1024 * 1024)).toFixed(2)} MB` - : kb(bytes); + const packed = sizeText; const describe = entry => { const value = change(entry.head?.packedBytes, entry.base?.packedBytes); const deps = added.filter(dep => dep.package === entry.name); @@ -442,7 +443,7 @@ function notRunLine({jobs, a11yReport, visualVerdict, touched = null}) { labels.length === 1 ? labels[0] : `${labels.slice(0, -1).join(', ')} or ${labels.at(-1)}`; - return `Not run on this PR: add ${list} to run ${labels.length === 1 ? 'it' : 'a full suite'}.\n\n`; + return `Not run on this PR: add ${list} to run ${labels.length === 1 ? 'it' : 'them'}.\n\n`; } /** The compact report body, between the heading and the footer. */ @@ -454,7 +455,6 @@ function buildCompactReport({ a11yReport = null, visualVerdict = null, packageSize = null, - visualImageUrl = null, }) { const origin = preview?.state === 'ready' ? preview.url : preview?.aliasOrigin || ''; @@ -464,7 +464,7 @@ function buildCompactReport({ touched, a11yReport, visualVerdict, - })}${buildVisualDeltas({verdict: visualVerdict, imageBase: visualImageUrl})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict, touched})}`; + })}${buildVisualDeltas({verdict: visualVerdict})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict, touched})}`; } module.exports = { diff --git a/.github/scripts/lib/pr-status.test.mjs b/.github/scripts/lib/pr-status.test.mjs index 5fcf097a948d4..f319bbbd8f257 100644 --- a/.github/scripts/lib/pr-status.test.mjs +++ b/.github/scripts/lib/pr-status.test.mjs @@ -178,6 +178,7 @@ describe('buildComponentTable', () => { it('lists a modified component folder with no size', () => { const table = buildComponentTable({ analysis: { + baseSha: 'a'.repeat(40), modifiedComponents: ['Card'], componentStats: {Card: {package: '@astryxdesign/lab', linesOfCode: 9}}, }, @@ -186,6 +187,38 @@ describe('buildComponentTable', () => { expect(table).toContain('| lab/Card | — | 9 (new) |'); }); + it('says not measured, never new, when main has no sizes for the merge base', () => { + const table = buildComponentTable({ + analysis: { + componentSizes: { + 'core/Card': {importGzipBytes: 2048, cssGzipBytes: null}, + }, + componentStats: {Card: {linesOfCode: 9}}, + }, + origin: '', + }); + expect(table).toContain( + '| Card | 2.0 kB (not measured) | 9 (not measured) |', + ); + }); + + it('never shows a pass for a visual run that failed', () => { + const table = buildComponentTable({ + analysis: { + baseSha: 'a'.repeat(40), + componentSizes: { + 'core/Card': {importGzipBytes: 2048, cssGzipBytes: null}, + }, + baseComponentSizes: { + 'core/Card': {importGzipBytes: 2048, cssGzipBytes: null}, + }, + }, + origin: '', + visualVerdict: {status: 'failed', changes: []}, + }); + expect(table).toContain('| ✗ · — · — |'); + }); + it('renders nothing when no component changed', () => { expect(buildComponentTable({analysis: {}, origin: ORIGIN})).toBe(''); }); @@ -201,19 +234,17 @@ describe('buildVisualDeltas', () => { diffPixels, }); - it('collapses the changed shots under their count, largest change first, with frames', () => { + it('collapses the changed shots under their count, largest change first', () => { const verdict = { status: 'changed', counts: {total: 20}, - changes: [change('a', 1), change('b', 50), change('c', 9, 'Chip')], + changes: [change('a', 1), change('b', 1500), change('c', 9, 'Chip')], added: ['core-dialog--default__neutral-light'], + removed: ['core-old--default__neutral-light'], }; - const section = buildVisualDeltas({ - verdict, - imageBase: 'https://raw.example/frames', - }); + const section = buildVisualDeltas({verdict}); expect(section).toMatch( - /^
\nVisual changes: 4 snapshots in 3 components<\/summary>/, + /^
\nVisual changes: 5 snapshots in 4 components<\/summary>/, ); expect(section.match(/^\| \S+ · [a-c] /gm)).toEqual([ '| Card · b ', @@ -221,38 +252,33 @@ describe('buildVisualDeltas', () => { '| Card · a ', ]); expect(section).toContain( - 'before frame', + '| Card · b (neutral, light) | changed (1,500 px) |', ); - // An added shot has an after frame only. expect(section).toContain( - '| — | after frame | — |', + '| core-dialog--default\\_\\_neutral-light | added |', ); + expect(section).toContain('| core-old--default\\_\\_neutral-light | removed |'); }); - it('frames only the shots publish-frames.mjs publishes, and counts the rest', () => { + it(`lists at most ${VISUAL_FRAME_LIMIT} shots and counts the rest`, () => { const changes = Array.from({length: VISUAL_FRAME_LIMIT + 3}, (_, i) => change(`s${i}`, 100 - i), ); const section = buildVisualDeltas({ verdict: {status: 'changed', counts: {total: 99}, changes}, - imageBase: 'https://raw.example/f/', }); - expect(section.match(/ { + it('is absent without changes, or when the visual run failed', () => { expect(buildVisualDeltas({verdict: {status: 'ok', changes: []}})).toBe(''); expect(buildVisualDeltas({verdict: null})).toBe(''); expect( buildVisualDeltas({ - verdict: { - status: 'changed', - changes: [change('a', 1)], - counts: {total: 1}, - }, + verdict: {status: 'failed', changes: [change('a', 1)]}, }), - ).toContain('| frame unavailable |'); + ).toBe(''); }); }); @@ -354,7 +380,7 @@ describe('notRunLine', () => { ], }), ).toBe( - 'Not run on this PR: add `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` to run a full suite.\n\n', + 'Not run on this PR: add `ci:run-visual`, `ci:run-a11y` or `ci:run-rtl` to run them.\n\n', ); }); diff --git a/.github/scripts/lib/report-text.js b/.github/scripts/lib/report-text.js index 111d929532694..9623fe830b4e8 100644 --- a/.github/scripts/lib/report-text.js +++ b/.github/scripts/lib/report-text.js @@ -45,9 +45,9 @@ function safeUrl(value) { */ function sizeText(bytes) { if (typeof bytes !== 'number' || !Number.isFinite(bytes)) return 'N/A'; - if (Math.abs(bytes) < 1024) return `${bytes}B`; - if (Math.abs(bytes) < 1024 * 1024) return `${(bytes / 1024).toFixed(1)}KB`; - return `${(bytes / (1024 * 1024)).toFixed(2)}MB`; + if (Math.abs(bytes) < 1024) return `${bytes} B`; + if (Math.abs(bytes) < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} kB`; + return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; } module.exports = {inline, num, safeUrl, sizeText}; diff --git a/.github/scripts/visual-gate/publish-frames.mjs b/.github/scripts/visual-gate/publish-frames.mjs deleted file mode 100644 index 89871033d9012..0000000000000 --- a/.github/scripts/visual-gate/publish-frames.mjs +++ /dev/null @@ -1,215 +0,0 @@ -#!/usr/bin/env node -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file Publishes the changed visual shots' frames for the PR report. - * @input `--input ` (publish-pr-report.mjs's output: verdict.json and - * re-encoded `before|after|diff/.png`), `--pr `, - * `--head-sha `, `--repository `, `--limit `, and a - * token in GITHUB_TOKEN. - * @output Commits up to `limit` (at most 12) changed shots' frames, PNG or - * WebP of at most 500 KB each by their own bytes, largest change first, - * to the pull request's `assets/pr-` branch (the path PR screenshots - * already use) under `visual//`, marking a branch it creates with - * MARKER so only such branches are pruned, and writes `image_base` (their - * raw URL prefix) to GITHUB_OUTPUT. Writes nothing when no shot changed. - * @position The PR Comment workflow's comment job, after the visual report is - * prepared and before the report renders. It writes only that branch, and - * only frames the trusted publisher re-encoded. - */ - -import fs from 'node:fs'; -import path from 'node:path'; -import process from 'node:process'; -import {fileURLToPath} from 'node:url'; - -const KEY = /^(?!\.+$)[A-Za-z0-9._-]{1,240}$/; -const KINDS = ['before', 'after', 'diff']; -/** Shots published per head, whatever `--limit` asks for. */ -export const MAX_SHOTS = 12; -export const MAX_FRAME_BYTES = 500 * 1024; - -/** A PNG or WebP no larger than MAX_FRAME_BYTES, by its own bytes. */ -export function isPublishableImage(bytes) { - if ( - !Buffer.isBuffer(bytes) || - bytes.length === 0 || - bytes.length > MAX_FRAME_BYTES - ) - return false; - const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); - if (bytes.subarray(0, 8).equals(png)) return true; - return ( - bytes.subarray(0, 4).toString('latin1') === 'RIFF' && - bytes.subarray(8, 12).toString('latin1') === 'WEBP' - ); -} - -/** - * The file that marks an assets branch as created by this script. The daily - * branch pruner deletes only marked branches, once their PR closes; a branch - * someone pushed by hand never gets the marker. - */ -export const MARKER = '.pr-report-frames'; - -/** The only ref this script writes: PR #'s assets branch. */ -export function assetsBranch(pr) { - if (!Number.isSafeInteger(pr) || pr <= 0) throw new Error('not a PR number'); - return `assets/pr-${pr}`; -} - -/** The frames to publish: changed shots by pixels changed, then added and removed. */ -export function framesToPublish(verdict, limit) { - if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed') - return []; - const changes = [...(verdict.changes || [])].sort( - (a, b) => Number(b.diffPixels ?? 0) - Number(a.diffPixels ?? 0), - ); - const shots = [ - ...changes.map(change => ({key: change.key, kinds: KINDS})), - ...(verdict.added || []).map(key => ({key, kinds: ['after']})), - ...(verdict.removed || []).map(key => ({key, kinds: ['before']})), - ].filter(shot => KEY.test(String(shot.key ?? ''))); - return shots.slice(0, Math.min(Number(limit) || 0, MAX_SHOTS)); -} - -async function api(request, method, route, body) { - const response = await request(method, route, body); - if (response.status === 404) return null; - if (response.status >= 300) - throw new Error(`${method} ${route} failed: ${response.status}`); - return response.json(); -} - -/** - * One commit on `assets/pr-` holding the frames, on top of the branch when - * it exists. Returns the raw URL prefix of the published frames, or null. - */ -export async function publishFrames({ - input, - pr, - headSha, - repository, - limit, - request, -}) { - const verdict = JSON.parse( - fs.readFileSync(path.join(input, 'verdict.json'), 'utf8'), - ); - const files = []; - for (const shot of framesToPublish(verdict, limit)) { - for (const kind of shot.kinds) { - const file = path.join(input, kind, `${shot.key}.png`); - if (!fs.existsSync(file)) continue; - const bytes = fs.readFileSync(file); - if (!isPublishableImage(bytes)) continue; - files.push({path: `visual/${headSha}/${kind}/${shot.key}.png`, bytes}); - } - } - if (files.length === 0) return null; - if (!/^[0-9a-f]{40}$/.test(String(headSha))) - throw new Error('not a head SHA'); - const branch = assetsBranch(pr); - const ref = await api( - request, - 'GET', - `/repos/${repository}/git/ref/heads/${branch}`, - ); - const parent = ref?.object?.sha ?? null; - const parentCommit = parent - ? await api(request, 'GET', `/repos/${repository}/git/commits/${parent}`) - : null; - const tree = []; - if (!parent) { - const marker = await api( - request, - 'POST', - `/repos/${repository}/git/blobs`, - { - content: Buffer.from( - 'Created by the PR report; pruned when the PR closes.\n', - ).toString('base64'), - encoding: 'base64', - }, - ); - tree.push({path: MARKER, mode: '100644', type: 'blob', sha: marker.sha}); - } - for (const entry of files) { - const blob = await api(request, 'POST', `/repos/${repository}/git/blobs`, { - content: entry.bytes.toString('base64'), - encoding: 'base64', - }); - tree.push({path: entry.path, mode: '100644', type: 'blob', sha: blob.sha}); - } - const newTree = await api(request, 'POST', `/repos/${repository}/git/trees`, { - ...(parentCommit ? {base_tree: parentCommit.tree.sha} : {}), - tree, - }); - const commit = await api( - request, - 'POST', - `/repos/${repository}/git/commits`, - { - message: `Visual frames for #${pr} at ${headSha.slice(0, 7)}`, - tree: newTree.sha, - parents: parent ? [parent] : [], - }, - ); - if (parent) - await api( - request, - 'PATCH', - `/repos/${repository}/git/refs/heads/${branch}`, - { - sha: commit.sha, - }, - ); - else - await api(request, 'POST', `/repos/${repository}/git/refs`, { - ref: `refs/heads/${branch}`, - sha: commit.sha, - }); - return `https://raw.githubusercontent.com/${repository}/${branch}/visual/${headSha}/`; -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - const flag = name => { - const index = process.argv.indexOf(`--${name}`); - return index === -1 ? null : process.argv[index + 1]; - }; - const pr = Number(flag('pr')); - const headSha = String(flag('head-sha') ?? ''); - const repository = String(flag('repository') ?? ''); - if (!Number.isSafeInteger(pr) || pr <= 0) - throw new Error('--pr must be a PR number'); - if (!/^[0-9a-f]{40}$/.test(headSha)) - throw new Error('--head-sha must be a full SHA'); - if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) - throw new Error('--repository must be owner/repo'); - const token = process.env.GITHUB_TOKEN; - const request = (method, route, body) => - fetch(`https://api.github.com${route}`, { - method, - headers: { - authorization: `Bearer ${token}`, - accept: 'application/vnd.github+json', - 'content-type': 'application/json', - }, - body: body ? JSON.stringify(body) : undefined, - }); - const imageBase = await publishFrames({ - input: path.resolve(flag('input')), - pr, - headSha, - repository, - limit: Number(flag('limit') ?? 12), - request, - }); - if (imageBase && process.env.GITHUB_OUTPUT) - fs.appendFileSync(process.env.GITHUB_OUTPUT, `image_base=${imageBase}\n`); - console.log( - imageBase - ? `Published frames under ${imageBase}` - : 'No changed shots to publish.', - ); -} diff --git a/.github/scripts/visual-gate/publish-frames.test.mjs b/.github/scripts/visual-gate/publish-frames.test.mjs deleted file mode 100644 index 4534108fdac43..0000000000000 --- a/.github/scripts/visual-gate/publish-frames.test.mjs +++ /dev/null @@ -1,214 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file publish-frames.test.mjs - * Pins which visual frames the PR report publishes (the largest changes - * first, then added and removed shots, at most `limit`, safe keys only) and - * how: one commit on `assets/pr-`, on top of the branch when it exists. - */ - -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import {afterEach, describe, expect, it} from 'vitest'; -import { - MAX_FRAME_BYTES, - MAX_SHOTS, - MARKER, - assetsBranch, - framesToPublish, - isPublishableImage, - publishFrames, -} from './publish-frames.mjs'; - -const PNG = Buffer.concat([ - Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), - Buffer.from('rest'), -]); - -const HEAD = 'a'.repeat(40); -const roots = []; -afterEach(() => { - for (const root of roots.splice(0)) - fs.rmSync(root, {recursive: true, force: true}); -}); - -function input(verdict) { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'frames-')); - roots.push(root); - fs.writeFileSync(path.join(root, 'verdict.json'), JSON.stringify(verdict)); - for (const kind of ['before', 'after', 'diff']) { - fs.mkdirSync(path.join(root, kind)); - for (const key of ['a', 'b', 'c', 'added']) - fs.writeFileSync(path.join(root, kind, `${key}.png`), PNG); - } - return root; -} - -function fakeGitHub({branchExists}) { - const calls = []; - const request = async (method, route, body) => { - calls.push({method, route, body}); - const json = data => ({status: 200, json: async () => data}); - if (method === 'GET' && route.includes('/git/ref/')) - return branchExists - ? json({object: {sha: 'parent'}}) - : {status: 404, json: async () => ({})}; - if (method === 'GET' && route.includes('/git/commits/')) - return json({tree: {sha: 'parent-tree'}}); - if (route.endsWith('/git/blobs')) - return json({sha: `blob-${calls.length}`}); - if (route.endsWith('/git/trees')) return json({sha: 'tree'}); - if (route.endsWith('/git/commits')) return json({sha: 'commit'}); - return json({}); - }; - return {calls, request}; -} - -describe('framesToPublish', () => { - it('takes the largest changes first, then added and removed shots, up to the limit', () => { - const verdict = { - changes: [ - {key: 'a', diffPixels: 1}, - {key: 'b', diffPixels: 9}, - {key: '../escape', diffPixels: 99}, - ], - added: ['added'], - removed: ['gone'], - }; - expect(framesToPublish(verdict, 3).map(shot => shot.key)).toEqual([ - 'b', - 'a', - 'added', - ]); - expect(framesToPublish({...verdict, status: 'skipped'}, 3)).toEqual([]); - }); -}); - -describe('image and ref limits', () => { - it('accepts only PNG or WebP bytes no larger than 500 KB', () => { - expect(isPublishableImage(PNG)).toBe(true); - const webp = Buffer.concat([ - Buffer.from('RIFF'), - Buffer.alloc(4), - Buffer.from('WEBPVP8 '), - ]); - expect(isPublishableImage(webp)).toBe(true); - expect(isPublishableImage(Buffer.from(''))).toBe( - false, - ); - expect(isPublishableImage(Buffer.from('GIF89a'))).toBe(false); - expect( - isPublishableImage(Buffer.concat([PNG, Buffer.alloc(MAX_FRAME_BYTES)])), - ).toBe(false); - }); - - it('publishes at most 12 shots, whatever the limit asks', () => { - const changes = Array.from({length: 30}, (_, i) => ({ - key: `s${i}`, - diffPixels: i, - })); - expect(framesToPublish({changes}, 100)).toHaveLength(MAX_SHOTS); - expect(MAX_SHOTS).toBe(12); - }); - - it('writes only assets/pr-', () => { - expect(assetsBranch(42)).toBe('assets/pr-42'); - for (const bad of [0, -1, 1.5, NaN, '42/../main']) - expect(() => assetsBranch(bad)).toThrow(); - }); -}); - -describe('publishFrames', () => { - it("commits the frames on top of the PR's existing assets branch", async () => { - const {calls, request} = fakeGitHub({branchExists: true}); - const base = await publishFrames({ - input: input({changes: [{key: 'b', diffPixels: 2}]}), - pr: 9171, - headSha: HEAD, - repository: 'meta/astryx', - limit: 12, - request, - }); - expect(base).toBe( - `https://raw.githubusercontent.com/meta/astryx/assets/pr-9171/visual/${HEAD}/`, - ); - const tree = calls.find(call => call.route.endsWith('/git/trees')).body; - expect(tree.base_tree).toBe('parent-tree'); - expect(tree.tree.map(entry => entry.path)).not.toContain(MARKER); - expect(tree.tree.map(entry => entry.path)).toEqual([ - `visual/${HEAD}/before/b.png`, - `visual/${HEAD}/after/b.png`, - `visual/${HEAD}/diff/b.png`, - ]); - expect( - calls.find(call => call.route.endsWith('/git/commits')).body.parents, - ).toEqual(['parent']); - expect(calls.at(-1)).toMatchObject({ - method: 'PATCH', - route: '/repos/meta/astryx/git/refs/heads/assets/pr-9171', - body: {sha: 'commit'}, - }); - }); - - it('skips a frame that is not an image, and touches no other ref', async () => { - const root = input({changes: [{key: 'a', diffPixels: 1}]}); - fs.writeFileSync(path.join(root, 'diff', 'a.png'), ''); - const {calls, request} = fakeGitHub({branchExists: true}); - await publishFrames({ - input: root, - pr: 7, - headSha: HEAD, - repository: 'meta/astryx', - limit: 12, - request, - }); - const tree = calls.find(call => call.route.endsWith('/git/trees')).body; - expect(tree.tree.map(entry => entry.path)).toEqual([ - `visual/${HEAD}/before/a.png`, - `visual/${HEAD}/after/a.png`, - ]); - const refs = calls.filter(call => call.route.includes('/git/ref')); - expect( - refs.every( - call => - call.route.includes('assets/pr-7') || - call.route.endsWith('/git/refs'), - ), - ).toBe(true); - }); - - it('creates the branch when it does not exist, and writes nothing without changes', async () => { - const created = fakeGitHub({branchExists: false}); - await publishFrames({ - input: input({added: ['added']}), - pr: 5, - headSha: HEAD, - repository: 'meta/astryx', - limit: 12, - request: created.request, - }); - // Only a branch this script creates carries the pruning marker. - const createdTree = created.calls.find(call => - call.route.endsWith('/git/trees'), - ).body; - expect(createdTree.tree.map(entry => entry.path)).toContain(MARKER); - expect(created.calls.at(-1)).toMatchObject({ - method: 'POST', - route: '/repos/meta/astryx/git/refs', - body: {ref: 'refs/heads/assets/pr-5'}, - }); - const none = fakeGitHub({branchExists: true}); - await expect( - publishFrames({ - input: input({changes: []}), - pr: 5, - headSha: HEAD, - repository: 'meta/astryx', - limit: 12, - request: none.request, - }), - ).resolves.toBeNull(); - expect(none.calls).toEqual([]); - }); -}); diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs index 4c1518c455941..35ca0688e2d95 100644 --- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs +++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs @@ -67,6 +67,8 @@ describe('PR report and deployment workflow contracts', () => { // The comment job reads this head's preview state once; no job runs on a // deployment event, and none waits for one. expect(value.jobs.comment.permissions.deployments).toBe('read'); + // It writes no repository contents. + expect(value.jobs.comment.permissions.contents).toBeUndefined(); expect(workflow('pr-comment.yml')).toContain( 'validateAnalysisMetadata(metadata, {', ); diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml index a506c93133ce5..4cc59a63bc871 100644 --- a/.github/workflows/pr-comment.yml +++ b/.github/workflows/pr-comment.yml @@ -200,11 +200,8 @@ jobs: permissions: pull-requests: write # actions: the artifacts and the source run's job list; deployments: - # this head's Vercel preview state; contents: the changed visual frames - # on the pull request's assets branch (publish-frames.mjs writes only - # that branch). + # this head's Vercel preview state. actions: read - contents: write deployments: read # The artifact downloads are optional (a run may not produce them), so # they are not failures of this job. @@ -215,7 +212,6 @@ jobs: steps.identity.outcome == 'failure' || steps.cross-check.outcome == 'failure' || steps.visual.outcome == 'failure' || - steps.frames.outcome == 'failure' || steps.publish.outcome == 'failure' }} steps: - name: Checkout trusted default-branch code @@ -385,25 +381,6 @@ jobs: --run-id "$RUN_ID" \ --run-attempt "$RUN_ATTEMPT" - # The report shows changed shots as before/after/diff thumbnails. Their - # re-encoded frames go to the pull request's assets branch, the path PR - # screenshots already use; nothing is written when no shot changed. - - name: Publish changed visual frames - id: frames - continue-on-error: true - if: steps.identity.outputs.valid == 'true' && steps.visual.outcome == 'success' - env: - GITHUB_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ steps.identity.outputs.pr_number }} - HEAD_SHA: ${{ steps.identity.outputs.head_sha }} - run: | - node .github/scripts/visual-gate/publish-frames.mjs \ - --input trusted-visual \ - --pr "$PR_NUMBER" \ - --head-sha "$HEAD_SHA" \ - --repository "$GITHUB_REPOSITORY" \ - --limit 12 - - name: Generate and post PR comment id: publish continue-on-error: true @@ -420,7 +397,6 @@ jobs: RUN_ID: ${{ steps.identity.outputs.run_id }} RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }} SOURCE_CONCLUSION: ${{ steps.identity.outputs.source_conclusion }} - VISUAL_IMAGE_URL: ${{ steps.frames.outputs.image_base }} with: retries: 3 script: | @@ -432,7 +408,6 @@ jobs: github, core, context, - visualImageUrl: process.env.VISUAL_IMAGE_URL || null, expectedIdentity: { prNumber: Number(process.env.PR_NUMBER), headSha: process.env.HEAD_SHA, diff --git a/scripts/prune-branches.sh b/scripts/prune-branches.sh index b20ebdc594b31..f5ad24b161c67 100755 --- a/scripts/prune-branches.sh +++ b/scripts/prune-branches.sh @@ -2,7 +2,6 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # scripts/prune-branches.sh — Delete remote branches with merged/closed PRs -# (an `assets/pr-` branch the PR report created goes with PR #) # # Usage: # ./scripts/prune-branches.sh # dry run (default) @@ -91,15 +90,7 @@ stale_cutoff="$3" repo="$4" safe_name=$(echo "$branch" | tr '/' '_') -# An `assets/pr-` branch the PR report created (it carries the -# `.pr-report-frames` marker) goes with PR #. Every other branch, including -# assets branches pushed by hand, keeps the rule below. -if [[ "$branch" =~ ^assets/pr-([0-9]+)$ ]] && git cat-file -e "origin/$branch:.pr-report-frames" 2>/dev/null; then - state=$(gh pr view "${BASH_REMATCH[1]}" --repo "$repo" --json state --jq '.state' 2>/dev/null || echo "error") - [ "$state" = "error" ] && state="OPEN" -else - state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error") -fi +state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error") if [ "$state" = "MERGED" ] || [ "$state" = "CLOSED" ]; then echo "$branch" > "$tmp_dir/delete/$safe_name" diff --git a/scripts/prune-branches.test.mjs b/scripts/prune-branches.test.mjs deleted file mode 100644 index d621e20a95127..0000000000000 --- a/scripts/prune-branches.test.mjs +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * @file prune-branches.test.mjs - * Pins that the daily branch pruner deletes an `assets/pr-` branch the PR - * report created (it carries `.pr-report-frames`) once PR # is merged or - * closed, keeps it while the PR is open, and does not delete an assets branch - * without the marker, such as screenshots pushed by hand, for its PR closing. - */ - -import {execFileSync} from 'node:child_process'; -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; -import {fileURLToPath} from 'node:url'; -import {afterEach, describe, expect, it} from 'vitest'; - -const SCRIPT = path.join( - path.dirname(fileURLToPath(import.meta.url)), - 'prune-branches.sh', -); -const roots = []; -afterEach(() => { - for (const root of roots.splice(0)) - fs.rmSync(root, {recursive: true, force: true}); -}); - -function setup(prStates) { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'prune-')); - roots.push(root); - const git = (cwd, ...args) => - execFileSync('git', args, {cwd, encoding: 'utf8'}); - const origin = path.join(root, 'origin.git'); - const work = path.join(root, 'work'); - git(root, 'init', '-q', '--bare', origin); - git(root, 'init', '-q', '-b', 'main', work); - git(work, 'config', 'user.email', 'test@test.co'); - git(work, 'config', 'user.name', 'Test'); - git(work, 'commit', '-q', '--allow-empty', '-m', 'base'); - git(work, 'remote', 'add', 'origin', origin); - // assets/pr-3 is pushed by hand: no marker. - git(work, 'push', '-q', 'origin', 'main', 'main:assets/pr-3'); - fs.writeFileSync(path.join(work, '.pr-report-frames'), 'marker\n'); - git(work, 'add', '.pr-report-frames'); - git(work, 'commit', '-q', '-m', 'frames'); - git(work, 'push', '-q', 'origin', 'HEAD:assets/pr-1', 'HEAD:assets/pr-2'); - // A stand-in for gh: `pr view ` answers from prStates; `pr list` finds no PR. - const bin = path.join(root, 'bin'); - fs.mkdirSync(bin); - fs.writeFileSync( - path.join(bin, 'gh'), - `#!/usr/bin/env bash\nif [ "$1 $2" = "pr view" ]; then\n case "$3" in\n${Object.entries( - prStates, - ) - .map(([n, state]) => ` ${n}) echo ${state} ;;`) - .join('\n')}\n esac\nelse\n echo none\nfi\n`, - {mode: 0o755}, - ); - return {work, bin}; -} - -describe('prune-branches.sh', () => { - it("deletes a closed PR's marked assets branch, keeps an open PR's, and never an unmarked one", () => { - const {work, bin} = setup({1: 'MERGED', 2: 'OPEN', 3: 'CLOSED'}); - const out = execFileSync('bash', [SCRIPT], { - cwd: work, - encoding: 'utf8', - env: {...process.env, PATH: `${bin}:${process.env.PATH}`}, - }).replace(/\u001b\[[0-9;]*m/g, ''); - const willDelete = out.slice( - out.indexOf('Merged/closed PR (will delete):'), - ); - expect(willDelete).toContain('assets/pr-1'); - expect(willDelete).not.toContain('assets/pr-2'); - // assets/pr-3 has no marker: its PR is closed, yet it is not deleted. - expect(out).not.toMatch(/^\s+assets\/pr-3$/m); - expect(out).toContain('0 branches (no PR'); - }); -}); From 4557523f290223f44f941d2bd1730392afccc2a3 Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 12:58:03 -0700 Subject: [PATCH 14/15] ci: the PR report shows before/after/diff thumbnails from frames a frames-only job publishes to assets/pr-; the pruner removes only branches it created once their PR closes --- .github/scripts/generate-pr-comment.js | 5 +- .github/scripts/lib/pr-preview.mjs | 33 +++ .github/scripts/lib/pr-preview.test.mjs | 88 ++++++- .github/scripts/lib/pr-status.js | 33 ++- .github/scripts/lib/pr-status.test.mjs | 34 ++- .../scripts/visual-gate/publish-frames.mjs | 226 ++++++++++++++++++ .../visual-gate/publish-frames.test.mjs | 215 +++++++++++++++++ .../visual-gate/workflow-concurrency.test.mjs | 7 +- .github/workflows/pr-comment.yml | 99 +++++++- scripts/prune-branches.sh | 11 +- scripts/prune-branches.test.mjs | 79 ++++++ 11 files changed, 813 insertions(+), 17 deletions(-) create mode 100644 .github/scripts/visual-gate/publish-frames.mjs create mode 100644 .github/scripts/visual-gate/publish-frames.test.mjs create mode 100644 scripts/prune-branches.test.mjs diff --git a/.github/scripts/generate-pr-comment.js b/.github/scripts/generate-pr-comment.js index 17df462c0a988..354a689e24738 100644 --- a/.github/scripts/generate-pr-comment.js +++ b/.github/scripts/generate-pr-comment.js @@ -3,7 +3,7 @@ /** * @description Generates the PR comment from the run's analysis results - * @input --analysis --base-stats --package-size --a11y --visual --preview --jobs --touched --source-conclusion --run-url + * @input --analysis --base-stats --package-size --a11y --visual --visual-frames --preview --jobs --touched --source-conclusion --run-url * @output Formatted markdown comment body to stdout, compact by design: this * head's preview first, the modified components (preview links, size, lines * of code, a mark per browser suite that ran; three rows visible, the rest @@ -67,6 +67,8 @@ const preview = readOptionalJson('preview'); const jobs = readOptionalJson('jobs'); const touched = readOptionalJson('touched'); const packageSize = readOptionalJson('package-size'); +// The visual frames publish-frames.mjs put on this PR's assets branch. +const visualFrames = readOptionalJson('visual-frames'); const visualVerdict = readOptionalJson('visual'); let a11yReport = null; try { @@ -127,6 +129,7 @@ const report = buildCompactReport({ a11yReport, visualVerdict, packageSize, + visualFrames, }); const footerLinksStr = runUrl ? ` | ${extLink('View full report', runUrl)}` diff --git a/.github/scripts/lib/pr-preview.mjs b/.github/scripts/lib/pr-preview.mjs index 44914fe836eed..a5076c469bf22 100644 --- a/.github/scripts/lib/pr-preview.mjs +++ b/.github/scripts/lib/pr-preview.mjs @@ -112,6 +112,35 @@ export async function headPreview({ : {state: 'unknown', sha}; } +/** + * The frames publish-frames.mjs published, written for the generator, only + * when their URL is this pull request's assets branch for this exact head and + * each frame is a `/.png` path; otherwise null. + */ +export function trustedFrames({frames, identity, inputs}) { + const expected = `https://raw.githubusercontent.com/${identity.baseRepository}/assets/pr-${identity.prNumber}/visual/${identity.headSha}/`; + let published; + try { + published = JSON.parse(frames?.published || '[]'); + } catch { + return null; + } + if ( + frames?.imageBase !== expected || + !Array.isArray(published) || + published.length === 0 || + !published.every(entry => + /^(before|after|diff)\/(?!\.+\.png$)[A-Za-z0-9._-]{1,240}\.png$/.test( + String(entry), + ), + ) + ) + return null; + const file = path.join(inputs, 'visual-frames.json'); + fs.writeFileSync(file, JSON.stringify({imageBase: expected, published})); + return file; +} + /** * The source run's jobs (latest attempt) with each step's conclusion, so the * report can say which jobs ran and which skipped their work by path. Null @@ -601,6 +630,7 @@ export async function reconcilePrComment({ baseStatsPath = 'pr-size-base/size-base.json', packageSizePath = 'pr-analysis/package-size.json', touchedPath = 'touched-components/touched-components.json', + visualFrames = null, a11yPath = 'a11y/a11y-report.json', visualPath = 'trusted-visual/verdict.json', createIfMissing = true, @@ -664,6 +694,8 @@ export async function reconcilePrComment({ repo, runId: identity.sourceRunId, }); + // Frames only from this PR's assets branch for this exact head. + const framesPath = trustedFrames({frames: visualFrames, identity, inputs}); const previewPath = path.join(inputs, 'preview.json'); const jobsPath = path.join(inputs, 'jobs.json'); if (preview) fs.writeFileSync(previewPath, JSON.stringify(preview)); @@ -692,6 +724,7 @@ export async function reconcilePrComment({ // checks, or with the ci:run-a11y label); the comment says so. ...(fs.existsSync(a11yPath) ? ['--a11y', a11yPath] : []), ...(fs.existsSync(visualPath) ? ['--visual', visualPath] : []), + ...(framesPath ? ['--visual-frames', framesPath] : []), ...(trustedHeadReport({ reportPath: touchedPath, label: 'touched-components report', diff --git a/.github/scripts/lib/pr-preview.test.mjs b/.github/scripts/lib/pr-preview.test.mjs index 98b81806a30cc..ca80630e06880 100644 --- a/.github/scripts/lib/pr-preview.test.mjs +++ b/.github/scripts/lib/pr-preview.test.mjs @@ -15,6 +15,7 @@ import { confirmSourceRunIdentity, PR_ANALYSIS_MARKER, reconcilePrComment, + trustedFrames, resolveWorkflowRunPullRequest, validateAnalysisMetadata, } from './pr-preview.mjs'; @@ -492,6 +493,7 @@ describe('trusted PR preview identity', () => { expect(writeCapableJobs).toEqual([ 'spec-only-reconcile', + 'frames', 'comment', 'report', ]); @@ -561,7 +563,13 @@ describe('trusted PR preview identity', () => { describe('PR Comment keeps its outcome off main', () => { // workflow_run attaches these jobs to main's latest commit, so a failing step // here would turn main red for a pull request's problem. - const LANE = ['resolve', 'spec-only-reconcile', 'comment', 'report']; + const LANE = [ + 'resolve', + 'spec-only-reconcile', + 'frames', + 'comment', + 'report', + ]; const OPTIONAL = new Set(['actions/download-artifact@v8']); it.each(LANE)('never fails the run from the %s job', name => { @@ -588,10 +596,20 @@ describe('PR Comment keeps its outcome off main', () => { it('reports the outcome on the head of the run the event names', () => { const report = PR_COMMENT_WORKFLOW.jobs.report; - expect(report.needs).toEqual(['resolve', 'spec-only-reconcile', 'comment']); + expect(report.needs).toEqual([ + 'resolve', + 'spec-only-reconcile', + 'frames', + 'comment', + ]); const [step] = report.steps; expect(step.env.HEAD_SHA).toBe('${{ github.event.workflow_run.head_sha }}'); - for (const name of ['resolve', 'spec-only-reconcile', 'comment']) { + for (const name of [ + 'resolve', + 'spec-only-reconcile', + 'frames', + 'comment', + ]) { expect(step.env.FAILED).toContain( `needs.${name}.outputs.failed == 'true'`, ); @@ -1378,3 +1396,67 @@ describe('PR comment preview reconciliation', () => { }, ); }); + +describe('visual frames', () => { + const identity = { + prNumber: 9171, + headSha: 'a'.repeat(40), + baseRepository: 'meta/astryx', + }; + const imageBase = `https://raw.githubusercontent.com/meta/astryx/assets/pr-9171/visual/${'a'.repeat(40)}/`; + const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'frames-')); + + it("passes frames only from this pull request's assets branch for this head", () => { + const published = JSON.stringify(['before/b.png', 'diff/b.png']); + const file = trustedFrames({ + frames: {imageBase, published}, + identity, + inputs: tmp(), + }); + expect(JSON.parse(fs.readFileSync(file, 'utf8'))).toEqual({ + imageBase, + published: ['before/b.png', 'diff/b.png'], + }); + for (const bad of [ + imageBase.replace('pr-9171', 'pr-1'), + imageBase.replace('a'.repeat(40), 'b'.repeat(40)), + 'https://evil.example/visual/', + ]) + expect( + trustedFrames({ + frames: {imageBase: bad, published}, + identity, + inputs: tmp(), + }), + ).toBeNull(); + }); + + it('refuses frame paths that are not before/after/diff PNGs', () => { + for (const published of [ + '["../x.png"]', + '["other/b.png"]', + '[]', + 'not json', + ]) + expect( + trustedFrames({ + frames: {imageBase, published}, + identity, + inputs: tmp(), + }), + ).toBeNull(); + }); + + it('gives only the frames job contents: write, with no persisted credentials', () => { + const {jobs} = PR_COMMENT_WORKFLOW; + for (const [name, job] of Object.entries(jobs)) + expect(job.permissions?.contents === 'write', name).toBe( + name === 'frames', + ); + const checkout = jobs.frames.steps.find(step => + String(step.uses).startsWith('actions/checkout@'), + ); + expect(checkout.with).toEqual({ref: 'main', 'persist-credentials': false}); + expect(jobs.comment.needs).toEqual(['resolve', 'frames']); + }); +}); diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js index c79ccc60f6752..76f841c9c6bea 100644 --- a/.github/scripts/lib/pr-status.js +++ b/.github/scripts/lib/pr-status.js @@ -309,11 +309,13 @@ function buildComponentTable({ } /** - * The changed shots, largest change first: up to VISUAL_FRAME_LIMIT rows - * naming each shot and its change, inside one disclosure whose summary - * carries the counts. The run's visual report holds the frames. + * The changed shots, largest change first: up to VISUAL_FRAME_LIMIT rows, + * inside one disclosure whose summary carries the counts. With `frames` + * (publish-frames.mjs's `imageBase` and the `/.png` it published), + * each row shows its before/after/diff thumbnails, and only published frames; + * without it, each row names its change. */ -function buildVisualDeltas({verdict}) { +function buildVisualDeltas({verdict, frames = null}) { if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed') return ''; const all = [ @@ -347,8 +349,24 @@ function buildVisualDeltas({verdict}) { .join(', '); return variant ? `${story} (${variant})` : story; }; - const header = '| Component · story | Change |\n|---|---|\n'; - const line = row => `| ${label(row)} | ${changeText(row)} |`; + const published = new Set(frames?.published || []); + const base = + frames?.imageBase && published.size > 0 + ? `${String(frames.imageBase).replace(/\/+$/, '')}/` + : null; + const frame = (row, kind) => { + if (!row.kinds.includes(kind)) return '—'; + const file = `${kind}/${row.key}.png`; + if (!published.has(file)) return 'not published'; + return `${kind} frame`; + }; + const header = base + ? '| Component · story | Before | After | Diff |\n|---|---|---|---|\n' + : '| Component · story | Change |\n|---|---|\n'; + const line = row => + base + ? `| ${label(row)} | ${frame(row, 'before')} | ${frame(row, 'after')} | ${frame(row, 'diff')} |` + : `| ${label(row)} | ${changeText(row)} |`; // A shot key is `---__-`. const componentOf = row => String( @@ -455,6 +473,7 @@ function buildCompactReport({ a11yReport = null, visualVerdict = null, packageSize = null, + visualFrames = null, }) { const origin = preview?.state === 'ready' ? preview.url : preview?.aliasOrigin || ''; @@ -464,7 +483,7 @@ function buildCompactReport({ touched, a11yReport, visualVerdict, - })}${buildVisualDeltas({verdict: visualVerdict})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict, touched})}`; + })}${buildVisualDeltas({verdict: visualVerdict, frames: visualFrames})}${packagesLine(packageSize)}${notRunLine({jobs, a11yReport, visualVerdict, touched})}`; } module.exports = { diff --git a/.github/scripts/lib/pr-status.test.mjs b/.github/scripts/lib/pr-status.test.mjs index f319bbbd8f257..b5d1182805664 100644 --- a/.github/scripts/lib/pr-status.test.mjs +++ b/.github/scripts/lib/pr-status.test.mjs @@ -257,7 +257,9 @@ describe('buildVisualDeltas', () => { expect(section).toContain( '| core-dialog--default\\_\\_neutral-light | added |', ); - expect(section).toContain('| core-old--default\\_\\_neutral-light | removed |'); + expect(section).toContain( + '| core-old--default\\_\\_neutral-light | removed |', + ); }); it(`lists at most ${VISUAL_FRAME_LIMIT} shots and counts the rest`, () => { @@ -271,6 +273,36 @@ describe('buildVisualDeltas', () => { expect(section).toContain("3 more are in the run's visual report."); }); + it('shows thumbnails for the frames publish-frames.mjs published, and only those', () => { + const section = buildVisualDeltas({ + verdict: { + status: 'changed', + changes: [change('b', 50), change('a', 1)], + added: ['core-dialog--default__neutral-light'], + }, + frames: { + imageBase: 'https://raw.example/frames', + published: [ + 'before/b.png', + 'after/b.png', + 'diff/b.png', + 'after/core-dialog--default__neutral-light.png', + ], + }, + }); + expect(section).toContain('| Component · story | Before | After | Diff |'); + expect(section).toContain( + 'before frame', + ); + expect(section).toContain( + '| Card · a (neutral, light) | not published | not published | not published |', + ); + // An added shot has an after frame only. + expect(section).toContain( + '| — | after frame | — |', + ); + }); + it('is absent without changes, or when the visual run failed', () => { expect(buildVisualDeltas({verdict: {status: 'ok', changes: []}})).toBe(''); expect(buildVisualDeltas({verdict: null})).toBe(''); diff --git a/.github/scripts/visual-gate/publish-frames.mjs b/.github/scripts/visual-gate/publish-frames.mjs new file mode 100644 index 0000000000000..3ea7b6eed8891 --- /dev/null +++ b/.github/scripts/visual-gate/publish-frames.mjs @@ -0,0 +1,226 @@ +#!/usr/bin/env node +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Publishes the changed visual shots' frames for the PR report. + * @input `--input ` (publish-pr-report.mjs's output: verdict.json and + * re-encoded `before|after|diff/.png`), `--pr `, + * `--head-sha `, `--repository `, `--limit `, and a + * token in GITHUB_TOKEN. + * @output Commits up to `limit` (at most 12) changed shots' frames, PNG or + * WebP of at most 500 KB each by their own bytes, largest change first, + * to the pull request's `assets/pr-` branch (the path PR screenshots + * already use) under `visual//`, marking a branch it creates with + * MARKER so only such branches are pruned, and writes `image_base` (their + * raw URL prefix) to GITHUB_OUTPUT. Writes nothing when no shot changed. + * @position The PR Comment workflow's comment job, after the visual report is + * prepared and before the report renders. It writes only that branch, and + * only frames the trusted publisher re-encoded. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import {fileURLToPath} from 'node:url'; + +const KEY = /^(?!\.+$)[A-Za-z0-9._-]{1,240}$/; +const KINDS = ['before', 'after', 'diff']; +/** Shots published per head, whatever `--limit` asks for. */ +export const MAX_SHOTS = 12; +export const MAX_FRAME_BYTES = 500 * 1024; + +/** A PNG or WebP no larger than MAX_FRAME_BYTES, by its own bytes. */ +export function isPublishableImage(bytes) { + if ( + !Buffer.isBuffer(bytes) || + bytes.length === 0 || + bytes.length > MAX_FRAME_BYTES + ) + return false; + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + if (bytes.subarray(0, 8).equals(png)) return true; + return ( + bytes.subarray(0, 4).toString('latin1') === 'RIFF' && + bytes.subarray(8, 12).toString('latin1') === 'WEBP' + ); +} + +/** + * The file that marks an assets branch as created by this script. The daily + * branch pruner deletes only marked branches, once their PR closes; a branch + * someone pushed by hand never gets the marker. + */ +export const MARKER = '.pr-report-frames'; + +/** The only ref this script writes: PR #'s assets branch. */ +export function assetsBranch(pr) { + if (!Number.isSafeInteger(pr) || pr <= 0) throw new Error('not a PR number'); + return `assets/pr-${pr}`; +} + +/** The frames to publish: changed shots by pixels changed, then added and removed. */ +export function framesToPublish(verdict, limit) { + if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed') + return []; + const changes = [...(verdict.changes || [])].sort( + (a, b) => Number(b.diffPixels ?? 0) - Number(a.diffPixels ?? 0), + ); + const shots = [ + ...changes.map(change => ({key: change.key, kinds: KINDS})), + ...(verdict.added || []).map(key => ({key, kinds: ['after']})), + ...(verdict.removed || []).map(key => ({key, kinds: ['before']})), + ].filter(shot => KEY.test(String(shot.key ?? ''))); + return shots.slice(0, Math.min(Number(limit) || 0, MAX_SHOTS)); +} + +async function api(request, method, route, body) { + const response = await request(method, route, body); + if (response.status === 404) return null; + if (response.status >= 300) + throw new Error(`${method} ${route} failed: ${response.status}`); + return response.json(); +} + +/** + * One commit on `assets/pr-` holding the frames, on top of the branch when + * it exists. Returns the raw URL prefix of the published frames and each + * published frame's `/.png`, or null when none was published. + */ +export async function publishFrames({ + input, + pr, + headSha, + repository, + limit, + request, +}) { + const verdict = JSON.parse( + fs.readFileSync(path.join(input, 'verdict.json'), 'utf8'), + ); + const files = []; + for (const shot of framesToPublish(verdict, limit)) { + for (const kind of shot.kinds) { + const file = path.join(input, kind, `${shot.key}.png`); + if (!fs.existsSync(file)) continue; + const bytes = fs.readFileSync(file); + if (!isPublishableImage(bytes)) continue; + files.push({ + frame: `${kind}/${shot.key}.png`, + path: `visual/${headSha}/${kind}/${shot.key}.png`, + bytes, + }); + } + } + if (files.length === 0) return null; + if (!/^[0-9a-f]{40}$/.test(String(headSha))) + throw new Error('not a head SHA'); + const branch = assetsBranch(pr); + const ref = await api( + request, + 'GET', + `/repos/${repository}/git/ref/heads/${branch}`, + ); + const parent = ref?.object?.sha ?? null; + const parentCommit = parent + ? await api(request, 'GET', `/repos/${repository}/git/commits/${parent}`) + : null; + const tree = []; + if (!parent) { + const marker = await api( + request, + 'POST', + `/repos/${repository}/git/blobs`, + { + content: Buffer.from( + 'Created by the PR report; pruned when the PR closes.\n', + ).toString('base64'), + encoding: 'base64', + }, + ); + tree.push({path: MARKER, mode: '100644', type: 'blob', sha: marker.sha}); + } + for (const entry of files) { + const blob = await api(request, 'POST', `/repos/${repository}/git/blobs`, { + content: entry.bytes.toString('base64'), + encoding: 'base64', + }); + tree.push({path: entry.path, mode: '100644', type: 'blob', sha: blob.sha}); + } + const newTree = await api(request, 'POST', `/repos/${repository}/git/trees`, { + ...(parentCommit ? {base_tree: parentCommit.tree.sha} : {}), + tree, + }); + const commit = await api( + request, + 'POST', + `/repos/${repository}/git/commits`, + { + message: `Visual frames for #${pr} at ${headSha.slice(0, 7)}`, + tree: newTree.sha, + parents: parent ? [parent] : [], + }, + ); + if (parent) + await api( + request, + 'PATCH', + `/repos/${repository}/git/refs/heads/${branch}`, + { + sha: commit.sha, + }, + ); + else + await api(request, 'POST', `/repos/${repository}/git/refs`, { + ref: `refs/heads/${branch}`, + sha: commit.sha, + }); + return { + imageBase: `https://raw.githubusercontent.com/${repository}/${branch}/visual/${headSha}/`, + published: files.map(entry => entry.frame), + }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const flag = name => { + const index = process.argv.indexOf(`--${name}`); + return index === -1 ? null : process.argv[index + 1]; + }; + const pr = Number(flag('pr')); + const headSha = String(flag('head-sha') ?? ''); + const repository = String(flag('repository') ?? ''); + if (!Number.isSafeInteger(pr) || pr <= 0) + throw new Error('--pr must be a PR number'); + if (!/^[0-9a-f]{40}$/.test(headSha)) + throw new Error('--head-sha must be a full SHA'); + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) + throw new Error('--repository must be owner/repo'); + const token = process.env.GITHUB_TOKEN; + const request = (method, route, body) => + fetch(`https://api.github.com${route}`, { + method, + headers: { + authorization: `Bearer ${token}`, + accept: 'application/vnd.github+json', + 'content-type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }); + const result = await publishFrames({ + input: path.resolve(flag('input')), + pr, + headSha, + repository, + limit: Number(flag('limit') ?? 12), + request, + }); + if (result && process.env.GITHUB_OUTPUT) + fs.appendFileSync( + process.env.GITHUB_OUTPUT, + `image_base=${result.imageBase}\npublished=${JSON.stringify(result.published)}\n`, + ); + console.log( + result + ? `Published ${result.published.length} frames under ${result.imageBase}` + : 'No changed shots to publish.', + ); +} diff --git a/.github/scripts/visual-gate/publish-frames.test.mjs b/.github/scripts/visual-gate/publish-frames.test.mjs new file mode 100644 index 0000000000000..216e516d08d1b --- /dev/null +++ b/.github/scripts/visual-gate/publish-frames.test.mjs @@ -0,0 +1,215 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file publish-frames.test.mjs + * Pins which visual frames the PR report publishes (the largest changes + * first, then added and removed shots, at most `limit`, safe keys only) and + * how: one commit on `assets/pr-`, on top of the branch when it exists. + */ + +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import {afterEach, describe, expect, it} from 'vitest'; +import { + MAX_FRAME_BYTES, + MAX_SHOTS, + MARKER, + assetsBranch, + framesToPublish, + isPublishableImage, + publishFrames, +} from './publish-frames.mjs'; + +const PNG = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + Buffer.from('rest'), +]); + +const HEAD = 'a'.repeat(40); +const roots = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, {recursive: true, force: true}); +}); + +function input(verdict) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'frames-')); + roots.push(root); + fs.writeFileSync(path.join(root, 'verdict.json'), JSON.stringify(verdict)); + for (const kind of ['before', 'after', 'diff']) { + fs.mkdirSync(path.join(root, kind)); + for (const key of ['a', 'b', 'c', 'added']) + fs.writeFileSync(path.join(root, kind, `${key}.png`), PNG); + } + return root; +} + +function fakeGitHub({branchExists}) { + const calls = []; + const request = async (method, route, body) => { + calls.push({method, route, body}); + const json = data => ({status: 200, json: async () => data}); + if (method === 'GET' && route.includes('/git/ref/')) + return branchExists + ? json({object: {sha: 'parent'}}) + : {status: 404, json: async () => ({})}; + if (method === 'GET' && route.includes('/git/commits/')) + return json({tree: {sha: 'parent-tree'}}); + if (route.endsWith('/git/blobs')) + return json({sha: `blob-${calls.length}`}); + if (route.endsWith('/git/trees')) return json({sha: 'tree'}); + if (route.endsWith('/git/commits')) return json({sha: 'commit'}); + return json({}); + }; + return {calls, request}; +} + +describe('framesToPublish', () => { + it('takes the largest changes first, then added and removed shots, up to the limit', () => { + const verdict = { + changes: [ + {key: 'a', diffPixels: 1}, + {key: 'b', diffPixels: 9}, + {key: '../escape', diffPixels: 99}, + ], + added: ['added'], + removed: ['gone'], + }; + expect(framesToPublish(verdict, 3).map(shot => shot.key)).toEqual([ + 'b', + 'a', + 'added', + ]); + expect(framesToPublish({...verdict, status: 'skipped'}, 3)).toEqual([]); + }); +}); + +describe('image and ref limits', () => { + it('accepts only PNG or WebP bytes no larger than 500 KB', () => { + expect(isPublishableImage(PNG)).toBe(true); + const webp = Buffer.concat([ + Buffer.from('RIFF'), + Buffer.alloc(4), + Buffer.from('WEBPVP8 '), + ]); + expect(isPublishableImage(webp)).toBe(true); + expect(isPublishableImage(Buffer.from(''))).toBe( + false, + ); + expect(isPublishableImage(Buffer.from('GIF89a'))).toBe(false); + expect( + isPublishableImage(Buffer.concat([PNG, Buffer.alloc(MAX_FRAME_BYTES)])), + ).toBe(false); + }); + + it('publishes at most 12 shots, whatever the limit asks', () => { + const changes = Array.from({length: 30}, (_, i) => ({ + key: `s${i}`, + diffPixels: i, + })); + expect(framesToPublish({changes}, 100)).toHaveLength(MAX_SHOTS); + expect(MAX_SHOTS).toBe(12); + }); + + it('writes only assets/pr-', () => { + expect(assetsBranch(42)).toBe('assets/pr-42'); + for (const bad of [0, -1, 1.5, NaN, '42/../main']) + expect(() => assetsBranch(bad)).toThrow(); + }); +}); + +describe('publishFrames', () => { + it("commits the frames on top of the PR's existing assets branch", async () => { + const {calls, request} = fakeGitHub({branchExists: true}); + const base = await publishFrames({ + input: input({changes: [{key: 'b', diffPixels: 2}]}), + pr: 9171, + headSha: HEAD, + repository: 'meta/astryx', + limit: 12, + request, + }); + expect(base).toEqual({ + imageBase: `https://raw.githubusercontent.com/meta/astryx/assets/pr-9171/visual/${HEAD}/`, + published: ['before/b.png', 'after/b.png', 'diff/b.png'], + }); + const tree = calls.find(call => call.route.endsWith('/git/trees')).body; + expect(tree.base_tree).toBe('parent-tree'); + expect(tree.tree.map(entry => entry.path)).not.toContain(MARKER); + expect(tree.tree.map(entry => entry.path)).toEqual([ + `visual/${HEAD}/before/b.png`, + `visual/${HEAD}/after/b.png`, + `visual/${HEAD}/diff/b.png`, + ]); + expect( + calls.find(call => call.route.endsWith('/git/commits')).body.parents, + ).toEqual(['parent']); + expect(calls.at(-1)).toMatchObject({ + method: 'PATCH', + route: '/repos/meta/astryx/git/refs/heads/assets/pr-9171', + body: {sha: 'commit'}, + }); + }); + + it('skips a frame that is not an image, and touches no other ref', async () => { + const root = input({changes: [{key: 'a', diffPixels: 1}]}); + fs.writeFileSync(path.join(root, 'diff', 'a.png'), ''); + const {calls, request} = fakeGitHub({branchExists: true}); + await publishFrames({ + input: root, + pr: 7, + headSha: HEAD, + repository: 'meta/astryx', + limit: 12, + request, + }); + const tree = calls.find(call => call.route.endsWith('/git/trees')).body; + expect(tree.tree.map(entry => entry.path)).toEqual([ + `visual/${HEAD}/before/a.png`, + `visual/${HEAD}/after/a.png`, + ]); + const refs = calls.filter(call => call.route.includes('/git/ref')); + expect( + refs.every( + call => + call.route.includes('assets/pr-7') || + call.route.endsWith('/git/refs'), + ), + ).toBe(true); + }); + + it('creates the branch when it does not exist, and writes nothing without changes', async () => { + const created = fakeGitHub({branchExists: false}); + await publishFrames({ + input: input({added: ['added']}), + pr: 5, + headSha: HEAD, + repository: 'meta/astryx', + limit: 12, + request: created.request, + }); + // Only a branch this script creates carries the pruning marker. + const createdTree = created.calls.find(call => + call.route.endsWith('/git/trees'), + ).body; + expect(createdTree.tree.map(entry => entry.path)).toContain(MARKER); + expect(created.calls.at(-1)).toMatchObject({ + method: 'POST', + route: '/repos/meta/astryx/git/refs', + body: {ref: 'refs/heads/assets/pr-5'}, + }); + const none = fakeGitHub({branchExists: true}); + await expect( + publishFrames({ + input: input({changes: []}), + pr: 5, + headSha: HEAD, + repository: 'meta/astryx', + limit: 12, + request: none.request, + }), + ).resolves.toBeNull(); + expect(none.calls).toEqual([]); + }); +}); diff --git a/.github/scripts/visual-gate/workflow-concurrency.test.mjs b/.github/scripts/visual-gate/workflow-concurrency.test.mjs index 35ca0688e2d95..a83fb51cd49e8 100644 --- a/.github/scripts/visual-gate/workflow-concurrency.test.mjs +++ b/.github/scripts/visual-gate/workflow-concurrency.test.mjs @@ -63,7 +63,10 @@ describe('PR report and deployment workflow contracts', () => { ); expect(value.jobs[job].concurrency['cancel-in-progress']).toBe(false); } - expect(value.jobs.comment.needs).toBe('resolve'); + expect(value.jobs.comment.needs).toEqual(['resolve', 'frames']); + expect(value.jobs.frames.concurrency.group).toBe( + 'pr-frames-${{ needs.resolve.outputs.pr_number }}', + ); // The comment job reads this head's preview state once; no job runs on a // deployment event, and none waits for one. expect(value.jobs.comment.permissions.deployments).toBe('read'); @@ -78,7 +81,7 @@ describe('PR report and deployment workflow contracts', () => { const value = workflow('pr-comment.yml'); const reconcile = value.slice( value.indexOf(' spec-only-reconcile:'), - value.indexOf(' comment:'), + value.indexOf(' frames:'), ); expect(reconcile).toContain('reconcilePrComment'); expect(reconcile).toContain('createIfMissing: false'); diff --git a/.github/workflows/pr-comment.yml b/.github/workflows/pr-comment.yml index 4cc59a63bc871..52e966680f90f 100644 --- a/.github/workflows/pr-comment.yml +++ b/.github/workflows/pr-comment.yml @@ -183,8 +183,96 @@ jobs: 'The current change has no visual or preview-producing scope. Visual checks are not applicable; preview links are not shown.', }); - comment: + # The changed visual frames behind the report's thumbnails, on the pull + # request's assets branch. The only job here that writes repository + # contents: it runs trusted code from main (never pull-request code), keeps + # no credentials in its checkout, and publish-frames.mjs writes only + # assets/pr-, and only validated PNG/WebP frames. + frames: needs: resolve + concurrency: + group: pr-frames-${{ needs.resolve.outputs.pr_number }} + cancel-in-progress: false + if: >- + always() && + github.event.action == 'completed' && + needs.resolve.outputs.valid == 'true' && + needs.resolve.outputs.spec_only != 'true' && + needs.resolve.outputs.tooling_only != 'true' + runs-on: ubuntu-24.04-arm + permissions: + actions: read + contents: write + outputs: + image_base: ${{ steps.frames.outputs.image_base }} + published: ${{ steps.frames.outputs.published }} + failed: >- + ${{ steps.checkout.outcome == 'failure' || + steps.setup.outcome == 'failure' || + steps.visual.outcome == 'failure' || + steps.frames.outcome == 'failure' }} + steps: + - name: Checkout trusted default-branch code + id: checkout + continue-on-error: true + uses: actions/checkout@v7 + with: + ref: main + persist-credentials: false + + - name: Setup Node and pnpm + id: setup + continue-on-error: true + if: steps.checkout.outcome == 'success' + uses: ./.github/actions/setup + + - name: Download canonical visual report + id: visual-download + if: steps.setup.outcome == 'success' + uses: actions/download-artifact@v8 + with: + name: visual-pr-report + path: visual-pr-report + run-id: ${{ needs.resolve.outputs.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + continue-on-error: true + + # The same re-encoding the comment job applies: only JSON and + # re-encoded PNGs from the exact CI attempt. + - name: Prepare canonical visual report + id: visual + continue-on-error: true + if: steps.visual-download.outcome == 'success' + env: + HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + RUN_ID: ${{ needs.resolve.outputs.run_id }} + RUN_ATTEMPT: ${{ needs.resolve.outputs.run_attempt }} + run: | + node .github/scripts/visual-gate/publish-pr-report.mjs \ + --input visual-pr-report \ + --output trusted-visual \ + --head-sha "$HEAD_SHA" \ + --run-id "$RUN_ID" \ + --run-attempt "$RUN_ATTEMPT" + + - name: Publish changed visual frames + id: frames + continue-on-error: true + if: steps.visual.outcome == 'success' + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ needs.resolve.outputs.pr_number }} + HEAD_SHA: ${{ needs.resolve.outputs.head_sha }} + run: | + node .github/scripts/visual-gate/publish-frames.mjs \ + --input trusted-visual \ + --pr "$PR_NUMBER" \ + --head-sha "$HEAD_SHA" \ + --repository "$GITHUB_REPOSITORY" \ + --limit 12 + + comment: + needs: [resolve, frames] concurrency: group: pr-report-${{ needs.resolve.outputs.pr_number }} cancel-in-progress: false @@ -397,6 +485,8 @@ jobs: RUN_ID: ${{ steps.identity.outputs.run_id }} RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }} SOURCE_CONCLUSION: ${{ steps.identity.outputs.source_conclusion }} + VISUAL_IMAGE_BASE: ${{ needs.frames.outputs.image_base }} + VISUAL_PUBLISHED: ${{ needs.frames.outputs.published }} with: retries: 3 script: | @@ -420,6 +510,10 @@ jobs: sourceRunAttempt: Number(process.env.RUN_ATTEMPT), sourceConclusion: process.env.SOURCE_CONCLUSION, }, + visualFrames: { + imageBase: process.env.VISUAL_IMAGE_BASE, + published: process.env.VISUAL_PUBLISHED, + }, }); # The CI-completion lane runs on workflow_run, so GitHub attaches its jobs to @@ -428,7 +522,7 @@ jobs: # continues on error, and this job reports their outcome as the PR Comment # status on the pull request's own head. report: - needs: [resolve, spec-only-reconcile, comment] + needs: [resolve, spec-only-reconcile, frames, comment] if: >- always() && github.event_name == 'workflow_run' && @@ -449,6 +543,7 @@ jobs: FAILED: >- ${{ needs.resolve.outputs.failed == 'true' || needs.spec-only-reconcile.outputs.failed == 'true' || + needs.frames.outputs.failed == 'true' || needs.comment.outputs.failed == 'true' || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }} diff --git a/scripts/prune-branches.sh b/scripts/prune-branches.sh index f5ad24b161c67..b20ebdc594b31 100755 --- a/scripts/prune-branches.sh +++ b/scripts/prune-branches.sh @@ -2,6 +2,7 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # scripts/prune-branches.sh — Delete remote branches with merged/closed PRs +# (an `assets/pr-` branch the PR report created goes with PR #) # # Usage: # ./scripts/prune-branches.sh # dry run (default) @@ -90,7 +91,15 @@ stale_cutoff="$3" repo="$4" safe_name=$(echo "$branch" | tr '/' '_') -state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error") +# An `assets/pr-` branch the PR report created (it carries the +# `.pr-report-frames` marker) goes with PR #. Every other branch, including +# assets branches pushed by hand, keeps the rule below. +if [[ "$branch" =~ ^assets/pr-([0-9]+)$ ]] && git cat-file -e "origin/$branch:.pr-report-frames" 2>/dev/null; then + state=$(gh pr view "${BASH_REMATCH[1]}" --repo "$repo" --json state --jq '.state' 2>/dev/null || echo "error") + [ "$state" = "error" ] && state="OPEN" +else + state=$(gh pr list --repo "$repo" --head "$branch" --state all --json state --jq '.[0].state // "none"' 2>/dev/null || echo "error") +fi if [ "$state" = "MERGED" ] || [ "$state" = "CLOSED" ]; then echo "$branch" > "$tmp_dir/delete/$safe_name" diff --git a/scripts/prune-branches.test.mjs b/scripts/prune-branches.test.mjs new file mode 100644 index 0000000000000..d621e20a95127 --- /dev/null +++ b/scripts/prune-branches.test.mjs @@ -0,0 +1,79 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file prune-branches.test.mjs + * Pins that the daily branch pruner deletes an `assets/pr-` branch the PR + * report created (it carries `.pr-report-frames`) once PR # is merged or + * closed, keeps it while the PR is open, and does not delete an assets branch + * without the marker, such as screenshots pushed by hand, for its PR closing. + */ + +import {execFileSync} from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {afterEach, describe, expect, it} from 'vitest'; + +const SCRIPT = path.join( + path.dirname(fileURLToPath(import.meta.url)), + 'prune-branches.sh', +); +const roots = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, {recursive: true, force: true}); +}); + +function setup(prStates) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'prune-')); + roots.push(root); + const git = (cwd, ...args) => + execFileSync('git', args, {cwd, encoding: 'utf8'}); + const origin = path.join(root, 'origin.git'); + const work = path.join(root, 'work'); + git(root, 'init', '-q', '--bare', origin); + git(root, 'init', '-q', '-b', 'main', work); + git(work, 'config', 'user.email', 'test@test.co'); + git(work, 'config', 'user.name', 'Test'); + git(work, 'commit', '-q', '--allow-empty', '-m', 'base'); + git(work, 'remote', 'add', 'origin', origin); + // assets/pr-3 is pushed by hand: no marker. + git(work, 'push', '-q', 'origin', 'main', 'main:assets/pr-3'); + fs.writeFileSync(path.join(work, '.pr-report-frames'), 'marker\n'); + git(work, 'add', '.pr-report-frames'); + git(work, 'commit', '-q', '-m', 'frames'); + git(work, 'push', '-q', 'origin', 'HEAD:assets/pr-1', 'HEAD:assets/pr-2'); + // A stand-in for gh: `pr view ` answers from prStates; `pr list` finds no PR. + const bin = path.join(root, 'bin'); + fs.mkdirSync(bin); + fs.writeFileSync( + path.join(bin, 'gh'), + `#!/usr/bin/env bash\nif [ "$1 $2" = "pr view" ]; then\n case "$3" in\n${Object.entries( + prStates, + ) + .map(([n, state]) => ` ${n}) echo ${state} ;;`) + .join('\n')}\n esac\nelse\n echo none\nfi\n`, + {mode: 0o755}, + ); + return {work, bin}; +} + +describe('prune-branches.sh', () => { + it("deletes a closed PR's marked assets branch, keeps an open PR's, and never an unmarked one", () => { + const {work, bin} = setup({1: 'MERGED', 2: 'OPEN', 3: 'CLOSED'}); + const out = execFileSync('bash', [SCRIPT], { + cwd: work, + encoding: 'utf8', + env: {...process.env, PATH: `${bin}:${process.env.PATH}`}, + }).replace(/\u001b\[[0-9;]*m/g, ''); + const willDelete = out.slice( + out.indexOf('Merged/closed PR (will delete):'), + ); + expect(willDelete).toContain('assets/pr-1'); + expect(willDelete).not.toContain('assets/pr-2'); + // assets/pr-3 has no marker: its PR is closed, yet it is not deleted. + expect(out).not.toMatch(/^\s+assets\/pr-3$/m); + expect(out).toContain('0 branches (no PR'); + }); +}); From b7c3bb4613ed1a67e7a0715fb49e7e31789d87ef Mon Sep 17 00:00:00 2001 From: Cindy Zhang Date: Sat, 31 Oct 2026 15:39:15 -0700 Subject: [PATCH 15/15] ci: frames follow the report's own shot selection; drop the unused visual-format renderer --- .github/scripts/lib/pr-status.js | 17 +- .github/scripts/lib/visual-format.js | 128 ------- .github/scripts/lib/visual-format.test.mjs | 340 ------------------ .../scripts/visual-gate/publish-frames.mjs | 39 +- .../visual-gate/publish-frames.test.mjs | 30 +- 5 files changed, 63 insertions(+), 491 deletions(-) delete mode 100644 .github/scripts/lib/visual-format.js delete mode 100644 .github/scripts/lib/visual-format.test.mjs diff --git a/.github/scripts/lib/pr-status.js b/.github/scripts/lib/pr-status.js index 2cc5de8f80a3a..ba4c358b16606 100644 --- a/.github/scripts/lib/pr-status.js +++ b/.github/scripts/lib/pr-status.js @@ -289,10 +289,16 @@ function buildComponentTable({analysis, origin, a11yReport, visualVerdict}) { * each row shows its before/after/diff thumbnails, and only published frames; * without it, each row names its change. */ -function buildVisualDeltas({verdict, frames = null}) { +/** + * The changed shots in report order: changed by pixels changed, largest first, + * then added, then removed. The report lists the first VISUAL_FRAME_LIMIT, + * and publish-frames.mjs publishes frames for the same ones, so thumbnails + * and rows cannot drift apart. + */ +function visualShots(verdict) { if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed') - return ''; - const all = [ + return []; + return [ ...[...(verdict.changes || [])] .sort((a, b) => Number(b.diffPixels ?? 0) - Number(a.diffPixels ?? 0)) .map(change => ({...change, kinds: ['before', 'after', 'diff']})), @@ -303,6 +309,10 @@ function buildVisualDeltas({verdict, frames = null}) { kinds: ['before'], })), ]; +} + +function buildVisualDeltas({verdict, frames = null}) { + const all = visualShots(verdict); if (all.length === 0) return ''; const rows = all.slice(0, VISUAL_FRAME_LIMIT); const unshown = all.length - rows.length; @@ -463,6 +473,7 @@ function buildCompactReport({ } module.exports = { + visualShots, SUITES, VISUAL_FRAME_LIMIT, baseJobName, diff --git a/.github/scripts/lib/visual-format.js b/.github/scripts/lib/visual-format.js deleted file mode 100644 index 54501059c4094..0000000000000 --- a/.github/scripts/lib/visual-format.js +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -/** - * Shared visual-gate formatting for PR enrichment. - * Used by generate-pr-comment.js and test-pr-enrichment.js. - */ - -const {inline, num, safeUrl} = require('./report-text'); - -/** - * Build the visual-regression section of the PR comment. - * - * The section has to survive four states, and the awkward one is `skipped`: - * a check that quietly vanishes on the PRs most likely to break something - * visually is worse than no check, so a skip states its reason explicitly. - * - * @param {object|null} verdict - the gate's verdict.json, or null when the job did not run - * @param {string} [reportUrl] - immutable published report for this run - * @param {string} [imageUrl] - immutable raw-image base for inline evidence - * @returns {string} markdown - */ -function buildVisualSection(verdict, reportUrl, imageUrl) { - if (!verdict) return ''; - - // Verdict fields are report data: render them as literal inline text (and - // counts as numbers), never as markup — see report-text.js. - const safeReportUrl = reportUrl ? safeUrl(reportUrl) : null; - const link = safeReportUrl - ? ` View the report` - : ''; - const reportBase = safeReportUrl - ? `${safeReportUrl.replace(/\/+$/, '')}/` - : null; - // Inline images use raw branch URLs so a delayed Pages deployment cannot - // break evidence that is already present on gh-pages. - const imageBase = safeUrl(imageUrl) - ? `${safeUrl(imageUrl).replace(/\/+$/, '')}/` - : reportBase; - - if (verdict.status === 'skipped') { - return `### Visual Regression\n\n**Status:** Skipped — ${inline(verdict.reason)}${link}\n\n`; - } - if (verdict.status === 'failed') { - return `### Visual Regression\n\n**Status:** ${num(verdict.counts?.failed)} shot(s) could not be captured.${link}\n\n`; - } - const added = verdict.added ?? []; - const removed = verdict.removed ?? []; - if (!verdict.changes || verdict.changes.length === 0) { - if (added.length > 0 || removed.length > 0) { - const frames = imageBase - ? [ - ...added.map(key => ({key, kind: 'after'})), - ...removed.map(key => ({key, kind: 'before'})), - ] - .slice(0, 3) - .map(({key, kind}) => { - const safeKey = encodeURIComponent(String(key)); - const label = - kind === 'after' ? 'Added — After' : 'Removed — Before'; - return `

${label}
${label} visual regression frame

`; - }) - .join('\n') - : ''; - return `### Visual Regression\n\n**${added.length} added · ${removed.length} removed.**${link}\n\n${frames}\n\n`; - } - const compared = num(verdict.counts?.total); - return `### Visual Regression\n\n**Status:** No visual change across ${compared} compared shot(s).\n\n`; - } - - const rows = verdict.changes - .slice(0, 20) - .map( - change => - `| ${inline(change.component || change.title)} | ${inline(change.name)} | ${inline(change.theme)} | ${inline(change.mode)} | ${num(change.diffPixels).toLocaleString()} |`, - ) - .join('\n'); - - // Put the evidence where the decision is made. A link to an artifact ZIP is - // not a visual review: it makes the reviewer download, unzip, and match file - // names before they can answer whether the after is correct. The static - // report is published beside the PR preview, and its first three deltas - // render directly in the comment; the full report carries the rest + wipe UI. - const evidence = imageBase - ? verdict.changes - .slice(0, 3) - .map(change => { - const key = encodeURIComponent(String(change.key ?? '')); - if (!key) return ''; - const label = `${inline(change.component || change.title)} — ${inline(change.name)} — ${inline(change.theme)} ${inline(change.mode)}`; - return ` -
-${label} - - - - - - - -
BeforeAfterDiff
Before visual regression frameAfter visual regression framePixel difference frame
-
`; - }) - .filter(Boolean) - .join('\n') - : ''; - const more = - verdict.changes.length > 20 - ? `\n\n_and ${verdict.changes.length - 20} more._` - : ''; - - return `### Visual Regression - -**${verdict.changes.length} of ${num(verdict.counts?.total)} shot(s) changed.**${link} - -A change here is a question, not a failure: check whether the *after* is the -picture you intended. Record that review on the PR. Baseline maintenance is an -explicit dispatch of CI; this report never rewrites the baseline or adds a release gate. - -| component | story | theme | mode | pixels | -|---|---|---|---|---| -${rows}${more} - -${evidence} - -`; -} - -module.exports = {buildVisualSection}; diff --git a/.github/scripts/lib/visual-format.test.mjs b/.github/scripts/lib/visual-format.test.mjs deleted file mode 100644 index d5ba7f1719aaa..0000000000000 --- a/.github/scripts/lib/visual-format.test.mjs +++ /dev/null @@ -1,340 +0,0 @@ -// Copyright (c) Meta Platforms, Inc. and affiliates. - -import {describe, expect, it} from 'vitest'; -import {createRequire} from 'node:module'; - -const {buildVisualSection} = createRequire(import.meta.url)( - './visual-format.js', -); - -const verdict = (overrides = {}) => ({ - status: 'pass', - counts: { - total: 16, - unchanged: 16, - changed: 0, - added: 0, - removed: 0, - failed: 0, - }, - changes: [], - ...overrides, -}); - -describe('buildVisualSection', () => { - it('renders nothing when the job did not run', () => { - expect(buildVisualSection(null)).toBe(''); - }); - - it('says so plainly when nothing moved', () => { - expect(buildVisualSection(verdict())).toContain( - 'No visual change across 16 compared shot(s)', - ); - }); - - it('shows added-only evidence instead of claiming there was no visual change', () => { - const section = buildVisualSection( - verdict({ - status: 'changed', - counts: { - total: 20, - unchanged: 14, - changed: 0, - added: 1, - removed: 0, - failed: 0, - }, - added: ['new-shot'], - removed: [], - }), - 'https://example.com/report/', - 'https://example.com/images/', - ); - expect(section).toContain('1 added · 0 removed'); - expect(section).toContain('View the report'); - expect(section).toContain('https://example.com/images/after/new-shot.png'); - expect(section).not.toContain('No visual change'); - }); - - it('shows the before image for a removed-only bundle', () => { - const section = buildVisualSection( - verdict({ - status: 'changed', - counts: { - total: 0, - unchanged: 0, - changed: 0, - added: 0, - removed: 1, - failed: 0, - }, - added: [], - removed: ['old-shot'], - }), - 'https://example.com/report/', - 'https://example.com/images/', - ); - expect(section).toContain('0 added · 1 removed'); - expect(section).toContain('https://example.com/images/before/old-shot.png'); - }); - - it('states the reason for a skip, so a broad PR does not look like a pass', () => { - const section = buildVisualSection( - verdict({ - status: 'skipped', - reason: '900 shots exceeds the 240-shot budget', - }), - 'https://example.com/report/', - ); - expect(section).toContain('Skipped'); - expect(section).toContain('900 shots exceeds the 240-shot budget'); - expect(section).toContain('View the report'); - expect(section).not.toContain('/accept-visual'); - }); - - it('renders verdict strings as one-line literal text', () => { - const section = buildVisualSection( - verdict({ - status: 'skipped', - reason: 'first line\n## not a heading, just text', - }), - ); - expect(section).toContain( - 'first line ## not a heading, just <em>text</em>', - ); - expect(section).not.toContain('\n## not a heading'); - }); - - it('keeps a change row a single table row whatever the field contents', () => { - const section = buildVisualSection( - verdict({ - counts: { - total: 2, - unchanged: 1, - changed: 1, - added: 0, - removed: 0, - failed: 0, - }, - changes: [ - { - component: 'Cell | with pipes', - name: 'story\nnewline', - theme: 'y2k', - mode: 'light', - diffPixels: '1234', - }, - ], - }), - ); - expect(section).toContain( - '| Cell \\| with pipes | story newline | y2k | light | 1,234 |', - ); - }); - - it('drops a report link that is not a plain https URL', () => { - const good = buildVisualSection( - verdict({status: 'failed', counts: {total: 1, failed: 1}}), - 'https://example.com/report', - ); - expect(good).toContain('href="https://example.com/report"'); - const bad = buildVisualSection( - verdict({status: 'failed', counts: {total: 1, failed: 1}}), - 'report.html" onmouseover="x', - ); - expect(bad).not.toContain('View the report'); - }); - - it('lists each changed shot with the theme and mode it changed in', () => { - const section = buildVisualSection( - verdict({ - status: 'changed', - counts: { - total: 16, - unchanged: 14, - changed: 2, - added: 0, - removed: 0, - failed: 0, - }, - changes: [ - { - key: 'a', - component: 'Button', - name: 'Primary', - theme: 'y2k', - mode: 'light', - diffPixels: 1126, - }, - { - key: 'b', - component: 'Button', - name: 'Primary', - theme: 'y2k', - mode: 'dark', - diffPixels: 1401, - }, - ], - }), - ); - expect(section).toContain('2 of 16 shot(s) changed'); - expect(section).toContain('| Button | Primary | y2k | light | 1,126 |'); - }); - - it('frames a change as a question rather than a failure', () => { - const section = buildVisualSection( - verdict({ - status: 'changed', - counts: { - total: 2, - unchanged: 1, - changed: 1, - added: 0, - removed: 0, - failed: 0, - }, - changes: [ - { - key: 'a', - component: 'B', - name: 'S', - theme: 't', - mode: 'light', - diffPixels: 5, - }, - ], - }), - ); - expect(section).toMatch(/question, not a failure/); - }); - - it('caps the table and says how many were left out', () => { - const changes = Array.from({length: 25}, (_, index) => ({ - key: `k${index}`, - component: 'C', - name: 'S', - theme: 't', - mode: 'light', - diffPixels: index, - })); - const section = buildVisualSection( - verdict({status: 'changed', counts: {total: 25, changed: 25}, changes}), - ); - expect(section).toContain('and 5 more'); - }); - - it('embeds before, after, and diff images for changed shots', () => { - const section = buildVisualSection( - verdict({ - status: 'changed', - counts: { - total: 2, - unchanged: 1, - changed: 1, - added: 0, - removed: 0, - failed: 0, - }, - context: {runId: '123456', runAttempt: '2'}, - changes: [ - { - key: 'core-button--primary__y2k-light', - component: 'Button', - name: 'Primary', - theme: 'y2k', - mode: 'light', - diffPixels: 12, - }, - ], - }), - 'https://evidence.example/pr/123/visual/head/run/', - 'https://evidence.example/pr/123/visual/head/run/', - ); - expect(section).toContain( - 'https://evidence.example/pr/123/visual/head/run/before/core-button--primary__y2k-light.png', - ); - expect(section).toContain( - 'https://evidence.example/pr/123/visual/head/run/after/core-button--primary__y2k-light.png', - ); - expect(section).toContain( - 'https://evidence.example/pr/123/visual/head/run/diff/core-button--primary__y2k-light.png', - ); - expect(section).toContain('BeforeAfterDiff'); - expect(section).toContain('Record that review on the PR'); - expect(section).not.toContain('/accept-visual'); - }); - - it('does not offer acceptance before immutable evidence is published', () => { - const section = buildVisualSection( - verdict({ - status: 'changed', - counts: { - total: 2, - unchanged: 1, - changed: 1, - added: 0, - removed: 0, - failed: 0, - }, - context: {runId: '123456', runAttempt: '2'}, - changes: [ - { - key: 'a', - component: 'Button', - name: 'Primary', - theme: 'neutral', - mode: 'light', - diffPixels: 12, - }, - ], - }), - ); - expect(section).not.toContain('/accept-visual'); - expect(section).not.toContain('repository maintainer'); - }); - - it('embeds at most three deltas so the PR comment stays readable', () => { - const changes = Array.from({length: 5}, (_, index) => ({ - key: `shot-${index}`, - component: 'C', - name: 'S', - theme: 't', - mode: 'light', - diffPixels: 1, - })); - const section = buildVisualSection( - verdict({status: 'changed', counts: {total: 5, changed: 5}, changes}), - 'https://example.com/visual/', - ); - expect((section.match(/
/g) ?? []).length).toBe(3); - expect(section).not.toContain('/before/shot-3.png'); - }); - - it('reports a capture failure distinctly from a change', () => { - const section = buildVisualSection( - verdict({status: 'failed', counts: {total: 4, failed: 2, changed: 0}}), - ); - expect(section).toContain('2 shot(s) could not be captured'); - }); - - it('links the report when one was published', () => { - const section = buildVisualSection( - verdict({ - status: 'changed', - counts: {total: 1, changed: 1}, - changes: [ - { - key: 'a', - component: 'B', - name: 'S', - theme: 't', - mode: 'light', - diffPixels: 1, - }, - ], - }), - 'https://example.com/report/', - ); - expect(section).toContain('https://example.com/report/'); - }); -}); diff --git a/.github/scripts/visual-gate/publish-frames.mjs b/.github/scripts/visual-gate/publish-frames.mjs index 3ea7b6eed8891..123ea201d2562 100644 --- a/.github/scripts/visual-gate/publish-frames.mjs +++ b/.github/scripts/visual-gate/publish-frames.mjs @@ -12,21 +12,26 @@ * to the pull request's `assets/pr-` branch (the path PR screenshots * already use) under `visual//`, marking a branch it creates with * MARKER so only such branches are pruned, and writes `image_base` (their - * raw URL prefix) to GITHUB_OUTPUT. Writes nothing when no shot changed. - * @position The PR Comment workflow's comment job, after the visual report is - * prepared and before the report renders. It writes only that branch, and - * only frames the trusted publisher re-encoded. + * raw URL prefix) and `published` (each frame's `/.png`) to + * GITHUB_OUTPUT. Writes nothing when no shot changed. + * @position The PR Comment workflow's frames job, after the visual report is + * prepared; the comment job renders the thumbnails. It writes only that + * branch, and only frames the trusted publisher re-encoded. */ import fs from 'node:fs'; +import {createRequire} from 'node:module'; import path from 'node:path'; import process from 'node:process'; import {fileURLToPath} from 'node:url'; +// The report's own shot order and limit (lib/pr-status.js). +const {visualShots, VISUAL_FRAME_LIMIT} = createRequire(import.meta.url)( + '../lib/pr-status.js', +); const KEY = /^(?!\.+$)[A-Za-z0-9._-]{1,240}$/; -const KINDS = ['before', 'after', 'diff']; -/** Shots published per head, whatever `--limit` asks for. */ -export const MAX_SHOTS = 12; +/** Shots published per head, whatever `--limit` asks for: the report's rows. */ +export const MAX_SHOTS = VISUAL_FRAME_LIMIT; export const MAX_FRAME_BYTES = 500 * 1024; /** A PNG or WebP no larger than MAX_FRAME_BYTES, by its own bytes. */ @@ -58,19 +63,15 @@ export function assetsBranch(pr) { return `assets/pr-${pr}`; } -/** The frames to publish: changed shots by pixels changed, then added and removed. */ +/** + * The frames to publish: the report's rows (visualShots), the first `limit` + * of them and at most MAX_SHOTS, and only shots whose key is a safe name. + */ export function framesToPublish(verdict, limit) { - if (!verdict || verdict.status === 'skipped' || verdict.status === 'failed') - return []; - const changes = [...(verdict.changes || [])].sort( - (a, b) => Number(b.diffPixels ?? 0) - Number(a.diffPixels ?? 0), - ); - const shots = [ - ...changes.map(change => ({key: change.key, kinds: KINDS})), - ...(verdict.added || []).map(key => ({key, kinds: ['after']})), - ...(verdict.removed || []).map(key => ({key, kinds: ['before']})), - ].filter(shot => KEY.test(String(shot.key ?? ''))); - return shots.slice(0, Math.min(Number(limit) || 0, MAX_SHOTS)); + return visualShots(verdict) + .slice(0, Math.min(Number(limit) || 0, MAX_SHOTS)) + .filter(shot => KEY.test(String(shot.key ?? ''))) + .map(shot => ({key: shot.key, kinds: shot.kinds})); } async function api(request, method, route, body) { diff --git a/.github/scripts/visual-gate/publish-frames.test.mjs b/.github/scripts/visual-gate/publish-frames.test.mjs index 216e516d08d1b..4605b8a8efed1 100644 --- a/.github/scripts/visual-gate/publish-frames.test.mjs +++ b/.github/scripts/visual-gate/publish-frames.test.mjs @@ -66,7 +66,7 @@ function fakeGitHub({branchExists}) { } describe('framesToPublish', () => { - it('takes the largest changes first, then added and removed shots, up to the limit', () => { + it('takes the largest changes first, then added and removed shots, up to the limit, and never an unsafe key', () => { const verdict = { changes: [ {key: 'a', diffPixels: 1}, @@ -76,15 +76,43 @@ describe('framesToPublish', () => { added: ['added'], removed: ['gone'], }; + // The unsafe key keeps its report row (shown as not published). expect(framesToPublish(verdict, 3).map(shot => shot.key)).toEqual([ 'b', 'a', + ]); + expect(framesToPublish(verdict, 5).map(shot => shot.key)).toEqual([ + 'b', + 'a', 'added', + 'gone', ]); expect(framesToPublish({...verdict, status: 'skipped'}, 3)).toEqual([]); }); }); +describe('the same shots as the report', () => { + it("publishes exactly the report's rows, in its order", async () => { + const {createRequire} = await import('node:module'); + const {visualShots, VISUAL_FRAME_LIMIT} = createRequire(import.meta.url)( + '../lib/pr-status.js', + ); + const verdict = { + status: 'changed', + changes: Array.from({length: 20}, (_, i) => ({ + key: `s${i}`, + diffPixels: i, + })), + added: ['new-shot'], + }; + expect(framesToPublish(verdict, 100).map(shot => shot.key)).toEqual( + visualShots(verdict) + .slice(0, VISUAL_FRAME_LIMIT) + .map(shot => shot.key), + ); + }); +}); + describe('image and ref limits', () => { it('accepts only PNG or WebP bytes no larger than 500 KB', () => { expect(isPublishableImage(PNG)).toBe(true);