diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 5cf295a..e3230c7 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -23,6 +23,9 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install workflow verification dependencies + run: python3 -m pip install --requirement requirements-validation.txt + - name: Compare desired state with GitHub env: GH_TOKEN: ${{ secrets.ATRINIK_SETTINGS_TOKEN }} diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000..e75109f --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,27 @@ +name: Dependency review + +on: + pull_request: + branches: [main] + types: [opened, synchronize, reopened, edited] + merge_group: + types: [checks_requested] + +permissions: + contents: read + +jobs: + dependency-review: + name: Dependency review + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: high + fail-on-scopes: runtime,development,unknown + warn-only: false + comment-summary-in-pr: never + show-openssf-scorecard: false + retry-on-snapshot-warnings: true + retry-on-snapshot-warnings-timeout: 120 diff --git a/.github/workflows/publish-planning.yml b/.github/workflows/publish-planning.yml index 32ae3d6..914f3ee 100644 --- a/.github/workflows/publish-planning.yml +++ b/.github/workflows/publish-planning.yml @@ -18,6 +18,9 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install workflow verification dependencies + run: python3 -m pip install --requirement requirements-validation.txt + - name: Verify settings credential inventory env: GH_TOKEN: ${{ secrets.ATRINIK_SETTINGS_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index c53e0d5..a3f436c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -225,3 +225,68 @@ `release_policy: none` and are omitted from `release_tags`. - Update this `AGENTS.md` in the same change when major rework alters governance ownership, policy scope, publisher behavior, required checks, or validation. + +- `config/security-merge-protection.json` owns the dependency-review and CodeQL + target sets, per-gate `pending`/`enforced` activation intent, and default-setup + required language minima. Keep both sets and activation keys equal to the + required-CI inventory. Union required languages with live default-setup + languages, fail on unreadable language state, and patch only real drift. +- Keep `06 - CodeQL security` and `07 - Dependency review` repository-owned in + both policy scopes; Team has no organization code-scanning rule support. + Pending gates only probe readiness. Promote intent through source review after + read-only `bin/security-readiness` proves landed successful producers, before + the first authorized live installation. Enforced gates remain required when + deleted or producers fail; audit fails on absence and publication restores + protection without fresh initial qualification. Keep + aggregate checks, zero approvals, and owner pull-request-only bypass intact. +- Advanced CodeQL readiness parses executable YAML with the declared validation + dependency: bind classification outputs, matrix, scoped config, official + full-SHA init/analyze actions, and the always-running aggregate. Reject + duplicate keys, disconnected or disabled steps, and failure suppression. + Require ordinary pull-request triggers covering the live default branch and + opened/synchronize/reopened events without path filters. + Producer helpers remain trusted; preserve owner-controlled action updates, + category names and paths without a duplicate version/path catalog. +- `config/actions-sha-pinning.json` persists SHA rollout as `pending` with no + canary or `enforced` with an accepted PR coordinate. Promote it through normal + source review only after ordinary PR-bound action-update evidence passes. + Pending command-line canaries only probe readiness; they never enable SHA + enforcement. Enforced policy uses its stored proof to restore drift and fails + if proof cannot be revalidated; audit reports disabled enforced policy as + drift. Preserve existing true settings. Allow action names with `@*`; keep + full SHAs in producer workflows instead of duplicate pin catalogs. Canary + legacy-check absence requires verified protection evidence; a bare or + ambiguous 404 is unreadable. Enumerate and validate every ruleset page before + publisher identity, absence, update or retirement decisions. +- Adoption of existing security gates into enforced source intent requires the + effective default-branch API to prove the repository-owned ruleset and desired + tool/context. Never infer activation from target membership or silently change + source intent. Retire target and activation entries together. Retired targets + lose only exact repository-owned managed security rules; + preserve archived, foreign, and modified same-name rules. Re-read and validate + the exact rule payload after the repository metadata fence immediately before + deletion. Audit retained named security rules on non-target repositories as + drift even when disabled, modified or scoped to another ref. Keep repository + workflow overrides read-only and unable to approve PRs, with delta-aware writes. +- Repository-scope `03 - Required CI` retirement requires reviewed exact prior + repository/ruleset IDs and contexts in `config/retired-required-ci.json`. + Delete only the unchanged Actions-bound template after fresh identity/archive + and payload checks. Preserve modified/foreign rules and missing-provenance + rules with actionable `SKIP`; retain records for idempotent reruns. + +- Before updating an existing security gate, re-read and revalidate its identity + and recognized payload, retaining concurrent modifications. Bind the final + metadata default branch to the branch used for pending-gate qualification + or existing effective-gate proof; changed branches must not activate a gate. + Pending existing-gate proof also requires unchanged enforcement and ref + applicability in the final rule read. Enforced intent retains recognized + restoration. Audit full legacy + protection on active default and declared maintenance branches; any presence + is drift under the canonical ruleset contract. Only verified unprotected + responses count as absent; unreadable evidence fails closed and audit never + removes protections. Compare the complete effective default and maintenance branch policy with + declared allowed rule types and parameters; undeclared integrity or restriction types are drift, + even when every desired rule is present. Only the exact repository-disabled REST response may + fall back to a fixed read-only GraphQL query with matching repository/branch + identity, active state and explicit null legacy-rule association. Missing or + conflicting evidence fails closed; preserve historical rules on other refs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 35f85ae..5145d20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,6 +129,68 @@ after apply and again before separately authorized exact branch deletion; never combine either operation with tag, release, asset, default-branch, or unrelated policy changes. +Security merge controls use repository-level rules in both publisher scopes. +Keep `config/security-merge-protection.json` aligned with `required_ci` and +preserve ordinary aggregates. New dependency gates require the reviewed +workflow on the default branch and a successful corresponding PR run; CodeQL +gates require successful current-head analyses, including required languages. +Record successful initial qualification in the policy PR and promote the gate's +`activation` entry from `pending` to `enforced` through source review before its +first live installation. Pending publication only probes readiness. Enforced +intent survives missing rules and later producer failures, allowing restoration +without fresh initial proof; it does not claim live installation success. Remove +activation entries with retired targets, preserving exact owned-rule rollback. +Re-read the exact security-rule payload after the repository metadata fence +immediately before deletion, retaining concurrent modifications. Audit retained +named security rules on non-target repositories as drift regardless of their +enforcement or ref applicability. +For an existing repository-owned `03 - Required CI` rule, also record exact repository +and ruleset IDs and its prior contexts in `config/retired-required-ci.json`. +Validate retained/modified/foreign rules and fresh identity checks; keep records +for idempotent retirement. Missing provenance must retain the rule. +Tests must cover missing/failed producers, unreadable evidence, archived targets, language unions, both scopes, +and zero security mutations on a converged rerun. SHA enforcement must remain +pending until an ordinary action-update canary proves successful PR-bound +current-head CI without an allowlist edit and the persistent rollout source is +promoted to `enforced` with that accepted PR. Later disabled enforcement is drift, +not pending rollout; ordinary publication revalidates the recorded proof before +restoring it. Test rule retirement, effective default-branch applicability, and +repository workflow permission overrides. Action allowances use owner/action wildcards; +full commit pins belong in workflows, not redundant version assertions. +Canary legacy-check absence requires verified protection evidence; bare or +ambiguous 404 responses fail closed. Publisher identity and absence decisions +require a complete validated ruleset inventory across all pages, including +retirement and ordinary repository/organization rules. + +Advanced CodeQL readiness must validate executable YAML wiring from classifier +invocation and outputs through the dynamic analysis matrix, scoped generated +configuration, official full-SHA CodeQL actions, and the unconditional aggregate. +Require the live default branch's ordinary pull-request trigger without path +filters, including opened, synchronize and reopened events. Use the existing +validation dependency with duplicate-key rejection. Test +comments-only markers, disconnected inputs, disabled or failure-suppressed steps, +checkout revision drift, and ambiguous YAML; preserve full-SHA updates, formatting +changes, additional partitions, and category renames. Classification path +semantics remain in trusted producer helpers; do not add a central source-path or +action-version catalog or claim resistance to malicious repository writers. + +Security gate updates must re-read and validate ownership and the recognized +payload immediately before PUT, preserving concurrent modifications. Bind the +final metadata default branch to the branch used for pending-gate qualification +or existing effective-gate proof; a changed branch must not activate a gate. +Pending existing-gate proof requires unchanged enforcement and ref applicability +in the final rule read; enforced intent retains recognized restoration. Audit +legacy protection on active default and maintenance branches as drift under the +canonical ruleset contract; distinguish verified unprotected responses from +unreadable evidence, and never remove protection during audit. Compare the +complete effective default and maintenance branch policy with declared allowed +rule types and parameters; additional +undeclared integrity or restriction types are drift. The exact +repository-disabled REST response may use the fixed read-only GraphQL fallback; +require matching repository identity and branch, active state and an explicit +null legacy-rule association. Errors and missing or conflicting fields fail +closed; preserve unrelated historical rules. + Crates.io publication contracts bind exact repository/workflow/environment and official auth-action identities with full immutable workflow SHA references; preserve the reviewed `rust-lang/crates-io-auth-action@*` allowance for normal diff --git a/README.md b/README.md index 7d65efc..8399d2c 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,10 @@ read-only and are skipped on later runs. remain outside GitHub and this repository. - GitHub Actions defaults to read-only, cannot approve pull requests, and may use only Atrinik, GitHub, Codecov coverage, explicitly allowed Docker - actions, and the exact reviewed crates.io authentication action commit. + actions, and `rust-lang/crates-io-auth-action@*`. Workflow references remain + full commit SHAs; the action allowlist permits routine pin updates. The + publisher also restores per-repository workflow permission overrides to + read-only with PR approvals disabled, applying only real drift. - Historical repositories listed in `config/repositories.json` are archived. The five former standalone classic component repositories are already archived read-only after their history, active work, issues, and release @@ -147,10 +150,202 @@ separate checked procedure below. Codecov must be installed for the listed repositories so their OIDC-authenticated coverage uploads and badges remain available. +## Staged security merge protection + +`config/security-merge-protection.json` targets the 21 repositories in +`required_ci`, including `service-updater`; `.github` and `web-platform` are +excluded. Its `activation` map records each target gate's reviewed `pending` or +`enforced` intent. `enforced` means protection is required, including when the +live rule is missing; it is not evidence of a successful installation. Existing +aggregate checks and zero required approvals remain intact. +The existing publisher restores `service-updater`'s already inventoried PR, +`Updater validation`/title CI, and release-tag protections. + +Both policy scopes install repository-owned `06 - CodeQL security` rules: +GitHub Team does not support organization code-scanning rules. CodeQL blocks +high and critical security findings (`high_or_higher`), with ordinary quality +alerts disabled (`none`). A separate repository-owned `07 - Dependency review` +requires the GitHub Actions `Dependency review` job. Both permit only +organization owners to bypass through a pull request; bypass can merge findings +and must be a deliberate owner decision. Direct pushes gain no exception. + +The dependency workflow runs on pull requests to `main` when opened, updated, +reopened, or edited, including a base-branch retarget to `main`. It also runs +for merge queue groups when checks are requested. The action reads each event's +base and head revisions directly, with read-only contents permission and no +checkout. It rejects new high/critical +vulnerable runtime, development, or unknown-scope dependencies and fails on +review errors. +It relies on GitHub's dependency graph and submitted snapshots; unsupported or +missing manifests cannot provide comprehensive dependency coverage. This is +separate from Dependabot's scheduled update and alert capabilities. + +Dependency review uses ordinary pull-request CI and trusts repository writers. +The required check context and GitHub Actions app binding do not attest immutable +workflow bytes. A writer can modify the producer within the same PR, including +replacing its review with a successful no-op. The high/critical gate remains in +place for ordinary introduced vulnerabilities in supported dependency data; +it does not defend against malicious repository writers or an owner using the +pull-request bypass. This policy adds no human approval requirement. + +GitHub's [Dependabot alert limitations](https://docs.github.com/en/code-security/concepts/supply-chain-security/dependabot-alerts#limitations) +exclude advisory alerts for Actions referenced by commit SHA. SHA pinning and +a passing dependency review therefore do not prove that an Action commit is +free of known vulnerabilities; review upstream advisories when updating pins. +The high/critical dependency gate remains enabled for vulnerabilities GitHub +reports through its supported dependency data. + +Default CodeQL setup similarly analyzes supported languages and its detected build +coverage; enabling it alone does not prove a successful analysis. Rust in +`content-toolkit` is an explicit minimum, unioned with its existing languages. +Legacy `javascript`/`typescript` aliases use `javascript-typescript` in the +PATCH payload, preserving that coverage and other live language entries. +Read failures never replace that live language set with an incomplete set. + +GitHub's [code-scanning merge-protection limitations](https://docs.github.com/en/code-security/concepts/code-scanning/merge-protection) +exclude Dependabot pull requests analyzed by default setup and merge queue +groups. Dependency review runs for queue groups and remains a required check +for those Dependabot PRs; it does not inherit those CodeQL exceptions. CodeQL merge protection also depends +on the alert's affected lines appearing in the PR diff. Ordinary quality alerts +remain excluded by `alerts_threshold: none`. + +Classic readiness parses the executable workflow with the declared PyYAML +validation dependency. It binds classification outputs to the analysis matrix, +regenerates each selected scope's configuration, and follows that output into +the official CodeQL initialization and analysis steps and the always-running +validation aggregate. Disabled steps, failure suppression, disconnected inputs, +and ambiguous YAML fail closed. The current producer uses `build-mode: none`; +full-SHA action updates remain producer-owned, and classifier source owns path +selection and category names. This structural check trusts repository writers +and their Python helpers; it does not attest that arbitrary writer-controlled +classifier or aggregate code behaves honestly. It adds no approval requirement. + +Rollout is owned by an organization administrator: + +1. Merge the dependency workflow PRs and let ordinary pull requests prove their + `Dependency review` jobs. Merge this policy and review `bin/publish`'s plan. +2. Apply only with separate live-settings authorization. Existing governance, + the wildcard action allowance, and the Rust language correction can converge + while security gates remain `PENDING`. Pending gates only probe + `bin/security-readiness`; even a successful probe cannot activate protection + or silently promote source intent. Dependency readiness + requires the reviewed workflow contract on the default branch, a successful + latest PR run/job, and the same workflow at the check suite's executed + revision. Synthetic merge revisions must bind the associated PR's base and + head. CodeQL requires current-head coverage for every configured default-setup + language plus policy minima. Classic instead requires its successful full + advanced producer and all current source-derived analysis partitions. The + advanced inventory independently requires Classic’s client, server, and core + C/C++, Python, and GitHub Actions scope/language pairs, so reducing producer + declarations or executed jobs cannot silently reduce required coverage; + changed-path PR subsets cannot establish readiness. The bounded newest-first + query stops once current coverage is proven; retired categories and old + analysis history do not require manual cleanup. +3. After successful initial producer qualification, promote that gate's + `activation` entry to `enforced` through source review and merge, retaining + the qualification evidence in the policy PR. Adoption of existing protection + instead requires read-only evidence of an active, repository-owned gate with + the intended tool/context applying to the live default branch. Desired target + membership alone never proves activation. Review the next plan and apply only + with live-settings authorization. Persisted intent precedes the first write, + so interrupted installation or later deletion cannot revert to pending. + The publisher creates or restores enforced protection without requiring a + currently healthy producer; audit reports missing or modified protection as + drift even when producer evidence fails or cannot be read. Scope migrations + preserve these repository-owned gates. A converged security gate or language + configuration causes no mutation or security-configuration reattachment. + Existing-rule repair accepts only recognized enforcement and ref-scope drift + on the exact repository-owned rule. Changed bypass, tool, context, severity, + or unknown policy fields are retained and reported for explicit review. + Retiring a target removes its required-CI, security target, and activation + entries together through source review. The publisher removes only matching + repository-owned security rules from active retired targets, preserving foreign or modified same-name + rules for explicit review and leaving archived repositories untouched. It re-reads + the exact rule payload after the repository metadata fence immediately before + deletion to retain concurrent modifications. Audit reports retained named + security rules on non-target repositories as drift even when disabled, + modified or scoped to another ref. +4. Desired organization `sha_pinning_required` is true. Its durable rollout + contract is `config/actions-sha-pinning.json`: the initial `pending` phase + has no accepted canary and never enables the field. Verify an ordinary + action-pin update PR with `bin/security-readiness sha OWNER/REPOSITORY/PR`. + Qualifying changes update action pins only. Reusable workflow references are + [excluded from GitHub's action SHA enforcement](https://docs.github.com/en/organizations/managing-organization-settings/disabling-or-limiting-github-actions-for-your-organization), so changing one cannot qualify + a canary, even alongside an action update. Unchanged reusable references and + real action subpaths remain supported. + The optional publisher `--sha-pinning-canary` argument only reports readiness + while pending. After successful PR evidence, promote the source to + `{"phase":"enforced","canary":"OWNER/REPOSITORY/PR"}` through the normal + review and merge process before the first separately authorized live apply. + This records the accepted PR, with no duplicate action-version inventory. + The verifier checks the active conservative organization Actions mode, + repository enablement, exact PR changes, and successful PR-bound current-head + checks. These include configured CI, desired Dependency review for targeted + repositories, and all effective ruleset and legacy branch-protection status + contexts, with trusted app identities. Legacy-check absence requires verified + protection evidence; bare or ambiguous 404 responses fail closed. This proves current policy and PR-run + evidence; it does not attest historical organization settings. + Ordinary subsequent publication uses that stored canary. It first converges + selected mode, then allowed-action patterns, then revalidates the canary + before restoring a disabled SHA setting. A dry run reports the required + staging without claiming that planned settings are active. Unreadable or + failed proof blocks SHA restoration while retaining conservative policy + repairs; the audit reports disabled enforced policy + as drift, never as initial rollout pending. A live true setting is always + preserved. Full SHA pinning does not prove an allowed action is trustworthy. + +Advanced CodeQL qualification also requires ordinary pull-request triggers +covering the live default branch and opened, synchronize and reopened events, +without workflow path filters. Successful manual or scheduled analysis cannot +qualify a missing or retargeted pull-request producer. Before updating an +existing security gate, publication rechecks its ownership and recognized +payload immediately before the write, retaining concurrent policy changes. +Pending-gate qualification and existing effective-gate proof bind the same +default branch as the final metadata fence; a changed branch cannot activate +a gate using proof from the earlier branch. Pending existing-gate proof also +requires unchanged enforcement and ref applicability in the final rule read; +concurrent disabling or retargeting invalidates that proof. Enforced intent +retains its recognized restoration behavior. + +Rulesets are the canonical branch protection contract. Audit also reads legacy +protection on every active default branch and declared maintenance branch. Any +legacy protection is drift requiring separate reviewed manual remediation; +unreadable responses fail closed. Only a verified unprotected-branch response +counts as absent. For the exact repository-disabled REST response, a fixed +read-only GraphQL query may instead prove that the identified active repository +and exact existing branch have an explicitly null legacy-rule association. +Missing, conflicting or unreadable evidence remains drift. Historical rules on +other branches remain untouched. Audit compares the complete effective default and maintenance branch +policy with declared allowed rule types and parameters; extra integrity or restriction types remain +drift even when every desired rule is present. The audit never removes branch protections. + +Publisher identity, absence, update and retirement decisions use the complete +validated ruleset inventory across all pages, including ordinary organization +and repository rules. Unreadable pages cannot count as absent rules. + +Repository-scope retirement of an existing `03 - Required CI` rule also needs a +reviewed `config/retired-required-ci.json` record with its exact repository and +ruleset IDs and previous required contexts. Remove the repository from +`required_ci` and both security target/activation inventories in the same source +change. Publication deletes only the unchanged GitHub Actions-bound template +after fresh repository identity, archive-state and rule-payload checks. Missing +provenance or modified/foreign rules are retained with an actionable `SKIP`; +keep retirement records for idempotent reruns. This does not delete branches or +ordinary branch protections. + +Read-only preflight failures and unfinished producers are explicit pending +controls, never successful rollout evidence. The audit reports pending controls +separately and rejects unreadable evidence. No paid subscription is introduced. +Rollback requires reviewed desired-state changes and separately authorized +apply; reverting a producer while its required gate remains active can prevent +merges. Do not remove a gate merely because a vulnerable PR fails it. + ## Usage -Requirements: `git`, `gh`, `jq`, and an authenticated organization-owner -account. +Requirements: `git`, `gh`, `jq`, `python3`, the packages in +`requirements-validation.txt`, and an authenticated organization-owner account. +Install the declared validation dependencies before local publisher, audit, or +advanced CodeQL readiness checks; the owning workflows install them too. ```sh gh auth refresh -h github.com -s admin:org,project,repo @@ -323,9 +518,9 @@ to update the automation branch, also authorizes Git-reference and release API operations. GitHub likewise has no separate create-or-update-PR permission that excludes review APIs: `pull requests: write` covers both. The App credential therefore cannot by itself prove that tag, release, or review calls are -technically impossible. Because the App owns its generated pull request, it -cannot provide the distinct human approval required by the ordinary gate, and -it has no bypass. The consumer must never review, approve, merge, tag, publish, +technically impossible. Ordinary pull requests require zero human approvals; +required checks and the pull-request process still apply, and the App has no +bypass. The consumer must never review, approve, merge, tag, publish, dispatch a release, write the default branch, or change repository settings. Its reviewed code, exact branch and changed-path checks, ordinary pull-request gate, protected release-tag rules, workflow contract tests, and audit trail @@ -348,10 +543,11 @@ Provision or rotate the key as an Atrinik organization owner: `update-content.yml` and require one App-authored disposable pull request to receive ordinary `Classic validation`, `CodeQL validation`, and `Conventional PR title` checks. -4. Confirm the App-authored pull request cannot satisfy its own human approval - or merge gate, close the disposable pull request, remove its branch, and - only then delete the previous private key. A later scheduled run must also - succeed before rotation is considered complete. +4. Confirm the App-authored pull request receives the required checks, the App + has no bypass, and the consumer does not approve or merge the pull request. + Close the disposable pull request, remove its branch, and only then delete + the previous private key. A later scheduled run must also succeed before + rotation is considered complete. For suspected disclosure or misuse, disable the Classic updater workflow, suspend installation `153045686`, delete the repository secret, and revoke the @@ -715,7 +911,14 @@ publisher fails closed if a live repository ID differs from the inventory. Do not add a repository to `config/codeql-advanced-setup.json` merely because default setup needs tuning. The inventory is reserved for a reviewed advanced CodeQL workflow that needs repository-specific build, component, or path -coverage. Both security configurations must remain identical except for their +coverage. Its `required_partitions` records semantic scope/language minima, +while the current producer owns display names and upload categories. Readiness +reads literal partition declarations without executing repository Python and +requires a distinct successful job and matching upload for every declaration, +including additions. Changing action versions needs no coverage-inventory edit. +Remove the matching minima when removing an advanced repository; an empty +inventory with an empty minima object is valid. Both security configurations +must remain identical except for their name, description, and `code_scanning_options.allow_advanced` value. Moving a repository to `archive` is an executable policy change: the next @@ -1436,7 +1639,8 @@ credential-free preparation verifies exact released versions and registry checksums, resolves dependencies, updates Cargo locks, and runs aggregate validation before requesting an App token in the isolated PR-writing job. They maintain only their owned `automation/released-crate-dependencies` branch and -pull request; ordinary required checks and human review still govern merging. +pull request; ordinary required checks and the pull-request process still govern +merging, with zero required human approvals. Missing registry readiness produces no migration PR. They never upload crates, approve a pull request, merge it, or loosen branch protection. Prove initial App permissions with a disposable/owned PR and current ordinary checks, without diff --git a/bin/audit b/bin/audit index ebcb69b..657a0fd 100755 --- a/bin/audit +++ b/bin/audit @@ -23,10 +23,12 @@ trap 'rm -rf "${temporary}"' EXIT "${root}/bin/validate" >/dev/null github_api() { - gh api -H "X-GitHub-Api-Version: ${api_version}" "$@" + gh api --method GET -H "X-GitHub-Api-Version: ${api_version}" "$@" } declare -a findings=() +declare -a pending=() +declare -A ruleset_cache=() record_finding() { findings+=("$1") @@ -61,6 +63,14 @@ is_temporary_security_advisory_repository() { "${repositories_config}" >/dev/null } +# Capture before parsing: process substitutions otherwise hide API failures. +if ! github_api --paginate --slurp "orgs/${organization}/repos?per_page=100&type=all" | + jq -e 'add | if type == "array" then . else error("invalid inventory") end' \ + >"${temporary}/inventory.json"; then + echo "error: organization repository inventory could not be read" >&2 + exit 1 +fi + declare -A live_archived=() while IFS=$'\t' read -r \ repository archived private visibility disabled fork has_issues has_projects has_wiki; do @@ -71,24 +81,11 @@ while IFS=$'\t' read -r \ continue fi live_archived["${repository}"]=${archived} -done < <( - github_api --paginate "orgs/${organization}/repos?per_page=100&type=all" \ - --jq ' - .[] | - [ - .name, - (.archived | tostring), - (.private | tostring), - .visibility, - (.disabled | tostring), - (.fork | tostring), - (.has_issues | tostring), - (.has_projects | tostring), - (.has_wiki | tostring) - ] | - @tsv - ' -) +done < <(jq -r ' + .[] | [.name, (.archived | tostring), (.private | tostring), .visibility, + (.disabled | tostring), (.fork | tostring), (.has_issues | tostring), + (.has_projects | tostring), (.has_wiki | tostring)] | @tsv +' "${temporary}/inventory.json") declare -A expected_active=() declare -A expected_archived=() @@ -140,19 +137,6 @@ for repository in "${!expected_archived[@]}"; do fi done -expected_rule_name() { - local repository=$1 - local base=$2 - - if jq -e --arg repository "${repository}" \ - '.repositories | index($repository) != null' "${advisory_windows}" \ - >/dev/null; then - printf '%s - classic\n' "${base}" - else - printf '%s\n' "${base}" - fi -} - has_configured_repository() { local property=$1 local repository=$2 @@ -175,6 +159,316 @@ has_release_tags() { >/dev/null } +audit_actions() { + local current + for policy in permissions selected-actions workflow; do + local endpoint="orgs/${organization}/actions/permissions" + local config="${root}/config/actions-permissions.json" + case ${policy} in + selected-actions) endpoint+=/selected-actions; config="${root}/config/actions-selected.json" ;; + workflow) endpoint+=/workflow; config="${root}/config/actions-workflow.json" ;; + esac + if ! current=$(github_api "${endpoint}"); then + record_finding "organization Actions ${policy} could not be read: ${organization}" + continue + fi + if ! jq -e --slurpfile desired "${config}" ' + . as $current | $desired[0] | to_entries | all(.[]; + if .key == "sha_pinning_required" then true + elif .key == "patterns_allowed" then + (.value | sort) == ($current.patterns_allowed | sort) + else $current[.key] == .value end) + ' <<<"${current}" >/dev/null; then + record_finding "organization Actions ${policy} drift: ${organization}" + fi + if [[ ${policy} == permissions ]] && + jq -e '.sha_pinning_required == true' "${config}" >/dev/null; then + case $(jq -r '.sha_pinning_required | if type == "boolean" then tostring else "unreadable" end' <<<"${current}") in + true) ;; + false) + case $(jq -r '.phase' "${root}/config/actions-sha-pinning.json") in + pending) pending+=("Actions SHA pinning deferred pending canary proof: ${organization}") ;; + enforced) record_finding "Actions SHA pinning enforcement drift: ${organization}" ;; + *) record_finding "Actions SHA pinning rollout phase could not be read: ${organization}" ;; + esac + ;; + *) record_finding "Actions SHA pinning state could not be read: ${organization}" ;; + esac + fi + done +} + +audit_actions + +# The branch endpoint proves effective applicability; inherited names alone do not. +# Details additionally prove bypass and ref/repository scope, including tag policy. +# Match the entire publisher-owned repository scope. Being included in a wider +# scope does not prove convergence: a widened tag rule can constrain services +# with no release contract. Repository-level rulesets have only ref conditions. +organization_ruleset_scope() { + local name=$1 repository=$2 + jq -cn --arg name "${name}" --arg repository "${repository}" \ + --slurpfile repositories "${repositories_config}" \ + --slurpfile windows "${advisory_windows}" ' + {include: [], exclude: [], protected: false} | + if $name == "01 - Default branch integrity" or + $name == "01 - Default branch linear history" then + .include = ["~ALL"] | .exclude = $windows[0].repositories + elif $name == "01 - Default branch integrity - classic" or + $name == "01 - Default branch linear history - classic" or + $name == "02 - Changes through pull requests - classic" then + .include = ["classic"] + elif $name == "02 - Changes through pull requests" then + .include = ($repositories[0].pull_request_gate - $windows[0].repositories) + elif ($name | startswith("03 - Required CI - ")) then + .include = [$repository] + elif $name == "04 - Immutable release tags" then + .include = $repositories[0].release_tags + else error("unknown managed ruleset scope") end' +} + +ruleset_normalization=' +# Omitted request fields are expanded by GitHub to these exact defaults. +# Preserve nondefault, malformed and unknown fields as observable drift. +def normalize_rules: + map(if .type == "pull_request" then + .parameters |= ( + if .required_reviewers == [] then del(.required_reviewers) else . end | + if .require_extra_approval_for_unattributed_changes == true then + del(.require_extra_approval_for_unattributed_changes) else . end | + if (.allowed_merge_methods | type) == "array" and + (.allowed_merge_methods | sort) == ["merge", "rebase", "squash"] then + del(.allowed_merge_methods) else . end) + elif .type == "required_status_checks" then + .parameters |= ( + if .do_not_enforce_on_create == false then del(.do_not_enforce_on_create) else . end | + .required_status_checks |= sort_by(.context, .integration_id)) + else . end) | sort_by(.type); +' + +read_effective_rules() { + local repository=$1 branch=$2 encoded + encoded=$(jq -rn --arg branch "${branch}" '$branch | @uri') + github_api --paginate --slurp \ + "repos/${organization}/${repository}/rules/branches/${encoded}?per_page=100" | jq -e ' + if type == "array" and length > 0 and all(.[]; type == "array") then add + else error("invalid effective-rule pages") end | + if type == "array" and all(.[]; type == "object" and + (.type | type == "string" and length > 0)) then . + else error("invalid effective-rule inventory") end' +} + +audit_effective_policy() { + local repository=$1 branch=$2 effective=$3 desired=$4 + # API provenance is diagnostic metadata. Every policy field, including + # unknown future parameters, must match the declared contract. CI and + # Dependency review may contribute separate rules with the same parameters. + if ! jq -e --argjson desired "${desired}" "${ruleset_normalization}"' + def policy_fields: map(del(.ruleset_id, .ruleset_source, .ruleset_source_type)); + def without_checks: + if .type == "required_status_checks" then .parameters.required_status_checks = [] + else . end; + def checks: + [.[] | select(.type == "required_status_checks") | .parameters.required_status_checks[]] | + unique_by(.context, .integration_id) | sort_by(.context, .integration_id); + ($desired | normalize_rules) as $wanted | + (policy_fields | normalize_rules) as $current | + ([$current[].type] | unique) == ([$wanted[].type] | unique) and + all($current[]; . as $rule | + any($wanted[]; without_checks == ($rule | without_checks))) and + ($current | checks) == ($wanted | checks) + ' <<<"${effective}" >/dev/null 2>&1; then + record_finding "effective branch policy drift requires manual review: ${organization}/${repository}/${branch}" + fi +} + +check_ruleset() { + local repository=$1 name=$2 template=$3 list=$4 + local entry id source endpoint detail wanted scope + entry=$(jq -c --arg name "${name}" '[.[] | select(.name == $name)]' <<<"${list}") + if [[ $(jq length <<<"${entry}") != 1 ]]; then + record_finding "missing or duplicate inherited ruleset '${name}': ${organization}/${repository}" + return + fi + id=$(jq -r '.[0].id' <<<"${entry}") + source=$(jq -r '.[0].source_type' <<<"${entry}") + if [[ ${name} == '06 - CodeQL security' || ${name} == '07 - Dependency review' ]] && + [[ ${source} != Repository ]]; then + record_finding "managed security ruleset must be repository-owned '${name}': ${organization}/${repository}" + return + fi + case ${source} in + Organization) endpoint="orgs/${organization}/rulesets/${id}" ;; + Repository) endpoint="repos/${organization}/${repository}/rulesets/${id}" ;; + *) record_finding "ruleset source could not be read '${name}': ${organization}/${repository}"; return ;; + esac + if [[ ${name} == '04 - Immutable release tags' && ${source} == Repository ]] && + ! has_release_tags "${repository}"; then + record_finding "unexpected managed release-tag protection: ${organization}/${repository}" + return + fi + if [[ ! -v ruleset_cache["${endpoint}"] ]]; then + if ! detail=$(github_api "${endpoint}"); then + record_finding "ruleset contents could not be read '${name}': ${organization}/${repository}" + return + fi + ruleset_cache["${endpoint}"]=${detail} + fi + detail=${ruleset_cache["${endpoint}"]} + wanted=$(cat "${template}") + if [[ ${name} == '03 - Required CI - '* ]]; then + wanted=$(jq --argjson checks "${expected_checks}" \ + '.rules[0].parameters.required_status_checks = $checks' <<<"${wanted}") + # Dependency review is governed by its separate rollout ruleset. + wanted=$(jq '.rules[0].parameters.required_status_checks |= map(select(.context != "Dependency review"))' <<<"${wanted}") + fi + if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ + "${advisory_windows}" >/dev/null && [[ ${name} == '03 - Required CI - '* ]]; then + wanted=$(jq '.bypass_actors[].bypass_mode = "always"' <<<"${wanted}") + fi + if [[ ${source} == Organization ]]; then + scope=$(organization_ruleset_scope "${name}" "${repository}") + wanted=$(jq --argjson scope "${scope}" '.conditions.repository_name = $scope' <<<"${wanted}") + fi + if ! jq -e --argjson wanted "${wanted}" "${ruleset_normalization}"' + # GitHub returns null for OrganizationAdmin, whose desired actor ID is 1. + def normalize_bypass: + map(if .actor_type == "OrganizationAdmin" and .actor_id == null then + .actor_id = 1 else . end) | sort_by(.actor_type, .actor_id, .bypass_mode); + def normalize_conditions: + with_entries(if .key == "ref_name" or .key == "repository_name" then + .value.include |= sort | .value.exclude |= sort else . end); + .enforcement == $wanted.enforcement and .target == $wanted.target and + (.bypass_actors | normalize_bypass) == ($wanted.bypass_actors | normalize_bypass) and + (.conditions | normalize_conditions) == ($wanted.conditions | normalize_conditions) and + (.rules | normalize_rules) == ($wanted.rules | normalize_rules) + ' <<<"${detail}" >/dev/null; then + record_finding "ruleset scope or policy drift '${name}': ${organization}/${repository}" + fi +} + +# Rulesets are the declared protection contract in both publisher scopes. +# Legacy protection layers additional requirements outside the rules endpoint; +# retain it for manual review rather than silently treating rulesets as complete. +audit_legacy_protection() { + local repository=$1 branch=$2 metadata=$3 encoded response status=0 body + encoded=$(jq -rn --arg branch "${branch}" '$branch | @uri') + response=$(github_api --include \ + "repos/${organization}/${repository}/branches/${encoded}/protection" \ + 2>"${temporary}/legacy-protection.err") || status=$? + response=${response//$'\r'/} + body=${response#*$'\n\n'} + if [[ ${status} == 1 && ${response%%$'\n'*} =~ ^HTTP/[0-9.]+\ 404(\ |$) ]] && + jq -se 'length == 1 and (.[0] | type == "object" and + .message == "Branch not protected" and .status == "404")' \ + <<<"${body}" >/dev/null 2>&1; then + return + fi + # Some repositories report disabled REST protection despite retaining legacy + # rules for other branches. Only an exact-ref GraphQL read can prove absence. + if [[ ${status} == 1 && ${response%%$'\n'*} =~ ^HTTP/[0-9.]+\ 404(\ |$) ]] && + jq -se 'length == 1 and (.[0] | type == "object" and + .message == "Branch protection has been disabled on this repository." and + .status == "404")' <<<"${body}" >/dev/null 2>&1; then + audit_disabled_legacy_protection "${repository}" "${branch}" "${metadata}" + return + fi + if [[ ${status} == 0 && ${response%%$'\n'*} =~ ^HTTP/[0-9.]+\ 200(\ |$) ]] && + jq -se 'length == 1 and (.[0] | type == "object")' \ + <<<"${body}" >/dev/null 2>&1; then + record_finding "unexpected legacy branch protection requires manual review: ${organization}/${repository}/${branch}" + else + record_finding "legacy branch protection could not be read: ${organization}/${repository}/${branch}" + fi +} + +audit_disabled_legacy_protection() { + local repository=$1 branch=$2 metadata=$3 repository_id response + # GraphQL variables are literal; gh supplies their values as separate fields. + # shellcheck disable=SC2016 + local query='query LegacyBranchProtection($owner: String!, $name: String!, $qualifiedName: String!) { repository(owner: $owner, name: $name) { databaseId nameWithOwner isArchived ref(qualifiedName: $qualifiedName) { name branchProtectionRule { id } } } }' + # Fence rename/transfer/recreation against both the fresh REST metadata and + # the enumeration used to select this active repository. + if ! repository_id=$(jq -er --arg repository "${repository}" \ + --arg coordinate "${organization}/${repository}" \ + --slurpfile inventory "${temporary}/inventory.json" ' + . as $metadata | + [$inventory[0][] | select(.name == $repository)] as $entries | + if ($entries | length) == 1 and $entries[0].id == .id and + $entries[0].archived == false and .archived == false and + .name == $repository and .full_name == $coordinate and + (.id | type == "number" and . > 0 and floor == .) + then .id else error("repository identity drift") end + ' "${metadata}" 2>/dev/null) || + ! response=$(gh api --method POST -H "X-GitHub-Api-Version: ${api_version}" \ + graphql -f "query=${query}" -f "owner=${organization}" \ + -f "name=${repository}" -f "qualifiedName=refs/heads/${branch}" \ + 2>"${temporary}/legacy-protection.err") || + ! jq -se --argjson repository_id "${repository_id}" \ + --arg coordinate "${organization}/${repository}" --arg branch "${branch}" ' + length == 1 and (.[0] | type == "object" and (has("errors") | not) and + (.data.repository | type == "object" and + .databaseId == $repository_id and .nameWithOwner == $coordinate and + .isArchived == false and + (.ref | type == "object" and .name == $branch and + has("branchProtectionRule") and + (.branchProtectionRule == null or + (.branchProtectionRule | type == "object" and + (.id | type == "string" and length > 0)))))) + ' <<<"${response}" >/dev/null 2>&1; then + record_finding "legacy branch protection could not be read: ${organization}/${repository}/${branch}" + elif ! jq -e '.data.repository.ref.branchProtectionRule == null' \ + <<<"${response}" >/dev/null; then + record_finding "unexpected legacy branch protection requires manual review: ${organization}/${repository}/${branch}" + fi +} + +audit_security() { + local repository=$1 desired="${root}/config/code-security.json" setup="${root}/config/codeql-default-setup.json" + local current languages + if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ + "${root}/config/codeql-advanced-setup.json" >/dev/null; then + desired="${root}/config/code-security-advanced.json" + setup="${root}/config/codeql-default-setup-advanced.json" + fi + if current=$(github_api "repos/${organization}/${repository}/code-security-configuration"); then + if ! jq -e --slurpfile desired "${desired}" ' + .status == "enforced" and (.configuration | . as $config | + . + {code_security: (.code_security // + (if .advanced_security == "enabled" or .advanced_security == "code_security" then "enabled" else "disabled" end)), + secret_protection: (.secret_protection // + (if .advanced_security == "enabled" or .advanced_security == "secret_protection" then "enabled" else "disabled" end))} | + . as $normalized | $desired[0] | to_entries | all(.[]; $normalized[.key] == .value)) + ' <<<"${current}" >/dev/null; then + record_finding "security configuration attachment or baseline drift: ${organization}/${repository}" + fi + else + record_finding "security configuration could not be read: ${organization}/${repository}" + fi + if current=$(github_api "repos/${organization}/${repository}/code-scanning/default-setup"); then + languages=$(jq -c --arg repository "${repository}" '.codeql_required_languages[$repository] // []' \ + "${root}/config/security-merge-protection.json") + if ! jq -e --slurpfile desired "${setup}" --argjson languages "${languages}" ' + . as $current | ($desired[0] | to_entries | all(.[]; $current[.key] == .value)) and + (if $desired[0].state == "configured" then + $current.languages | if type == "array" then all(.[]; type == "string") else false end + else true end) and + (($languages - ($current.languages // [])) | length == 0) + ' <<<"${current}" >/dev/null; then + record_finding "CodeQL setup or required language drift: ${organization}/${repository}" + fi + else + record_finding "CodeQL setup could not be read: ${organization}/${repository}" + fi + if ! github_api "repos/${organization}/${repository}/vulnerability-alerts" >/dev/null; then + record_finding "Dependabot alerts disabled or unreadable: ${organization}/${repository}" + fi + if ! current=$(github_api "repos/${organization}/${repository}/private-vulnerability-reporting") || + ! jq -e '.enabled == true' <<<"${current}" >/dev/null; then + record_finding "private vulnerability reporting disabled or unreadable: ${organization}/${repository}" + fi +} + for repository in "${!expected_active[@]}"; do [[ -v live_archived["${repository}"] ]] || continue [[ ${live_archived["${repository}"]} == false ]] || continue @@ -185,6 +479,16 @@ for repository in "${!expected_active[@]}"; do continue fi + if current=$(github_api "repos/${organization}/${repository}/actions/permissions/workflow"); then + if ! jq -e --slurpfile desired "${root}/config/actions-workflow.json" ' + . as $current | $desired[0] | to_entries | all(.[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + record_finding "repository Actions workflow permissions drift: ${organization}/${repository}" + fi + else + record_finding "repository Actions workflow permissions could not be read: ${organization}/${repository}" + fi + if ! jq -e --slurpfile desired "${repository_defaults}" ' .archived == false and .default_branch == $desired[0].default_branch and @@ -214,32 +518,177 @@ for repository in "${!expected_active[@]}"; do record_finding "repository custom properties could not be read: ${organization}/${repository}" fi - if rulesets=$(github_api \ - "repos/${organization}/${repository}/rulesets?includes_parents=true" | \ - jq -r '.[].name' | sort); then - required_rulesets=( - "$(expected_rule_name "${repository}" "01 - Default branch integrity")" - "$(expected_rule_name "${repository}" "01 - Default branch linear history")" - ) - if has_configured_repository '.pull_request_gate' "${repository}"; then - required_rulesets+=( - "$(expected_rule_name "${repository}" "02 - Changes through pull requests")" - ) + rulesets='[]' + if ! rulesets=$(github_api --paginate --slurp \ + "repos/${organization}/${repository}/rulesets?includes_parents=true&per_page=100" | jq -e ' + if type == "array" and length > 0 and all(.[]; type == "array") then add + else error("invalid ruleset pages") end | + if type == "array" and all(.[]; + type == "object" and (.name | type == "string" and length > 0)) then . + else error("invalid ruleset inventory") end'); then + record_finding "inherited rulesets could not be read: ${organization}/${repository}" + rulesets='[]' + fi + # Retired/non-target security gates must be absent from the listing, even + # when disabled or scoped away from the default branch. Presence is manual + # drift regardless of ownership or payload; this audit never removes a rule. + for security_kind in dependency_review code_scanning; do + case ${security_kind} in + dependency_review) security_name='07 - Dependency review' ;; + code_scanning) security_name='06 - CodeQL security' ;; + esac + if ! jq -e --arg repository "${repository}" --arg kind "${security_kind}" \ + '.[$kind] | index($repository) != null' \ + "${root}/config/security-merge-protection.json" >/dev/null && + jq -e --arg name "${security_name}" 'any(.[]; .name == $name)' \ + <<<"${rulesets}" >/dev/null; then + record_finding "unexpected retained security ruleset '${security_name}' requires manual review: ${organization}/${repository}" fi - if has_required_ci "${repository}"; then - required_rulesets+=("03 - Required CI - ${repository}") + done + branch=$(jq -r '.default_branch' "${repository_defaults}") + audit_legacy_protection "${repository}" "${branch}" "${metadata_file}" + while IFS= read -r maintenance; do + maintenance_branch=$(jq -r .branch <<<"${maintenance}") + audit_legacy_protection "${repository}" "${maintenance_branch}" "${metadata_file}" + if ! maintenance_effective=$(read_effective_rules "${repository}" "${maintenance_branch}"); then + record_finding "effective maintenance-branch rules could not be read: ${organization}/${repository}/${maintenance_branch}" + maintenance_effective='[]' fi - if has_release_tags "${repository}"; then - required_rulesets+=("04 - Immutable release tags") + maintenance_desired=$(jq --argjson checks "$(jq -c .required_ci <<<"${maintenance}")" ' + .rules | map(if .type == "required_status_checks" then + if ($checks | length) == 0 then empty + else .parameters.required_status_checks = + ($checks | map({context: ., integration_id: 15368})) end + else . end) + ' "${root}/config/rulesets/maintenance-branch.json") + audit_effective_policy "${repository}" "${maintenance_branch}" "${maintenance_effective}" "${maintenance_desired}" + done < <(jq -c --arg repository "${repository}" ' + .maintenance_branches[] | select(.repository == $repository) + ' "${repositories_config}") + effective='[]' + if ! effective=$(read_effective_rules "${repository}" "${branch}"); then + record_finding "effective default-branch rules could not be read: ${organization}/${repository}" + effective='[]' + fi + expected_checks=$(jq -c --arg repository "${repository}" \ + '(.required_ci[$repository] // []) | map({context: ., integration_id: 15368})' "${repositories_config}") + dependency_required=false + if jq -e --arg repository "${repository}" '.dependency_review | index($repository) != null' \ + "${root}/config/security-merge-protection.json" >/dev/null; then + if jq -e --arg repository "${repository}" \ + '.activation.dependency_review[$repository] == "enforced"' \ + "${root}/config/security-merge-protection.json" >/dev/null || + jq -e 'any(.[]; .type == "required_status_checks" and + any(.parameters.required_status_checks[]; .context == "Dependency review"))' \ + <<<"${effective}" >/dev/null || + jq -e 'any(.[]; .name == "07 - Dependency review")' <<<"${rulesets}" >/dev/null; then + dependency_required=true + elif "${root}/bin/security-readiness" dependency "${organization}/${repository}" \ + >"${temporary}/${repository}.readiness" 2>&1; then + pending+=("Dependency review merge protection awaits reviewed activation intent: ${organization}/${repository}") + else + readiness_status=$? + if ((readiness_status == 1)); then + pending+=("Dependency review merge protection awaits landed successful producer: ${organization}/${repository}") + else + record_finding "Dependency review readiness could not be read: ${organization}/${repository}" + fi + fi + if ${dependency_required}; then + expected_checks=$(jq '. + [{context: "Dependency review", integration_id: 15368}]' <<<"${expected_checks}") + check_ruleset "${repository}" '07 - Dependency review' \ + "${root}/config/rulesets/dependency-review.json" "${rulesets}" "${metadata_file}" + fi + fi + if ! jq -e --argjson checks "${expected_checks}" ' + ([.[] | select(.type == "required_status_checks") | .parameters.required_status_checks[]] | + unique_by(.context, .integration_id) | sort_by(.context, .integration_id)) == + ($checks | sort_by(.context, .integration_id)) and + all(.[] | select(.type == "required_status_checks"); .parameters.strict_required_status_checks_policy == true) + ' <<<"${effective}" >/dev/null; then + record_finding "effective required CI contexts or app drift: ${organization}/${repository}" + fi + desired_effective=$(jq --argjson checks "${expected_checks}" ' + if ($checks | length) == 0 then [] + else .rules | map(.parameters.required_status_checks = $checks) end + ' "${root}/config/rulesets/required-ci.json") + for policy in default-branch-integrity default-branch-linear-history; do + template="${root}/config/rulesets/${policy}.json" + if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ + "${advisory_windows}" >/dev/null; then + template="${root}/config/rulesets/${policy}-classic.json" + fi + check_ruleset "${repository}" "$(jq -r .name "${template}")" "${template}" "${rulesets}" "${metadata_file}" + desired_effective=$(jq --argjson prior "${desired_effective}" '$prior + .rules' "${template}") + if ! jq -e --slurpfile desired "${template}" \ + '. as $effective | all($desired[0].rules[]; . as $rule | any($effective[]; .type == $rule.type))' \ + <<<"${effective}" >/dev/null; then + record_finding "effective default-branch integrity drift: ${organization}/${repository}" + fi + done + if has_configured_repository '.pull_request_gate' "${repository}"; then + template="${root}/config/rulesets/pull-request-gate.json" + if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ + "${advisory_windows}" >/dev/null; then template="${root}/config/rulesets/pull-request-gate-classic.json"; fi + check_ruleset "${repository}" "$(jq -r .name "${template}")" "${template}" "${rulesets}" "${metadata_file}" + desired_effective=$(jq --argjson prior "${desired_effective}" '$prior + .rules' "${template}") + if ! jq -e --slurpfile desired "${template}" \ + "${ruleset_normalization}"'([.[] | select(.type == "pull_request") | {type, parameters}] | + normalize_rules) == ($desired[0].rules | normalize_rules)' \ + <<<"${effective}" >/dev/null; then + record_finding "effective pull-request policy drift: ${organization}/${repository}" + fi + elif jq -e 'any(.[]; .type == "pull_request")' <<<"${effective}" >/dev/null; then + record_finding "unexpected effective pull-request gate: ${organization}/${repository}" + fi + if has_required_ci "${repository}"; then + check_ruleset "${repository}" "03 - Required CI - ${repository}" \ + "${root}/config/rulesets/required-ci.json" "${rulesets}" "${metadata_file}" + fi + # Parent listings can contain correctly scoped but nonapplicable rulesets. + # Inspect the exact managed name on nonrelease repositories too, so widening + # its complete scope fails while foreign, unmanaged tag rules remain unowned. + if has_release_tags "${repository}" || + jq -e 'any(.[]; .name == "04 - Immutable release tags")' <<<"${rulesets}" >/dev/null; then + check_ruleset "${repository}" '04 - Immutable release tags' \ + "${root}/config/rulesets/immutable-release-tags.json" "${rulesets}" "${metadata_file}" + fi + if jq -e --arg repository "${repository}" '.code_scanning | index($repository) != null' \ + "${root}/config/security-merge-protection.json" >/dev/null; then + codeql_required=false + if jq -e --arg repository "${repository}" \ + '.activation.code_scanning[$repository] == "enforced"' \ + "${root}/config/security-merge-protection.json" >/dev/null || + jq -e 'any(.[]; .type == "code_scanning")' <<<"${effective}" >/dev/null || + jq -e 'any(.[]; .name == "06 - CodeQL security")' <<<"${rulesets}" >/dev/null; then + codeql_required=true + elif "${root}/bin/security-readiness" codeql "${organization}/${repository}" \ + >"${temporary}/${repository}.codeql-readiness" 2>&1; then + pending+=("CodeQL merge protection awaits reviewed activation intent: ${organization}/${repository}") + else + readiness_status=$? + if ((readiness_status == 1)); then + pending+=("CodeQL merge protection awaits successful analysis: ${organization}/${repository}") + else + record_finding "CodeQL readiness could not be read: ${organization}/${repository}" + fi fi - for ruleset in "${required_rulesets[@]}"; do - if ! grep -Fxq -- "${ruleset}" <<<"${rulesets}"; then - record_finding "missing inherited ruleset '${ruleset}': ${organization}/${repository}" + if ${codeql_required}; then + check_ruleset "${repository}" '06 - CodeQL security' \ + "${root}/config/rulesets/code-scanning.json" "${rulesets}" "${metadata_file}" + desired_effective=$(jq --argjson prior "${desired_effective}" '$prior + .rules' \ + "${root}/config/rulesets/code-scanning.json") + if ! jq -e --slurpfile desired "${root}/config/rulesets/code-scanning.json" \ + '[.[] | select(.type == "code_scanning") | {type, parameters}] == $desired[0].rules' \ + <<<"${effective}" >/dev/null; then + record_finding "effective CodeQL merge protection drift: ${organization}/${repository}" fi - done - else - record_finding "inherited rulesets could not be read: ${organization}/${repository}" + fi + elif jq -e 'any(.[]; .type == "code_scanning")' <<<"${effective}" >/dev/null; then + record_finding "unexpected effective CodeQL merge protection: ${organization}/${repository}" fi + audit_effective_policy "${repository}" "${branch}" "${effective}" "${desired_effective}" + audit_security "${repository}" if ! jq -e ' .security_and_analysis.secret_scanning.status == "enabled" and @@ -251,10 +700,18 @@ for repository in "${!expected_active[@]}"; do fi done +if ((${#pending[@]})); then + printf 'PENDING %s\n' "${pending[@]}" | sort -u +fi + if ((${#findings[@]})); then printf 'DRIFT %s\n' "${findings[@]}" | sort -u echo "Governance audit failed: ${#findings[@]} finding(s)." >&2 exit 1 fi -echo "Governance audit passed: ${#expected_active[@]} active repositories checked." +if ((${#pending[@]})); then + echo "Governance audit baseline passed with ${#pending[@]} pending rollout(s): ${#expected_active[@]} active repositories checked." +else + echo "Governance audit passed: ${#expected_active[@]} active repositories checked." +fi diff --git a/bin/publish b/bin/publish index 597483f..f669bff 100755 --- a/bin/publish +++ b/bin/publish @@ -7,6 +7,7 @@ policy_scope=${ATRINIK_POLICY_SCOPE:-auto} api_version=2026-03-10 apply=false retire_maintenance= +sha_pinning_canary= while (($#)); do case $1 in @@ -14,6 +15,14 @@ while (($#)); do apply=true shift ;; + --sha-pinning-canary) + if (($# < 2)) || [[ -z $2 ]]; then + echo "error: --sha-pinning-canary requires OWNER/REPOSITORY/PR" >&2 + exit 2 + fi + sha_pinning_canary=$2 + shift 2 + ;; --retire-maintenance) if (($# < 2)) || [[ -z $2 ]]; then echo "error: --retire-maintenance requires REPOSITORY/BRANCH" >&2 @@ -23,7 +32,7 @@ while (($#)); do shift 2 ;; *) - echo "usage: $0 [--apply] [--retire-maintenance REPOSITORY/BRANCH]" >&2 + echo "usage: $0 [--apply] [--retire-maintenance REPOSITORY/BRANCH] [--sha-pinning-canary OWNER/REPOSITORY/PR]" >&2 exit 2 ;; esac @@ -37,7 +46,7 @@ auto | organization | repository) ;; ;; esac -for command in gh git jq sort; do +for command in gh git jq sort python3; do if ! command -v "${command}" >/dev/null 2>&1; then echo "error: ${command} is required" >&2 exit 1 @@ -51,6 +60,11 @@ advisory_merge_windows_config=${root}/config/advisory-merge-windows.json codeql_advanced_config=${root}/config/codeql-advanced-setup.json immutable_releases_config=${root}/config/immutable-releases.json retired_maintenance_config=${root}/config/retired-maintenance-branches.json +security_merge_config=${root}/config/security-merge-protection.json +if [[ -n ${sha_pinning_canary} && ${sha_pinning_canary} != "${organization}/"* ]]; then + echo "error: SHA canary must belong to ${organization}" >&2 + exit 2 +fi temporary_files=() trap 'rm -f "${temporary_files[@]}"' EXIT @@ -60,6 +74,31 @@ github_api() { gh api -H "X-GitHub-Api-Version: ${api_version}" "$@" } +# Capture every page before any caller treats a ruleset as present or absent. +# gh 2.46 emits one JSON array per page; --slurp is unavailable on that host. +list_rulesets() { + local endpoint=$1 pages + pages=$(github_api "${endpoint}" --paginate) || { + echo "error: cannot read complete ruleset inventory: ${endpoint}" >&2 + return 1 + } + jq -ecs ' + if length > 0 and all(.[]; + type == "array" and all(.[]; + type == "object" and + (.id | type == "number" and . > 0 and floor == .) and + (.name | type == "string" and length > 0))) then + add | + if (map(.id) | length == (unique | length)) and + (map(.name) | length == (unique | length)) then . + else error("duplicate ruleset identity or name") end + else error("invalid ruleset page") end + ' <<<"${pages}" || { + echo "error: invalid or ambiguous complete ruleset inventory: ${endpoint}" >&2 + return 1 + } +} + run_api() { local method=$1 local endpoint=$2 @@ -163,7 +202,7 @@ retire_maintenance_branch_ruleset() { exit 1 fi - rulesets=$(github_api "orgs/${organization}/rulesets") + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 while IFS=$'\t' read -r preserved_id preserved_name; do if ! jq -e --argjson id "${preserved_id}" --arg name "${preserved_name}" ' any(.[]; .id == $id and .name == $name and .enforcement == "active") @@ -236,7 +275,7 @@ retire_maintenance_branch_ruleset() { run_api DELETE "orgs/${organization}/rulesets/${ruleset_id}" if ${apply}; then - rulesets=$(github_api "orgs/${organization}/rulesets") + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 if jq -e --argjson id "${ruleset_id}" 'any(.[]; .id == $id)' \ <<<"${rulesets}" >/dev/null; then echo "error: retirement ruleset still exists after apply" >&2 @@ -332,13 +371,11 @@ is_temporary_security_advisory_repository() { upsert_repository_ruleset() { local repository=$1 local input=$2 - local name id + local name id rulesets name=$(jq -r '.name' "${input}") - id=$( - github_api "repos/${organization}/${repository}/rulesets?includes_parents=false" | - jq -r --arg name "${name}" 'map(select(.name == $name))[0].id // empty' - ) + rulesets=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 + id=$(jq -r --arg name "${name}" 'map(select(.name == $name))[0].id // empty' <<<"${rulesets}") if [[ -n ${id} ]]; then run_api PUT "repos/${organization}/${repository}/rulesets/${id}" "${input}" @@ -349,13 +386,11 @@ upsert_repository_ruleset() { upsert_organization_ruleset() { local input=$1 - local name id + local name id rulesets name=$(jq -r '.name' "${input}") - id=$( - github_api "orgs/${organization}/rulesets" | - jq -r --arg name "${name}" 'map(select(.name == $name))[0].id // empty' - ) + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 + id=$(jq -r --arg name "${name}" 'map(select(.name == $name))[0].id // empty' <<<"${rulesets}") if [[ -n ${id} ]]; then run_api PUT "orgs/${organization}/rulesets/${id}" "${input}" @@ -461,9 +496,10 @@ maintenance_ruleset_is_desired() { } delete_stale_repository_maintenance_rulesets() { - local repository ruleset name id + local repository ruleset name id rulesets for repository in "${active_repositories[@]}"; do + rulesets=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -471,24 +507,21 @@ delete_stale_repository_maintenance_rulesets() { continue fi if ! maintenance_ruleset_is_desired "${repository}" "${name}"; then - run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" || return 1 fi - done < <( - github_api \ - "repos/${organization}/${repository}/rulesets?includes_parents=false" | - jq -c '.[]' - ) + done < <(jq -c '.[]' <<<"${rulesets}") done } delete_stale_repository_advisory_rulesets() { - local repository ruleset name id window_active + local repository ruleset name id window_active rulesets while IFS= read -r repository; do window_active=false if repository_has_advisory_merge_window "${repository}"; then window_active=true fi + rulesets=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -498,7 +531,7 @@ delete_stale_repository_advisory_rulesets() { "01 - Default branch linear history" | \ "02 - Changes through pull requests") run_api DELETE \ - "repos/${organization}/${repository}/rulesets/${id}" + "repos/${organization}/${repository}/rulesets/${id}" || return 1 ;; esac else @@ -507,22 +540,19 @@ delete_stale_repository_advisory_rulesets() { "01 - Default branch linear history - ${repository}" | \ "02 - Changes through pull requests - ${repository}") run_api DELETE \ - "repos/${organization}/${repository}/rulesets/${id}" + "repos/${organization}/${repository}/rulesets/${id}" || return 1 ;; esac fi - done < <( - github_api \ - "repos/${organization}/${repository}/rulesets?includes_parents=false" | - jq -c '.[]' - ) + done < <(jq -c '.[]' <<<"${rulesets}") done < <(jq -r '.pull_request_gate[]' "${repositories_config}") } delete_managed_repository_rulesets() { local repository=$1 - local ruleset name id + local ruleset name id rulesets + rulesets=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -536,18 +566,16 @@ delete_managed_repository_rulesets() { "03 - Required CI" | \ "04 - Immutable release tags" | \ "05 - Maintenance branch - "*) - run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" || return 1 ;; esac - done < <( - github_api "repos/${organization}/${repository}/rulesets?includes_parents=false" | - jq -c '.[]' - ) + done < <(jq -c '.[]' <<<"${rulesets}") } delete_managed_organization_rulesets() { - local ruleset name id + local ruleset name id rulesets + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -561,10 +589,69 @@ delete_managed_organization_rulesets() { "04 - Immutable release tags" | \ "03 - Required CI - "* | \ "05 - Maintenance branch - "*) - run_api DELETE "orgs/${organization}/rulesets/${id}" + run_api DELETE "orgs/${organization}/rulesets/${id}" || return 1 ;; esac - done < <(github_api "orgs/${organization}/rulesets" | jq -c '.[]') + done < <(jq -c '.[]' <<<"${rulesets}") +} + +retired_required_ci_matches() { + local repository=$1 id=$2 template=$3 current=$4 + jq -e --argjson id "${id}" --arg source "${organization}/${repository}" ' + .id == $id and .source_type == "Repository" and .source == $source + ' <<<"${current}" >/dev/null && security_ruleset_matches "${template}" "${current}" +} + +delete_retired_repository_required_ci() { + local repository listed record id template current metadata fresh + for repository in "${active_repositories[@]}"; do + [[ -v active_repository_ids["${repository}"] ]] || continue + if jq -e --arg repository "${repository}" '.required_ci | has($repository)' \ + "${repositories_config}" >/dev/null; then + continue + fi + listed=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 + listed=$(jq -c '[.[] | select(.name == "03 - Required CI")]' <<<"${listed}") + [[ $(jq length <<<"${listed}") != 0 ]] || continue + record=$(jq -c --arg repository "${repository}" \ + '[.retirements[] | select(.repository == $repository)]' "${root}/config/retired-required-ci.json") + if [[ $(jq length <<<"${listed}") != 1 || $(jq length <<<"${record}") != 1 ]]; then + echo "SKIP ${organization}/${repository} 03 - Required CI: retained; requires one reviewed config/retired-required-ci.json record and unique live rule" + continue + fi + record=$(jq -c '.[0]' <<<"${record}") + id=$(jq -r '.ruleset_id' <<<"${record}") + if ! jq -e --argjson id "${id}" '.[0].id == $id' <<<"${listed}" >/dev/null; then + echo "SKIP ${organization}/${repository} 03 - Required CI: recorded ruleset identity differs; retained" + continue + fi + template=$(mktemp) + temporary_files+=("${template}") + jq --argjson record "${record}" ' + .rules[0].parameters.required_status_checks = + ($record.required_ci | map({context: ., integration_id: 15368})) + ' "${root}/config/rulesets/required-ci.json" >"${template}" + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || return 1 + if ! retired_required_ci_matches "${repository}" "${id}" "${template}" "${current}"; then + echo "SKIP ${organization}/${repository} 03 - Required CI: modified or foreign rule; retained" + continue + fi + metadata=$(github_api "repos/${organization}/${repository}") || return 1 + if ! jq -e --argjson record "${record}" --argjson enumerated "${active_repository_ids["${repository}"]}" \ + --arg full_name "${organization}/${repository}" ' + .id == $record.repository_id and .id == $enumerated and + .full_name == $full_name and .archived == false + ' <<<"${metadata}" >/dev/null; then + echo "SKIP ${organization}/${repository} 03 - Required CI: repository identity or archive state changed; retained" + continue + fi + fresh=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || return 1 + if ! retired_required_ci_matches "${repository}" "${id}" "${template}" "${fresh}"; then + echo "SKIP ${organization}/${repository} 03 - Required CI: rule changed during retirement preflight; retained" + continue + fi + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" || return 1 + done } publish_repository_rulesets() { @@ -578,7 +665,7 @@ publish_repository_rulesets() { ruleset_config=${root}/config/rulesets/default-branch-linear-history-classic.json fi upsert_repository_ruleset "${repository}" \ - "${ruleset_config}" + "${ruleset_config}" || return 1 done for repository in "${active_repositories[@]}"; do @@ -587,7 +674,7 @@ publish_repository_rulesets() { ruleset_config=${root}/config/rulesets/default-branch-integrity-classic.json fi upsert_repository_ruleset "${repository}" \ - "${ruleset_config}" + "${ruleset_config}" || return 1 done while IFS= read -r repository; do @@ -596,7 +683,7 @@ publish_repository_rulesets() { ruleset_config=${root}/config/rulesets/pull-request-gate-classic.json fi upsert_repository_ruleset "${repository}" \ - "${ruleset_config}" + "${ruleset_config}" || return 1 done < <(jq -r '.pull_request_gate[]' "${repositories_config}") while IFS= read -r repository; do @@ -617,12 +704,12 @@ publish_repository_rulesets() { ($repositories[0].required_ci[$repository] | map({context: ., integration_id: $app_id})) ' "${root}/config/rulesets/required-ci.json" >"${ci_payload}" - upsert_repository_ruleset "${repository}" "${ci_payload}" + upsert_repository_ruleset "${repository}" "${ci_payload}" || return 1 done < <(jq -r '.required_ci | keys[]' "${repositories_config}") while IFS= read -r repository; do upsert_repository_ruleset "${repository}" \ - "${root}/config/rulesets/immutable-release-tags.json" + "${root}/config/rulesets/immutable-release-tags.json" || return 1 done < <(jq -r '.release_tags[]' "${repositories_config}") while IFS= read -r maintenance; do @@ -632,21 +719,22 @@ publish_repository_rulesets() { input=$(make_maintenance_ruleset \ "${repository}" "${branch}" "${required_ci}" repository) temporary_files+=("${input}") - upsert_repository_ruleset "${repository}" "${input}" + upsert_repository_ruleset "${repository}" "${input}" || return 1 done < <(jq -c '.maintenance_branches[]' "${repositories_config}") - delete_stale_repository_maintenance_rulesets - delete_stale_repository_advisory_rulesets + delete_retired_repository_required_ci || return 1 + delete_stale_repository_maintenance_rulesets || return 1 + delete_stale_repository_advisory_rulesets || return 1 if [[ ${organization_plan} != free ]]; then - delete_managed_organization_rulesets + delete_managed_organization_rulesets || return 1 fi } publish_organization_rulesets() { local input repository ci_payload maintenance branch required_ci local -a desired_rulesets - local repositories ruleset name id desired + local repositories ruleset name id desired rulesets desired_rulesets=( "01 - Default branch integrity" @@ -667,12 +755,12 @@ publish_organization_rulesets() { "${root}/config/rulesets/default-branch-integrity-classic.json" \ "01 - Default branch integrity - classic" '["classic"]') temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 input=$(make_organization_ruleset \ "${root}/config/rulesets/default-branch-linear-history-classic.json" \ "01 - Default branch linear history - classic" '["classic"]') temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 fi repositories=$(jq -c '.repositories' "${advisory_merge_windows_config}") @@ -681,13 +769,13 @@ publish_organization_rulesets() { "${root}/config/rulesets/default-branch-linear-history.json" \ "01 - Default branch linear history" '["~ALL"]' "${repositories}") temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 input=$(make_organization_ruleset \ "${root}/config/rulesets/default-branch-integrity.json" \ "01 - Default branch integrity" '["~ALL"]' "${repositories}") temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 # Establish the Classic-specific policy before excluding it from the shared # pull-request policy. @@ -696,7 +784,7 @@ publish_organization_rulesets() { "${root}/config/rulesets/pull-request-gate-classic.json" \ "02 - Changes through pull requests - classic" '["classic"]') temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 fi repositories=$(jq -c \ @@ -707,14 +795,14 @@ publish_organization_rulesets() { "${root}/config/rulesets/pull-request-gate.json" \ "02 - Changes through pull requests" "${repositories}") temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 repositories=$(jq -c '.release_tags' "${repositories_config}") input=$(make_organization_ruleset \ "${root}/config/rulesets/immutable-release-tags.json" \ "04 - Immutable release tags" "${repositories}") temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 while IFS= read -r repository; do name="03 - Required CI - ${repository}" @@ -743,7 +831,7 @@ publish_organization_rulesets() { ($repositories[0].required_ci[$repository] | map({context: ., integration_id: $app_id})) ' "${root}/config/rulesets/required-ci.json" >"${ci_payload}" - upsert_organization_ruleset "${ci_payload}" + upsert_organization_ruleset "${ci_payload}" || return 1 done < <(jq -r '.required_ci | keys[]' "${repositories_config}") while IFS= read -r maintenance; do @@ -755,9 +843,10 @@ publish_organization_rulesets() { input=$(make_maintenance_ruleset \ "${repository}" "${branch}" "${required_ci}" organization) temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 done < <(jq -c '.maintenance_branches[]' "${repositories_config}") + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -779,12 +868,12 @@ publish_organization_rulesets() { fi done if ! ${desired}; then - run_api DELETE "orgs/${organization}/rulesets/${id}" + run_api DELETE "orgs/${organization}/rulesets/${id}" || return 1 fi - done < <(github_api "orgs/${organization}/rulesets" | jq -c '.[]') + done < <(jq -c '.[]' <<<"${rulesets}") for repository in "${active_repositories[@]}"; do - delete_managed_repository_rulesets "${repository}" + delete_managed_repository_rulesets "${repository}" || return 1 done } @@ -1000,35 +1089,433 @@ wait_for_code_security_configuration() { } converge_codeql_default_setup() { - local repository input desired_state desired_query current - local current_state current_query + local repository input desired_state desired_query current required_languages + local current_state current_query language_match for repository in "${active_repositories[@]}"; do input=${root}/config/codeql-default-setup.json if repository_uses_advanced_codeql "${repository}"; then input=${root}/config/codeql-default-setup-advanced.json fi - + required_languages=$(jq -c --arg repository "${repository}" \ + '.codeql_required_languages[$repository] // []' "${security_merge_config}") desired_state=$(jq -r '.state' "${input}") desired_query=$(jq -r '.query_suite // empty' "${input}") + current='{}' if current=$(github_api \ "repos/${organization}/${repository}/code-scanning/default-setup" \ 2>/dev/null); then current_state=$(jq -r '.state // empty' <<<"${current}") + if ! jq -e ' + (.state == "not-configured" and .languages == null) or + (.languages | type == "array" and all(.[]; type == "string" and length > 0)) + ' <<<"${current}" >/dev/null; then + echo "error: invalid existing CodeQL languages for ${repository}" >&2 + return 1 + fi current_query=$(jq -r '.query_suite // empty' <<<"${current}") - if [[ ${current_state} == "${desired_state}" ]] && + language_match=$(jq --argjson required "${required_languages}" \ + '($required - (.languages // [])) == []' <<<"${current}") + if [[ ${current_state} == "${desired_state}" && ${language_match} == true ]] && { [[ -z ${desired_query} ]] || [[ ${current_query} == "${desired_query}" ]]; }; then echo "KEEP /repos/${organization}/${repository}/code-scanning/default-setup is ${desired_state}" continue fi + else + # A failed read is unknown state, including repositories with no policy + # language minimum or with advanced setup. Never publish from that state. + echo "error: cannot read existing CodeQL languages for ${repository}" >&2 + return 1 + fi + if [[ ${required_languages} != '[]' ]]; then + local language_payload + language_payload=$(mktemp) + temporary_files+=("${language_payload}") + jq --argjson current "${current}" --argjson required "${required_languages}" \ + '.languages = (($current.languages // []) + $required | + map(if . == "javascript" or . == "typescript" then + "javascript-typescript" else . end) | unique)' \ + "${input}" >"${language_payload}" + input=${language_payload} fi - run_api PATCH \ "repos/${organization}/${repository}/code-scanning/default-setup" \ "${input}" done } +security_ruleset_matches() { + local template=$1 current=$2 + jq -e --slurpfile desired "${template}" ' + # GitHub returns the organization-owner sentinel with a null ID. + def normalize: + .bypass_actors |= (map( + if .actor_type == "OrganizationAdmin" and .actor_id == null then + .actor_id = 1 + else . end) | sort_by(.actor_type, .actor_id, .bypass_mode)) | + # GitHub expands this omitted request parameter to false. Keep true, + # null and unknown parameters visible so modified gates are not retired. + .rules |= map(if .type == "required_status_checks" then + .parameters |= ( + if .do_not_enforce_on_create == false then del(.do_not_enforce_on_create) else . end | + .required_status_checks |= sort_by(.context, .integration_id)) + else . end); + normalize as $current | + all($desired[0] | normalize | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null +} + +security_ruleset_restoration_allowed() { + local repository=$1 id=$2 template=$3 current=$4 + jq -e --argjson id "${id}" --slurpfile desired "${template}" \ + --arg source "${organization}/${repository}" ' + .id == $id and .name == $desired[0].name and + .source_type == "Repository" and .source == $source and + (.enforcement | IN("active", "disabled", "evaluate")) + ' <<<"${current}" >/dev/null && + security_ruleset_matches "${template}" "$(jq --slurpfile desired "${template}" ' + .enforcement = $desired[0].enforcement | + .conditions.ref_name.include = $desired[0].conditions.ref_name.include | + .conditions.ref_name.exclude = $desired[0].conditions.ref_name.exclude + ' <<<"${current}")" +} + +security_gate_applies_to_default_branch() { + local repository=$1 kind=$2 id=$3 metadata branch effective + metadata=$(github_api "repos/${organization}/${repository}") || return 1 + jq -e --argjson id "${active_repository_ids["${repository}"]}" ' + .id == $id and .archived == false and + (.default_branch | type == "string" and length > 0) + ' <<<"${metadata}" >/dev/null || return 1 + branch=$(jq -r '.default_branch | @uri' <<<"${metadata}") + effective=$(github_api "repos/${organization}/${repository}/rules/branches/${branch}") || return 1 + jq -e --argjson id "${id}" --arg kind "${kind}" ' + any(.[]; .ruleset_id == $id and + if $kind == "code_scanning" then + .type == "code_scanning" and + any(.parameters.code_scanning_tools[]?; .tool == "CodeQL") + else + .type == "required_status_checks" and + any(.parameters.required_status_checks[]?; + .context == "Dependency review" and .integration_id == 15368) + end) + ' <<<"${effective}" >/dev/null || return 1 + printf '%s\n' "${branch}" +} + +owned_security_ruleset_matches() { + local repository=$1 id=$2 template=$3 current=$4 + jq -e --argjson id "${id}" --arg source "${organization}/${repository}" ' + .id == $id and .source_type == "Repository" and .source == $source + ' <<<"${current}" >/dev/null && security_ruleset_matches "${template}" "${current}" +} + +delete_removed_security_merge_rulesets() { + local repository kind template name existing rule id current metadata + for repository in "${active_repositories[@]}"; do + [[ -v active_repository_ids["${repository}"] ]] || continue + # The complete live inventory was checked against governed repositories + # before this function; archived repositories never enter this collection. + for kind in code_scanning dependency_review; do + if jq -e --arg kind "${kind}" --arg repository "${repository}" \ + '.[$kind] | index($repository) != null' "${security_merge_config}" >/dev/null; then + continue + fi + if [[ ${kind} == code_scanning ]]; then + template=${root}/config/rulesets/code-scanning.json + else + template=${root}/config/rulesets/dependency-review.json + fi + name=$(jq -r '.name' "${template}") + existing=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 + while IFS= read -r rule; do + id=$(jq -r '.id' <<<"${rule}") + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || return 1 + if owned_security_ruleset_matches "${repository}" "${id}" "${template}" "${current}"; then + metadata=$(github_api "repos/${organization}/${repository}") || return 1 + if jq -e --argjson id "${active_repository_ids["${repository}"]}" \ + --arg full_name "${organization}/${repository}" ' + .id == $id and .full_name == $full_name and .archived == false + ' <<<"${metadata}" >/dev/null; then + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || { + echo "error: security gate final retirement read failed; retained: ${repository} ${name}" >&2 + return 1 + } + if ! owned_security_ruleset_matches "${repository}" "${id}" "${template}" "${current}"; then + echo "SKIP ${organization}/${repository} ${name}: rule changed during retirement preflight; retained" + continue + fi + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" || return 1 + else + echo "SKIP ${organization}/${repository} ${name}: repository identity or archive state changed; retained" + fi + else + echo "SKIP ${organization}/${repository} ${name}: unrecognized ownership or payload; retained" + fi + done < <(jq -c --arg name "${name}" '.[] | select(.name == $name)' <<<"${existing}") + done + done +} + +# These repository rules remain repository-owned even in organization scope: +# GitHub Team does not support organization-level code-scanning rules. +publish_security_merge_rulesets() { + local repository kind template name existing id current reason status readiness_mode phase metadata effective_branch proven_current + delete_removed_security_merge_rulesets || return 1 + for kind in code_scanning dependency_review; do + if [[ ${kind} == code_scanning ]]; then + template=${root}/config/rulesets/code-scanning.json + readiness_mode=codeql + else + template=${root}/config/rulesets/dependency-review.json + readiness_mode=dependency + fi + name=$(jq -r '.name' "${template}") + while IFS= read -r repository; do + if [[ ! -v active_repository_ids["${repository}"] ]]; then + echo "SKIP ${organization}/${repository} is not active; ${name}" + continue + fi + phase=$(jq -er --arg kind "${kind}" --arg repository "${repository}" \ + '.activation[$kind][$repository] | select(. == "pending" or . == "enforced")' \ + "${security_merge_config}") || { + echo "error: invalid security gate activation intent: ${repository} ${kind}" >&2 + return 1 + } + existing=$(list_rulesets \ + "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 + if ! jq -e --arg name "${name}" \ + '[.[] | select(.name == $name)] | length <= 1' <<<"${existing}" >/dev/null; then + echo "error: ambiguous managed ruleset ${name} on ${repository}" >&2 + return 1 + fi + id=$(jq -r --arg name "${name}" \ + '.[] | select(.name == $name) | .id' <<<"${existing}") + current='{}' + if [[ -n ${id} ]]; then + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") + if ! security_ruleset_restoration_allowed "${repository}" "${id}" "${template}" "${current}"; then + echo "error: security gate ownership or managed rule identity changed; retained: ${repository} ${name}" >&2 + return 1 + fi + if security_ruleset_matches "${template}" "${current}"; then + echo "KEEP /repos/${organization}/${repository}/rulesets/${id} ${name}" + continue + fi + fi + # Pending qualification cannot create protection before reviewed intent is + # persisted. Existing effective gates still survive producer failures. + effective_branch= + proven_current=${current} + if [[ ${phase} == pending ]] && { [[ -z ${id} ]] || + ! effective_branch=$(security_gate_applies_to_default_branch "${repository}" "${kind}" "${id}"); }; then + status=0 + reason=$("${root}/bin/security-readiness" "${readiness_mode}" \ + "${organization}/${repository}") || status=$? + if ((status == 0)); then + echo "READY ${organization}/${repository} ${name}: promote reviewed activation intent before installation" + else + echo "PENDING ${organization}/${repository} ${name}: ${reason}" + fi + continue + fi + # Enforced source is durable intent, not a claim about current producer + # health. Restore it only on the same active repository and owned gate. + metadata=$(github_api "repos/${organization}/${repository}") || return 1 + if ! jq -e --argjson id "${active_repository_ids["${repository}"]}" \ + --arg name "${organization}/${repository}" ' + .id == $id and .full_name == $name and .archived == false and + (.default_branch | type == "string" and length > 0) + ' <<<"${metadata}" >/dev/null; then + echo "error: security gate repository identity or archive state changed: ${repository}" >&2 + return 1 + fi + if [[ ${phase} == pending ]] && + [[ $(jq -r '.default_branch | @uri' <<<"${metadata}") != "${effective_branch}" ]]; then + echo "error: default branch changed after existing security gate proof; retained: ${repository} ${name}" >&2 + return 1 + fi + if [[ -n ${id} ]]; then + # Re-read after producer and repository checks: a concurrent policy edit + # must not be overwritten with the earlier restoration decision. + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || { + echo "error: security gate final read failed; retained: ${repository} ${name}" >&2 + return 1 + } + if ! security_ruleset_restoration_allowed "${repository}" "${id}" "${template}" "${current}"; then + echo "error: security gate changed during restoration preflight; retained: ${repository} ${name}" >&2 + return 1 + fi + # Pending intent may preserve a proven effective gate, but cannot + # restore protection revoked after that proof. Only enforced intent + # authorizes enforcement or ref-scope restoration without qualification. + if [[ ${phase} == pending ]] && ! jq -e --argjson proven "${proven_current}" ' + def protection: + {enforcement, ref_name: (.conditions.ref_name | + .include |= sort | .exclude |= sort)}; + protection == ($proven | protection) + ' <<<"${current}" >/dev/null; then + echo "error: existing security gate protection changed after effective proof; retained: ${repository} ${name}" >&2 + return 1 + fi + if security_ruleset_matches "${template}" "${current}"; then + echo "KEEP /repos/${organization}/${repository}/rulesets/${id} ${name}" + continue + fi + run_api PUT "repos/${organization}/${repository}/rulesets/${id}" "${template}" + else + run_api POST "repos/${organization}/${repository}/rulesets" "${template}" + fi + done < <(jq -r --arg kind "${kind}" '.[$kind][]' "${security_merge_config}") + done +} + +validate_sha_canary_target() { + local canary=$1 owner repository number metadata + IFS=/ read -r owner repository number <<<"${canary}" + if [[ ${owner} != "${organization}" || ! ${number} =~ ^[1-9][0-9]*$ || + ! ${repository} =~ ^[A-Za-z0-9_.-]+$ ]] || + [[ ! -v active_repository_ids["${repository}"] ]]; then + echo "error: SHA canary must identify an active governed repository in ${organization}" >&2 + return 1 + fi + metadata=$(github_api "repos/${organization}/${repository}") + if ! jq -e --argjson id "${active_repository_ids["${repository}"]}" \ + --arg name "${organization}/${repository}" ' + .id == $id and .full_name == $name and .archived == false + ' <<<"${metadata}" >/dev/null; then + echo "error: SHA canary repository identity or archive state changed" >&2 + return 1 + fi +} + +converge_actions_permissions() { + local current selected input reason phase canary staging_needed=false + local permissions_filter='type == "object" and + (.enabled_repositories | IN("all", "none", "selected")) and + (.allowed_actions | IN("all", "local_only", "selected")) and + (.sha_pinning_required == null or (.sha_pinning_required | type == "boolean"))' + current=$(github_api "orgs/${organization}/actions/permissions") || return 1 + if ! jq -e "${permissions_filter}" <<<"${current}" >/dev/null; then + echo "error: cannot read current organization Actions permissions" >&2 + return 1 + fi + phase=$(jq -r '.phase' "${root}/config/actions-sha-pinning.json") + canary=$(jq -r '.canary // empty' "${root}/config/actions-sha-pinning.json") + if [[ ${phase} == pending && -n ${sha_pinning_canary} ]]; then + validate_sha_canary_target "${sha_pinning_canary}" || return 1 + elif [[ ${phase} == enforced ]] && + ! jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then + validate_sha_canary_target "${canary}" || return 1 + fi + + # GitHub requires selected mode before its selected-actions endpoint can be + # updated. Stage both conservative controls before proving SHA compatibility. + # Omit SHA entirely: this preserves an existing value without enabling it + # from a stale read if another actor changes the field during staging. + input=$(mktemp) + temporary_files+=("${input}") + jq 'del(.sha_pinning_required)' "${root}/config/actions-permissions.json" >"${input}" + if jq -e --slurpfile desired "${input}" ' + . as $current | all($desired[0] | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + echo "KEEP /orgs/${organization}/actions/permissions" + else + staging_needed=true + run_api PUT "orgs/${organization}/actions/permissions" "${input}" || return 1 + fi + # This endpoint may require selected mode. A dry run with a mode change + # can plan its update, but cannot claim to have read the staged live policy. + if ! ${apply} && [[ $(jq -r '.allowed_actions' <<<"${current}") != selected ]]; then + selected=null + else + selected=$(github_api "orgs/${organization}/actions/permissions/selected-actions") || return 1 + if ! jq -e ' + (.github_owned_allowed | type == "boolean") and + (.verified_allowed | type == "boolean") and + (.patterns_allowed | type == "array" and all(.[]; type == "string")) + ' <<<"${selected}" >/dev/null; then + echo "error: cannot read current selected Actions policy" >&2 + return 1 + fi + fi + if jq -e --slurpfile desired "${root}/config/actions-selected.json" ' + .github_owned_allowed == $desired[0].github_owned_allowed and + .verified_allowed == $desired[0].verified_allowed and + (.patterns_allowed | sort) == ($desired[0].patterns_allowed | sort) + ' <<<"${selected}" >/dev/null; then + echo "KEEP /orgs/${organization}/actions/permissions/selected-actions" + else + staging_needed=true + run_api PUT "orgs/${organization}/actions/permissions/selected-actions" \ + "${root}/config/actions-selected.json" || return 1 + fi + if ${apply}; then + # Base the SHA decision on live post-staging state: another writer may + # disable it while conservative mode or pattern updates are in flight. + current=$(github_api "orgs/${organization}/actions/permissions") || return 1 + if ! jq -e "${permissions_filter}" <<<"${current}" >/dev/null || + ! jq -e --slurpfile desired "${input}" ' + . as $current | all($desired[0] | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + echo "error: organization Actions permissions unreadable or changed during staging" >&2 + return 1 + fi + fi + # An explicitly requested pending-phase probe is read-only and independent + # of whether SHA pinning is already enabled. Do not let preservation skip it. + if [[ ${phase} == pending && -n ${sha_pinning_canary} ]]; then + validate_sha_canary_target "${sha_pinning_canary}" || return 1 + if reason=$("${root}/bin/security-readiness" sha "${sha_pinning_canary}"); then + echo "READY canary evidence: ${reason}; source rollout phase remains pending" + else + echo "PENDING canary evidence: ${reason}" + fi + fi + if jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then + return 0 + fi + if [[ ${phase} == enforced ]]; then + if ! ${apply} && ${staging_needed}; then + echo "PLAN SHA pinning proof and restoration after Actions policy staging is applied" + return 0 + fi + validate_sha_canary_target "${canary}" || return 1 + if ! reason=$("${root}/bin/security-readiness" sha "${canary}"); then + echo "error: enforced SHA pinning cannot be restored: ${reason}" >&2 + return 1 + fi + echo "READY enforced SHA pinning: ${reason}" + run_api PUT "orgs/${organization}/actions/permissions" \ + "${root}/config/actions-permissions.json" || return 1 + else + echo "PENDING SHA pinning: promote the reviewed source rollout phase and accepted canary after readiness" + fi +} + +converge_repository_workflow_permissions() { + local repository current + for repository in "${active_repositories[@]}"; do + [[ -v active_repository_ids["${repository}"] ]] || continue + current=$(github_api "repos/${organization}/${repository}/actions/permissions/workflow") + if ! jq -e 'type == "object" and + (.default_workflow_permissions | IN("read", "write")) and + (.can_approve_pull_request_reviews | type == "boolean") + ' <<<"${current}" >/dev/null; then + echo "error: cannot read repository workflow permissions: ${repository}" >&2 + return 1 + fi + if jq -e --slurpfile desired "${root}/config/actions-workflow.json" ' + . as $current | all($desired[0] | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + echo "KEEP /repos/${organization}/${repository}/actions/permissions/workflow" + else + run_api PUT "repos/${organization}/${repository}/actions/permissions/workflow" \ + "${root}/config/actions-workflow.json" + fi + done +} + publish_organization_security() { local advanced_id baseline_id defaults_payload defaults local advanced_repository_count baseline_repository_count @@ -1357,10 +1844,7 @@ preflight_new_crate_immutable_selection converge_immutable_release_policy converge_organization_settings "${organization_metadata}" -run_api PUT "orgs/${organization}/actions/permissions" \ - "${root}/config/actions-permissions.json" -run_api PUT "orgs/${organization}/actions/permissions/selected-actions" \ - "${root}/config/actions-selected.json" +converge_actions_permissions run_api PUT "orgs/${organization}/actions/permissions/workflow" \ "${root}/config/actions-workflow.json" @@ -1369,6 +1853,8 @@ for repository in "${active_repositories[@]}"; do "${root}/config/repository-defaults.json" done +converge_repository_workflow_permissions + if [[ ${policy_scope} == organization ]]; then publish_organization_rulesets publish_organization_security @@ -1377,6 +1863,8 @@ else publish_repository_security fi +publish_security_merge_rulesets + while IFS= read -r repository; do if ! repository_archived=$( github_api "repos/${organization}/${repository}" --jq '.archived' 2>/dev/null diff --git a/bin/security-readiness b/bin/security-readiness new file mode 100755 index 0000000..a00982e --- /dev/null +++ b/bin/security-readiness @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +exec python3 "${root}/lib/security_readiness.py" "$@" diff --git a/bin/validate b/bin/validate index 9697042..f50643f 100755 --- a/bin/validate +++ b/bin/validate @@ -51,6 +51,62 @@ jq -e ' } ' "${organization_config}" >/dev/null +jq -e --slurpfile repositories "${repositories_config}" \ + --slurpfile advanced "${advanced_codeql_config}" ' + . as $policy | + (keys == ["activation", "code_scanning", "codeql_required_languages", "dependency_review"]) and + (.activation | type == "object" and keys == ["code_scanning", "dependency_review"]) and + all(["code_scanning", "dependency_review"][]; . as $kind | + ($policy.activation[$kind] | type == "object" and keys == $policy[$kind]) and + all($policy.activation[$kind][]; IN("pending", "enforced"))) and + .dependency_review == ($repositories[0].required_ci | keys) and + .code_scanning == .dependency_review and + (.codeql_required_languages | type == "object") and + all(.codeql_required_languages | to_entries[]; + .key as $repository | + ($repositories[0].required_ci | has($repository)) and + ($advanced[0].repositories | index($repository)) == null and + (.value | type == "array" and length > 0 and . == (sort | unique)) and + all(.value[]; IN("c-cpp", "csharp", "go", "java-kotlin", + "javascript-typescript", "python", "ruby", "rust", "swift"))) and + .codeql_required_languages["content-toolkit"] == ["rust"] +' "${root}/config/security-merge-protection.json" >/dev/null + +jq -e '.sha_pinning_required == true' \ + "${root}/config/actions-permissions.json" >/dev/null + +jq -e --slurpfile repositories "${repositories_config}" ' + (keys == ["canary", "phase"]) and + if .phase == "pending" then .canary == null + elif .phase == "enforced" then + (.canary | type == "string" and test("^atrinik/[A-Za-z0-9_.-]+/[1-9][0-9]*$")) and + ((.canary | split("/")[1]) as $repository | + ($repositories[0].required_ci | has($repository))) + else false end +' "${root}/config/actions-sha-pinning.json" >/dev/null + +for security_ruleset in code-scanning dependency-review; do + jq -e ' + .target == "branch" and .enforcement == "active" and + .conditions == {ref_name: {include: ["~DEFAULT_BRANCH"], exclude: []}} and + .bypass_actors == [{actor_id: 1, actor_type: "OrganizationAdmin", + bypass_mode: "pull_request"}] + ' "${root}/config/rulesets/${security_ruleset}.json" >/dev/null +done +jq -e ' + .name == "06 - CodeQL security" and + .rules == [{type: "code_scanning", parameters: {code_scanning_tools: [{ + tool: "CodeQL", security_alerts_threshold: "high_or_higher", alerts_threshold: "none" + }]}}] +' "${root}/config/rulesets/code-scanning.json" >/dev/null +jq -e ' + .name == "07 - Dependency review" and + .rules == [{type: "required_status_checks", parameters: { + required_status_checks: [{context: "Dependency review", integration_id: 15368}], + strict_required_status_checks_policy: true + }}] +' "${root}/config/rulesets/dependency-review.json" >/dev/null + jq -e ' .bypass_actors == [] and [.rules[].type] == ["deletion", "non_fast_forward"] @@ -160,6 +216,26 @@ jq -e ' ($branches | length) == ($branches | unique | length)) ' "${repositories_config}" >/dev/null +jq -e --slurpfile repositories "${repositories_config}" ' + (keys == ["retirements"]) and (.retirements | type == "array") and + all(.retirements[]; + (keys == ["repository", "repository_id", "required_ci", "ruleset_id"]) and + (.repository | type == "string" and length > 0) and + (.repository_id | type == "number" and floor == . and . > 0) and + (.ruleset_id | type == "number" and floor == . and . > 0) and + (.required_ci | type == "array" and length > 0) and + all(.required_ci[]; type == "string" and test("[^[:space:]]")) and + ((.required_ci | length) == (.required_ci | unique | length)) and + (.repository as $repository | + ($repositories[0].required_ci | has($repository) | not) and + ($repositories[0].pull_request_gate | index($repository)) != null and + ($repositories[0].archive | index($repository)) == null) + ) and + ((.retirements | map(.repository) | length) == (.retirements | map(.repository) | unique | length)) and + ((.retirements | map(.repository_id) | length) == (.retirements | map(.repository_id) | unique | length)) and + ((.retirements | map(.ruleset_id) | length) == (.retirements | map(.ruleset_id) | unique | length)) +' "${root}/config/retired-required-ci.json" >/dev/null + jq -e \ --slurpfile repositories "${repositories_config}" ' (keys == ["retirements"]) and @@ -221,7 +297,18 @@ jq -e \ jq -e \ --slurpfile repositories "${repositories_config}" ' - (keys == ["repositories"]) and + (keys == ["repositories", "required_partitions"]) and + (.required_partitions | type == "object") and + ((.required_partitions | keys) == (.repositories | sort)) and + all(.required_partitions[]; + type == "array" and length > 0 and + all(.[]; + (keys == ["language", "scope"]) and + (.scope | type == "string" and test("^[a-z][a-z0-9+-]*$")) and + (.language | type == "string" and test("^[a-z][a-z0-9+-]*$")) + ) and + ((map([.scope, .language]) | length) == (map([.scope, .language]) | unique | length)) + ) and (.repositories | type == "array") and ((.repositories | length) == (.repositories | unique | length)) and all( @@ -852,12 +939,8 @@ jq -e \ .bootstrap_policy == "owner-local-api-token-once-then-oidc-only" and (.authentication_action | type == "string" and test("^rust-lang/crates-io-auth-action@[0-9a-f]{40}$")) and - (.authentication_action as $action | - ($actions[0].patterns_allowed | map(select(startswith("rust-lang/")))) as $patterns | - ($patterns | length) > 0 and - all($patterns[]; test("^rust-lang/crates-io-auth-action@([0-9a-f]{40}|\\*)$")) and - (($patterns | index($action)) != null or - ($patterns | index("rust-lang/crates-io-auth-action@*")) != null)) and + (($actions[0].patterns_allowed | map(select(startswith("rust-lang/")))) == + ["rust-lang/crates-io-auth-action@*"]) and .provider == "github" and $actions[0].github_owned_allowed == true and ($repositories[0].pull_request_gate | index($name)) != null and diff --git a/config/actions-permissions.json b/config/actions-permissions.json index 9f8c2fe..8ed7a80 100644 --- a/config/actions-permissions.json +++ b/config/actions-permissions.json @@ -1,4 +1,5 @@ { "enabled_repositories": "all", - "allowed_actions": "selected" + "allowed_actions": "selected", + "sha_pinning_required": true } diff --git a/config/actions-sha-pinning.json b/config/actions-sha-pinning.json new file mode 100644 index 0000000..8e5861e --- /dev/null +++ b/config/actions-sha-pinning.json @@ -0,0 +1,4 @@ +{ + "phase": "pending", + "canary": null +} diff --git a/config/codeql-advanced-setup.json b/config/codeql-advanced-setup.json index 114f975..808abec 100644 --- a/config/codeql-advanced-setup.json +++ b/config/codeql-advanced-setup.json @@ -1,5 +1,29 @@ { "repositories": [ "classic" - ] + ], + "required_partitions": { + "classic": [ + { + "scope": "client", + "language": "c-cpp" + }, + { + "scope": "server", + "language": "c-cpp" + }, + { + "scope": "core-cpp", + "language": "c-cpp" + }, + { + "scope": "python", + "language": "python" + }, + { + "scope": "actions", + "language": "actions" + } + ] + } } diff --git a/config/retired-required-ci.json b/config/retired-required-ci.json new file mode 100644 index 0000000..80b8b75 --- /dev/null +++ b/config/retired-required-ci.json @@ -0,0 +1,3 @@ +{ + "retirements": [] +} diff --git a/config/rulesets/code-scanning.json b/config/rulesets/code-scanning.json new file mode 100644 index 0000000..a3a0f14 --- /dev/null +++ b/config/rulesets/code-scanning.json @@ -0,0 +1,34 @@ +{ + "name": "06 - CodeQL security", + "target": "branch", + "enforcement": "active", + "bypass_actors": [ + { + "actor_id": 1, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "rules": [ + { + "type": "code_scanning", + "parameters": { + "code_scanning_tools": [ + { + "tool": "CodeQL", + "security_alerts_threshold": "high_or_higher", + "alerts_threshold": "none" + } + ] + } + } + ] +} diff --git a/config/rulesets/dependency-review.json b/config/rulesets/dependency-review.json new file mode 100644 index 0000000..2fe155b --- /dev/null +++ b/config/rulesets/dependency-review.json @@ -0,0 +1,34 @@ +{ + "name": "07 - Dependency review", + "target": "branch", + "enforcement": "active", + "bypass_actors": [ + { + "actor_id": 1, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "rules": [ + { + "type": "required_status_checks", + "parameters": { + "required_status_checks": [ + { + "context": "Dependency review", + "integration_id": 15368 + } + ], + "strict_required_status_checks_policy": true + } + } + ] +} diff --git a/config/security-merge-protection.json b/config/security-merge-protection.json new file mode 100644 index 0000000..72a10c3 --- /dev/null +++ b/config/security-merge-protection.json @@ -0,0 +1,101 @@ +{ + "dependency_review": [ + "agent-integrations", + "atrinik", + "classic", + "client", + "content", + "content-toolkit", + "deploy-control", + "devcontainer", + "editor", + "github-settings", + "metaserver-worker", + "observatory", + "playtester", + "protocol", + "renderer", + "resources", + "server", + "service-updater", + "sound", + "tools", + "website" + ], + "code_scanning": [ + "agent-integrations", + "atrinik", + "classic", + "client", + "content", + "content-toolkit", + "deploy-control", + "devcontainer", + "editor", + "github-settings", + "metaserver-worker", + "observatory", + "playtester", + "protocol", + "renderer", + "resources", + "server", + "service-updater", + "sound", + "tools", + "website" + ], + "codeql_required_languages": { + "content-toolkit": [ + "rust" + ] + }, + "activation": { + "code_scanning": { + "agent-integrations": "pending", + "atrinik": "pending", + "classic": "pending", + "client": "pending", + "content": "pending", + "content-toolkit": "pending", + "deploy-control": "pending", + "devcontainer": "pending", + "editor": "pending", + "github-settings": "pending", + "metaserver-worker": "pending", + "observatory": "pending", + "playtester": "pending", + "protocol": "pending", + "renderer": "pending", + "resources": "pending", + "server": "pending", + "service-updater": "pending", + "sound": "pending", + "tools": "pending", + "website": "pending" + }, + "dependency_review": { + "agent-integrations": "pending", + "atrinik": "pending", + "classic": "pending", + "client": "pending", + "content": "pending", + "content-toolkit": "pending", + "deploy-control": "pending", + "devcontainer": "pending", + "editor": "pending", + "github-settings": "pending", + "metaserver-worker": "pending", + "observatory": "pending", + "playtester": "pending", + "protocol": "pending", + "renderer": "pending", + "resources": "pending", + "server": "pending", + "service-updater": "pending", + "sound": "pending", + "tools": "pending", + "website": "pending" + } + } +} diff --git a/lib/security_readiness.py b/lib/security_readiness.py new file mode 100644 index 0000000..e34618a --- /dev/null +++ b/lib/security_readiness.py @@ -0,0 +1,1216 @@ +#!/usr/bin/env python3 +"""Read-only preflight for security controls; 0 ready, 1 deferred, 2 unreadable.""" + +import argparse +import ast +import base64 +import binascii +import difflib +import fnmatch +import json +from pathlib import Path +import re +import shlex +import subprocess +import sys +from urllib.parse import quote, urlencode + + +ROOT = Path(__file__).resolve().parent.parent +VERSION = "2026-03-10" +ACTION_APP = 15368 +SHA = re.compile(r"[0-9a-fA-F]{40}\Z") +COORDINATE = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z") +SIMPLE_NAME = r"(?:[A-Za-z][A-Za-z0-9 ._/-]*|'[A-Za-z][A-Za-z0-9 ._/-]*'|\"[A-Za-z][A-Za-z0-9 ._/-]*\")" +USES = re.compile( + r"(?P[ \t]*(?:-[ \t]+)?uses:[ \t]+)" + r"(?P[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+)@" + r"(?P[0-9a-fA-F]{40})(?P[ \t]*)" + r"(?P#[ \t]*v?[0-9]+(?:\.[0-9A-Za-z_-]+)*(?:[ \t]*)?)?" + r"(?P\r?\n)?\Z" +) + + +class Deferred(Exception): + pass + + +class EvidenceError(Exception): + pass + + +class APIError(EvidenceError): + def __init__(self, endpoint, status=None, response=None): + self.status = status + self.response = response + super().__init__(f"API unreadable: {endpoint.split('?')[0]}" + + (f" (HTTP {status})" if status else "")) + + +def api(endpoint): + """Use GET explicitly: query parameters must never select gh's POST default.""" + try: + result = subprocess.run( + ["gh", "api", "--method", "GET", "-H", f"X-GitHub-Api-Version: {VERSION}", + "-H", "Accept: application/vnd.github+json", endpoint], + capture_output=True, text=True, timeout=30, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise APIError(endpoint) from exc + if result.returncode: + status = re.search(r"HTTP (\d{3})", result.stderr) + try: + response = json.loads(result.stdout) + except json.JSONDecodeError: + response = None + raise APIError(endpoint, int(status[1]) if status else None, response) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise EvidenceError("API returned invalid JSON") from exc + + +def config(name): + return json.loads((ROOT / "config" / name).read_text()) + + +def require(condition, reason): + if not condition: + raise Deferred(reason) + + +def repository(coordinate): + metadata = api(f"repos/{coordinate}") + if metadata["full_name"].lower() != coordinate.lower(): + raise EvidenceError("repository identity changed") + require(metadata["archived"] is False, "repository archived") + if not isinstance(metadata["default_branch"], str) or not metadata["default_branch"]: + raise EvidenceError("default branch missing") + return metadata + + +def contents(coordinate, path, ref): + endpoint = f"repos/{coordinate}/contents/{path}?{urlencode({'ref': ref})}" + try: + value = api(endpoint) + except APIError as exc: + if exc.status == 404: + raise Deferred(f"{path} missing at required revision") from exc + raise + if value.get("type") != "file" or value.get("encoding") != "base64": + raise EvidenceError("workflow content is not an encoded file") + try: + # GitHub wraps content with newlines; reject other non-base64 bytes. + return base64.b64decode(value["content"].replace("\n", ""), validate=True) + except (binascii.Error, ValueError) as exc: + raise EvidenceError("workflow content encoding invalid") from exc + + +def pages(endpoint, key=None): + """Bound enumeration; missing or oversized evidence cannot imply readiness.""" + results = [] + for page in range(1, 11): + value = api(endpoint + ("&" if "?" in endpoint else "?") + + urlencode({"per_page": 100, "page": page})) + entries = value[key] if key else value + if not isinstance(entries, list): + raise EvidenceError("API collection malformed") + results.extend(entries) + if len(entries) < 100: + if key and "total_count" in value and len(results) != value["total_count"]: + raise EvidenceError("API collection changed during enumeration") + return results + raise Deferred("evidence exceeds bounded enumeration") + + +def latest_dependency_run(coordinate): + try: + value = api(f"repos/{coordinate}/actions/workflows/dependency-review.yml/runs" + "?event=pull_request&per_page=1") + except APIError as exc: + if exc.status == 404: + raise Deferred("dependency-review workflow not landed") from exc + raise + require(bool(value["workflow_runs"]), "dependency-review PR run missing") + return value["workflow_runs"][0] + + +def dependency_contract(data): + """Allow producer-owned immutable pin bumps without weakening the contract. + + Keep indentation and every policy value intact. Ignore blank lines, standalone + comments and trailing whitespace; normalize only this official action's SHA + and its optional same-line version annotation. + """ + lines = [] + pins = 0 + for line in data.decode("utf-8").splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + match = USES.fullmatch(line) + if match and match["action"] == "actions/dependency-review-action": + pins += 1 + line = match["prefix"] + "actions/dependency-review-action@" + if re.fullmatch("name: " + SIMPLE_NAME, line.rstrip()): + line = "name: Dependency review" + lines.append(line.rstrip()) + require(pins == 1, "dependency-review workflow requires one official full-SHA action") + normalized = [] + index = 0 + while index < len(lines): + line = lines[index] + step_name = re.fullmatch(r"( *)- name: " + SIMPLE_NAME, line) + if step_name and index + 1 < len(lines): + following = lines[index + 1] + if following == step_name[1] + " uses: actions/dependency-review-action@": + line = step_name[1] + "- uses: actions/dependency-review-action@" + index += 1 + # Producers may spell the same one-element branch filter as a block + # list or a flow list. Preserve key indentation and reject extra items. + flow = re.fullmatch(r"( *)branches: *\[ *(?:main|'main'|\"main\") *\]", line) + block = re.fullmatch(r"( *)branches:", line) + if flow: + line = flow[1] + "branches: [main]" + elif block and index + 1 < len(lines): + item = re.fullmatch(r"( *)- +(?:main|'main'|\"main\")", lines[index + 1]) + if item and len(item[1]) > len(block[1]): + line = block[1] + "branches: [main]" + index += 1 + # Activity order and flow/block spelling do not change which PR or + # merge-group events trigger review. Keep duplicates and unknown events visible to the + # contract comparison rather than silently broadening the trigger. + activity_flow = re.fullmatch(r"( *)types: *\[(.*)\]", line) + activity_block = re.fullmatch(r"( *)types:", line) + activities = None + if activity_flow: + activities = [item.strip() for item in activity_flow[2].split(",")] + indentation = activity_flow[1] + elif activity_block: + activities = [] + indentation = activity_block[1] + while index + 1 < len(lines): + item = re.fullmatch(r"( *)- +(.+)", lines[index + 1]) + if not item or len(item[1]) <= len(indentation): + break + activities.append(item[2]) + index += 1 + if activities is not None: + values = [] + for activity in activities: + value = re.fullmatch(r"(?:([a-z_]+)|'([a-z_]+)'|\"([a-z_]+)\")", activity) + require(value is not None, "dependency-review activity types unsupported") + values.append(next(part for part in value.groups() if part is not None)) + line = indentation + "types: [" + ", ".join(sorted(values)) + "]" + normalized.append(line) + index += 1 + # YAML mapping key order does not change the PR trigger. Reorder only the + # two recognized, already-normalized keys; retain every value and any + # additional or duplicated key for the exact contract comparison below. + for index in range(len(normalized) - 2): + if (normalized[index] == " pull_request:" + and normalized[index + 1].startswith(" types: [") + and normalized[index + 2] == " branches: [main]"): + normalized[index + 1:index + 3] = [normalized[index + 2], normalized[index + 1]] + return normalized + + +def pull_identity(pr): + return {"number": pr["number"], "state": pr["state"], "merged": pr["merged"], "draft": pr["draft"], + "head": (pr["head"]["sha"], pr["head"]["repo"]["full_name"]), + "base": (pr["base"]["sha"], pr["base"]["ref"], pr["base"]["repo"]["full_name"])} + + +def merged_run_pull(coordinate, run, default_branch): + """Recover only an unambiguous, fully fetched merged PR after refs disappear.""" + require(run["event"] == "pull_request" + and re.fullmatch(r"\.github/workflows/[^/]+\.ya?ml", run["path"]) is not None, + "merged PR association requires a PR workflow") + require(bool(SHA.fullmatch(run["head_sha"])), "merged PR association head missing") + associated = pages(f"repos/{coordinate}/commits/{run['head_sha']}/pulls") + require(len(associated) == 1, "merged PR association missing or ambiguous") + pr = api(f"repos/{coordinate}/pulls/{associated[0]['number']}") + require(pr["number"] == associated[0]["number"] and pr["merged"] is True + and pr["state"] == "closed" and pr["head"]["sha"] == run["head_sha"] + and pr["head"]["repo"]["full_name"].lower() == run["head_repository"]["full_name"].lower() + and pr["base"]["repo"]["full_name"].lower() == coordinate.lower() + and pr["base"]["ref"] == default_branch, + "merged PR association identity does not match run/default branch") + return pr + + +def association_references(coordinate, run, suite, default_branch, recovered): + if run["pull_requests"] and suite["pull_requests"]: + return run["pull_requests"], suite["pull_requests"] + pr = merged_run_pull(coordinate, run, default_branch) + for references in (run["pull_requests"], suite["pull_requests"]): + require(all(item["number"] == pr["number"] + and item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["sha"] == pr["base"]["sha"] + and item["base"]["ref"] == pr["base"]["ref"] for item in references), + "nonempty PR association conflicts with merged PR") + recovered.append((run, default_branch, pull_identity(pr))) + return [pr], [pr] + + +def verify_recovered_associations(coordinate, recovered): + for run, default_branch, identity in recovered: + require(pull_identity(merged_run_pull(coordinate, run, default_branch)) == identity, + "merged PR association changed during preflight") + + +def dependency(coordinate): + metadata = repository(coordinate) + path = ".github/workflows/dependency-review.yml" + desired = dependency_contract((ROOT / path).read_bytes()) + landed = contents(coordinate, path, metadata["default_branch"]) + require(dependency_contract(landed) == desired, + "dependency-review workflow not landed with desired enforcement contract") + run = latest_dependency_run(coordinate) + require(run["event"] == "pull_request" and run["status"] == "completed" + and run["conclusion"] == "success", "latest dependency-review PR run not successful") + require(run["path"] == path, "dependency-review run workflow path changed") + suite_endpoint = f"repos/{coordinate}/check-suites/{run['check_suite_id']}" + suite = api(suite_endpoint) + require(suite["id"] == run["check_suite_id"] and suite["app"]["id"] == ACTION_APP + and suite["status"] == "completed" and suite["conclusion"] == "success", + "dependency-review execution check suite not successful") + executed = suite["head_sha"] + require(bool(SHA.fullmatch(executed)), "dependency-review executed SHA missing") + merge_parents = None + if executed != run["head_sha"]: + merge_commit = api(f"repos/{coordinate}/commits/{executed}") + merge_parents = [parent["sha"] for parent in merge_commit["parents"]] + recovered, pulls = [], [] + references, suite_references = association_references( + coordinate, run, suite, metadata["default_branch"], recovered) + for reference in references: + pr = api(f"repos/{coordinate}/pulls/{reference['number']}") + require(pr["base"]["repo"]["full_name"].lower() == coordinate.lower() + and pr["base"]["ref"] == metadata["default_branch"], + "dependency-review PR does not target default branch") + require(pr["head"]["sha"] == run["head_sha"], "dependency-review PR head changed") + require(reference["head"]["sha"] == pr["head"]["sha"] + and reference["base"]["sha"] == pr["base"]["sha"] + and reference["base"]["ref"] == pr["base"]["ref"], + "dependency-review run PR association is stale") + require(any(item["number"] == pr["number"] + and item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["sha"] == pr["base"]["sha"] + and item["base"]["ref"] == pr["base"]["ref"] + for item in suite_references), + "dependency-review execution PR association missing or stale") + if merge_parents is not None: + require(len(merge_parents) == 2 and set(merge_parents) + == {pr["head"]["sha"], pr["base"]["sha"]}, + "dependency-review executed merge does not match current PR head/base") + pulls.append(pr) + require(contents(coordinate, path, executed) == landed, + "dependency-review run used different workflow bytes") + jobs = pages(f"repos/{coordinate}/actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs", "jobs") + matches = [job for job in jobs if job["name"] == "Dependency review"] + require(len(matches) == 1 and matches[0]["status"] == "completed" + and matches[0]["conclusion"] == "success", "Dependency review job not successful") + fresh = latest_dependency_run(coordinate) + run_keys = ("id", "run_attempt", "status", "conclusion", "event", "head_sha", + "check_suite_id", "pull_requests", "path", "head_repository") + require(all(fresh[key] == run[key] for key in run_keys), + "dependency-review run changed during preflight") + fresh_suite = api(suite_endpoint) + suite_keys = ("id", "head_sha", "app", "status", "conclusion", "pull_requests") + require(all(fresh_suite[key] == suite[key] for key in suite_keys), + "dependency-review execution check suite changed during preflight") + for pr in pulls: + fresh_pr = api(f"repos/{coordinate}/pulls/{pr['number']}") + require(fresh_pr["head"]["sha"] == pr["head"]["sha"] + and fresh_pr["base"]["sha"] == pr["base"]["sha"] + and fresh_pr["base"]["ref"] == pr["base"]["ref"] + and fresh_pr["base"]["repo"]["full_name"] == pr["base"]["repo"]["full_name"] + and all(fresh_pr[key] == pr[key] for key in ("state", "draft", "merged")), + "dependency-review PR changed during preflight") + verify_recovered_associations(coordinate, recovered) + require(contents(coordinate, path, metadata["default_branch"]) == landed, + "default-branch dependency-review workflow changed during preflight") + require(repository(coordinate)["default_branch"] == metadata["default_branch"], + "repository default branch changed during preflight") + return "workflow enforcement contract landed; latest PR job successful with same workflow bytes" + + +def pin_changes(before, after, selected): + """Compare complete files, so API's potentially truncated patches grant nothing.""" + old = before.decode("utf-8").splitlines(keepends=True) + new = after.decode("utf-8").splitlines(keepends=True) + changed = 0 + for tag, old_start, old_end, new_start, new_end in difflib.SequenceMatcher( + None, old, new, autojunk=False).get_opcodes(): + if tag == "equal": + continue + require(tag == "replace" and old_end - old_start == new_end - new_start, + "canary changes more than action pins") + for left, right in zip(old[old_start:old_end], new[new_start:new_end]): + left_match, right_match = USES.fullmatch(left), USES.fullmatch(right) + require(bool(left_match and right_match), "canary changes more than action pins") + for key in ("prefix", "action", "space", "newline"): + require(left_match[key] == right_match[key], "canary changes action structure") + require(left_match["sha"] != right_match["sha"], "canary contains unrelated comment changes") + action = right_match["action"] + # GitHub's action SHA requirement excludes reusable workflow refs. + # The canary admits only action-pin changes, including in mixed PRs. + require(re.fullmatch(r"[^/]+/[^/]+/\.github/workflows/[^/]+\.ya?ml", action, + flags=re.IGNORECASE) is None, + "canary changes a reusable workflow reference, not only action pins") + identity = f"{action}@{right_match['sha']}" + require((selected["github_owned_allowed"] and action.split("/")[0].lower() + in {"actions", "github"}) + or any(fnmatch.fnmatchcase(identity, pattern) + for pattern in selected["patterns_allowed"]), + "new action pin is outside desired selected-actions patterns") + changed += 1 + require(changed > 0, "canary has no full-SHA action update") + return changed + + +def merge_check_run(coordinate, check, pr, merges, proofs, recovered): + """Establish PR association before selecting the latest check, even if failing.""" + if check["head_sha"] != pr["head"]["sha"] and check["head_sha"] not in merges: + return False + suite_id = check["check_suite"]["id"] + if suite_id not in proofs: + endpoint = f"repos/{coordinate}/actions/runs?check_suite_id={suite_id}&per_page=1" + value = api(endpoint) + runs = value["workflow_runs"] + valid = bool(len(runs) == 1 and runs[0]["event"] == "pull_request" + and runs[0]["head_sha"] == pr["head"]["sha"] + and runs[0]["check_suite_id"] == suite_id) + suite, bases = None, set() + if valid: + suite = api(f"repos/{coordinate}/check-suites/{suite_id}") + references, suite_references = association_references( + coordinate, runs[0], suite, pr["base"]["ref"], recovered) + association_bases = [ + {item["base"]["sha"] for item in refs if item["number"] == pr["number"] + and item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["ref"] == pr["base"]["ref"]} + for refs in (references, suite_references)] + bases = association_bases[0] & association_bases[1] + # Open PRs require the current base. Merged PRs retain the common + # recorded base from their run and suite, even after main advances. + if not pr["merged"]: + bases &= {pr["base"]["sha"]} + valid = suite["id"] == suite_id and suite["app"]["id"] == ACTION_APP and bool(bases) + proofs[suite_id] = {"endpoint": endpoint, "run": runs[0] if valid else None, + "suite": suite, "bases": bases, "parents": {}} + proof = proofs[suite_id] + if proof["run"] is None: + return False + if check["head_sha"] != pr["head"]["sha"]: + if check["head_sha"] not in proof["parents"]: + commit = api(f"repos/{coordinate}/commits/{check['head_sha']}") + proof["parents"][check["head_sha"]] = [item["sha"] for item in commit["parents"]] + parents = proof["parents"][check["head_sha"]] + return len(parents) == 2 and any(set(parents) == {base, pr["head"]["sha"]} + for base in proof["bases"]) + return True + + +def canary_actions_policy(coordinate, metadata): + organization = coordinate.split('/')[0] + if type(metadata["id"]) is not int or metadata["id"] < 1: + raise EvidenceError("canary repository identity malformed") + current = api(f"orgs/{organization}/actions/permissions") + if (current["allowed_actions"] not in {"all", "local_only", "selected"} + or current["enabled_repositories"] not in {"all", "none", "selected"}): + raise EvidenceError("organization Actions permissions malformed") + require(current["allowed_actions"] == "selected", "organization Actions selected policy is not active") + require(current["enabled_repositories"] != "none", "organization Actions are disabled") + membership = [] + if current["enabled_repositories"] == "selected": + membership = pages(f"orgs/{organization}/actions/permissions/repositories", "repositories") + require(any(item["id"] == metadata["id"] and item["full_name"].lower() == coordinate.lower() + for item in membership), "canary repository is not enabled for organization Actions") + repository_policy = api(f"repos/{coordinate}/actions/permissions") + if (not isinstance(repository_policy["enabled"], bool) + or repository_policy["allowed_actions"] not in {"all", "local_only", "selected"}): + raise EvidenceError("repository Actions permissions malformed") + require(repository_policy["enabled"] and repository_policy["allowed_actions"] != "local_only", + "canary repository does not allow external Actions") + # Repository 'all' cannot relax the organization's selected-actions ceiling. + selected = api(f"orgs/{organization}/actions/permissions/selected-actions") + desired = config("actions-selected.json") + require(desired["github_owned_allowed"] is True and desired["verified_allowed"] is False, + "desired selected-actions policy is not conservative") + for value in (selected, desired): + if (not isinstance(value["github_owned_allowed"], bool) + or not isinstance(value["verified_allowed"], bool) + or not isinstance(value["patterns_allowed"], list) + or not all(isinstance(item, str) for item in value["patterns_allowed"])): + raise EvidenceError("selected-actions policy malformed") + require(selected["github_owned_allowed"] is True and selected["verified_allowed"] is False + and sorted(selected["patterns_allowed"]) == sorted(desired["patterns_allowed"]), + "live selected-actions policy differs from desired policy") + return {"mode": current["allowed_actions"], "enabled": current["enabled_repositories"], + "membership": sorted((item["id"], item["full_name"]) for item in membership), + "repository": repository_policy, "selected": selected} + + +def disabled_legacy_absent(coordinate, metadata): + """Prove absence only for the exact REST repository-disabled response.""" + branch = metadata["default_branch"] + fresh = repository(coordinate) + if (type(metadata["id"]) is not int or metadata["id"] <= 0 + or fresh["id"] != metadata["id"] or type(fresh["id"]) is not int + or fresh["full_name"] != coordinate or metadata["full_name"] != coordinate + or fresh["default_branch"] != branch): + raise EvidenceError("legacy protection repository identity/default branch changed") + # This fixed query is read-only. GraphQL requires POST; variables remain + # separate literal fields and cannot add operations to the query. + query = ("query LegacyBranchProtection($owner: String!, $name: String!, $qualifiedName: String!) " + "{ repository(owner: $owner, name: $name) { databaseId nameWithOwner isArchived " + "ref(qualifiedName: $qualifiedName) { name branchProtectionRule { id } } } }") + owner, name = coordinate.split("/") + try: + result = subprocess.run( + ["gh", "api", "--method", "POST", "-H", f"X-GitHub-Api-Version: {VERSION}", + "graphql", "-f", f"query={query}", "-f", f"owner={owner}", "-f", f"name={name}", + "-f", f"qualifiedName=refs/heads/{branch}"], + capture_output=True, text=True, timeout=30, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise EvidenceError("legacy protection GraphQL unreadable") from exc + if result.returncode: + raise EvidenceError("legacy protection GraphQL unreadable") + try: + response = json.loads(result.stdout) + repo = response["data"]["repository"] + ref = repo["ref"] + valid = ("errors" not in response and type(repo["databaseId"]) is int + and repo["databaseId"] == metadata["id"] and repo["nameWithOwner"] == coordinate + and repo["isArchived"] is False and ref["name"] == branch + and "branchProtectionRule" in ref and ref["branchProtectionRule"] is None) + except (json.JSONDecodeError, KeyError, TypeError): + valid = False + if not valid: + raise EvidenceError("legacy protection absence not proven for repository/default branch") + + +def canary_legacy_checks(coordinate, metadata): + branch = quote(metadata["default_branch"], safe="") + try: + protection = api(f"repos/{coordinate}/branches/{branch}/protection") + except APIError as exc: + body = exc.response + if exc.status != 404 or not isinstance(body, dict) or body.get("status") != "404": + raise + if body.get("message") == "Branch not protected": + return None + if body.get("message") == "Branch protection has been disabled on this repository.": + disabled_legacy_absent(coordinate, metadata) + return None + raise + if not isinstance(protection, dict) or "required_status_checks" not in protection: + raise EvidenceError("legacy branch protection malformed") + return protection["required_status_checks"] + + +def canary_required_checks(coordinate, metadata): + branch = metadata["default_branch"] + rules = pages(f"repos/{coordinate}/rules/branches/{quote(branch, safe='')}") + entries = [] + for rule in rules: + if rule["type"] == "required_status_checks": + entries.extend((item["context"], item["integration_id"]) + for item in rule["parameters"]["required_status_checks"]) + legacy = canary_legacy_checks(coordinate, metadata) + if legacy is not None: + if (not isinstance(legacy, dict) or not isinstance(legacy["checks"], list) + or not isinstance(legacy["contexts"], list) + or not all(isinstance(value, str) and value for value in legacy["contexts"])): + raise EvidenceError("legacy required status checks malformed") + entries.extend((item["context"], item["app_id"]) for item in legacy["checks"]) + entries.extend((context, None) for context in legacy["contexts"]) + for context, app in entries: + if not isinstance(context, str) or not context or (app is not None and type(app) is not int): + raise EvidenceError("effective required status check malformed") + require(app in (None, -1, ACTION_APP), + f"effective required check is bound to a non-Actions app: {context}") + repository_name = coordinate.split('/')[1] + configured = config("repositories.json")["required_ci"].get(repository_name) + require(bool(configured), "canary has no configured required contexts") + if not isinstance(configured, list) or not all(isinstance(value, str) and value for value in configured): + raise EvidenceError("configured required checks malformed") + required = set(configured) | {context for context, _ in entries} + if repository_name in config("security-merge-protection.json")["dependency_review"]: + required.add("Dependency review") + return (list(dict.fromkeys(configured)) + sorted(required - set(configured)), + sorted(json.dumps(item) for item in entries)) + + +def sha(coordinate, number): + metadata = repository(coordinate) + pr = api(f"repos/{coordinate}/pulls/{number}") + require(pr["merged"] is True or (pr["state"] == "open" and not pr["draft"]), + "action-update canary must be merged or open and review-ready") + require(pr["base"]["repo"]["full_name"].lower() == coordinate.lower() + and pr["base"]["ref"] == metadata["default_branch"], + "canary does not target default branch") + require(bool(SHA.fullmatch(pr["head"]["sha"])), "canary head SHA missing") + actions_policy = canary_actions_policy(coordinate, metadata) + selected = config("actions-selected.json") + required, required_evidence = canary_required_checks(coordinate, metadata) + files = pages(f"repos/{coordinate}/pulls/{number}/files") + require(bool(files) and len(files) == pr["changed_files"], "canary file evidence incomplete") + updates = 0 + for item in files: + path = item["filename"] + require(item["status"] == "modified" and + re.fullmatch(r"\.github/workflows/[^/]+\.ya?ml", path) is not None, + "canary contains changes outside existing workflows") + updates += pin_changes(contents(coordinate, path, pr["base"]["sha"]), + contents(pr["head"]["repo"]["full_name"], path, pr["head"]["sha"]), selected) + merge = pr.get("merge_commit_sha") if not pr["merged"] else None + merges, recovered = set(), [] + if merge: + require(bool(SHA.fullmatch(merge)), "canary merge SHA malformed") + commit = api(f"repos/{coordinate}/commits/{merge}") + require({parent["sha"] for parent in commit["parents"]} + == {pr["base"]["sha"], pr["head"]["sha"]}, "canary merge SHA is stale") + merges.add(merge) + if pr["merged"]: + # After squash/rebase merge, merge_commit_sha identifies publication, + # not the temporary merge commit that PR Actions checks evaluated. + runs = pages(f"repos/{coordinate}/actions/runs?event=pull_request&head_sha={pr['head']['sha']}", + "workflow_runs") + for run in runs: + if run["event"] != "pull_request" or run["head_sha"] != pr["head"]["sha"]: + continue + suite = api(f"repos/{coordinate}/check-suites/{run['check_suite_id']}") + references, _ = association_references( + coordinate, run, suite, metadata["default_branch"], recovered) + references = [item for item in references if item["number"] == number] + if not references: + continue + candidate = suite["head_sha"] + if candidate == pr["head"]["sha"]: + continue + require(bool(SHA.fullmatch(candidate)), "historical canary merge SHA malformed") + commit = api(f"repos/{coordinate}/commits/{candidate}") + require(any(item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["ref"] == metadata["default_branch"] + and {parent["sha"] for parent in commit["parents"]} + == {item["base"]["sha"], pr["head"]["sha"]} for item in references), + "historical canary merge does not match PR head/base") + merges.add(candidate) + checks = [] + for ref in [pr["head"]["sha"], *sorted(merges)]: + checks.extend(pages(f"repos/{coordinate}/commits/{ref}/check-runs?filter=all&app_id={ACTION_APP}", + "check_runs")) + proofs, selected_checks = {}, [] + for context in required: + candidates = [check for check in checks if check["name"] == context + and check["app"]["id"] == ACTION_APP + and check["app"]["slug"] == "github-actions" + and merge_check_run(coordinate, check, pr, merges, proofs, recovered)] + require(bool(candidates), f"required check missing for canary head: {context}") + # Check IDs order creation, including queued checks with no started_at. + # Scheduling/start time must not let an older green check hide a newer + # PR-associated check or suite that has not completed successfully. + latest = max(candidates, key=lambda check: check["id"]) + run = proofs[latest["check_suite"]["id"]]["run"] + require(run["status"] == "completed" and run["conclusion"] == "success", + f"required PR workflow run not successful for canary head: {context}") + require(latest["status"] == "completed" and latest["conclusion"] == "success", + f"required check not successful for canary head: {context}") + selected_checks.append(latest) + run_keys = ("id", "run_attempt", "event", "head_sha", "check_suite_id", "status", "conclusion", + "pull_requests", "path", "head_repository") + check_keys = ("id", "name", "head_sha", "status", "conclusion", "app", "check_suite", "started_at") + fresh_checks = [] + for ref in [pr["head"]["sha"], *sorted(merges)]: + fresh_checks.extend(pages(f"repos/{coordinate}/commits/{ref}/check-runs?filter=all&app_id={ACTION_APP}", + "check_runs")) + def check_snapshot(values): + return sorted(json.dumps({key: check[key] for key in check_keys}, sort_keys=True) + for check in values if check["name"] in required) + require(check_snapshot(fresh_checks) == check_snapshot(checks), + "canary required check collection changed during preflight") + for suite_id in {check["check_suite"]["id"] for check in selected_checks}: + proof = proofs[suite_id] + fresh_runs = api(proof["endpoint"])["workflow_runs"] + require(len(fresh_runs) == 1 and all(fresh_runs[0][key] == proof["run"][key] for key in run_keys), + "canary PR workflow-run evidence changed during preflight") + fresh_suite = api(f"repos/{coordinate}/check-suites/{suite_id}") + suite_keys = ("id", "head_sha", "app", "status", "conclusion", "pull_requests") + require(all(fresh_suite[key] == proof["suite"][key] for key in suite_keys), + "canary check-suite evidence changed during preflight") + verify_recovered_associations(coordinate, recovered) + for check in selected_checks: + fresh_check = api(f"repos/{coordinate}/check-runs/{check['id']}") + require(all(fresh_check[key] == check[key] for key in check_keys), + "canary required check evidence changed during preflight") + fresh = api(f"repos/{coordinate}/pulls/{number}") + require(fresh["head"]["sha"] == pr["head"]["sha"] + and fresh["base"]["sha"] == pr["base"]["sha"] + and fresh.get("merge_commit_sha") == pr.get("merge_commit_sha") + and fresh["state"] == pr["state"] and fresh["merged"] == pr["merged"] + and (fresh["merged"] or not fresh["draft"]), "canary changed during preflight") + require(canary_actions_policy(coordinate, metadata) == actions_policy, + "live Actions permissions changed during preflight") + require(canary_required_checks(coordinate, metadata) + == (required, required_evidence), "effective required checks changed during preflight") + fresh_metadata = repository(coordinate) + require(fresh_metadata["default_branch"] == metadata["default_branch"] + and fresh_metadata["id"] == metadata["id"], "repository identity/default branch changed during preflight") + return f"ordinary action-update PR #{number}: {updates} allowed full-SHA updates; required checks passed" + + +def canonical_language(value): + return {"javascript/typescript": "javascript-typescript", "javascript": "javascript-typescript", + "typescript": "javascript-typescript"}.get(value, value) + + +def analysis_language(analysis): + environment = json.loads(analysis.get("environment") or "{}") + language = environment.get("language") + if not language: + match = re.search(r"(?:^|/)language:([a-z0-9+-]+)(?:/|$)", analysis["category"]) + language = match[1] if match else None + return canonical_language(language), environment + + +def successful_analysis(analysis, head): + require(analysis["commit_sha"] == head, "latest CodeQL analysis does not cover current default head") + require(analysis["error"] in ("", None) and analysis["rules_count"] > 0, + "latest CodeQL analysis failed or contains no executed rules") + + +def newest_analysis_pages(coordinate, branch): + endpoint = f"repos/{coordinate}/code-scanning/analyses?" + urlencode( + {"ref": f"refs/heads/{branch}", "tool_name": "CodeQL"}) + for page in range(1, 11): + entries = api(endpoint + f"&per_page=100&page={page}") + if not isinstance(entries, list): + raise EvidenceError("CodeQL analysis collection malformed") + yield entries + if len(entries) < 100: + return + + +def configured_codeql_languages(setup): + values = setup["languages"] + if not isinstance(values, list) or not all(isinstance(item, str) and item for item in values): + raise EvidenceError("CodeQL default-setup languages malformed") + return {canonical_language(value) for value in values} + + +def default_codeql_coverage(coordinate, branch, head, languages): + """Select only the newest canonical analyses within the bounded history.""" + found = {} + for entries in newest_analysis_pages(coordinate, branch): + for analysis in entries: + if analysis["tool"]["name"] != "CodeQL" or analysis["ref"] != f"refs/heads/{branch}": + continue + language, _ = analysis_language(analysis) + # Default setup owns one canonical category per configured language. + # Retired advanced partitions, even in that language, are historical. + category_language = canonical_language(analysis["category"].removeprefix("/language:")) + if (not analysis["category"].startswith("/language:") + or category_language not in languages or category_language in found): + continue + successful_analysis(analysis, head) + require(language == category_language, "default CodeQL analysis language does not match its category") + found[language] = analysis + if set(found) == languages: + return found + raise Deferred("required configured CodeQL language analysis missing within bounded evidence") + + +def default_codeql(coordinate, branch, head, minima): + endpoint = f"repos/{coordinate}/code-scanning/default-setup" + setup = api(endpoint) + require(setup["state"] == "configured", "CodeQL default setup not configured") + languages = configured_codeql_languages(setup) + require(bool(languages), "CodeQL default setup has no configured languages") + require(set(minima) <= languages, "required CodeQL languages not configured") + coverage = default_codeql_coverage(coordinate, branch, head, languages) + fresh = api(endpoint) + require(fresh["state"] == setup["state"] + and configured_codeql_languages(fresh) == languages, + "CodeQL default setup changed during preflight") + require(default_codeql_coverage(coordinate, branch, head, languages) == coverage, + "CodeQL default-setup analysis coverage changed during preflight") + return "latest canonical default-setup analyses cover every configured language on current head" + + +def classifier_full_events(source): + """Read the owner's full-event contract without executing fetched Python.""" + tree = ast.parse(source) + assignments = [node for node in tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "FULL_EVENTS" for target in node.targets)] + require(len(assignments) == 1, "advanced CodeQL classifier full-event contract missing") + events = ast.literal_eval(assignments[0].value) + mains = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "main"] + require(len(mains) == 1, "advanced CodeQL classifier main contract missing") + full = False + for node in ast.walk(mains[0]): + if (isinstance(node, ast.If) and isinstance(node.test, ast.Compare) + and ast.unparse(node.test) == "arguments.event in FULL_EVENTS"): + full = any(isinstance(item, ast.Assign) and isinstance(item.value, ast.Constant) + and item.value.value is True and any(isinstance(target, ast.Name) + and target.id == "full" for target in item.targets) for item in node.body) + require(full and {"push", "schedule", "workflow_dispatch"} <= set(events), + "advanced CodeQL classifier does not select full analysis for full events") + return set(events) & {"push", "schedule", "workflow_dispatch"} + + +def advanced_partition_minima(policy, repository_name): + """Validate source-owned coverage even when preflight is called directly.""" + inventories = policy.get("required_partitions") + if (not isinstance(inventories, dict) or set(inventories) != set(policy["repositories"]) + or not isinstance(inventories.get(repository_name), list) + or not inventories[repository_name]): + raise EvidenceError("advanced CodeQL required partition inventory malformed/missing") + minima = set() + for item in inventories[repository_name]: + if (not isinstance(item, dict) or set(item) != {"scope", "language"} + or any(not isinstance(item[key], str) + or not re.fullmatch(r"[a-z][a-z0-9+-]*", item[key]) for key in item)): + raise EvidenceError("advanced CodeQL required partition identity malformed") + identity = (item["scope"], canonical_language(item["language"])) + if identity in minima: + raise EvidenceError("advanced CodeQL required partition identity duplicated") + minima.add(identity) + return minima + + +def classifier_partitions(source, minima): + """Inspect literal matrix declarations; never execute repository Python.""" + tree = ast.parse(source) + functions = [node for node in tree.body if isinstance(node, ast.FunctionDef) + and node.name == "write_outputs"] + require(len(functions) == 1, "advanced CodeQL partition declarations missing") + function = functions[0] + assignments = [node for node in ast.walk(function) if isinstance(node, ast.Name) + and node.id == "entries" and isinstance(node.ctx, ast.Store)] + initializers = [node for node in function.body if isinstance(node, ast.Assign) + and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == "entries" and isinstance(node.value, ast.List) + and not node.value.elts] + require(len(assignments) == len(initializers) == 1, + "advanced CodeQL partition declaration shape unsupported") + declarations, identities, categories = {}, set(), set() + for node in ast.walk(function): + if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) and node.func.value.id == "entries"): + continue + require(node.func.attr == "append" and len(node.args) == 1 and not node.keywords, + "advanced CodeQL partition declaration shape unsupported") + try: + item = ast.literal_eval(node.args[0]) + except (ValueError, TypeError): + raise Deferred("advanced CodeQL partition declaration is not literal") from None + require(isinstance(item, dict) and set(item) == {"display", "category", "scope", "language", "run"} + and item["run"] is True and all(isinstance(item[key], str) and item[key] + for key in ("display", "category", "scope", "language")), + "advanced CodeQL partition declaration malformed") + identity = (item["scope"], canonical_language(item["language"])) + require(item["display"] not in declarations and item["category"] not in categories + and identity not in identities, "advanced CodeQL partition declarations duplicated") + declarations[item["display"]] = item + categories.add(item["category"]) + identities.add(identity) + require(bool(declarations) and minima <= identities, + "advanced CodeQL producer omits required partition coverage") + return declarations + + +def latest_full_codeql_run(coordinate, branch, head, events): + value = api(f"repos/{coordinate}/actions/workflows/codeql.yml/runs?head_sha={head}&per_page=100") + candidates = [run for run in value["workflow_runs"] if run["event"] in events + and run["head_sha"] == head and run["head_branch"] == branch] + require(bool(candidates), "current-head full-event CodeQL workflow run missing") + run = candidates[0] + require(run["path"] == ".github/workflows/codeql.yml" and run["status"] == "completed" + and run["conclusion"] == "success", "latest full-event CodeQL workflow run not successful") + return run + + +def advanced_codeql_coverage(coordinate, branch, head, minima, selected, declarations): + """Select current producer uploads, ignoring retired categories and history.""" + found, categories, languages = {}, set(), set() + for entries in newest_analysis_pages(coordinate, branch): + for analysis in entries: + if (analysis["tool"]["name"] != "CodeQL" or analysis["ref"] != f"refs/heads/{branch}" + or analysis.get("analysis_key") != ".github/workflows/codeql.yml:analyze"): + continue + category = analysis["category"] + if category in categories: + continue + categories.add(category) + language, environment = analysis_language(analysis) + display = environment.get("display") + if display not in selected: + continue + job = selected[display] + declaration = declarations[display] + if analysis["created_at"] < job["started_at"]: + continue + require(display not in found, "advanced CodeQL selected display has ambiguous upload categories") + successful_analysis(analysis, head) + category_language = re.search(r"(?:^|/)language:([a-z0-9+-]+)$", category) + require(environment.get("category") == category == declaration["category"] + and environment.get("scope") == declaration["scope"] + and language == canonical_language(declaration["language"]) + and category_language is not None + and canonical_language(category_language[1]) == language + and job["started_at"] <= analysis["created_at"] <= job["completed_at"], + "advanced CodeQL analysis upload does not belong to selected full-run job") + found[display] = analysis + languages.add(language) + if set(found) == set(selected): + break + if set(found) == set(selected): + require({canonical_language(value) for value in minima} <= languages, + "required CodeQL language analysis missing") + return found + raise Deferred("required advanced CodeQL partition upload missing within bounded evidence") + + +def advanced_workflow_wiring(source, default_branch): + """Check executable dataflow, not the honesty of repository-owned Python. + + The supported ordinary-job shape is deliberately bounded. Producer-owned + action SHAs and classifier path/category declarations are not central pins. + """ + try: + import yaml + except ImportError as exc: + raise EvidenceError("install requirements-validation.txt for CodeQL workflow verification") from exc + + class UniqueLoader(yaml.SafeLoader): + # Actions uses `on` as a mapping key. Preserve scalar booleans as text, + # as in the existing publication verifier, and validate them explicitly. + yaml_implicit_resolvers = { + key: [(tag, regex) for tag, regex in values if tag != "tag:yaml.org,2002:bool"] + for key, values in yaml.SafeLoader.yaml_implicit_resolvers.items() + } + + def unique_mapping(loader, node, deep=False): + result = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in result: + raise ValueError(f"duplicate YAML key: {key}") + result[key] = loader.construct_object(value_node, deep=deep) + return result + + UniqueLoader.add_constructor("tag:yaml.org,2002:map", unique_mapping) + try: + workflow = yaml.load(source, Loader=UniqueLoader) + except (yaml.YAMLError, ValueError, TypeError) as exc: + raise EvidenceError("advanced CodeQL workflow YAML malformed or ambiguous") from exc + + def check(condition, detail): + require(condition, "advanced CodeQL executable workflow contract: " + detail) + + def expression(value, expected, wrapped=True): + if not isinstance(value, str): + return False + value = value.strip() + if value.startswith("${{") and value.endswith("}}"): + value = value[3:-2].strip() + elif wrapped: + return False + # Ignore token spacing, preserving string contents (including spaces). + def tokens(text): + return re.findall(r"'(?:[^']|'')*'|[A-Za-z_][A-Za-z_0-9]*|==|\S", text) + return tokens(value) == tokens(expected) + + def execution_settings(value, context): + check(isinstance(value, dict), context + " must be a mapping") + check(not value.get("env") and not value.get("defaults"), context + " execution overrides") + check(value.get("continue-on-error", "false") == "false", context + " suppresses failure") + check(not any(key in value for key in ("shell", "working-directory", "container", "services")), + context + " changes execution context") + + def step(value, context, condition=None): + execution_settings(value, context) + check(set(value) <= {"name", "id", "uses", "with", "run", "env", "if", "continue-on-error"}, + context + " unsupported step execution") + check(("if" not in value) if condition is None else expression(value.get("if"), condition, False), + context + " condition disconnected") + + def action(value, name): + check(isinstance(value.get("uses"), str) + and re.fullmatch(re.escape(name) + r"@[0-9a-fA-F]{40}", value["uses"]) is not None + and "run" not in value, "expected official full-SHA " + name) + + def checkout(value): + step(value, "checkout") + action(value, "actions/checkout") + inputs = value.get("with", {}) + check(isinstance(inputs, dict) + and set(inputs) <= {"fetch-depth", "ref", "repository", "persist-credentials"}, + "checkout must use the current complete workspace") + check("ref" not in inputs or expression(inputs["ref"], "github.sha"), + "checkout revision differs from run") + check("repository" not in inputs or expression(inputs["repository"], "github.repository"), + "checkout repository differs from run") + + def command(value, prefix): + check(isinstance(value.get("run"), str) and "uses" not in value and "with" not in value, + "helper must execute a run step") + try: + # Retain shell quoting as well as argument values: single-quoted or + # escaped $EVENT_NAME is a literal, not the bound event environment. + source = value["run"].replace("\\\r\n", "").replace("\\\n", "") + raw = shlex.split(source, comments=True, posix=False) + decoded = [shlex.split(token) for token in raw] + check(all(len(item) == 1 for item in decoded), "helper shell arguments unsupported") + tokens = [item[0] for item in decoded] + except ValueError: + raw, tokens = [], [] + check(tokens[:len(prefix)] == prefix, "helper invocation disconnected") + arguments, raw_arguments = tokens[len(prefix):], raw[len(prefix):] + check(len(arguments) % 2 == 0, "helper arguments malformed") + options, quoted_options = {}, {} + for index in range(0, len(arguments), 2): + key, argument = arguments[index:index + 2] + check(key.startswith("--") and key not in options, "helper arguments ambiguous") + options[key] = argument + quoted_options[key] = raw_arguments[index + 1] + return options, quoted_options + + def classifier(value, scoped=False): + expected_env = {"BASE_SHA": "github.event.pull_request.base.sha", + "EVENT_NAME": "github.event_name", "HEAD_SHA": "github.event.pull_request.head.sha"} + environment = value.get("env") + check(isinstance(environment, dict) and set(environment) == set(expected_env) + and all(expression(environment[key], expected) for key, expected in expected_env.items()), + "classifier event/revision environment disconnected") + # Environment is validated above; every other execution override is rejected. + step({key: item for key, item in value.items() if key != "env"}, "classifier") + check(value.get("id") == "classify", "classifier step identity disconnected") + options, quoted = command(value, ["python3", "tools/ci/classify_changes.py"]) + expected = {"--event": "EVENT_NAME", "--base": "BASE_SHA", "--head": "HEAD_SHA", + "--github-output": "GITHUB_OUTPUT"} + check(set(options) == set(expected) | {"--codeql-config"} | ({"--codeql-scope"} if scoped else set()), + "classifier arguments disconnected") + check(all(quoted[key] in ("$" + variable, "${" + variable + "}", + '"$' + variable + '"', '"${' + variable + '}"') + for key, variable in expected.items()), "classifier event/output arguments disconnected") + path = options["--codeql-config"] + check(re.fullmatch(r"(?:[A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\.ya?ml", path) is not None + and ".." not in path.split("/"), "classifier config destination unsupported") + if scoped: + check(expression(options["--codeql-scope"], "matrix.scope"), "classifier scope disconnected") + return path + + execution_settings(workflow, "workflow") + check(isinstance(default_branch, str) and bool(default_branch), "default branch missing") + events = workflow.get("on") + if isinstance(events, str): + events = {events: None} + elif isinstance(events, list): + check(all(isinstance(event, str) for event in events) + and len(events) == len(set(events)), "workflow events malformed") + events = dict.fromkeys(events) + check(isinstance(events, dict) and "pull_request" in events, + "pull_request trigger missing") + pull_request = events["pull_request"] + if pull_request is None: + pull_request = {} + # Do not partially interpret GitHub glob/exclusion or workflow path filters: + # they can leave a required producer pending. Path selection belongs in the + # trusted classifier after the workflow has started for the pull request. + check(isinstance(pull_request, dict) and set(pull_request) <= {"branches", "types"}, + "pull_request filters unsupported") + if "branches" in pull_request: + branches = pull_request["branches"] + check(isinstance(branches, list) and bool(branches) + and all(isinstance(branch, str) and branch + and not any(char in branch for char in "*?[]!+\\") for branch in branches) + and default_branch in branches, "pull_request does not cover the default branch") + activities = {"opened", "synchronize", "reopened"} + if "types" in pull_request: + # Values need an explicit enum: the Actions loader preserves YAML + # booleans as text, so an invalid `false` must not become an event name. + supported = activities | {"assigned", "unassigned", "labeled", "unlabeled", "edited", "closed", + "converted_to_draft", "locked", "unlocked", "enqueued", "dequeued", + "milestoned", "demilestoned", "ready_for_review", "review_requested", + "review_request_removed", "auto_merge_enabled", "auto_merge_disabled"} + types = pull_request["types"] + check(isinstance(types, list) and all(isinstance(value, str) and value in supported for value in types) + and len(types) == len(set(types)) and activities <= set(types), + "pull_request activity coverage incomplete") + jobs = workflow.get("jobs") + check(isinstance(jobs, dict) and set(jobs) == {"classify", "analyze", "codeql-validation"}, + "requires ordinary classification, analysis and validation jobs") + for name, job in jobs.items(): + execution_settings(job, name) + check(set(job) <= {"name", "runs-on", "timeout-minutes", "outputs", "steps", "if", "needs", + "strategy", "continue-on-error"}, name + " unsupported job execution") + runner = job.get("runs-on") + check((isinstance(runner, str) and bool(runner)) or + (isinstance(runner, list) and bool(runner) and + all(isinstance(item, str) and item for item in runner)), name + " runner missing") + steps = job.get("steps") + check(isinstance(steps, list) and all(isinstance(item, dict) for item in steps), + name + " executable steps missing") + identifiers = [item["id"] for item in steps if "id" in item] + check(all(isinstance(item, str) for item in identifiers) + and len(identifiers) == len(set(identifiers)), name + " duplicate step identities") + classify, analyze, validation = (jobs[name] for name in ("classify", "analyze", "codeql-validation")) + check(classify.get("name") == "CodeQL classification" + and not any(key in classify for key in ("if", "needs", "strategy")) + and len(classify["steps"]) == 2, "classification job execution disconnected") + outputs = classify.get("outputs", {}) + check(isinstance(outputs, dict) and set(outputs) == {"matrix", "run"} + and expression(outputs["matrix"], "steps.classify.outputs.codeql_matrix") + and expression(outputs["run"], "steps.classify.outputs.codeql_run"), "classifier outputs disconnected") + checkout(classify["steps"][0]) + config_path = classifier(classify["steps"][1]) + + check(analyze.get("needs") in ("classify", ["classify"]) + and expression(analyze.get("if"), "needs.classify.outputs.run == 'true'", False), + "analysis dependency/condition disconnected") + name = analyze.get("name", "") + match = re.fullmatch(r"CodeQL analysis \((\$\{\{.*\}\})\)", name) if isinstance(name, str) else None + check(match is not None and expression(match[1], "matrix.display"), "analysis display disconnected") + strategy = analyze.get("strategy", {}) + check(isinstance(strategy, dict) and set(strategy) == {"fail-fast", "matrix"} + and strategy["fail-fast"] == "false" + and expression(strategy["matrix"], "fromJSON(needs.classify.outputs.matrix)"), + "analysis matrix disconnected") + check("outputs" not in analyze and len(analyze["steps"]) == 4, "analysis step order unsupported") + checkout(analyze["steps"][0]) + check(classifier(analyze["steps"][1], scoped=True) == config_path, "analysis config destination disconnected") + initialize, analyze_step = analyze["steps"][2:] + step(initialize, "CodeQL init", "matrix.run == true") + action(initialize, "github/codeql-action/init") + inputs = initialize.get("with", {}) + check(isinstance(inputs, dict) and set(inputs) == {"build-mode", "config-file", "languages"} + and inputs["build-mode"] == "none" + and expression(inputs["config-file"], "steps.classify.outputs.codeql_config") + and expression(inputs["languages"], "matrix.language"), "CodeQL init/config/language disconnected") + step(analyze_step, "CodeQL analysis", "matrix.run == true") + action(analyze_step, "github/codeql-action/analyze") + inputs = analyze_step.get("with", {}) + check(isinstance(inputs, dict) and set(inputs) == {"category"} + and expression(inputs["category"], "matrix.category"), "CodeQL analysis category disconnected") + + check(validation.get("name") == "CodeQL validation" + and expression(validation.get("if"), "always()", False) + and isinstance(validation.get("needs"), list) and sorted(validation["needs"]) == ["analyze", "classify"] + and not any(key in validation for key in ("strategy", "outputs")) + and len(validation["steps"]) == 2, "validation dependency/condition disconnected") + checkout(validation["steps"][0]) + aggregate = validation["steps"][1] + step(aggregate, "validation aggregate") + options, _ = command(aggregate, ["python3", "tools/ci/require_checks.py", "optional"]) + check(set(options) == {"--label", "--classifier-result", "--required", "--result"} + and options["--label"] == "CodeQL" + and expression(options["--classifier-result"], "needs.classify.result") + and expression(options["--required"], "needs.classify.outputs.run") + and expression(options["--result"], "needs.analyze.result"), "validation result arguments disconnected") + + +def advanced_codeql(coordinate, branch, head, minima, partition_minima): + sources = {} + for path in (".github/workflows/codeql.yml", "tools/ci/classify_changes.py", "tools/ci/require_checks.py"): + sources[path] = contents(coordinate, path, branch) + require(contents(coordinate, path, head) == sources[path], "advanced CodeQL producer source is stale") + advanced_workflow_wiring(sources[".github/workflows/codeql.yml"], branch) + declarations = classifier_partitions(sources["tools/ci/classify_changes.py"], partition_minima) + events = classifier_full_events(sources["tools/ci/classify_changes.py"]) + run = latest_full_codeql_run(coordinate, branch, head, events) + jobs_endpoint = f"repos/{coordinate}/actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs" + jobs = pages(jobs_endpoint, "jobs") + require(bool(jobs) and all(job["head_sha"] == head and job["status"] == "completed" + and job["conclusion"] == "success" for job in jobs), + "advanced CodeQL full-run jobs not all successful on current head") + require(len([job for job in jobs if job["name"] == "CodeQL classification"]) == 1, + "advanced CodeQL classification job missing") + validation = [job for job in jobs if job["name"] == "CodeQL validation"] + require(len(validation) == 1, "advanced CodeQL validation job missing") + expected_url = f"https://api.github.com/repos/{coordinate}/check-runs/" + check_url = validation[0]["check_run_url"] + require(check_url.startswith(expected_url) and check_url[len(expected_url):].isdigit(), + "advanced CodeQL validation check identity missing") + check_endpoint = f"repos/{coordinate}/check-runs/{check_url[len(expected_url):]}" + check = api(check_endpoint) + require(check["name"] == "CodeQL validation" and check["head_sha"] == head + and check["app"]["id"] == ACTION_APP and check["app"]["slug"] == "github-actions" + and check["check_suite"]["id"] == run["check_suite_id"] + and check["status"] == "completed" and check["conclusion"] == "success", + "advanced CodeQL validation check is not the trusted current-head producer") + selected = {} + for job in jobs: + match = re.fullmatch(r"CodeQL analysis \((.+)\)", job["name"]) + if match: + require(match[1] not in selected, "advanced CodeQL analysis jobs duplicated") + selected[match[1]] = job + require(set(selected) == set(declarations), + "advanced CodeQL full run does not cover every declared partition") + coverage = advanced_codeql_coverage(coordinate, branch, head, minima, selected, declarations) + fresh = latest_full_codeql_run(coordinate, branch, head, events) + require(all(fresh[key] == run[key] for key in + ("id", "run_attempt", "head_sha", "head_branch", "event", "path", "check_suite_id")), + "advanced CodeQL full run changed during preflight") + require(api(check_endpoint) == check, "advanced CodeQL validation check changed during preflight") + require(pages(jobs_endpoint, "jobs") == jobs, "advanced CodeQL full-run jobs changed during preflight") + for path, source in sources.items(): + require(contents(coordinate, path, branch) == source, + "advanced CodeQL producer source changed during preflight") + require(advanced_codeql_coverage(coordinate, branch, head, minima, selected, declarations) == coverage, + "advanced CodeQL analysis coverage changed during preflight") + return "current full-event CodeQL producer covers policy minima and every declared partition upload" + + +def codeql(coordinate): + metadata = repository(coordinate) + branch = metadata["default_branch"] + branch_endpoint = f"repos/{coordinate}/branches/{quote(branch, safe='')}" + head = api(branch_endpoint)["commit"]["sha"] + require(bool(SHA.fullmatch(head)), "default-branch head missing") + required = config("security-merge-protection.json")["codeql_required_languages"].get( + coordinate.split('/')[1], []) + advanced = config("codeql-advanced-setup.json") + if coordinate.split('/')[1] in advanced["repositories"]: + partition_minima = advanced_partition_minima(advanced, coordinate.split('/')[1]) + reason = advanced_codeql(coordinate, branch, head, required, partition_minima) + else: + reason = default_codeql(coordinate, branch, head, [canonical_language(value) for value in required]) + require(api(branch_endpoint)["commit"]["sha"] == head, "default branch changed during CodeQL preflight") + require(repository(coordinate)["default_branch"] == branch, + "repository default branch changed during preflight") + return reason + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("dependency", "sha", "codeql")) + parser.add_argument("target", help="OWNER/REPO, or OWNER/REPO/PR for sha") + args = parser.parse_args(argv) + try: + parts = args.target.split("/") + coordinate = "/".join(parts[:2]) + if not COORDINATE.fullmatch(coordinate) or len(parts) != (3 if args.mode == "sha" else 2): + raise EvidenceError("invalid repository or PR coordinate") + if args.mode == "sha": + if not parts[2].isdigit() or int(parts[2]) < 1: + raise EvidenceError("invalid PR number") + reason = sha(coordinate, int(parts[2])) + else: + reason = {"dependency": dependency, "codeql": codeql}[args.mode](coordinate) + print(f"ready: {reason}") + return 0 + except Deferred as exc: + print(f"deferred: {exc}") + return 1 + except (EvidenceError, OSError, ValueError, KeyError, TypeError, AttributeError, SyntaxError) as exc: + reason = str(exc) if isinstance(exc, EvidenceError) else "local or API evidence malformed/unreadable" + print(f"error: {reason}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/requirements-validation.txt b/requirements-validation.txt index 39990fd..ad78371 100644 --- a/requirements-validation.txt +++ b/requirements-validation.txt @@ -1,2 +1,2 @@ -# YAML parsing for exact-main executable publication workflow verification. +# YAML parsing for executable publication and CodeQL workflow verification. PyYAML==6.0.3 diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py new file mode 100755 index 0000000..6b8e82b --- /dev/null +++ b/tests/audit-fixtures.py @@ -0,0 +1,551 @@ +#!/usr/bin/env python3 +"""Read-only GitHub API double for the effective-governance audit tests.""" +import copy +import fnmatch +import json +import os +import sys +from pathlib import Path +from urllib.parse import unquote + +root = Path(os.environ['AUDIT_CONFIG_ROOT']) +scenario = os.environ.get('GH_TEST_SCENARIO', 'pass') + + +def config(name): + return json.loads((root / 'config' / name).read_text()) + + +def reject(message): + print(message, file=sys.stderr) + sys.exit(2) + + +args = sys.argv[1:] +if not args or args.pop(0) != 'api': + reject('API only') +slurp = False +include = False +endpoint = None +method = None +fields = {} +while args: + arg = args.pop(0) + if arg == '--method': + method = args.pop(0) + elif arg == '-H': + args.pop(0) + elif arg == '--paginate': + pass + elif arg == '--slurp': + slurp = True + elif arg == '--include': + include = True + elif arg == '-f': + key, separator, value = args.pop(0).partition('=') + if not separator or key in fields: + reject('invalid or duplicated GraphQL variable') + fields[key] = value + elif arg.startswith('-'): + reject(f'unexpected or mutating API flag: {arg}') + else: + endpoint = arg +if endpoint is None: + reject('endpoint missing') +query = 'query LegacyBranchProtection($owner: String!, $name: String!, $qualifiedName: String!) { repository(owner: $owner, name: $name) { databaseId nameWithOwner isArchived ref(qualifiedName: $qualifiedName) { name branchProtectionRule { id } } } }' +if endpoint == 'graphql': + if (method != 'POST' or set(fields) != {'query', 'owner', 'name', 'qualifiedName'} + or fields['query'] != query or fields['owner'] != 'atrinik' + or fields['name'] != 'observatory' + or fields['qualifiedName'] not in ('refs/heads/main', 'refs/heads/release/next#candidate') + or not scenario.startswith(('legacy-disabled-', 'legacy-maintenance-disabled-'))): + reject('unexpected or mutating GraphQL request') +elif method != 'GET' or fields: + reject('audit attempted API mutation') +with open(os.environ['AUDIT_API_LOG'], 'a', encoding='utf-8') as log: + log.write(endpoint + '\n') + +properties = config('repository-properties.json') +repositories = config('repositories.json') +policy = config('security-merge-protection.json') +active = sorted(name for name, value in properties['repositories'].items() + if value['lifecycle'] != 'archived') + + +def rulesets(repository): + specs = [('default-branch-integrity', None), ('default-branch-linear-history', None)] + if repository in repositories['pull_request_gate']: + specs.append(('pull-request-gate', None)) + if repository in repositories['required_ci']: + specs.append(('required-ci', '03 - Required CI - ' + repository)) + if repository in repositories['release_tags'] or scenario in ( + 'inherited-tag-unaffected', 'tag-extra-nonrelease', 'tag-all-nonrelease', 'tag-repository-nonrelease'): + specs.append(('immutable-release-tags', None)) + if repository in policy['code_scanning'] and scenario not in ('pending-codeql', 'ready-missing-codeql', 'codeql-readiness-unreadable', 'enforced-missing-codeql', 'enforced-unreadable-codeql'): + specs.append(('code-scanning', None)) + if repository in policy['dependency_review'] and scenario not in ('pending', 'ready-missing', 'dependency-readiness-unreadable', 'enforced-missing-dependency', 'enforced-unreadable-dependency'): + specs.append(('dependency-review', None)) + if scenario == 'foreign-tag': + specs.append(('immutable-release-tags', 'Foreign tag retention')) + retained_template = None + if scenario.startswith(('retained-', 'retired-')): + kind = scenario.split('-')[1] + retained_template = {'codeql': 'code-scanning', 'dependency': 'dependency-review'}.get(kind) + if retained_template: + specs.append((retained_template, None)) + if scenario.endswith('-duplicate'): + specs.append((retained_template, None)) + result = [] + for index, (template, name) in enumerate(specs, 1): + item = config('rulesets/' + template + '.json') + if name: + item['name'] = name + if template == 'required-ci': + item['rules'][0]['parameters']['required_status_checks'] = [ + {'context': context, 'integration_id': 15368} + for context in repositories['required_ci'][repository]] + # Actual ruleset GET responses expand omitted request parameters. + for rule in item['rules']: + if rule['type'] == 'pull_request': + rule['parameters'].update(required_reviewers=[], + require_extra_approval_for_unattributed_changes=True, + allowed_merge_methods=['merge', 'squash', 'rebase']) + deviations = { + 'api-required-reviewers': ('required_reviewers', [{'reviewer': {'id': 42, 'type': 'Team'}, 'minimum_approvals': 1}]), + 'api-extra-approval': ('require_extra_approval_for_unattributed_changes', False), + 'api-restricted-merges': ('allowed_merge_methods', ['squash']), + 'api-duplicate-merges': ('allowed_merge_methods', ['merge', 'squash', 'rebase', 'merge']), + 'api-malformed-reviewers': ('required_reviewers', None), + 'api-malformed-approval': ('require_extra_approval_for_unattributed_changes', 'true'), + 'api-malformed-merges': ('allowed_merge_methods', 'merge,squash,rebase'), + 'api-unknown-pr-field': ('unknown_future_control', False), + } + if scenario in deviations: + key, value = deviations[scenario] + rule['parameters'][key] = value + if scenario == 'api-default-order': + rule['parameters']['allowed_merge_methods'].reverse() + if rule['type'] == 'required_status_checks': + rule['parameters']['do_not_enforce_on_create'] = False + if scenario == 'api-create-exemption': + rule['parameters']['do_not_enforce_on_create'] = True + if scenario == 'api-malformed-create-exemption': + rule['parameters']['do_not_enforce_on_create'] = None + if scenario == 'api-defaults-omitted': + for rule in item['rules']: + for key in ('required_reviewers', 'require_extra_approval_for_unattributed_changes', + 'allowed_merge_methods', 'do_not_enforce_on_create'): + rule.get('parameters', {}).pop(key, None) + item.update(id=active.index(repository) * 10 + index, + source_type='Repository' if scenario == 'repository-rulesets' or + template in ('code-scanning', 'dependency-review') else 'Organization') + if ((scenario == 'organization-codeql-owner' and template == 'code-scanning') or + (scenario == 'organization-dependency-owner' and template == 'dependency-review')): + item['source_type'] = 'Organization' + if scenario == 'tag-repository-nonrelease' and template == 'immutable-release-tags': + item['source_type'] = 'Repository' + # Match GitHub's actual owner representation, including repository rulesets. + for actor in item['bypass_actors']: + if actor['actor_type'] == 'OrganizationAdmin': + actor['actor_id'] = None + if scenario == 'owner-id-drift' and repository == 'observatory' and template == 'pull-request-gate': + item['bypass_actors'][0]['actor_id'] = 2 + if item['source_type'] == 'Organization': + if template in ('default-branch-integrity', 'default-branch-linear-history'): + included = ['~ALL'] + excluded = config('advisory-merge-windows.json')['repositories'] + elif template == 'pull-request-gate': + windows = config('advisory-merge-windows.json')['repositories'] + included = [name for name in repositories['pull_request_gate'] if name not in windows] + excluded = [] + elif template == 'immutable-release-tags': + included, excluded = repositories['release_tags'], [] + elif template == 'code-scanning': + included, excluded = policy['code_scanning'], [] + elif template == 'dependency-review': + included, excluded = policy['dependency_review'], [] + else: + included, excluded = [repository], [] + item['conditions']['repository_name'] = { + 'include': list(included), 'exclude': list(excluded), 'protected': False} + if scenario == 'scope-order' and 'repository_name' in item['conditions']: + item['conditions']['repository_name']['include'].reverse() + item['conditions']['repository_name']['exclude'].reverse() + if scenario == 'repository-scope-extra' and template == 'dependency-review': + item['conditions']['repository_name'] = {'include': ['~ALL'], 'exclude': [], 'protected': False} + if scenario in ('tag-all-scope', 'tag-all-nonrelease') and template == 'immutable-release-tags': + item['conditions']['repository_name']['include'] = ['~ALL'] + if scenario in ('tag-extra-repo', 'tag-extra-nonrelease') and template == 'immutable-release-tags': + item['conditions']['repository_name']['include'].append('.github') + if scenario == 'tag-extra-exclude' and template == 'immutable-release-tags': + item['conditions']['repository_name']['exclude'] = ['foreign-repository'] + if scenario == 'pr-all-scope' and template == 'pull-request-gate': + item['conditions']['repository_name']['include'] = ['~ALL'] + if scenario == 'default-extra-exclude' and template == 'default-branch-integrity': + item['conditions']['repository_name']['exclude'].append('foreign-repository') + # Keep inherited names while making actual targeting ineffective. + if scenario == 'wrong-pr-scope' and repository == '.github' and template == 'pull-request-gate': + item['conditions']['repository_name']['include'] = ['client'] + if scenario == 'pending-wrong-dependency-scope' and repository == 'observatory' and template == 'dependency-review': + item['conditions']['repository_name'] = {'include': ['observatory'], 'exclude': ['observatory'], 'protected': False} + if scenario == 'missing-tag-scope' and repository == 'service-updater' and template == 'immutable-release-tags': + item['conditions']['repository_name']['exclude'] = ['service-updater'] + if scenario == 'tag-ref-drift' and repository == 'service-updater' and template == 'immutable-release-tags': + item['conditions']['ref_name']['include'] = ['refs/tags/unrelated*'] + if scenario == 'integrity-bypass' and repository == 'observatory' and template == 'default-branch-integrity': + item['bypass_actors'] = [{'actor_type': 'OrganizationAdmin', 'actor_id': 1, 'bypass_mode': 'always'}] + if scenario == 'pr-bypass' and repository == 'observatory' and template == 'pull-request-gate': + item['bypass_actors'][0]['bypass_mode'] = 'always' + if scenario == 'codeql-threshold' and repository == 'observatory' and template == 'code-scanning': + item['rules'][0]['parameters']['code_scanning_tools'][0]['security_alerts_threshold'] = 'critical' + if scenario == 'codeql-tool' and repository == 'observatory' and template == 'code-scanning': + item['rules'][0]['parameters']['code_scanning_tools'][0]['tool'] = 'other-scanner' + if scenario == 'codeql-quality' and repository == 'observatory' and template == 'code-scanning': + item['rules'][0]['parameters']['code_scanning_tools'][0]['alerts_threshold'] = 'errors' + if template == retained_template: + if scenario.endswith(('-disabled', '-foreign-name', '-duplicate')): + item['enforcement'] = 'disabled' + if scenario.endswith('-moved'): + item['conditions']['ref_name']['include'] = ['refs/heads/other'] + if scenario.endswith('-modified'): + item['rules'] = [{'type': 'creation'}] + item['enforcement'] = 'disabled' + if scenario.endswith('-foreign-owner'): + item['source_type'] = 'Organization' + item['enforcement'] = 'disabled' + if scenario.endswith('-foreign-name'): + item['name'] = 'Foreign security policy' + result.append(item) + return result + + +payload = None +if endpoint == 'graphql': + variant = scenario.removeprefix('legacy-maintenance-').removeprefix('legacy-').removeprefix('disabled-') + repository = {'databaseId': active.index('observatory') + 1, 'nameWithOwner': 'atrinik/observatory', + 'isArchived': False, 'ref': {'name': fields['qualifiedName'].removeprefix('refs/heads/'), + 'branchProtectionRule': None}} + payload = {'data': {'repository': repository}} + if variant == 'present': + repository['ref']['branchProtectionRule'] = {'id': 'BPR_fixture'} + elif variant == 'wrong-id': + repository['databaseId'] += 100 + elif variant == 'wrong-owner': + repository['nameWithOwner'] = 'foreign/observatory' + elif variant == 'wrong-name': + repository['ref']['name'] = 'master' + elif variant == 'archived': + repository['isArchived'] = True + elif variant == 'missing-field': + del repository['ref']['branchProtectionRule'] + elif variant == 'null-ref': + repository['ref'] = None + elif variant == 'null-repository': + payload['data']['repository'] = None + elif variant == 'missing-data': + del payload['data'] + elif variant == 'malformed-rule': + repository['ref']['branchProtectionRule'] = False + elif variant == 'errors': + payload['errors'] = [{'message': 'read denied'}] + elif variant == 'unreadable': + reject('GraphQL request denied') + elif variant == 'non-json': + print('unavailable') + sys.exit(0) +elif endpoint.startswith('orgs/atrinik/repos?'): + if scenario == 'inventory-unreadable': + reject('inventory read denied') + payload = [dict(name=name, id=active.index(name) + 1 if name in active else 1000, + archived=value['lifecycle'] == 'archived', private=False, + visibility='public', disabled=False, fork=False, has_issues=True, + has_projects=True, has_wiki=True) + for name, value in properties['repositories'].items()] + if scenario == 'legacy-disabled-inventory-id': + next(item for item in payload if item['name'] == 'observatory')['id'] += 100 + if scenario == 'unmanaged': + payload.append(dict(name='unmanaged', archived=False, private=False, visibility='public', + disabled=False, fork=False, has_issues=True, has_projects=True, has_wiki=True)) + if scenario == 'temporary': + payload.append(dict(name='classic-ghsa-8533-3vg8-r287', archived=False, private=True, + visibility='private', disabled=False, fork=False, + has_issues=False, has_projects=False, has_wiki=False)) +elif endpoint == 'orgs/atrinik/properties/schema': + payload = properties['definitions'] + if scenario == 'schema-drift': + payload[0]['description'] = 'drift' +elif endpoint.startswith('orgs/atrinik/actions/permissions'): + suffix = endpoint.removeprefix('orgs/atrinik/actions/permissions') + filename = {'': 'actions-permissions.json', '/workflow': 'actions-workflow.json', + '/selected-actions': 'actions-selected.json'}[suffix] + if scenario == 'actions-unreadable': + reject('Actions read denied') + payload = config(filename) + if filename == 'actions-permissions.json': + payload['sha_pinning_required'] = scenario not in ('pending', 'sha-enforced-off', 'phase-unreadable') + if scenario == 'sha-invalid-type': + payload['sha_pinning_required'] = 'false' + if scenario == 'sha-unreadable': + del payload['sha_pinning_required'] + if filename == 'actions-workflow.json': + if scenario == 'actions-write-token': + payload['default_workflow_permissions'] = 'write' + if scenario == 'actions-approval': + payload['can_approve_pull_request_reviews'] = True + if filename == 'actions-selected.json' and scenario == 'actions-allowlist': + payload['patterns_allowed'].append('*') +elif endpoint.startswith('orgs/atrinik/rulesets/'): + identifier = int(endpoint.rsplit('/', 1)[1]) + payload = next((item for repository in active for item in rulesets(repository) + if item['id'] == identifier), None) + if payload is None or scenario == 'ruleset-unreadable': + reject('ruleset read denied') +elif endpoint.startswith('repos/atrinik/'): + repository, _, path = endpoint.removeprefix('repos/atrinik/').partition('/') + if repository not in active: + reject('audit attempted to inspect archived or foreign repository') + target = repository == 'observatory' + if path == 'actions/permissions/workflow': + if scenario == 'repository-actions-unreadable' and target: + reject('repository workflow permissions read denied') + payload = config('actions-workflow.json') + if scenario == 'repository-write-token' and target: + payload['default_workflow_permissions'] = 'write' + if scenario == 'repository-approval' and target: + payload['can_approve_pull_request_reviews'] = True + elif path.startswith('branches/') and path.endswith('/protection'): + if not include: + reject('legacy protection requires an observable HTTP status') + branch = path.removeprefix('branches/').removesuffix('/protection') + if branch not in ('main', 'release%2F2.x', 'release%2Fnext%23candidate'): + reject('unexpected or unencoded branch: ' + branch) + code, payload, result = 404, {'message': 'Branch not protected', 'status': '404'}, 1 + selected = target and branch == 'main' + if scenario.startswith('legacy-maintenance-'): + selected = target and branch == 'release%2Fnext%23candidate' + if selected: + variant = scenario.removeprefix('legacy-maintenance-').removeprefix('legacy-') + policies = { + 'extra-context': {'required_status_checks': {'checks': [ + {'context': 'Unconfigured check', 'app_id': 15368}]}}, + 'wrong-app': {'required_status_checks': {'checks': [ + {'context': 'Conventional PR title', 'app_id': 99}]}}, + 'approval': {'required_pull_request_reviews': {'required_approving_review_count': 1}}, + 'force-push': {'allow_force_pushes': {'enabled': True}}, + 'matching': {'required_status_checks': {'checks': [ + {'context': context, 'app_id': 15368} + for context in repositories['required_ci'].get(repository, [])]}}, + 'empty': {}, + } + if variant.startswith('disabled-'): + payload['message'] = 'Branch protection has been disabled on this repository.' + elif variant in policies: + code, payload, result = 200, policies[variant], 0 + elif variant == 'forbidden': + code, payload = 403, {'message': 'Resource not accessible by integration', 'status': '403'} + elif variant == 'not-found': + payload['message'] = 'Not Found' + elif variant == 'wrong-status': + payload['status'] = '403' + elif variant == 'mismatched-http': + code = 403 + elif variant == 'transport-error': + reject('connection failed') + elif variant == 'wrong-exit': + result = 2 + elif variant == 'success-error': + code, result = 200, 0 + elif variant == 'non-json': + payload = 'service unavailable' + elif variant == 'success-non-json': + code, payload, result = 200, 'service unavailable', 0 + elif variant == 'null': + payload = None + elif variant == 'array': + payload = [payload] + print(f'HTTP/2.0 {code} Fixture\r\nContent-Type: application/json\r\n\r') + print(payload if isinstance(payload, str) else json.dumps(payload)) + sys.exit(result) + elif path == 'properties/values': + payload = [{'property_name': key, 'value': value} + for key, value in properties['repositories'][repository].items()] + if scenario == 'properties-drift' and target: + payload[0]['value'] = 'drift' + elif path.startswith('rulesets?'): + if scenario == 'retained-list-unreadable': + reject('ruleset list read denied') + if scenario == 'retained-list-non-json': + print('unavailable') + sys.exit(0) + payload = [{'name': item['name'], 'id': item['id'], 'source_type': item['source_type']} + for item in rulesets(repository)] + if scenario == 'retained-list-object': + payload = {'message': 'unreadable'} + elif scenario == 'retained-list-missing-name': + payload.append({'id': 999, 'source_type': 'Repository'}) + elif scenario == 'retained-list-null-item': + payload.append(None) + elif scenario == 'retained-list-partial-page': + print(json.dumps([payload, None])) + sys.exit(0) + elif path.startswith('rulesets/'): + identifier = int(path.rsplit('/', 1)[1]) + payload = next((item for item in rulesets(repository) if item['id'] == identifier), None) + if payload is None: + reject('repository ruleset missing') + elif path.startswith('rules/branches/'): + branch = unquote(path.removeprefix('rules/branches/').partition('?')[0]) + if scenario == 'effective-unreadable' and target: + reject('effective branch read denied') + payload = [] + for item in rulesets(repository): + scope = item['conditions'].get('repository_name', {'include': [repository], 'exclude': []}) + included = any(pattern == '~ALL' or fnmatch.fnmatchcase(repository, pattern) for pattern in scope['include']) + excluded = any(pattern == '~ALL' or fnmatch.fnmatchcase(repository, pattern) for pattern in scope['exclude']) + refs = item['conditions']['ref_name'] + branch_included = any((pattern == '~DEFAULT_BRANCH' and branch == 'main') or + fnmatch.fnmatchcase('refs/heads/' + branch, pattern) for pattern in refs['include']) + branch_excluded = any((pattern == '~DEFAULT_BRANCH' and branch == 'main') or + fnmatch.fnmatchcase('refs/heads/' + branch, pattern) for pattern in refs['exclude']) + if (item['target'] == 'branch' and item['enforcement'] == 'active' + and included and not excluded and branch_included and not branch_excluded): + payload.extend(copy.deepcopy(item['rules'])) + for maintenance in repositories['maintenance_branches']: + if maintenance['repository'] == repository and maintenance['branch'] == branch: + rules = config('rulesets/maintenance-branch.json')['rules'] + for rule in rules: + if rule['type'] == 'required_status_checks': + if not maintenance['required_ci']: + continue + rule['parameters']['required_status_checks'] = [ + {'context': context, 'integration_id': 15368} + for context in maintenance['required_ci']] + payload.append(rule) + for rule in payload: + if target and rule['type'] == 'required_status_checks': + if scenario == 'wrong-app': + rule['parameters']['required_status_checks'][0]['integration_id'] = 9 + if scenario == 'wrong-context': + rule['parameters']['required_status_checks'][0]['context'] += ' drift' + if scenario == 'strict-ci': + rule['parameters']['strict_required_status_checks_policy'] = False + if target and rule['type'] == 'pull_request': + if scenario == 'approval-count': + rule['parameters']['required_approving_review_count'] = 1 + if scenario == 'thread-resolution': + rule['parameters']['required_review_thread_resolution'] = False + if scenario == 'effective-integrity' and target: + payload = [item for item in payload if item['type'] != 'deletion'] + if scenario == 'unexpected-codeql' and repository == '.github': + payload.extend(config('rulesets/code-scanning.json')['rules']) + if scenario == 'unexpected-pr' and repository == 'web-platform': + payload.append(config('rulesets/pull-request-gate.json')['rules'][0]) + if scenario.startswith('closure-'): + _, selected_branch, variant = scenario.split('-', 2) + selected = (selected_branch == 'default' and branch == 'main') or ( + selected_branch == 'maintenance' and branch == 'release/next#candidate') + if selected: + extra = { + 'signatures': {'type': 'required_signatures'}, + 'deployments': {'type': 'required_deployments', 'parameters': {'required_deployment_environments': ['production']}}, + 'update': {'type': 'update', 'parameters': {'update_allows_fetch_and_merge': False}}, + 'unknown': {'type': 'future_unconfigured_rule', 'parameters': {'enabled': True}}, + } + if variant in extra: + payload.append(extra[variant]) + elif variant == 'integrity-parameter': + next(rule for rule in payload if rule['type'] == 'deletion')['parameters'] = {'unexpected': True} + elif variant == 'status-parameter': + next(rule for rule in payload if rule['type'] == 'required_status_checks')['parameters']['unexpected'] = True + elif variant == 'top-field': + payload[0]['unknown_policy_field'] = True + elif variant == 'missing-type': + payload.append({'parameters': {}}) + elif variant == 'null-item': + payload.append(None) + elif variant == 'malformed-parameters': + next(rule for rule in payload if rule['type'] == 'deletion')['parameters'] = [] + elif variant == 'null-page': + print(json.dumps([payload, None])) + sys.exit(0) + elif variant == 'unreadable': + reject('effective branch read denied') + elif variant == 'non-json': + print('unavailable') + sys.exit(0) + elif variant in ('metadata-organization', 'metadata-repository'): + for rule in payload: + rule.update(ruleset_id=99, ruleset_source='atrinik' if variant.endswith('organization') else 'atrinik/observatory', + ruleset_source_type='Organization' if variant.endswith('organization') else 'Repository') + elif variant == 'signatures-inherited': + payload.append({'type': 'required_signatures', 'ruleset_id': 99, + 'ruleset_source': 'atrinik', 'ruleset_source_type': 'Organization'}) + elif path == 'code-security-configuration': + advanced = repository in config('codeql-advanced-setup.json')['repositories'] + if scenario == 'configuration-unreadable' and target: + reject('configuration read denied') + payload = {'status': 'enforced', 'configuration': config( + 'code-security-advanced.json' if advanced else 'code-security.json')} + if scenario == 'legacy-security': + payload['configuration'].pop('code_security') + payload['configuration'].pop('secret_protection') + payload['configuration']['advanced_security'] = 'enabled' + if scenario == 'configuration-detached' and target: + payload['status'] = 'attached' + if scenario == 'dependency-graph' and target: + payload['configuration']['dependency_graph'] = 'disabled' + if scenario == 'advanced-config' and repository == 'classic': + payload['configuration'] = config('code-security.json') + elif path == 'code-scanning/default-setup': + advanced = repository in config('codeql-advanced-setup.json')['repositories'] + if scenario == 'setup-unreadable' and target: + reject('setup read denied') + payload = config('codeql-default-setup-advanced.json' if advanced else 'codeql-default-setup.json') + payload['languages'] = policy['codeql_required_languages'].get(repository, []) + if target: + malformed_languages = { + 'languages-null': None, + 'languages-scalar': 'javascript-typescript', + 'languages-object': {'language': 'javascript-typescript'}, + 'languages-number': [7], + 'languages-mixed': ['javascript-typescript', None], + } + if scenario in malformed_languages: + payload['languages'] = malformed_languages[scenario] + elif scenario == 'languages-missing': + del payload['languages'] + elif scenario == 'languages-readable': + payload['languages'] = ['javascript-typescript', 'future-language'] + elif scenario == 'languages-empty': + payload['languages'] = [] + if scenario == 'advanced-null-languages' and repository == 'classic': + payload['languages'] = None + if scenario == 'missing-rust' and repository == 'content-toolkit': + payload['languages'] = ['javascript-typescript'] + if scenario == 'advanced-setup' and repository == 'classic': + payload['state'] = 'configured' + if scenario == 'default-setup' and target: + payload['state'] = 'not-configured' + elif path == 'vulnerability-alerts': + if scenario == 'dependabot-alerts' and target: + reject('alerts disabled') + sys.exit(0) # GitHub's enabled response is HTTP 204, without JSON. + elif path == 'private-vulnerability-reporting': + payload = {'enabled': not (scenario == 'private-reporting' and target)} + elif path == '': + payload = dict(config('repository-defaults.json'), id=active.index(repository) + 1, + name=repository, full_name='atrinik/' + repository, archived=False, security_and_analysis={ + name: {'status': 'enabled'} for name in ( + 'secret_scanning', 'secret_scanning_push_protection', + 'secret_scanning_validity_checks', 'dependabot_security_updates')}) + if scenario == 'legacy-disabled-metadata-name' and target: + payload['full_name'] = 'foreign/observatory' + if scenario == 'merge-drift' and target: + payload['allow_merge_commit'] = True + if scenario == 'secret-baseline' and target: + payload['security_and_analysis']['secret_scanning']['status'] = 'disabled' + else: + reject('unexpected repository endpoint: ' + endpoint) +else: + reject('unexpected endpoint: ' + endpoint) +print(json.dumps([payload] if slurp else payload)) diff --git a/tests/audit.sh b/tests/audit.sh index d34373c..f8c3223 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -5,228 +5,323 @@ set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) temporary=$(mktemp -d) trap 'rm -rf "${temporary}"' EXIT -mkdir "${temporary}/bin" - -cat >"${temporary}/bin/gh" <<'EOF' +mkdir -p "${temporary}/mock" "${temporary}/full/bin" +cp -R "${root}/config" "${temporary}/full/config" +cp "${root}/bin/audit" "${temporary}/full/bin/audit" +# Schema validation has its own policy tests; these isolate live API evidence. +printf '#!/usr/bin/env bash\nexit 0\n' >"${temporary}/full/bin/validate" +cat >"${temporary}/full/bin/security-readiness" <<'EOF' #!/usr/bin/env bash - -set -euo pipefail - -[[ ${1:-} == api ]] || exit 1 -shift -jq_filter= -endpoint= -while (($#)); do - case $1 in - -H) - shift 2 - ;; - --paginate) - shift - ;; - --jq) - jq_filter=$2 - shift 2 - ;; - *) - endpoint=$1 - shift - ;; - esac +printf '%s\n' "$1" >>"${AUDIT_READINESS_LOG}" +case ${GH_TEST_SCENARIO:-pass} in + pending|pending-codeql|pending-wrong-dependency-scope|enforced-missing-dependency|enforced-missing-codeql) exit 1 ;; + readiness-unreadable|dependency-readiness-unreadable|codeql-readiness-unreadable|enforced-unreadable-dependency|enforced-unreadable-codeql) exit 2 ;; + *) exit 0 ;; +esac +EOF +chmod +x "${temporary}/full/bin/validate" "${temporary}/full/bin/security-readiness" +cp "${root}/tests/audit-fixtures.py" "${temporary}/mock/gh" +chmod +x "${temporary}/mock/gh" +# Exercise both reviewed rollout phases using test-owned desired state. +printf '%s\n' '{"phase":"pending","canary":null}' \ + >"${temporary}/full/config/actions-sha-pinning.json" +jq '.sha_pinning_required = true' "${temporary}/full/config/actions-permissions.json" \ + >"${temporary}/actions.json" +mv "${temporary}/actions.json" "${temporary}/full/config/actions-permissions.json" +jq '.pull_request_gate |= ((. + [".github"]) | unique)' "${temporary}/full/config/repositories.json" \ + >"${temporary}/repositories.json" +mv "${temporary}/repositories.json" "${temporary}/full/config/repositories.json" +for repository in .github classic observatory content-toolkit service-updater web-platform; do + fixture="${temporary}/fixture-${repository}" + cp -R "${temporary}/full" "${fixture}" + jq --arg repository "${repository}" \ + '.repositories |= with_entries(select(.key == $repository))' \ + "${temporary}/full/config/repository-properties.json" >"${fixture}/config/repository-properties.json" done -emit() { - local payload=$1 - if [[ -n ${jq_filter} ]]; then - jq -r "${jq_filter}" <<<"${payload}" - else - printf '%s\n' "${payload}" - fi -} +cp -R "${temporary}/fixture-observatory" "${temporary}/fixture-legacy-branches" +jq '.maintenance_branches = [ + {repository: "observatory", branch: "release/2.x", required_ci: []}, + {repository: "observatory", branch: "release/next#candidate", required_ci: []}, + {repository: "classic", branch: "foreign/maintenance", required_ci: []} +]' "${temporary}/full/config/repositories.json" \ + >"${temporary}/fixture-legacy-branches/config/repositories.json" -config_root=${AUDIT_CONFIG_ROOT:?} -scenario=${GH_TEST_SCENARIO:-pass} +cp -R "${temporary}/fixture-observatory" "${temporary}/fixture-retired-observatory" +jq 'del(.required_ci.observatory)' "${temporary}/full/config/repositories.json" \ + >"${temporary}/fixture-retired-observatory/config/repositories.json" -case ${endpoint} in -orgs/atrinik/properties/schema) - if [[ ${scenario} == schema-drift ]]; then - jq -n --slurpfile config \ - "${config_root}/config/repository-properties.json" \ - '$config[0].definitions | map(. + {description: "drift"})' - else - jq -n --slurpfile config \ - "${config_root}/config/repository-properties.json" \ - '$config[0].definitions' - fi - ;; -orgs/atrinik/repos\?*) - payload=$(jq -n --slurpfile config "${config_root}/config/repository-properties.json" ' - [ - $config[0].repositories | - to_entries[] | - { - name: .key, - archived: (.value.lifecycle == "archived"), - private: false, - visibility: "public", - disabled: false, - fork: false, - has_issues: true, - has_projects: true, - has_wiki: true - } - ] | - if $ENV.GH_TEST_SCENARIO == "unmanaged" then - . + [{ - name: "unmanaged", - archived: false, - private: false, - visibility: "public", - disabled: false, - fork: false, - has_issues: true, - has_projects: true, - has_wiki: true - }] - elif $ENV.GH_TEST_SCENARIO == "temporary" then - . + [{ - name: "classic-ghsa-8533-3vg8-r287", - archived: false, - private: true, - visibility: "private", - disabled: false, - fork: false, - has_issues: false, - has_projects: false, - has_wiki: false - }] - else - . - end - ') - emit "${payload}" - ;; -repos/atrinik/*) - path=${endpoint#repos/atrinik/} - case ${path} in - */properties/values) - repository=${path%/properties/values} - if [[ ${scenario} == properties-drift && ${repository} == observatory ]]; then - jq -n ' - [ - {property_name: "component_role", value: ["unclassified"]}, - {property_name: "lifecycle", value: "seed"}, - {property_name: "provider_set", value: ["unclassified"]}, - {property_name: "release_policy", value: "none"} - ] - ' - else - jq -n --slurpfile config "${config_root}/config/repository-properties.json" \ - --arg repository "${repository}" ' - $config[0].repositories[$repository] | - to_entries | - map({property_name: .key, value: .value}) - ' - fi - ;; - */rulesets\?includes_parents=true) - repository=${path%/rulesets?includes_parents=true} - jq -n --slurpfile repositories "${config_root}/config/repositories.json" \ - --arg repository "${repository}" ' - [ - "01 - Default branch integrity", - "01 - Default branch linear history" - ] + - (if ($repositories[0].pull_request_gate | index($repository)) != null then - ["02 - Changes through pull requests"] - else [] end) + - (if ($repositories[0].required_ci | has($repository)) then - ["03 - Required CI - " + $repository] - else [] end) + - (if ($repositories[0].release_tags | index($repository)) != null then - ["04 - Immutable release tags"] - else [] end) | - map({name: .}) - ' - ;; - *) - repository=${path} - allow_merge_commit=false - if [[ ${scenario} == merge-drift && ${repository} == observatory ]]; then - allow_merge_commit=true - fi - jq -n --slurpfile defaults "${config_root}/config/repository-defaults.json" \ - --arg repository "${repository}" --argjson allow_merge_commit ${allow_merge_commit} ' - { - id: 1, - name: $repository, - archived: false, - default_branch: $defaults[0].default_branch, - allow_merge_commit: $allow_merge_commit, - allow_rebase_merge: $defaults[0].allow_rebase_merge, - allow_squash_merge: $defaults[0].allow_squash_merge, - delete_branch_on_merge: $defaults[0].delete_branch_on_merge, - squash_merge_commit_title: $defaults[0].squash_merge_commit_title, - squash_merge_commit_message: $defaults[0].squash_merge_commit_message, - security_and_analysis: { - secret_scanning: {status: "enabled"}, - secret_scanning_push_protection: {status: "enabled"}, - secret_scanning_validity_checks: {status: "enabled"}, - dependabot_security_updates: {status: "enabled"} - } - } - ' - ;; - esac - ;; -*) - echo "unexpected endpoint: ${endpoint}" >&2 - exit 1 - ;; -esac -EOF -chmod +x "${temporary}/bin/gh" +cp -R "${temporary}/fixture-legacy-branches" "${temporary}/fixture-effective-branches" +jq '(.maintenance_branches[] | select(.branch == "release/next#candidate") | .required_ci) = ["Conventional PR title"]' \ + "${temporary}/fixture-legacy-branches/config/repositories.json" \ + >"${temporary}/fixture-effective-branches/config/repositories.json" run_audit() { - local scenario=$1 + local scenario=$1 fixture=${2:-focused} repository=observatory + if [[ ${fixture} == focused ]]; then + case ${scenario} in + closure-*) repository=effective-branches ;; + retained-*) repository=.github ;; + retired-*) repository=retired-observatory ;; + legacy-maintenance-*|legacy-multiple) repository=legacy-branches ;; + wrong-pr-scope|unexpected-codeql|tag-extra-nonrelease|tag-all-nonrelease|tag-repository-nonrelease|inherited-tag-unaffected|foreign-tag) repository=.github ;; + advanced-config|advanced-setup|advanced-null-languages) repository=classic ;; + missing-rust) repository=content-toolkit ;; + missing-tag-scope|tag-ref-drift|tag-all-scope|tag-extra-repo|tag-extra-exclude) repository=service-updater ;; + unexpected-pr) repository=web-platform ;; + esac + fixture="fixture-${repository}" + fi + case ${scenario} in + sha-enforced-off|sha-enforced-on) + printf '%s\n' '{"phase":"enforced","canary":"atrinik/observatory/1"}' \ + >"${temporary}/${fixture}/config/actions-sha-pinning.json" + ;; + phase-unreadable) + printf '%s\n' '{}' >"${temporary}/${fixture}/config/actions-sha-pinning.json" + ;; + *) + printf '%s\n' '{"phase":"pending","canary":null}' \ + >"${temporary}/${fixture}/config/actions-sha-pinning.json" + ;; + esac + cp "${root}/config/security-merge-protection.json" "${temporary}/${fixture}/config/security-merge-protection.json" + case ${scenario} in + retired-*) + jq '.dependency_review -= ["observatory"] | .code_scanning -= ["observatory"] | + del(.activation.dependency_review.observatory, .activation.code_scanning.observatory)' \ + "${temporary}/${fixture}/config/security-merge-protection.json" >"${temporary}/activation.json" + mv "${temporary}/activation.json" "${temporary}/${fixture}/config/security-merge-protection.json" + ;; + enforced-missing-dependency|enforced-unreadable-dependency) + jq '.activation.dependency_review.observatory = "enforced"' \ + "${temporary}/${fixture}/config/security-merge-protection.json" >"${temporary}/activation.json" + mv "${temporary}/activation.json" "${temporary}/${fixture}/config/security-merge-protection.json" + ;; + enforced-missing-codeql|enforced-unreadable-codeql) + jq '.activation.code_scanning.observatory = "enforced"' \ + "${temporary}/${fixture}/config/security-merge-protection.json" >"${temporary}/activation.json" + mv "${temporary}/activation.json" "${temporary}/${fixture}/config/security-merge-protection.json" + ;; + esac + : >"${temporary}/${scenario}.readiness-log" GH_TEST_SCENARIO=${scenario} \ - PATH="${temporary}/bin:${PATH}" \ - AUDIT_CONFIG_ROOT="${root}" \ - "${root}/bin/audit" + AUDIT_READINESS_LOG="${temporary}/${scenario}.readiness-log" \ + PATH="${temporary}/mock:${PATH}" \ + AUDIT_CONFIG_ROOT="${temporary}/${fixture}" \ + AUDIT_API_LOG="${temporary}/api.log" \ + "${temporary}/${fixture}/bin/audit" } -output=$(run_audit pass) +output=$(run_audit pass full) grep -Fxq 'Governance audit passed: 23 active repositories checked.' <<<"${output}" - -temporary_output=$(run_audit temporary) -grep -Fxq 'Governance audit passed: 23 active repositories checked.' <<<"${temporary_output}" - -if run_audit unmanaged >"${temporary}/unmanaged.out" 2>"${temporary}/unmanaged.err"; then - echo "expected unmanaged repository audit to fail" >&2 +output=$(run_audit temporary full) +grep -Fxq 'Governance audit passed: 23 active repositories checked.' <<<"${output}" +output=$(run_audit legacy-security) +grep -Fq 'Governance audit passed:' <<<"${output}" +output=$(run_audit repository-rulesets) +grep -Fq 'Governance audit passed:' <<<"${output}" +for scenario in api-defaults-omitted api-default-order scope-order inherited-tag-unaffected foreign-tag sha-enforced-on languages-readable languages-empty advanced-null-languages; do + output=$(run_audit "${scenario}") + grep -Fq 'Governance audit passed:' <<<"${output}" +done +output=$(run_audit pending) +grep -Fq 'PENDING Actions SHA pinning deferred pending canary proof' <<<"${output}" +grep -Fq 'PENDING Dependency review merge protection awaits landed successful producer' <<<"${output}" +grep -Fq 'Governance audit baseline passed with' <<<"${output}" +if grep -Fq 'Governance audit passed:' <<<"${output}"; then + echo 'pending policy must not report full governance enforcement' >&2 exit 1 fi -grep -Fq 'active repository is not in the desired-state inventory: atrinik/unmanaged' \ - "${temporary}/unmanaged.out" -if run_audit merge-drift >"${temporary}/merge.out" 2>"${temporary}/merge.err"; then - echo "expected merge drift audit to fail" >&2 - exit 1 -fi -grep -Fq 'repository merge policy drift: atrinik/observatory' \ - "${temporary}/merge.out" +assert_drift() { + local scenario=$1 message=$2 + if run_audit "${scenario}" >"${temporary}/${scenario}.out" 2>"${temporary}/${scenario}.err"; then + echo "expected ${scenario} audit to fail" >&2 + exit 1 + fi + if ! grep -Fq "${message}" "${temporary}/${scenario}.out" "${temporary}/${scenario}.err"; then + cat "${temporary}/${scenario}.out" "${temporary}/${scenario}.err" >&2 + echo "missing expected ${scenario} diagnostic: ${message}" >&2 + exit 1 + fi +} -if run_audit properties-drift >"${temporary}/properties.out" 2>"${temporary}/properties.err"; then - echo "expected property drift audit to fail" >&2 +# Only the precise GitHub absence response is accepted; legacy policies are +# manual drift even when their contexts match the declared ruleset contract. +for scenario in extra-context wrong-app approval force-push matching empty success-error; do + assert_drift "legacy-${scenario}" 'unexpected legacy branch protection requires manual review: atrinik/observatory/main' +done +for scenario in forbidden not-found wrong-status mismatched-http transport-error wrong-exit non-json success-non-json null array; do + assert_drift "legacy-${scenario}" 'legacy branch protection could not be read: atrinik/observatory/main' +done +for scenario in extra-context approval; do + assert_drift "legacy-maintenance-${scenario}" 'unexpected legacy branch protection requires manual review: atrinik/observatory/release/next#candidate' +done +assert_drift legacy-maintenance-forbidden 'legacy branch protection could not be read: atrinik/observatory/release/next#candidate' +: >"${temporary}/api.log" +output=$(run_audit legacy-multiple) +grep -Fq 'Governance audit passed:' <<<"${output}" +for branch in main release%2F2.x release%2Fnext%23candidate; do + [[ $(grep -Fxc "repos/atrinik/observatory/branches/${branch}/protection" "${temporary}/api.log") == 1 ]] +done +if grep -Fq 'foreign' "${temporary}/api.log"; then + echo 'maintenance protection must remain scoped to its repository' >&2 exit 1 fi -grep -Fq 'repository custom-property drift: atrinik/observatory' \ - "${temporary}/properties.out" -if run_audit schema-drift >"${temporary}/schema.out" 2>"${temporary}/schema.err"; then - echo "expected property schema drift audit to fail" >&2 +# GitHub's disabled-protection response requires independent exact-ref proof; +# historical rules on another branch must neither create drift nor prove absence. +for scenario in legacy-disabled-absent legacy-maintenance-disabled-absent; do + output=$(run_audit "${scenario}") + grep -Fq 'Governance audit passed:' <<<"${output}" +done +assert_drift legacy-disabled-present 'unexpected legacy branch protection requires manual review: atrinik/observatory/main' +assert_drift legacy-maintenance-disabled-present 'unexpected legacy branch protection requires manual review: atrinik/observatory/release/next#candidate' +for scenario in wrong-id wrong-owner wrong-name archived missing-field null-ref null-repository missing-data malformed-rule errors unreadable non-json inventory-id metadata-name; do + assert_drift "legacy-disabled-${scenario}" 'legacy branch protection could not be read: atrinik/observatory/main' +done +assert_drift legacy-maintenance-disabled-wrong-name 'legacy branch protection could not be read: atrinik/observatory/release/next#candidate' + +# Retained named rules remain drift after target removal, regardless of whether +# their current rules apply to main. Unrelated names remain outside ownership. +for scope in retained retired; do + for kind in codeql dependency; do + case ${kind} in + codeql) name='06 - CodeQL security' ;; + dependency) name='07 - Dependency review' ;; + esac + for variant in disabled moved modified duplicate foreign-owner; do + assert_drift "${scope}-${kind}-${variant}" "unexpected retained security ruleset '${name}' requires manual review" + done + output=$(run_audit "${scope}-${kind}-foreign-name") + grep -Fq 'Governance audit passed:' <<<"${output}" + done + output=$(run_audit "${scope}-absent") + grep -Fq 'Governance audit passed:' <<<"${output}" +done +for variant in unreadable non-json object missing-name null-item partial-page; do + assert_drift "retained-list-${variant}" 'inherited rulesets could not be read' +done + +# Check the entire effective policy on main and exact maintenance refs. Desired +# CI, Dependency review and CodeQL coexist; API provenance is metadata only. +for branch in default maintenance; do + for variant in baseline metadata-organization metadata-repository; do + output=$(run_audit "closure-${branch}-${variant}") + grep -Fq 'Governance audit passed:' <<<"${output}" + done + for variant in signatures deployments update unknown integrity-parameter status-parameter top-field malformed-parameters signatures-inherited; do + assert_drift "closure-${branch}-${variant}" 'effective branch policy drift requires manual review' + done + for variant in missing-type null-item null-page unreadable non-json; do + assert_drift "closure-${branch}-${variant}" "effective ${branch}-branch rules could not be read" + done +done + +assert_drift unmanaged 'active repository is not in the desired-state inventory' +assert_drift merge-drift 'repository merge policy drift' +assert_drift properties-drift 'repository custom-property drift' +assert_drift schema-drift 'organization custom-property schema drift' +assert_drift inventory-unreadable 'organization repository inventory could not be read' +assert_drift wrong-pr-scope 'effective pull-request policy drift: atrinik/.github' +assert_drift missing-tag-scope "ruleset scope or policy drift '04 - Immutable release tags': atrinik/service-updater" +assert_drift tag-ref-drift "ruleset scope or policy drift '04 - Immutable release tags'" +for scenario in tag-all-scope tag-extra-repo tag-extra-exclude tag-all-nonrelease tag-extra-nonrelease; do + assert_drift "${scenario}" "ruleset scope or policy drift '04 - Immutable release tags'" +done +assert_drift tag-repository-nonrelease 'unexpected managed release-tag protection: atrinik/.github' +for scenario in api-required-reviewers api-extra-approval api-restricted-merges api-duplicate-merges \ + api-malformed-reviewers api-malformed-approval api-malformed-merges api-unknown-pr-field; do + assert_drift "${scenario}" "ruleset scope or policy drift '02 - Changes through pull requests'" +done +for scenario in api-create-exemption api-malformed-create-exemption; do + assert_drift "${scenario}" "ruleset scope or policy drift '03 - Required CI - observatory'" +done +assert_drift pr-all-scope "ruleset scope or policy drift '02 - Changes through pull requests'" +assert_drift default-extra-exclude "ruleset scope or policy drift '01 - Default branch integrity'" +assert_drift repository-scope-extra "ruleset scope or policy drift '07 - Dependency review'" +assert_drift organization-codeql-owner "managed security ruleset must be repository-owned '06 - CodeQL security'" +assert_drift organization-dependency-owner "managed security ruleset must be repository-owned '07 - Dependency review'" +assert_drift integrity-bypass "ruleset scope or policy drift '01 - Default branch integrity'" +assert_drift pr-bypass "ruleset scope or policy drift '02 - Changes through pull requests'" +assert_drift owner-id-drift "ruleset scope or policy drift '02 - Changes through pull requests'" +assert_drift pending-wrong-dependency-scope "ruleset scope or policy drift '07 - Dependency review'" +for scenario in wrong-app wrong-context strict-ci; do + assert_drift "${scenario}" 'effective required CI contexts or app drift' +done +for scenario in approval-count thread-resolution; do + assert_drift "${scenario}" 'effective pull-request policy drift' +done +assert_drift effective-integrity 'effective default-branch integrity drift' +assert_drift unexpected-pr 'effective pull-request policy drift: atrinik/web-platform' +assert_drift unexpected-codeql 'unexpected effective CodeQL merge protection: atrinik/.github' +for scenario in codeql-threshold codeql-tool codeql-quality; do + assert_drift "${scenario}" 'effective CodeQL merge protection drift' +done +for scenario in actions-write-token actions-approval; do + assert_drift "${scenario}" 'organization Actions workflow drift' +done +for scenario in repository-write-token repository-approval; do + assert_drift "${scenario}" 'repository Actions workflow permissions drift' +done +assert_drift repository-actions-unreadable 'repository Actions workflow permissions could not be read' +assert_drift actions-allowlist 'organization Actions selected-actions drift' +assert_drift actions-unreadable 'organization Actions permissions could not be read' +assert_drift sha-unreadable 'Actions SHA pinning state could not be read' +assert_drift sha-invalid-type 'Actions SHA pinning state could not be read' +assert_drift sha-enforced-off 'Actions SHA pinning enforcement drift' +assert_drift phase-unreadable 'Actions SHA pinning rollout phase could not be read' +assert_drift missing-rust 'CodeQL setup or required language drift: atrinik/content-toolkit' +for scenario in languages-missing languages-null languages-scalar languages-object languages-number languages-mixed; do + assert_drift "${scenario}" 'CodeQL setup or required language drift: atrinik/observatory' +done +for scenario in advanced-setup default-setup; do + assert_drift "${scenario}" 'CodeQL setup or required language drift' +done +for scenario in configuration-detached dependency-graph advanced-config; do + assert_drift "${scenario}" 'security configuration attachment or baseline drift' +done +assert_drift configuration-unreadable 'security configuration could not be read' +assert_drift setup-unreadable 'CodeQL setup could not be read' +assert_drift dependabot-alerts 'Dependabot alerts disabled or unreadable' +assert_drift private-reporting 'private vulnerability reporting disabled or unreadable' +assert_drift secret-baseline 'exposed security baseline drift' +assert_drift effective-unreadable 'effective default-branch rules could not be read' +assert_drift ruleset-unreadable 'ruleset contents could not be read' +for scenario in enforced-missing-dependency enforced-unreadable-dependency; do + assert_drift "${scenario}" 'effective required CI contexts or app drift' + [[ ! -s ${temporary}/${scenario}.readiness-log ]] + if grep -Fq 'PENDING Dependency review' "${temporary}/${scenario}.out"; then + echo 'enforced dependency protection must never fall back to pending' >&2 + exit 1 + fi +done +for scenario in enforced-missing-codeql enforced-unreadable-codeql; do + assert_drift "${scenario}" 'effective CodeQL merge protection drift' + [[ ! -s ${temporary}/${scenario}.readiness-log ]] + if grep -Fq 'PENDING CodeQL' "${temporary}/${scenario}.out"; then + echo 'enforced CodeQL protection must never fall back to pending' >&2 + exit 1 + fi +done +for scenario in ready-missing ready-missing-codeql; do + output=$(run_audit "${scenario}") + grep -Fq 'merge protection awaits reviewed activation intent' <<<"${output}" + grep -Fq 'Governance audit baseline passed with' <<<"${output}" +done +assert_drift dependency-readiness-unreadable 'Dependency review readiness could not be read' +assert_drift codeql-readiness-unreadable 'CodeQL readiness could not be read' +output=$(run_audit pending-codeql) +grep -Fq 'PENDING CodeQL merge protection awaits successful analysis' <<<"${output}" +grep -Fq 'Governance audit baseline passed with' <<<"${output}" +# An existing dependency gate survives a temporarily unreadable producer readiness +# API; read readiness is required only when first installing the context. +output=$(run_audit readiness-unreadable) +grep -Fq 'Governance audit passed:' <<<"${output}" + +if grep -Eq '^repos/atrinik/(legacy-|nawerhals)' "${temporary}/api.log"; then + echo 'audit must not inspect archived repositories' >&2 exit 1 fi -grep -Fq 'organization custom-property schema drift: atrinik' \ - "${temporary}/schema.out" - -echo "Governance drift audit tests passed." +echo 'Governance effective-policy audit tests passed.' diff --git a/tests/publish-maintenance-branch.sh b/tests/publish-maintenance-branch.sh index 6a50db7..28c24dd 100755 --- a/tests/publish-maintenance-branch.sh +++ b/tests/publish-maintenance-branch.sh @@ -14,11 +14,14 @@ source_root=${root} # Retain the original Classic rollout/recovery scenarios as a bounded fixture. root=${temporary}/classic-policy mkdir -p "${root}/bin" -cp "${source_root}/bin/publish" "${source_root}/bin/validate" "${root}/bin/" +cp "${source_root}/bin/publish" "${source_root}/bin/validate" \ + "${source_root}/bin/security-readiness" "${root}/bin/" +cp -R "${source_root}/lib" "${root}/lib" cp -R "${source_root}/config" "${source_root}/community-health" "${source_root}/.github" "${root}/" jq '.repositories |= map(select(.name == "classic"))' \ "${source_root}/config/immutable-releases.json" >"${root}/config/immutable-releases.json" mkdir "${temporary}/bin" +export GH_SELECTED_ACTIONS_POLICY=${root}/config/actions-selected.json cat >"${temporary}/bin/gh" <<'EOF' #!/usr/bin/env bash @@ -295,6 +298,15 @@ if [[ ${method} != GET ]]; then fi case "${endpoint}|${jq_filter}" in +repos/atrinik/*/actions/permissions/workflow\|*) + printf '{"default_workflow_permissions":"read","can_approve_pull_request_reviews":false}\n' + ;; +"orgs/atrinik/actions/permissions/selected-actions|") + cat "${GH_SELECTED_ACTIONS_POLICY}" + ;; +"orgs/atrinik/actions/permissions|") + printf '{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":false}\n' + ;; "orgs/atrinik|") if [[ ${GH_ORGANIZATION_SCENARIO:-converged} == drifted ]]; then jq -n '{ @@ -360,7 +372,7 @@ case "${endpoint}|${jq_filter}" in printf '{"status":"enforced","configuration":{"id":265376}}\n' ;; "repos/atrinik/protocol/code-scanning/default-setup|"* | "repos/atrinik/renderer/code-scanning/default-setup|"* | "repos/atrinik/content-toolkit/code-scanning/default-setup|"*) - printf '{"state":"configured","query_suite":"default"}\n' + printf '{"state":"configured","query_suite":"default","languages":["rust"]}\n' ;; "repos/atrinik/classic/immutable-releases|"*) if [[ ${GH_IMMUTABLE_VERIFY_FAILURE:-false} == true ]]; then @@ -474,17 +486,17 @@ repos/atrinik/*/environments/crates-io-release/variables\?per_page=100\&page=1\| if [[ ${security_scenario} == drifted || ${security_scenario} == create || ${GH_EMPTY_ADVANCED_INVENTORY:-false} == true ]]; then - printf '{"state":"configured","query_suite":"default"}\n' + printf '{"state":"configured","query_suite":"default","languages":["c-cpp"]}\n' else - printf '{"state":"not-configured","query_suite":"default"}\n' + printf '{"state":"not-configured","query_suite":"default","languages":null}\n' fi ;; "repos/atrinik/content/code-scanning/default-setup|"*) if [[ ${security_scenario} == drifted || ${security_scenario} == create ]]; then - printf '{"state":"not-configured"}\n' + printf '{"state":"not-configured","languages":null}\n' else - printf '{"state":"configured","query_suite":"default"}\n' + printf '{"state":"configured","query_suite":"default","languages":["python"]}\n' fi ;; "repos/atrinik/nawerhals|.archived") @@ -1214,8 +1226,9 @@ assert_idempotent_organization_security() { advisory_active_root=${temporary}/advisory-window-active mkdir -p "${advisory_active_root}/bin" -cp "${root}/bin/publish" "${root}/bin/validate" \ +cp "${root}/bin/publish" "${root}/bin/validate" "${root}/bin/security-readiness" \ "${advisory_active_root}/bin/" +cp -R "${root}/lib" "${advisory_active_root}/lib" cp -R "${root}/config" "${advisory_active_root}/config" cp -R "${root}/community-health" \ "${advisory_active_root}/community-health" @@ -1368,12 +1381,13 @@ assert_codeql_default_setup "${creation_log}" rollback_root=${temporary}/empty-inventory mkdir -p "${rollback_root}/bin" -cp "${root}/bin/publish" "${root}/bin/validate" "${rollback_root}/bin/" +cp "${root}/bin/publish" "${root}/bin/validate" "${root}/bin/security-readiness" "${rollback_root}/bin/" +cp -R "${root}/lib" "${rollback_root}/lib" cp -R "${root}/config" "${rollback_root}/config" cp -R "${root}/community-health" "${rollback_root}/community-health" mkdir -p "${rollback_root}/.github" cp -R "${root}/.github/workflows" "${rollback_root}/.github/workflows" -jq '.repositories = []' \ +jq '.repositories = [] | .required_partitions = {}' \ "${root}/config/codeql-advanced-setup.json" \ >"${rollback_root}/config/codeql-advanced-setup.json" rollback_log=${temporary}/empty-inventory.jsonl @@ -1457,8 +1471,9 @@ jq -s -e ' advisory_rollback_root=${temporary}/advisory-window-rollback mkdir -p "${advisory_rollback_root}/bin" -cp "${root}/bin/publish" "${root}/bin/validate" \ +cp "${root}/bin/publish" "${root}/bin/validate" "${root}/bin/security-readiness" \ "${advisory_rollback_root}/bin/" +cp -R "${root}/lib" "${advisory_rollback_root}/lib" cp -R "${root}/config" "${advisory_rollback_root}/config" cp -R "${root}/community-health" \ "${advisory_rollback_root}/community-health" @@ -1687,8 +1702,9 @@ crate_setup_root() { mkdir -p "${destination}/bin" "${destination}/lib" "${destination}/tests" cp "${source_root}/bin/publish" "${source_root}/bin/validate" \ - "${source_root}/bin/verify-manual-settings" "${destination}/bin/" - cp "${source_root}/lib/verify_crate_workflow.py" "${destination}/lib/" + "${source_root}/bin/verify-manual-settings" "${source_root}/bin/security-readiness" "${destination}/bin/" + cp "${source_root}/lib/verify_crate_workflow.py" "${source_root}/lib/security_readiness.py" \ + "${destination}/lib/" cp -R "${source_root}/config" "${source_root}/community-health" \ "${source_root}/.github" "${destination}/" cp -R "${source_root}/tests/fixtures" "${destination}/tests/" @@ -1784,7 +1800,8 @@ for mode in selected all; do GH_CRATE_IMMUTABLE_INVENTORY=true GH_CRATE_FIXTURE_ROOT=${source_root} \ PATH="${temporary}/bin:${PATH}" ATRINIK_POLICY_SCOPE=organization \ "${pending_setup_root}/bin/publish" --apply >"${temporary}/crate-${mode}-already-selected.out" - jq -s -e 'all(.[]; (.endpoint | contains("/contents/.github/workflows/") | not) and + jq -s -e 'all(.[]; + (.endpoint | test("/contents/\\.github/workflows/(publish-crate|release)\\.yml") | not) and (.endpoint | contains("/environments/crates-io-release") | not))' "${mode_log}" >/dev/null done diff --git a/tests/publish-maintenance-retirement.sh b/tests/publish-maintenance-retirement.sh index 587504c..7b40304 100755 --- a/tests/publish-maintenance-retirement.sh +++ b/tests/publish-maintenance-retirement.sh @@ -21,6 +21,9 @@ while (($#)); do -H | --header) shift 2 ;; + --paginate) + shift + ;; --method) method=$2 shift 2 diff --git a/tests/publish-retired-required-ci.sh b/tests/publish-retired-required-ci.sh new file mode 100755 index 0000000..bc128d4 --- /dev/null +++ b/tests/publish-retired-required-ci.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash + +# Production functions consume these fixture globals. +# shellcheck disable=SC2034 +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +source_root=${ATRINIK_PUBLISH_TEST_ROOT:-${root}} +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +sed -n '/^converge_organization_settings() {/,/^organization_metadata=/p' \ + "${source_root}/bin/publish" | sed '$d' >"${temporary}/functions.sh" +sed -n '/^list_rulesets() {/,/^}/p' "${source_root}/bin/publish" >>"${temporary}/functions.sh" +# shellcheck source=/dev/null +source "${temporary}/functions.sh" +root=${temporary}/fixture +mkdir -p "${root}/config" +cp -R "${source_root}/config/rulesets" "${root}/config/rulesets" +organization=atrinik +organization_plan=free +repositories_config=${root}/config/repositories.json +advisory_merge_windows_config=${root}/config/advisory-merge-windows.json +printf '{"repositories":[]}\n' >"${advisory_merge_windows_config}" +active_repositories=(web-platform) +declare -A active_repository_ids=([web-platform]=123) +temporary_files=() +log=${temporary}/api.jsonl +repository_has_advisory_merge_window() { return 1; } +upsert_repository_ruleset() { :; } +delete_stale_repository_maintenance_rulesets() { :; } +delete_stale_repository_advisory_rulesets() { :; } +github_api() { + jq -cn --arg endpoint "$1" '{method:"GET",endpoint:$endpoint}' >>"${log}" + case $1 in + 'repos/atrinik/web-platform/rulesets?includes_parents=false') + if [[ -e ${temporary}/deleted ]]; then echo '[]'; else cat "${temporary}/list.json"; fi ;; + repos/atrinik/web-platform) + cat "${temporary}/metadata.json" ;; + repos/atrinik/web-platform/rulesets/3) + if [[ -e ${temporary}/read-once && -e ${temporary}/changed.json ]]; then + cat "${temporary}/changed.json" + else + cat "${temporary}/rule.json" + fi + touch "${temporary}/read-once" ;; + *) echo "unexpected endpoint: $1" >&2; return 99 ;; + esac +} +run_api() { + [[ $1 == DELETE && $2 == repos/atrinik/web-platform/rulesets/3 ]] + if ${apply}; then + jq -cn --arg endpoint "$2" '{method:"DELETE",endpoint:$endpoint}' >>"${log}" + touch "${temporary}/deleted" + else + jq -cn --arg endpoint "$2" '{method:"PLAN",endpoint:$endpoint}' >>"${log}" + fi +} +reset_case() { + : >"${log}" + rm -f "${temporary}/deleted" "${temporary}/read-once" "${temporary}/changed.json" + apply=true + active_repository_ids=([web-platform]=123) + printf '{"pull_request_gate":["web-platform"],"required_ci":{},"release_tags":[],"maintenance_branches":[]}\n' >"${repositories_config}" + printf '{"retirements":[{"repository":"web-platform","repository_id":123,"ruleset_id":3,"required_ci":["Platform validation","Conventional PR title"]}]}\n' >"${root}/config/retired-required-ci.json" + printf '[{"id":1,"name":"01 - Default branch integrity"},{"id":3,"name":"03 - Required CI"},{"id":5,"name":"Foreign CI"}]\n' >"${temporary}/list.json" + printf '{"id":123,"full_name":"atrinik/web-platform","archived":false}\n' >"${temporary}/metadata.json" + jq '. + {id:3,source_type:"Repository",source:"atrinik/web-platform"} | + .bypass_actors[0].actor_id = null | + .rules[0].parameters += {do_not_enforce_on_create:false, + required_status_checks:[{context:"Conventional PR title",integration_id:15368}, + {context:"Platform validation",integration_id:15368}]}' \ + "${root}/config/rulesets/required-ci.json" >"${temporary}/rule.json" +} +assert_retained() { + publish_repository_rulesets >"${temporary}/output" + jq -se 'all(.[]; .method == "GET")' "${log}" >/dev/null + grep -Fq 'retained' "${temporary}/output" +} +reset_case +publish_repository_rulesets >"${temporary}/output" +jq -se '[.[] | select(.method == "DELETE")] == [{method:"DELETE",endpoint:"repos/atrinik/web-platform/rulesets/3"}]' "${log}" >/dev/null +# Converged retirement makes no second deletion. +: >"${log}" +publish_repository_rulesets >"${temporary}/output" +jq -se 'all(.[]; .method == "GET")' "${log}" >/dev/null +reset_case +jq 'del(.rules[0].parameters.do_not_enforce_on_create)' \ + "${temporary}/rule.json" >"${temporary}/implicit-default.json" +mv "${temporary}/implicit-default.json" "${temporary}/rule.json" +publish_repository_rulesets >"${temporary}/output" +jq -se 'any(.[]; .method == "DELETE")' "${log}" >/dev/null +reset_case +apply=false +publish_repository_rulesets >"${temporary}/output" +jq -se '[.[] | select(.method != "GET")] == [{method:"PLAN",endpoint:"repos/atrinik/web-platform/rulesets/3"}]' "${log}" >/dev/null +[[ ! -e ${temporary}/deleted ]] +for filter in \ + '.id = 999' '.source_type = "Organization"' '.source = "foreign/web-platform"' \ + '.name = "Foreign CI"' '.target = "tag"' '.enforcement = "disabled"' \ + '.bypass_actors[0].bypass_mode = "always"' '.bypass_actors[0].actor_id = 2' \ + '.conditions.ref_name.include = ["refs/heads/other"]' \ + '.rules[0].parameters.required_status_checks[0].integration_id = 9' \ + '.rules[0].parameters.required_status_checks[0].context = "Unknown check"' \ + '.rules[0].parameters.strict_required_status_checks_policy = false' \ + '.rules[0].parameters.do_not_enforce_on_create = true' \ + '.rules[0].parameters.do_not_enforce_on_create = null' \ + '.rules[0].parameters.unknown_option = false' '.rules += [{type:"deletion"}]'; do + reset_case + jq "${filter}" "${temporary}/rule.json" >"${temporary}/changed-rule.json" + mv "${temporary}/changed-rule.json" "${temporary}/rule.json" + assert_retained +done +for filter in '.id = 999' '.full_name = "foreign/web-platform"' '.archived = true'; do + reset_case + jq "${filter}" "${temporary}/metadata.json" >"${temporary}/changed-metadata.json" + mv "${temporary}/changed-metadata.json" "${temporary}/metadata.json" + assert_retained +done +reset_case +jq '.rules[0].parameters.required_status_checks[0].context = "Concurrent change"' \ + "${temporary}/rule.json" >"${temporary}/changed.json" +assert_retained +reset_case +jq '.retirements[0].repository_id = 999' "${root}/config/retired-required-ci.json" >"${temporary}/wrong-record.json" +mv "${temporary}/wrong-record.json" "${root}/config/retired-required-ci.json" +assert_retained +reset_case +printf '{"retirements":[]}\n' >"${root}/config/retired-required-ci.json" +assert_retained +grep -Fq 'config/retired-required-ci.json' "${temporary}/output" +reset_case +printf '[{"id":3,"name":"03 - Required CI"},{"id":4,"name":"03 - Required CI"}]\n' >"${temporary}/list.json" +if publish_repository_rulesets >"${temporary}/output" 2>&1; then + echo 'expected ambiguous live ruleset inventory to fail' >&2 + exit 1 +fi +jq -se 'all(.[]; .method == "GET")' "${log}" >/dev/null +grep -Fq 'ambiguous complete ruleset inventory' "${temporary}/output" +reset_case +printf '[{"id":4,"name":"03 - Required CI"}]\n' >"${temporary}/list.json" +assert_retained +# Still-active CI and repositories excluded from the active identity map are untouched. +reset_case +jq '.required_ci["web-platform"] = ["Platform validation"]' "${repositories_config}" >"${temporary}/active.json" +mv "${temporary}/active.json" "${repositories_config}" +publish_repository_rulesets >"${temporary}/output" +[[ ! -s ${log} ]] +reset_case +active_repository_ids=() +publish_repository_rulesets >"${temporary}/output" +[[ ! -s ${log} ]] +echo 'Required-CI retirement publisher tests passed.' diff --git a/tests/publish-ruleset-pagination.sh b/tests/publish-ruleset-pagination.sh new file mode 100755 index 0000000..610f85b --- /dev/null +++ b/tests/publish-ruleset-pagination.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash + +# Sourced production functions consume these fixture globals. +# shellcheck disable=SC2034 +set -euo pipefail +source_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +source_root=${ATRINIK_PUBLISH_TEST_ROOT:-${source_root}} +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +root=${temporary}/fixture +mkdir -p "${root}/config" +cp -R "${source_root}/config/." "${root}/config/" +sed -n '/^converge_organization_settings() {/,/^organization_metadata=/p' \ + "${source_root}/bin/publish" | sed '$d' >"${temporary}/functions.sh" +sed -n '/^list_rulesets() {/,/^}/p' "${source_root}/bin/publish" >>"${temporary}/functions.sh" +# shellcheck source=/dev/null +source "${temporary}/functions.sh" +organization=atrinik +organization_plan=free +apply=true +temporary_files=() +repositories_config=${root}/config/repositories.json +security_merge_config=${root}/config/security-merge-protection.json +advisory_merge_windows_config=${root}/config/advisory-merge-windows.json +api_log=${temporary}/api.jsonl +active_repositories=(content-toolkit) +declare -A active_repository_ids=([content-toolkit]=123) +# The first 30 entries are foreign policies. Managed entries occur only on the +# next page, matching GitHub's default page size and the reported duplicate bug. +jq -cn '[range(100;130) | {id:.,name:("Foreign policy " + tostring)}]' >"${temporary}/first-page.json" +github_api() { + local endpoint=$1 + shift + jq -cn --arg endpoint "${endpoint}" '{method:"GET",endpoint:$endpoint}' >>"${api_log}" + case ${endpoint} in + orgs/atrinik/rulesets | 'repos/atrinik/content-toolkit/rulesets?includes_parents=false') + [[ $# == 1 && $1 == --paginate ]] || { + echo 'inventory must use supported --paginate without --slurp' >&2 + return 99 + } + [[ ${page_state} != empty ]] || return 0 + cat "${temporary}/first-page.json" + case ${page_state} in + valid) cat "${temporary}/second-page.json" ;; + failed) echo 'gh: later ruleset page failed (HTTP 503)' >&2; return 1 ;; + malformed) echo '{' ;; + object) echo '{"message":"API error"}' ;; + null) echo 'null' ;; + nested) echo '[[{"id":8,"name":"Nested"}]]' ;; + invalid-id) echo '[{"id":null,"name":"Invalid"}]' ;; + fractional-id) echo '[{"id":8.5,"name":"Invalid"}]' ;; + invalid-name) echo '[{"id":8,"name":null}]' ;; + duplicate-id) echo '[{"id":100,"name":"Changed name"}]' ;; + duplicate-name) echo '[{"id":8,"name":"Foreign policy 100"}]' ;; + *) return 99 ;; + esac + ;; + repos/atrinik/content-toolkit) + echo '{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' ;; + repos/atrinik/content-toolkit/rulesets/3 | \ + repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) + cat "${temporary}/rule-${endpoint##*/}.json" ;; + *) echo "unexpected API endpoint: ${endpoint}" >&2; return 99 ;; + esac +} +run_api() { + local payload=null + [[ -z ${3:-} ]] || payload=$(cat "$3") + jq -cn --arg method "$1" --arg endpoint "$2" --argjson payload "${payload}" \ + '{method:$method,endpoint:$endpoint,payload:$payload}' >>"${api_log}" + [[ ${write_failed} != true ]] +} +reset_case() { + : >"${api_log}" + page_state=valid + write_failed=false + active_repositories=(content-toolkit) + active_repository_ids=([content-toolkit]=123) + echo '[]' >"${temporary}/second-page.json" + echo '{"repositories":[]}' >"${advisory_merge_windows_config}" + echo '{"pull_request_gate":["content-toolkit"],"required_ci":{"content-toolkit":["Toolkit validation"]},"release_tags":[],"maintenance_branches":[]}' >"${repositories_config}" + echo '{"dependency_review":["content-toolkit"],"code_scanning":["content-toolkit"],"activation":{"dependency_review":{"content-toolkit":"enforced"},"code_scanning":{"content-toolkit":"enforced"}}}' >"${security_merge_config}" + echo '{"retirements":[{"repository":"content-toolkit","repository_id":123,"ruleset_id":3,"required_ci":["Toolkit validation"]}]}' >"${root}/config/retired-required-ci.json" + for pair in '3 required-ci' '6 code-scanning' '7 dependency-review'; do + read -r id template <<<"${pair}" + jq --argjson id "${id}" '. + {id:$id,source_type:"Repository",source:"atrinik/content-toolkit"} | + if $id == 3 then .rules[0].parameters.required_status_checks = [{context:"Toolkit validation",integration_id:15368}] else . end' \ + "${root}/config/rulesets/${template}.json" >"${temporary}/rule-${id}.json" + done +} +assert_no_writes() { + jq -se 'all(.[]; .method == "GET")' "${api_log}" >/dev/null +} +# Ordinary repository/org upserts must update the existing off-page identity; +# complete absence alone permits POST. +for scope in repository organization; do + for found in true false; do + reset_case + if ${found}; then + echo '[{"id":31,"name":"01 - Default branch integrity"}]' >"${temporary}/second-page.json" + fi + if [[ ${scope} == repository ]]; then + upsert_repository_ruleset content-toolkit "${root}/config/rulesets/default-branch-integrity.json" + endpoint=repos/atrinik/content-toolkit/rulesets + else + upsert_organization_ruleset "${root}/config/rulesets/default-branch-integrity.json" + endpoint=orgs/atrinik/rulesets + fi + method=POST + if ${found}; then method=PUT; endpoint+=/31; fi + jq -se --arg method "${method}" --arg endpoint "${endpoint}" ' + [.[] | select(.method != "GET")] | length == 1 and .[0].method == $method and .[0].endpoint == $endpoint' "${api_log}" >/dev/null + done +done +# Security gate convergence, restoration, and retirement share full discovery +# in both scopes. Foreign same-name detail remains retained after discovery. +for policy_scope in organization repository; do + for state in converged disabled retired foreign; do + reset_case + echo '[{"id":6,"name":"06 - CodeQL security"},{"id":7,"name":"07 - Dependency review"}]' >"${temporary}/second-page.json" + if [[ ${state} == disabled ]]; then + for id in 6 7; do + jq '.enforcement = "disabled"' "${temporary}/rule-${id}.json" >"${temporary}/changed.json" + mv "${temporary}/changed.json" "${temporary}/rule-${id}.json" + done + elif [[ ${state} == retired || ${state} == foreign ]]; then + echo '{"dependency_review":[],"code_scanning":[],"activation":{"dependency_review":{},"code_scanning":{}}}' >"${security_merge_config}" + if [[ ${state} == foreign ]]; then + for id in 6 7; do + jq '.source = "foreign/content-toolkit"' "${temporary}/rule-${id}.json" >"${temporary}/changed.json" + mv "${temporary}/changed.json" "${temporary}/rule-${id}.json" + done + fi + fi + publish_security_merge_rulesets >"${temporary}/output" + case ${state} in + converged | foreign) assert_no_writes ;; + disabled | retired) + method=PUT + [[ ${state} != retired ]] || method=DELETE + jq -se --arg method "${method}" ' + [.[] | select(.method != "GET")] | length == 2 and + all(.[]; .method == $method) and + ([.[].endpoint] | sort) == ["repos/atrinik/content-toolkit/rulesets/6","repos/atrinik/content-toolkit/rulesets/7"]' "${api_log}" >/dev/null ;; + esac + done +done +# A recorded aggregate on a later page is retired only when its exact payload +# and repository provenance still match, leaving all foreign policies intact. +for state in owned foreign; do + reset_case + echo '[{"id":3,"name":"03 - Required CI"}]' >"${temporary}/second-page.json" + echo '{"pull_request_gate":[],"required_ci":{},"release_tags":[],"maintenance_branches":[]}' >"${repositories_config}" + if [[ ${state} == foreign ]]; then + jq '.source = "foreign/content-toolkit"' "${temporary}/rule-3.json" >"${temporary}/changed.json" + mv "${temporary}/changed.json" "${temporary}/rule-3.json" + fi + delete_retired_repository_required_ci >"${temporary}/output" + if [[ ${state} == foreign ]]; then assert_no_writes; else + jq -se '[.[] | select(.method != "GET")] | length == 1 and .[0].method == "DELETE" and .[0].endpoint == "repos/atrinik/content-toolkit/rulesets/3"' "${api_log}" >/dev/null + fi +done +# Scope migration and stale maintenance/advisory removal also see later pages. +for consumer in delete_managed_repository_rulesets delete_managed_organization_rulesets \ + delete_stale_repository_maintenance_rulesets delete_stale_repository_advisory_rulesets; do + reset_case + case ${consumer} in + delete_stale_repository_maintenance_rulesets) name='05 - Maintenance branch - content-toolkit - old' ;; + delete_stale_repository_advisory_rulesets) name='02 - Changes through pull requests - content-toolkit' ;; + *) name='01 - Default branch integrity' ;; + esac + jq -cn --arg name "${name}" '[{id:31,name:$name}]' >"${temporary}/second-page.json" + "${consumer}" content-toolkit + jq -se '[.[] | select(.method != "GET")] | length == 1 and .[0].method == "DELETE" and (.[0].endpoint | endswith("/31"))' "${api_log}" >/dev/null +done +# Propagating inventory failures through parent functions must also preserve +# API write failures instead of continuing a loop after a failed deletion. +for consumer in delete_managed_repository_rulesets delete_managed_organization_rulesets \ + delete_stale_repository_maintenance_rulesets delete_stale_repository_advisory_rulesets \ + delete_retired_repository_required_ci; do + reset_case + write_failed=true + case ${consumer} in + delete_stale_repository_maintenance_rulesets) name='05 - Maintenance branch - content-toolkit - old' ;; + delete_stale_repository_advisory_rulesets) name='02 - Changes through pull requests - content-toolkit' ;; + delete_retired_repository_required_ci) + name='03 - Required CI' + echo '{"required_ci":{}}' >"${repositories_config}" ;; + *) name='01 - Default branch integrity' ;; + esac + jq -cn --arg name "${name}" '[{id:3,name:$name},{id:32,name:"05 - Maintenance branch - content-toolkit - older"}]' >"${temporary}/second-page.json" + if "${consumer}" content-toolkit >"${temporary}/failure" 2>&1; then + echo "expected ${consumer} to propagate the failed DELETE" >&2 + exit 1 + fi + jq -se '[.[] | select(.method != "GET")] | length == 1 and .[0].method == "DELETE"' "${api_log}" >/dev/null +done +# The organization publisher's final stale-policy scan must not miss an +# off-page aggregate, nor mistake off-page desired rules for missing rules. +reset_case +active_repositories=() +echo '{"pull_request_gate":[],"required_ci":{},"release_tags":[],"maintenance_branches":[]}' >"${repositories_config}" +echo '[{"id":1,"name":"01 - Default branch integrity"},{"id":2,"name":"01 - Default branch linear history"},{"id":4,"name":"02 - Changes through pull requests"},{"id":5,"name":"04 - Immutable release tags"},{"id":31,"name":"03 - Required CI - retired"}]' >"${temporary}/second-page.json" +publish_organization_rulesets +jq -se 'all(.[]; .method != "POST") and + ([.[] | select(.method == "DELETE") | .endpoint] == ["orgs/atrinik/rulesets/31"])' "${api_log}" >/dev/null +# No inventory consumer may use an incomplete, malformed or ambiguous page +# stream as proof of absence, even when called from a shell conditional. +for invalid in failed malformed object null nested invalid-id fractional-id invalid-name duplicate-id duplicate-name empty; do + for consumer in upsert_repository_ruleset upsert_organization_ruleset \ + publish_repository_rulesets publish_organization_rulesets publish_security_merge_rulesets \ + delete_retired_repository_required_ci delete_removed_security_merge_rulesets \ + delete_managed_repository_rulesets delete_managed_organization_rulesets \ + delete_stale_repository_maintenance_rulesets delete_stale_repository_advisory_rulesets; do + reset_case + page_state=${invalid} + case ${consumer} in + upsert_repository_ruleset) arguments=(content-toolkit "${root}/config/rulesets/default-branch-integrity.json") ;; + upsert_organization_ruleset) arguments=("${root}/config/rulesets/default-branch-integrity.json") ;; + delete_retired_repository_required_ci) + arguments=() + echo '{"required_ci":{}}' >"${repositories_config}" ;; + delete_removed_security_merge_rulesets) + arguments=() + echo '{"code_scanning":[],"dependency_review":[]}' >"${security_merge_config}" ;; + *) arguments=(content-toolkit) ;; + esac + if "${consumer}" "${arguments[@]}" >"${temporary}/failure" 2>&1; then + echo "expected ${consumer} to reject ${invalid} later-page evidence" >&2 + exit 1 + fi + assert_no_writes + grep -Eq 'cannot read complete ruleset inventory|invalid or ambiguous complete ruleset inventory' "${temporary}/failure" + done +done +# Even an actionable first-page gate cannot be deleted while a later page is +# unreadable: pagination is a complete preflight, not a streaming mutation loop. +cp "${temporary}/first-page.json" "${temporary}/foreign-page.json" +for invalid in failed malformed; do + reset_case + page_state=${invalid} + jq '. + [{id:6,name:"06 - CodeQL security"},{id:7,name:"07 - Dependency review"}]' \ + "${temporary}/foreign-page.json" >"${temporary}/first-page.json" + echo '{"code_scanning":[],"dependency_review":[]}' >"${security_merge_config}" + if delete_removed_security_merge_rulesets >"${temporary}/failure" 2>&1; then + echo 'expected failed full enumeration to prevent first-page retirement' >&2 + exit 1 + fi + assert_no_writes +done +printf 'Ruleset pagination publisher tests passed.\n' diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh new file mode 100755 index 0000000..badb91b --- /dev/null +++ b/tests/publish-security-merge.sh @@ -0,0 +1,1142 @@ +#!/usr/bin/env bash + +# Sourced production functions consume the globals below. +# shellcheck disable=SC2034 +set -euo pipefail + +source_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +# An override permits testing the integrated candidate from an isolated worktree. +source_root=${ATRINIK_PUBLISH_TEST_ROOT:-${source_root}} +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +root=${temporary}/fixture +mkdir -p "${root}/bin" "${root}/config/rulesets" +cp "${source_root}/config/"*.json "${root}/config/" +cp "${source_root}/config/rulesets/"*.json "${root}/config/rulesets/" + +# Source production functions without executing validation, authentication or +# live publication. Every API call is replaced below by a fail-closed mock. +sed -n '/^converge_organization_settings() {/,/^organization_metadata=/p' \ + "${source_root}/bin/publish" | sed '$d' >"${temporary}/functions.sh" +sed -n '/^list_rulesets() {/,/^}/p' "${source_root}/bin/publish" >>"${temporary}/functions.sh" +# shellcheck source=/dev/null +source "${temporary}/functions.sh" +# Exercise the production scope dispatch, including its following gate call. +awk ' + /^if \[\[ \$\{policy_scope\} == organization \]\]; then$/ { capture = 1 } + capture { print } + capture && /^publish_security_merge_rulesets$/ { exit } +' "${source_root}/bin/publish" >"${temporary}/dispatch.sh" +[[ -s ${temporary}/dispatch.sh ]] + +organization=atrinik +apply=true +temporary_files=() +security_merge_config=${root}/config/security-merge-protection.json +codeql_advanced_config=${root}/config/codeql-advanced-setup.json +actions_sha_pinning_config=${root}/config/actions-sha-pinning.json +active_repositories=(content-toolkit) +declare -A active_repository_ids=([content-toolkit]=123) +api_log=${temporary}/api.jsonl +readiness_log=${temporary}/readiness.jsonl +export READINESS_LOG=${readiness_log} +export MOCK_API_LOG=${api_log} +export MOCK_ACTIONS_POLICY=${temporary}/actions-policy.json +export MOCK_ACTIONS_SELECTED=${root}/config/actions-selected.json +cat >"${root}/bin/security-readiness" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +jq -cn --arg mode "$1" --arg repository "$2" \ + '{mode: $mode, repository: $repository}' >>"${READINESS_LOG}" +if [[ $1 == sha ]]; then + jq -cn '{method: "PROOF", endpoint: "sha"}' >>"${MOCK_API_LOG}" + if [[ ${CHECK_POLICY_STAGED:-false} == true ]]; then + jq -e --slurpfile selected "${MOCK_ACTIONS_SELECTED}" ' + .permissions.allowed_actions == "selected" and + .permissions.enabled_repositories == "all" and + .selected == $selected[0] + ' "${MOCK_ACTIONS_POLICY}" >/dev/null || exit 98 + fi +fi +case ${READINESS_STATUS:-ready} in +ready) echo 'producer evidence ready'; exit 0 ;; +missing) echo 'producer is missing'; exit 1 ;; +failed) echo 'producer failed'; exit 1 ;; +stale) echo 'canary base changed during preflight'; exit 1 ;; +unreadable) echo 'producer evidence unreadable'; exit 2 ;; +*) exit 99 ;; +esac +MOCK +chmod +x "${root}/bin/security-readiness" + +# Minimal live response fixtures preserve the API shapes used by the publisher. +github_api() { + local endpoint=$1 + jq -cn --arg endpoint "${endpoint}" \ + '{method: "GET", endpoint: $endpoint}' >>"${api_log}" + case ${endpoint} in + repos/atrinik/content-toolkit) + if [[ -n ${gate_metadata_after_proof_filter} ]] && + jq -se 'any(.[]; .endpoint | contains("/rules/branches/"))' "${api_log}" >/dev/null; then + [[ ${gate_metadata_after_proof_filter} != unreadable ]] || return 1 + jq "${gate_metadata_after_proof_filter}" <<<"${repository_state}" + elif [[ -n ${gate_metadata_filter} ]]; then + [[ ${gate_metadata_filter} != unreadable ]] || return 1 + jq "${gate_metadata_filter}" <<<"${repository_state}" + else + printf '%s\n' "${repository_state}" + fi + ;; + repos/atrinik/content-toolkit/rules/branches/main | repos/atrinik/content-toolkit/rules/branches/release%2Fcurrent) + if [[ ${effective_state} == covering ]]; then + jq -n --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [$codeql[0].rules[] + {ruleset_id: 6}, + $dependency[0].rules[] + {ruleset_id: 7}]' + elif [[ ${effective_state} == different-id ]]; then + jq -n --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [$codeql[0].rules[] + {ruleset_id: 600}, + $dependency[0].rules[] + {ruleset_id: 700}]' + else + echo '[]' + fi + ;; + repos/atrinik/content-toolkit/actions/permissions/workflow) + [[ ${workflow_state} != unreadable ]] || return 1 + printf '%s\n' "${workflow_state}" + ;; + repos/atrinik/content-toolkit/code-scanning/default-setup) + if [[ ${codeql_state} == http-* ]]; then + echo "gh: default-setup read failed (HTTP ${codeql_state#http-})" >&2 + return 1 + fi + [[ ${codeql_state} != unreadable ]] || return 1 + printf '%s\n' "${codeql_state}" + ;; + 'repos/atrinik/content-toolkit/rulesets?includes_parents=false') + if [[ ${ruleset_state} == missing ]]; then + echo '[]' + else + jq -n --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [{id: 1, name: "01 - Default branch integrity"}, + {id: 3, name: "03 - Required CI"}, + {id: 6, name: $codeql[0].name}, {id: 7, name: $dependency[0].name}]' + fi + ;; + repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) + local template=code-scanning + [[ ${endpoint##*/} != 7 ]] || template=dependency-review + if [[ ${ruleset_state} == pending-branch-race ]]; then + local pending_filter=. pending_initial_filter=. + [[ ${endpoint##*/} != "${gate_update_id}" ]] || pending_initial_filter=${gate_initial_filter} + if [[ ${endpoint##*/} == "${gate_update_id}" ]] && + (( $(jq -s --arg endpoint "${endpoint}" '[.[] | select(.endpoint == $endpoint)] | length' "${api_log}") >= 2 )); then + [[ ${gate_update_filter} != unreadable ]] || return 1 + pending_filter=${gate_update_filter} + fi + jq --argjson id "${endpoint##*/}" --argjson target "${gate_update_id}" \ + --arg branch "$(jq -r '.default_branch' <<<"${repository_state}")" ' + . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + if $id == $target then .conditions.ref_name.include = [("refs/heads/" + $branch)] else . end' \ + "${root}/config/rulesets/${template}.json" | jq "${pending_initial_filter}" | jq "${pending_filter}" + return + elif [[ ${ruleset_state} == restoration-race || ${ruleset_state} == retirement-race ]]; then + local filter=. + if [[ ${endpoint##*/} == "${gate_update_id}" ]] && + (( $(jq -s --arg endpoint "${endpoint}" '[.[] | select(.endpoint == $endpoint)] | length' "${api_log}") >= 2 )); then + [[ ${gate_update_filter} != unreadable ]] || return 1 + filter=${gate_update_filter} + fi + jq --argjson id "${endpoint##*/}" --argjson target "${gate_update_id}" --arg state "${ruleset_state}" ' + . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + if $id == $target and $state == "restoration-race" then .enforcement = "disabled" else . end' \ + "${root}/config/rulesets/${template}.json" | jq "${filter}" + return + elif [[ ${ruleset_state} == api-default-* ]]; then + jq --argjson id "${endpoint##*/}" --arg state "${ruleset_state}" ' + . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + .bypass_actors[0].actor_id = null | + if $id == 7 then + .rules[0].parameters.do_not_enforce_on_create = + (if $state == "api-default-false" then false + elif $state == "api-default-true" then true else null end) | + if $state == "api-default-unknown" then + .rules[0].parameters.do_not_enforce_on_create = false | + .rules[0].parameters.unknown_future_control = false + else . end + else . end' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == exact || ${ruleset_state} == removal-owned ]]; then + jq --argjson id "${endpoint##*/}" \ + '. + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"}' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == removal-owner-null ]]; then + jq --argjson id "${endpoint##*/}" \ + '. + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + .bypass_actors[0].actor_id = null' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == removal-foreign ]]; then + jq --argjson id "${endpoint##*/}" \ + '. + {id: $id, source_type: "Organization", source: "atrinik"}' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == removal-id-drift ]]; then + jq '. + {id: 999, source_type: "Repository", source: "atrinik/content-toolkit"}' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == removal-name-collision ]]; then + jq --argjson id "${endpoint##*/}" \ + '. + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + .rules = [{type: "deletion"}, {type: "non_fast_forward"}]' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == owner-null ]]; then + jq '.bypass_actors[0].actor_id = null' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == foreign-repository-source ]]; then + jq '.source = "foreign/content-toolkit"' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == extra-rule ]]; then + jq '.rules += [{type: "deletion"}]' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == unknown-rule-parameter ]]; then + jq '.rules[0].parameters.unknown_future_control = true' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == unknown-condition ]]; then + jq '.conditions.unknown_future_scope = {}' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == active-wrong-ref ]]; then + jq '.conditions.ref_name.include = ["refs/heads/feature"]' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == active-drift ]]; then + jq '.conditions.ref_name.include = ["refs/heads/main"]' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == active-bypass-drift ]]; then + jq '.bypass_actors = []' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == unknown-ref-condition ]]; then + jq '.conditions.ref_name.unknown_future_scope = true' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == changed-severity ]]; then + if [[ ${template} == code-scanning ]]; then + jq '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' \ + "${root}/config/rulesets/${template}.json" + else + cat "${root}/config/rulesets/${template}.json" + fi + elif [[ ${ruleset_state} == active-unrelated-gate ]]; then + if [[ ${template} == code-scanning ]]; then + jq '.rules[0].parameters.code_scanning_tools[0].tool = "Another scanner"' \ + "${root}/config/rulesets/${template}.json" + else + jq '.rules[0].parameters.required_status_checks[0].integration_id = 999' \ + "${root}/config/rulesets/${template}.json" + fi + elif [[ ${ruleset_state} == active-without-gate ]]; then + jq '.rules = []' "${root}/config/rulesets/${template}.json" + else + jq '.enforcement = "disabled"' "${root}/config/rulesets/${template}.json" + fi | jq --argjson id "${endpoint##*/}" \ + '{id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} + .' + ;; + orgs/atrinik/actions/permissions) printf '%s\n' "${actions_state}" ;; + orgs/atrinik/actions/permissions/selected-actions) + # Model the API prerequisite: the dormant list cannot be read or changed + # until the publisher has staged conservative selected mode. + [[ $(jq -r '.allowed_actions' <<<"${actions_state}") == selected ]] || return 1 + [[ ${selected_state} != unreadable ]] || return 1 + printf '%s\n' "${selected_state}" + ;; + *) echo "unexpected mock API: ${endpoint}" >&2; return 99 ;; + esac +} + +run_api() { + if [[ $1 == DELETE ]]; then + jq -cn --arg endpoint "$2" '{method: "DELETE", endpoint: $endpoint}' >>"${api_log}" + return + fi + jq -cn --arg method "$1" --arg endpoint "$2" --slurpfile payload "$3" \ + '{method: $method, endpoint: $endpoint, payload: $payload[0]}' >>"${api_log}" + if ${apply}; then + case $2 in + orgs/atrinik/actions/permissions) + actions_state=$(jq --slurpfile payload "$3" '. * $payload[0]' <<<"${actions_state}") + ;; + orgs/atrinik/actions/permissions/selected-actions) + [[ $(jq -r '.allowed_actions' <<<"${actions_state}") == selected ]] || return 1 + selected_state=$(cat "$3") + case ${staging_race} in + disable | replace-repository) + actions_state=$(jq '.sha_pinning_required = false' <<<"${actions_state}") + if [[ ${staging_race} == replace-repository ]]; then + repository_state=$(jq '.id = 999' <<<"${repository_state}") + fi + ;; + malformed) + actions_state=$(jq '.sha_pinning_required = "false"' <<<"${actions_state}") + ;; + esac + ;; + esac + if [[ ${selected_state} != unreadable ]]; then + jq -n --argjson permissions "${actions_state}" --argjson selected "${selected_state}" \ + '{permissions:$permissions, selected:$selected}' >"${MOCK_ACTIONS_POLICY}" + fi + fi +} +reset_case() { + : >"${api_log}" + : >"${readiness_log}" + codeql_state='{"state":"configured","query_suite":"default","languages":["python","javascript-typescript"]}' + ruleset_state=missing + gate_update_id=6 + gate_update_filter=. + gate_initial_filter=. + gate_metadata_filter= + gate_metadata_after_proof_filter= + effective_state=covering + repository_state='{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' + workflow_state='{"default_workflow_permissions":"write","can_approve_pull_request_reviews":true}' + cp "${source_root}/config/security-merge-protection.json" "${security_merge_config}" + cp "${source_root}/config/codeql-advanced-setup.json" "${codeql_advanced_config}" + echo '{"phase":"pending","canary":null}' >"${actions_sha_pinning_config}" + actions_state='{"enabled_repositories":"selected","allowed_actions":"all","sha_pinning_required":false}' + selected_state=$(cat "${root}/config/actions-selected.json") + sha_pinning_canary= + staging_race= + apply=true + export CHECK_POLICY_STAGED=false + export READINESS_STATUS=ready +} +assert_no_writes() { + jq -se 'all(.[]; .method == "GET")' "${api_log}" >/dev/null +} + +reset_case +converge_codeql_default_setup >"${temporary}/languages.txt" +jq -se ' + [.[] | select(.method == "PATCH")] == [{ + method: "PATCH", + endpoint: "repos/atrinik/content-toolkit/code-scanning/default-setup", + payload: {state: "configured", query_suite: "default", + languages: ["javascript-typescript", "python", "rust"]} + }]' "${api_log}" >/dev/null +reset_case +codeql_state='{"state":"configured","query_suite":"default","languages":["actions","javascript","javascript-typescript","python","typescript"]}' +converge_codeql_default_setup >"${temporary}/languages-aliases.txt" +jq -se '[.[] | select(.method == "PATCH")][0].payload.languages == + ["actions", "javascript-typescript", "python", "rust"]' "${api_log}" >/dev/null +reset_case +codeql_state='{"state":"configured","query_suite":"default","languages":["rust","python"]}' +converge_codeql_default_setup >"${temporary}/languages-keep.txt" +assert_no_writes +reset_case +codeql_state=unreadable +if converge_codeql_default_setup >"${temporary}/languages-error.txt" 2>&1; then + echo 'expected unreadable language inventory to fail closed' >&2 + exit 1 +fi +assert_no_writes +reset_case +codeql_state='{"state":"configured","query_suite":"default","languages":null}' +if converge_codeql_default_setup >"${temporary}/languages-malformed.txt" 2>&1; then + echo 'expected malformed language inventory to fail closed' >&2 + exit 1 +fi +assert_no_writes + +# Read failures are fatal even without a language minimum, in both setup modes. +for setup_kind in default advanced; do + for http_status in 403 404 500; do + reset_case + jq '.codeql_required_languages = {}' "${security_merge_config}" >"${temporary}/no-minima.json" + cp "${temporary}/no-minima.json" "${security_merge_config}" + if [[ ${setup_kind} == advanced ]]; then + echo '{"repositories":["content-toolkit"]}' >"${codeql_advanced_config}" + fi + codeql_state=http-${http_status} + if converge_codeql_default_setup >"${temporary}/languages-${setup_kind}-${http_status}.txt" 2>&1; then + echo 'expected unreadable default setup to fail publication without a language minimum' >&2 + exit 1 + fi + grep -q 'cannot read existing CodeQL languages' "${temporary}/languages-${setup_kind}-${http_status}.txt" + assert_no_writes + done +done +reset_case +jq '.codeql_required_languages = {}' "${security_merge_config}" >"${temporary}/no-minima.json" +cp "${temporary}/no-minima.json" "${security_merge_config}" +codeql_state='{"state":"configured","query_suite":"default","languages":["python"]}' +converge_codeql_default_setup >"${temporary}/languages-readable-no-minimum.txt" +assert_no_writes +echo '{"repositories":["content-toolkit"]}' >"${codeql_advanced_config}" +codeql_state='{"state":"not-configured","languages":null}' +converge_codeql_default_setup >"${temporary}/languages-readable-advanced-null.txt" +assert_no_writes + +# Use the actual dispatch: security producer readiness follows base convergence +# for both scopes, and these Team-compatible gates always target repositories. +publish_organization_rulesets() { echo organization-rules >>"${temporary}/order"; } +publish_organization_security() { echo organization-security >>"${temporary}/order"; } +publish_repository_rulesets() { echo repository-rules >>"${temporary}/order"; } +publish_repository_security() { echo repository-security >>"${temporary}/order"; } +for policy_scope in organization repository; do + for producer in missing failed unreadable ready; do + reset_case + export READINESS_STATUS=${producer} + : >"${temporary}/order" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/${policy_scope}-${producer}.txt" + [[ $(cat "${temporary}/order") == "${policy_scope}-rules"$'\n'"${policy_scope}-security" ]] + assert_no_writes + if [[ ${producer} == ready ]]; then + [[ $(grep -c '^READY ' "${temporary}/${policy_scope}-${producer}.txt") == 2 ]] + fi + jq -se '[.[].mode] == ["codeql", "dependency"] and + all(.[]; .repository == "atrinik/content-toolkit")' "${readiness_log}" >/dev/null + jq -se 'all(.[]; .endpoint | startswith("repos/atrinik/content-toolkit/"))' \ + "${api_log}" >/dev/null + done +done + +# Reviewed enforced intent precedes initial installation and survives deletion, +# disablement, and ref drift independently of current producer health. +for policy_scope in organization repository; do + for producer in failed unreadable; do + for state in missing disabled active-wrong-ref; do + reset_case + ruleset_state=${state} + effective_state=absent + export READINESS_STATUS=${producer} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/enforced-${policy_scope}-${producer}-${state}.txt" + method=PUT + [[ ${state} != missing ]] || method=POST + jq -se --arg method "${method}" \ + --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [.[] | select(.method != "GET")] | + [.[].payload] == [$codeql[0], $dependency[0]] and + all(.[]; .method == $method)' "${api_log}" >/dev/null + [[ ! -s ${readiness_log} ]] + done + done +done + +# Changes made after the initial GET cannot be erased by restoration PUT. +for policy_scope in organization repository; do + for gate_update_target in 6 7; do + common_filters=( + '.id = 999' '.source_type = "Organization"' '.source = "foreign/content-toolkit"' + '.name = "Renamed gate"' '.target = "tag"' '.enforcement = "unknown"' + '.bypass_actors[0].bypass_mode = "always"' '.bypass_actors = []' + '.rules[0].parameters.unknown_future_control = true' + '.conditions.ref_name.unknown_future_scope = true' '.rules += [{type:"deletion"}]' + 'unreadable' + ) + if [[ ${gate_update_target} == 6 ]]; then + common_filters+=( + '.rules[0].parameters.code_scanning_tools[0].tool = "Other tool"' + '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' + ) + else + common_filters+=( + '.rules[0].parameters.required_status_checks[0].context = "Unrelated context"' + '.rules[0].parameters.required_status_checks[0].integration_id = 9' + '.rules[0].parameters.do_not_enforce_on_create = true' + '.rules[0].parameters.do_not_enforce_on_create = null' + ) + fi + for filter in "${common_filters[@]}"; do + reset_case + ruleset_state=restoration-race + gate_update_id=${gate_update_target} + gate_update_filter=${filter} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + if source "${temporary}/dispatch.sh" >"${temporary}/concurrent-gate.txt" 2>&1; then + echo "expected concurrent gate change to be retained: ${filter}" >&2 + exit 1 + fi + assert_no_writes + grep -Fq 'retained:' "${temporary}/concurrent-gate.txt" + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${gate_update_id}" \ + '[.[] | select(.endpoint == $endpoint)] | length == 2' "${api_log}" >/dev/null + done + # Unchanged ownership, explicit API defaults, and allowed enforcement/ref + # restoration still converge; a concurrent completed repair causes KEEP. + for filter in '.' \ + '.bypass_actors[0].actor_id = null | if .rules[0].type == "required_status_checks" then .rules[0].parameters.do_not_enforce_on_create = false else . end' \ + '.enforcement = "evaluate" | .conditions.ref_name.include = ["refs/heads/other"]' \ + '.enforcement = "active"'; do + reset_case + ruleset_state=restoration-race + gate_update_id=${gate_update_target} + gate_update_filter=${filter} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/safe-concurrent-gate.txt" + if [[ ${filter} == '.enforcement = "active"' ]]; then + assert_no_writes + else + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${gate_update_id}" ' + [.[] | select(.method != "GET")] | length == 1 and + .[0].method == "PUT" and .[0].endpoint == $endpoint' "${api_log}" >/dev/null + fi + done + for filter in '.archived = true' '.id = 999' '.full_name = "foreign/content-toolkit"' unreadable; do + reset_case + ruleset_state=restoration-race + gate_update_id=${gate_update_target} + gate_metadata_filter=${filter} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + if source "${temporary}/dispatch.sh" >"${temporary}/fresh-repository.txt" 2>&1; then + echo 'expected fresh repository identity/archive/read failure to block PUT' >&2 + exit 1 + fi + assert_no_writes + done + done +done + +# Pending existing-gate restoration is authorized only for the exact default +# branch whose effective protection was checked, even if another branch is valid. +for policy_scope in organization repository; do + for pending_gate_id in 6 7; do + for filter in '.default_branch = "new-main"' '.default_branch = "release/next"' \ + '.default_branch = null' 'del(.default_branch)' unreadable '.'; do + reset_case + ruleset_state=pending-branch-race + gate_update_id=${pending_gate_id} + gate_metadata_after_proof_filter=${filter} + export READINESS_STATUS=failed + jq '.activation |= with_entries(.value |= with_entries(.value = "pending"))' \ + "${security_merge_config}" >"${temporary}/pending.json" + cp "${temporary}/pending.json" "${security_merge_config}" + status=0 + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/pending-branch-fence.txt" 2>&1 || status=$? + if [[ ${filter} == '.' ]]; then + [[ ${status} == 0 ]] + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${pending_gate_id}" ' + [.[] | select(.method != "GET")] | length == 1 and + .[0].method == "PUT" and .[0].endpoint == $endpoint' "${api_log}" >/dev/null + else + [[ ${status} != 0 ]] + assert_no_writes + if [[ ${filter} == '.default_branch = "'* ]]; then + grep -Fq 'default branch changed after existing security gate proof; retained' \ + "${temporary}/pending-branch-fence.txt" + fi + fi + jq -se 'any(.[]; .endpoint == "repos/atrinik/content-toolkit/rules/branches/main")' \ + "${api_log}" >/dev/null + [[ ! -s ${readiness_log} ]] + cmp "${temporary}/pending.json" "${security_merge_config}" + done + done + # Durable enforced intent still restores missing gates on the current default. + reset_case + ruleset_state=missing + gate_metadata_filter='.default_branch = "new-main"' + export READINESS_STATUS=failed + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/enforced-current-default.txt" + jq -se '[.[] | select(.method != "GET")] | length == 2 and all(.[]; .method == "POST")' \ + "${api_log}" >/dev/null + [[ ! -s ${readiness_log} ]] +done + +# Pending grandfathering cannot restore a gate revoked after its effective +# proof. Enforced intent deliberately permits the same restoration changes. +for policy_scope in organization repository; do + for pending_gate_id in 6 7; do + for phase in pending enforced; do + for filter in '.enforcement = "disabled"' '.enforcement = "evaluate"' \ + '.conditions.ref_name.include = ["refs/heads/other"]' \ + '.conditions.ref_name.exclude = ["refs/heads/main"]' \ + 'unreadable' '.' \ + '.bypass_actors[0].actor_id = null | if .rules[0].type == "required_status_checks" then .rules[0].parameters.do_not_enforce_on_create = false else . end' \ + '.conditions.ref_name.include |= reverse | .conditions.ref_name.exclude |= reverse'; do + reset_case + ruleset_state=pending-branch-race + gate_update_id=${pending_gate_id} + gate_update_filter=${filter} + if [[ ${filter} == *'|= reverse'* ]]; then + gate_initial_filter='.conditions.ref_name.include += ["refs/heads/release/*"] | .conditions.ref_name.exclude = ["refs/heads/excluded-a", "refs/heads/excluded-b"]' + fi + export READINESS_STATUS=failed + jq --arg phase "${phase}" '.activation |= with_entries(.value |= with_entries(.value = $phase))' \ + "${security_merge_config}" >"${temporary}/proof-phase.json" + cp "${temporary}/proof-phase.json" "${security_merge_config}" + status=0 + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/pending-proof-fence.txt" 2>&1 || status=$? + if [[ ${filter} == unreadable ]] || { [[ ${phase} == pending ]] && + [[ ${filter} == '.enforcement ='* || ${filter} == '.conditions.ref_name.include ='* || + ${filter} == '.conditions.ref_name.exclude ='* ]]; }; then + [[ ${status} != 0 ]] + assert_no_writes + if [[ ${filter} != unreadable ]]; then + grep -Fq 'existing security gate protection changed after effective proof; retained' \ + "${temporary}/pending-proof-fence.txt" + fi + else + [[ ${status} == 0 ]] + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${pending_gate_id}" ' + [.[] | select(.method != "GET")] | length == 1 and + .[0].method == "PUT" and .[0].endpoint == $endpoint' "${api_log}" >/dev/null + fi + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${pending_gate_id}" ' + [.[] | select(.endpoint == $endpoint and .method == "GET")] | length == 2' "${api_log}" >/dev/null + if [[ ${phase} == pending ]]; then + jq -se 'any(.[]; .endpoint == "repos/atrinik/content-toolkit/rules/branches/main")' \ + "${api_log}" >/dev/null + fi + [[ ! -s ${readiness_log} ]] + cmp "${temporary}/proof-phase.json" "${security_merge_config}" + done + done + done +done + +# The two gate phases are independent; qualification does not mutate source. +reset_case +export READINESS_STATUS=failed +jq '.activation.code_scanning["content-toolkit"] = "enforced"' \ + "${security_merge_config}" >"${temporary}/mixed.json" +cp "${temporary}/mixed.json" "${security_merge_config}" +publish_security_merge_rulesets >"${temporary}/mixed.txt" +jq -se '[.[] | select(.method != "GET")] | length == 1 and + .[0].payload.name == "06 - CodeQL security"' "${api_log}" >/dev/null +jq -se '[.[].mode] == ["dependency"]' "${readiness_log}" >/dev/null +cmp "${temporary}/mixed.json" "${security_merge_config}" + +# Enforced restoration preserves foreign/ambiguous same-name rules and rejects +# archived or replaced repositories at the point of mutation. +for state in removal-foreign removal-id-drift removal-name-collision active-unrelated-gate active-without-gate unknown-rule-parameter unknown-condition unknown-ref-condition active-bypass-drift changed-severity foreign-repository-source extra-rule; do + reset_case + ruleset_state=${state} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + if publish_security_merge_rulesets >"${temporary}/enforced-collision-${state}.txt" 2>&1; then + echo 'expected enforced gate ownership collision to fail closed' >&2 + exit 1 + fi + assert_no_writes + [[ ! -s ${readiness_log} ]] +done +for kind in code_scanning dependency_review; do + for metadata in \ + '{"id":123,"full_name":"atrinik/content-toolkit","archived":true,"default_branch":"main"}' \ + '{"id":999,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' \ + '{"id":123,"full_name":"atrinik/renamed-toolkit","archived":false,"default_branch":"main"}' \ + '{"id":123,"full_name":"foreign/content-toolkit","archived":false,"default_branch":"main"}'; do + reset_case + export READINESS_STATUS=failed + repository_state=${metadata} + jq --arg kind "${kind}" '.activation[$kind]["content-toolkit"] = "enforced"' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + if publish_security_merge_rulesets >"${temporary}/enforced-metadata-${kind}.txt" 2>&1; then + echo 'expected enforced repository identity fence to fail closed' >&2 + exit 1 + fi + assert_no_writes + jq -se --arg mode "$([[ ${kind} == code_scanning ]] && echo codeql || echo dependency)" \ + 'all(.[]; .mode != $mode)' "${readiness_log}" >/dev/null + done +done + +reset_case +ruleset_state=exact +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/exact.txt" +assert_no_writes +[[ ! -s ${readiness_log} ]] +reset_case +ruleset_state=owner-null +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/owner-null.txt" +assert_no_writes +[[ ! -s ${readiness_log} ]] +reset_case +ruleset_state=active-drift +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/active-drift.txt" +jq -se '[.[] | select(.method != "GET")] | length == 2 and + all(.[]; .method == "PUT" and .payload.enforcement == "active")' \ + "${api_log}" >/dev/null +[[ ! -s ${readiness_log} ]] +ruleset_state=disabled +effective_state=absent +: >"${api_log}" +: >"${readiness_log}" +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/${ruleset_state}.txt" +assert_no_writes +jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null + +# An active rule on another ref does not grandfather a new default-branch gate. +for effective_state in absent different-id; do + : >"${api_log}" + : >"${readiness_log}" + ruleset_state=active-wrong-ref + export READINESS_STATUS=missing + publish_security_merge_rulesets >"${temporary}/wrong-ref-${effective_state}.txt" + assert_no_writes + jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null +done +reset_case +ruleset_state=active-drift +repository_state='{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"release/current"}' +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/encoded-ref.txt" +jq -se 'any(.[]; .endpoint == "repos/atrinik/content-toolkit/rules/branches/release%2Fcurrent") and + ([.[] | select(.method == "PUT")] | length == 2)' "${api_log}" >/dev/null +[[ ! -s ${readiness_log} ]] + +# Stale identities and archived metadata cannot grandfather a gate. +for repository_state in '{"id":999,"archived":false,"default_branch":"main"}' '{"id":123,"archived":true,"default_branch":"main"}'; do + : >"${api_log}" + : >"${readiness_log}" + ruleset_state=active-drift + effective_state=covering + export READINESS_STATUS=missing + publish_security_merge_rulesets >"${temporary}/metadata-drift.txt" + assert_no_writes + jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null +done + +# Exact repository-owned retired targets are removed; foreign collisions stay. +for removal in removal-owned removal-owner-null removal-foreign removal-id-drift removal-name-collision; do + reset_case + ruleset_state=${removal} + jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"] | del(.activation.code_scanning["content-toolkit"], .activation.dependency_review["content-toolkit"])' \ + "${security_merge_config}" >"${temporary}/removed.json" + cp "${temporary}/removed.json" "${security_merge_config}" + publish_security_merge_rulesets >"${temporary}/${removal}.txt" + if [[ ${removal} == removal-owned || ${removal} == removal-owner-null ]]; then + jq -se '[.[] | select(.method != "GET")] == [ + {method: "DELETE", endpoint: "repos/atrinik/content-toolkit/rulesets/6"}, + {method: "DELETE", endpoint: "repos/atrinik/content-toolkit/rulesets/7"} + ]' "${api_log}" >/dev/null + else + assert_no_writes + grep -q '^SKIP ' "${temporary}/${removal}.txt" + fi + [[ ! -s ${readiness_log} ]] +done + +# GitHub expands an omitted create-exemption flag to false. Both publisher +# scopes must keep exact active API payloads and retire only exact owned gates. +for policy_scope in organization repository; do + reset_case + ruleset_state=api-default-false + export READINESS_STATUS=failed + : >"${temporary}/order" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/${policy_scope}-api-default-keep.txt" + assert_no_writes + [[ ! -s ${readiness_log} ]] + grep -Fq 'KEEP /repos/atrinik/content-toolkit/rulesets/7 07 - Dependency review' \ + "${temporary}/${policy_scope}-api-default-keep.txt" + for default_state in false true null unknown; do + reset_case + ruleset_state=api-default-${default_state} + jq '.dependency_review -= ["content-toolkit"] | del(.activation.dependency_review["content-toolkit"])' \ + "${security_merge_config}" >"${temporary}/removed.json" + cp "${temporary}/removed.json" "${security_merge_config}" + : >"${temporary}/order" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/${policy_scope}-api-default-${default_state}.txt" + if [[ ${default_state} == false ]]; then + jq -se '[.[] | select(.method != "GET")] == [ + {method: "DELETE", endpoint: "repos/atrinik/content-toolkit/rulesets/7"} + ]' "${api_log}" >/dev/null + else + assert_no_writes + grep -Fq '07 - Dependency review: unrecognized ownership or payload; retained' \ + "${temporary}/${policy_scope}-api-default-${default_state}.txt" + fi + [[ ! -s ${readiness_log} ]] + done +done + +# The final retirement GET must still prove exact managed ownership/payload +# after the repository fence, including for changes that PUT may restore. +for policy_scope in organization repository; do + for retired_gate_id in 6 7; do + retired_kind=code_scanning + [[ ${retired_gate_id} != 7 ]] || retired_kind=dependency_review + retirement_filters=( + '.id = 999' '.source_type = "Organization"' '.source = "foreign/content-toolkit"' + '.name = "Renamed gate"' '.enforcement = "disabled"' '.enforcement = "evaluate"' + '.conditions.ref_name.include = ["refs/heads/other"]' + '.conditions.ref_name.exclude = ["~DEFAULT_BRANCH"]' + '.bypass_actors[0].bypass_mode = "always"' '.bypass_actors = []' + '.rules[0].parameters.unknown_future_control = true' '.rules += [{type:"deletion"}]' + 'unreadable' + ) + if [[ ${retired_gate_id} == 6 ]]; then + retirement_filters+=( + '.rules[0].parameters.code_scanning_tools[0].tool = "Other tool"' + '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' + ) + else + retirement_filters+=( + '.rules[0].parameters.required_status_checks[0].context = "Changed context"' + '.rules[0].parameters.required_status_checks[0].integration_id = 9' + '.rules[0].parameters.do_not_enforce_on_create = true' + '.rules[0].parameters.do_not_enforce_on_create = null' + ) + fi + for filter in "${retirement_filters[@]}" '.' \ + '.bypass_actors[0].actor_id = null | if .rules[0].type == "required_status_checks" then .rules[0].parameters.do_not_enforce_on_create = false else . end'; do + reset_case + ruleset_state=retirement-race + gate_update_id=${retired_gate_id} + gate_update_filter=${filter} + jq --arg kind "${retired_kind}" '.[$kind] -= ["content-toolkit"] | + del(.activation[$kind]["content-toolkit"])' "${security_merge_config}" >"${temporary}/removed.json" + cp "${temporary}/removed.json" "${security_merge_config}" + status=0 + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/retirement-race.txt" 2>&1 || status=$? + if [[ ${filter} == unreadable ]]; then + [[ ${status} != 0 ]] + assert_no_writes + grep -Fq 'final retirement read failed; retained' "${temporary}/retirement-race.txt" + elif [[ ${filter} == '.' || ${filter} == '.bypass_actors[0].actor_id = null |'* ]]; then + [[ ${status} == 0 ]] + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${retired_gate_id}" ' + [.[] | select(.method != "GET")] == [{method:"DELETE",endpoint:$endpoint}]' "${api_log}" >/dev/null + else + [[ ${status} == 0 ]] + assert_no_writes + grep -Fq 'rule changed during retirement preflight; retained' "${temporary}/retirement-race.txt" + fi + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${retired_gate_id}" ' + [to_entries[] | select(.value.endpoint == $endpoint and .value.method == "GET") | .key] as $reads | + ($reads | length) == 2 and + any(to_entries[]; .key > $reads[0] and .key < $reads[1] and + .value.endpoint == "repos/atrinik/content-toolkit")' "${api_log}" >/dev/null + [[ ! -s ${readiness_log} ]] + done + done +done + +# Retirements also fence a repository archived or replaced after enumeration. +for retired_state in '{"id":123,"archived":true}' '{"id":999,"archived":false}' \ + '{"id":123,"full_name":"foreign/content-toolkit","archived":false}'; do + reset_case + repository_state=${retired_state} + ruleset_state="removal-owned" + jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"] | del(.activation.code_scanning["content-toolkit"], .activation.dependency_review["content-toolkit"])' \ + "${security_merge_config}" >"${temporary}/removed.json" + cp "${temporary}/removed.json" "${security_merge_config}" + publish_security_merge_rulesets >"${temporary}/removed-state-changed.txt" + assert_no_writes + [[ ! -s ${readiness_log} ]] +done + +# Read errors must propagate even when publication is called conditionally. +reset_case +ruleset_state="removal-owned" +gate_metadata_filter=unreadable +jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"] | + del(.activation.code_scanning["content-toolkit"], .activation.dependency_review["content-toolkit"])' \ + "${security_merge_config}" >"${temporary}/removed.json" +cp "${temporary}/removed.json" "${security_merge_config}" +if publish_security_merge_rulesets >"${temporary}/retirement-metadata-unreadable.txt" 2>&1; then + echo 'expected unreadable retirement repository metadata to fail closed' >&2 + exit 1 +fi +assert_no_writes + +# An archived or missing repository is absent from the active identity map. +reset_case +active_repository_ids=() +publish_security_merge_rulesets >"${temporary}/inactive.txt" +[[ ! -s ${api_log} && ! -s ${readiness_log} ]] +active_repository_ids=([content-toolkit]=123) + +reset_case +converge_actions_permissions >"${temporary}/sha-deferred.txt" +jq -se '[.[] | select(.method == "PUT")][0].payload == + {enabled_repositories: "all", allowed_actions: "selected"}' "${api_log}" >/dev/null +[[ ! -s ${readiness_log} ]] +reset_case +actions_state='{"enabled_repositories":"selected","allowed_actions":"all","sha_pinning_required":true}' +converge_actions_permissions >"${temporary}/sha-retained.txt" +jq -se '[.[] | select(.method == "PUT")][0].payload | has("sha_pinning_required") | not' \ + "${api_log}" >/dev/null +[[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == true ]] +[[ ! -s ${readiness_log} ]] +for producer in failed ready; do + reset_case + sha_pinning_canary=atrinik/content-toolkit/99 + export READINESS_STATUS=${producer} + converge_actions_permissions >"${temporary}/sha-pending-${producer}.txt" + jq -se '[.[] | select(.method == "PUT")][0].payload | has("sha_pinning_required") | not' \ + "${api_log}" >/dev/null +done +reset_case +echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" +converge_actions_permissions >"${temporary}/sha-enforced-ready.txt" +jq -se --slurpfile desired "${root}/config/actions-permissions.json" ' + [.[] | select(.method == "PUT")][-1].payload == $desired[0]' "${api_log}" >/dev/null +jq -se '. == [{mode: "sha", repository: "atrinik/content-toolkit/99"}]' \ + "${readiness_log}" >/dev/null +for producer in failed unreadable; do + reset_case + echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" + export READINESS_STATUS=${producer} + if converge_actions_permissions >"${temporary}/sha-enforced-${producer}.txt" 2>&1; then + echo "expected enforced SHA evidence failure to fail closed: ${producer}" >&2 + exit 1 + fi + jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null +done +reset_case +echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" +actions_state=$(cat "${root}/config/actions-permissions.json") +export READINESS_STATUS=failed +converge_actions_permissions >"${temporary}/sha-keep.txt" +assert_no_writes +[[ ! -s ${readiness_log} ]] + +# Explicit pending canary probes still run when SHA is already enabled. +for producer in ready failed unreadable stale; do + reset_case + actions_state=$(cat "${root}/config/actions-permissions.json") + sha_pinning_canary=atrinik/content-toolkit/99 + export READINESS_STATUS=${producer} + converge_actions_permissions >"${temporary}/sha-enabled-probe-${producer}.txt" + jq -se '. == [{mode:"sha", repository:"atrinik/content-toolkit/99"}]' "${readiness_log}" >/dev/null + jq -se 'all(.[]; .method == "GET" or .method == "PROOF")' "${api_log}" >/dev/null + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == true ]] + if [[ ${producer} == ready ]]; then + grep -q '^READY canary evidence:' "${temporary}/sha-enabled-probe-${producer}.txt" + else + grep -q '^PENDING canary evidence:' "${temporary}/sha-enabled-probe-${producer}.txt" + fi +done +for invalid in foreign/content-toolkit/99 atrinik/unregistered/99 replaced; do + reset_case + actions_state=$(cat "${root}/config/actions-permissions.json") + sha_pinning_canary=${invalid} + if [[ ${invalid} == replaced ]]; then + sha_pinning_canary=atrinik/content-toolkit/99 + repository_state=$(jq '.id = 999' <<<"${repository_state}") + fi + if converge_actions_permissions >"${temporary}/sha-enabled-probe-invalid.txt" 2>&1; then + echo 'expected invalid requested canary identity to fail with SHA already enabled' >&2 + exit 1 + fi + assert_no_writes + [[ ! -s ${readiness_log} ]] + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == true ]] +done + +# Simultaneous mode, patterns and SHA drift repairs conservative policy first. +for producer in ready failed unreadable; do + reset_case + echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" + selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":["old/action@*"]}' + export CHECK_POLICY_STAGED=true + export READINESS_STATUS=${producer} + if converge_actions_permissions >"${temporary}/sha-staged-${producer}.txt" 2>&1; then + [[ ${producer} == ready ]] + else + [[ ${producer} != ready ]] + fi + jq -se ' + [.[] | select(.method == "PUT" or .method == "PROOF")] as $events | + $events[0].endpoint == "orgs/atrinik/actions/permissions" and + ($events[0].payload | has("sha_pinning_required") | not) and + $events[1].endpoint == "orgs/atrinik/actions/permissions/selected-actions" and + $events[2].method == "PROOF" + ' "${api_log}" >/dev/null + if [[ ${producer} == ready ]]; then + jq -se '[.[] | select(.method == "PUT" or .method == "PROOF")] | + length == 4 and .[3].payload.sha_pinning_required == true' "${api_log}" >/dev/null + # A second converged publication neither mutates nor re-proves retained SHA. + : >"${api_log}" + : >"${readiness_log}" + converge_actions_permissions >"${temporary}/sha-staged-converged.txt" + assert_no_writes + [[ ! -s ${readiness_log} ]] + else + jq -se '[.[] | select(.method == "PUT" or .method == "PROOF")] | length == 3' "${api_log}" >/dev/null + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == false ]] + fi +done + +# A disable concurrent with pattern staging must not reuse the initial true. +for producer in ready failed unreadable; do + reset_case + echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" + actions_state='{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":true}' + selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":[]}' + staging_race=disable + export READINESS_STATUS=${producer} + if converge_actions_permissions >"${temporary}/sha-concurrent-disable-${producer}.txt" 2>&1; then + [[ ${producer} == ready ]] + else + [[ ${producer} != ready ]] + fi + jq -se ' + to_entries as $events | + ([$events[] | select(.value.method == "PUT" and + .value.endpoint == "orgs/atrinik/actions/permissions/selected-actions")][0].key) as $staged | + ([$events[] | select(.key > $staged and .value.method == "GET" and + .value.endpoint == "orgs/atrinik/actions/permissions")][0].key) as $read | + ([$events[] | select(.key > $read and .value.method == "GET" and + .value.endpoint == "repos/atrinik/content-toolkit")][0].key) as $identity | + ([$events[] | select(.value.method == "PROOF")][0].key) as $proof | + $staged < $read and $read < $identity and $identity < $proof and + all($events[] | select(.key < $proof and .value.method == "PUT"); + .value.payload.sha_pinning_required != true) + ' "${api_log}" >/dev/null + if [[ ${producer} == ready ]]; then + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == true ]] + jq -se '.[-1].method == "PUT" and .[-1].payload.sha_pinning_required == true' "${api_log}" >/dev/null + else + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == false ]] + jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null + fi +done +for race in malformed replace-repository; do + reset_case + echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" + actions_state='{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":true}' + selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":[]}' + staging_race=${race} + if converge_actions_permissions >"${temporary}/sha-concurrent-${race}.txt" 2>&1; then + echo 'expected changed post-staging evidence to fail closed' >&2 + exit 1 + fi + [[ ! -s ${readiness_log} ]] + jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null +done +reset_case +actions_state='{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":true}' +selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":[]}' +staging_race=disable +converge_actions_permissions >"${temporary}/sha-concurrent-disable-pending.txt" +[[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == false ]] +[[ ! -s ${readiness_log} ]] +jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null + +reset_case +echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" +apply=false +converge_actions_permissions >"${temporary}/sha-stage-plan.txt" 2>&1 +grep -q 'PLAN SHA pinning proof' "${temporary}/sha-stage-plan.txt" +if grep -Eq 'jq:|error:' "${temporary}/sha-stage-plan.txt"; then + echo 'unexpected error in SHA staging dry run' >&2 + exit 1 +fi +[[ ! -s ${readiness_log} ]] +jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null +[[ $(jq -r '.allowed_actions' <<<"${actions_state}") == all ]] + +# An unreadable staged allowlist never reaches the canary or writes SHA. +reset_case +echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" +selected_state=unreadable +if converge_actions_permissions >"${temporary}/sha-staged-unreadable-policy.txt" 2>&1; then + echo 'expected unreadable staged Actions policy to fail closed' >&2 + exit 1 +fi +[[ ! -s ${readiness_log} ]] +jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null + +# Pending rollout may repair policy and probe, but never enables SHA pinning. +reset_case +selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":[]}' +sha_pinning_canary=atrinik/content-toolkit/99 +export CHECK_POLICY_STAGED=true +converge_actions_permissions >"${temporary}/sha-pending-staged.txt" +jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null + +# Malformed configured languages are unreadable for ordinary repositories too. +for languages in null '{}' '[false]' '[""]' '"rust"'; do + reset_case + jq '.codeql_required_languages = {}' "${security_merge_config}" >"${temporary}/no-minima.json" + cp "${temporary}/no-minima.json" "${security_merge_config}" + codeql_state=$(jq -cn --argjson languages "${languages}" \ + '{state:"configured", query_suite:"default", languages:$languages}') + if converge_codeql_default_setup >"${temporary}/malformed-languages.txt" 2>&1; then + echo 'expected malformed configured CodeQL languages to fail closed without minima' >&2 + exit 1 + fi + assert_no_writes +done +reset_case +jq '.codeql_required_languages = {}' "${security_merge_config}" >"${temporary}/no-minima.json" +cp "${temporary}/no-minima.json" "${security_merge_config}" +codeql_state='{"state":"configured","query_suite":"default","languages":[]}' +converge_codeql_default_setup >"${temporary}/readable-empty-languages.txt" +assert_no_writes +reset_case +codeql_state='{"state":"not-configured","languages":null}' +converge_codeql_default_setup >"${temporary}/unconfigured-languages.txt" +jq -se '[.[] | select(.method == "PATCH")][0].payload.languages == ["rust"]' "${api_log}" >/dev/null + +# Neither a foreign owner nor an unregistered repository can lend canary proof. +for phase in pending enforced; do + for rejected_canary in foreign/content-toolkit/99 atrinik/unregistered/99; do + reset_case + if [[ ${phase} == enforced ]]; then + jq -n --arg canary "${rejected_canary}" '{phase:"enforced", canary:$canary}' \ + >"${actions_sha_pinning_config}" + else + sha_pinning_canary=${rejected_canary} + fi + if converge_actions_permissions >"${temporary}/sha-identity-rejected.txt" 2>&1; then + echo 'expected foreign or unregistered canary to be rejected' >&2 + exit 1 + fi + assert_no_writes + [[ ! -s ${readiness_log} ]] + done +done +reset_case +repository_state='{"id":999,"full_name":"atrinik/content-toolkit","archived":false}' +sha_pinning_canary=atrinik/content-toolkit/99 +if converge_actions_permissions >"${temporary}/sha-replaced-repository.txt" 2>&1; then + echo 'expected a replaced canary repository to be rejected' >&2 + exit 1 +fi +assert_no_writes +[[ ! -s ${readiness_log} ]] + +reset_case +converge_repository_workflow_permissions >"${temporary}/workflow-drift.txt" +jq -se --slurpfile desired "${root}/config/actions-workflow.json" ' + [.[] | select(.method != "GET")] == [{method: "PUT", + endpoint: "repos/atrinik/content-toolkit/actions/permissions/workflow", payload: $desired[0]}] +' "${api_log}" >/dev/null +reset_case +workflow_state=$(cat "${root}/config/actions-workflow.json") +converge_repository_workflow_permissions >"${temporary}/workflow-noop.txt" +assert_no_writes +for workflow_state in unreadable '{}' '{"default_workflow_permissions":"read","can_approve_pull_request_reviews":"false"}'; do + : >"${api_log}" + if converge_repository_workflow_permissions >"${temporary}/workflow-error.txt" 2>&1; then + echo 'expected unreadable or malformed workflow permissions to fail closed' >&2 + exit 1 + fi + assert_no_writes +done + +rm -f "${temporary_files[@]}" +echo 'Security merge publisher tests passed.' diff --git a/tests/security-readiness.py b/tests/security-readiness.py new file mode 100644 index 0000000..81787a3 --- /dev/null +++ b/tests/security-readiness.py @@ -0,0 +1,1969 @@ +#!/usr/bin/env python3 +"""Deterministic, read-only security readiness contract tests.""" + +import base64 +import copy +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch +from urllib.parse import parse_qs, quote, urlparse + + +ROOT = Path(__file__).resolve().parent.parent +sys.dont_write_bytecode = True +SPEC = importlib.util.spec_from_file_location("security_readiness", ROOT / "lib/security_readiness.py") +readiness = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(readiness) +HEAD, BASE, MERGE = "a" * 40, "b" * 40, "c" * 40 +OLD, NEW = "d" * 40, "e" * 40 +WORKFLOW = ("name: Dependency review\n\non:\n pull_request:\n branches: [main]\n" + " types: [opened, synchronize, reopened, edited]\n" + " merge_group:\n types: [checks_requested]\n" + "permissions:\n contents: read\njobs:\n dependency-review:\n" + " name: Dependency review\n runs-on: ubuntu-24.04\n steps:\n" + f" - uses: actions/dependency-review-action@{OLD} # v5.0.0\n" + " with:\n fail-on-severity: high\n warn-only: false\n").encode() +# Snapshot of the actual content-toolkit producer, including its types-first +# mapping, block branch filter, named step, and complete enforcement settings. +# Immutable action versions remain producer-owned and are normalized by policy. +COMPONENT_WORKFLOW = b"""name: Dependency review + +on: + pull_request: + types: [opened, synchronize, reopened, edited] + branches: + - main + merge_group: + types: [checks_requested] + +permissions: + contents: read + +jobs: + dependency-review: + name: Dependency review + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Review dependency changes + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: high + fail-on-scopes: runtime,development,unknown + warn-only: false + comment-summary-in-pr: never + show-openssf-scorecard: false + retry-on-snapshot-warnings: true + retry-on-snapshot-warnings-timeout: 120 +""" +SELECTED = {"github_owned_allowed": True, "verified_allowed": False, + "patterns_allowed": ["rust-lang/crates-io-auth-action@*", "docker/setup-buildx-action@*"]} +# Executable Classic workflow snapshot; action pins remain producer-owned. +CODEQL_WORKFLOW = b"""name: CodeQL + +on: + push: + branches: + - main + pull_request: + branches: + - main + merge_group: + workflow_dispatch: + schedule: + - cron: "17 4 * * 2" + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: classic-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + classify: + name: CodeQL classification + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: ${{ steps.classify.outputs.codeql_matrix }} + run: ${{ steps.classify.outputs.codeql_run }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - id: classify + name: Select analysis languages and paths + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + python3 tools/ci/classify_changes.py \\ + --event "${EVENT_NAME}" \\ + --base "${BASE_SHA}" \\ + --head "${HEAD_SHA}" \\ + --github-output "${GITHUB_OUTPUT}" \\ + --codeql-config build/codeql-config.yml + + analyze: + name: CodeQL analysis (${{ matrix.display }}) + needs: classify + if: needs.classify.outputs.run == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.classify.outputs.matrix) }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - id: classify + name: Recreate the trusted path configuration + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + python3 tools/ci/classify_changes.py \\ + --event "${EVENT_NAME}" \\ + --base "${BASE_SHA}" \\ + --head "${HEAD_SHA}" \\ + --github-output "${GITHUB_OUTPUT}" \\ + --codeql-config build/codeql-config.yml \\ + --codeql-scope "${{ matrix.scope }}" + - name: Initialize CodeQL + if: matrix.run == true + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + build-mode: none + config-file: ${{ steps.classify.outputs.codeql_config }} + languages: ${{ matrix.language }} + - name: Analyze + if: matrix.run == true + uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + category: ${{ matrix.category }} + + codeql-validation: + name: CodeQL validation + if: always() + needs: + - classify + - analyze + runs-on: ubuntu-24.04 + timeout-minutes: 2 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Require every selected analysis + run: | + python3 tools/ci/require_checks.py optional \\ + --label CodeQL \\ + --classifier-result '${{ needs.classify.result }}' \\ + --required '${{ needs.classify.outputs.run }}' \\ + --result '${{ needs.analyze.result }}' +""" +PARTITIONS = [ + {"display": "C/C++ client", "scope": "client", "language": "c-cpp", + "category": "/partition:client/language:c-cpp", "run": True}, + {"display": "C/C++ server", "scope": "server", "language": "c-cpp", + "category": "/partition:server/language:c-cpp", "run": True}, + {"display": "C/C++ core", "scope": "core-cpp", "language": "c-cpp", + "category": "/partition:core/language:c-cpp", "run": True}, + {"display": "Python", "scope": "python", "language": "python", + "category": "/partition:core/language:python", "run": True}, + {"display": "GitHub Actions", "scope": "actions", "language": "actions", + "category": "/partition:core/language:actions", "run": True}, +] + + +def classifier_source(partitions): + source = """FULL_EVENTS = {'push', 'schedule', 'workflow_dispatch', 'merge_group'} +def main(): + if arguments.event in FULL_EVENTS: + full = True + paths = [] +def write_outputs(path, result, config): + entries = [] +""" + for item in partitions: + source += f" if result[{item['scope']!r}]:\n entries.append({item!r})\n" + source += " matrix = {'include': entries or [{'run': False, 'scope': 'union'}]}\n" + return source.encode() + + +CLASSIFIER = classifier_source(PARTITIONS) + + +def content(data): + return {"type": "file", "encoding": "base64", "content": base64.b64encode(data).decode()} + + +def check(name, ref=HEAD, identifier=1, conclusion="success", status="completed"): + return {"name": name, "head_sha": ref, "id": identifier, "status": status, + "conclusion": conclusion, "started_at": "2026-10-11T12:00:00Z", + "app": {"id": 15368, "slug": "github-actions"}, "check_suite": {"id": 99}} + + +class Fixture: + def __init__(self): + self.calls = [] + self.metadata = {"id": 42, "full_name": "atrinik/example", "default_branch": "main", "archived": False} + self.pr = {"number": 12, "state": "open", "draft": False, "merged": False, + "changed_files": 1, "merge_commit_sha": MERGE, + "base": {"ref": "main", "sha": BASE, "repo": {"full_name": "atrinik/example"}}, + "head": {"sha": HEAD, "repo": {"full_name": "atrinik/example"}}} + self.run = {"id": 9, "run_attempt": 2, "event": "pull_request", "status": "completed", + "conclusion": "success", "head_sha": HEAD, "check_suite_id": 99, + "path": ".github/workflows/dependency-review.yml", + "head_repository": {"full_name": "atrinik/example"}, + "pull_requests": [{"number": 12, "head": {"sha": HEAD}, + "base": {"sha": BASE, "ref": "main"}}]} + self.suite_runs = {} + self.associated_pulls = [{"number": 12}] + self.suite = {"id": 99, "head_sha": MERGE, "status": "completed", "conclusion": "success", + "app": {"id": 15368}, "pull_requests": copy.deepcopy(self.run["pull_requests"])} + self.jobs = [{"name": "Dependency review", "status": "completed", "conclusion": "success"}] + self.files = [{"filename": ".github/workflows/check.yml", "status": "modified"}] + self.before = f"name: Check\nsteps:\n - uses: rust-lang/crates-io-auth-action@{OLD} # v1\n".encode() + self.after = f"name: Check\nsteps:\n - uses: rust-lang/crates-io-auth-action@{NEW} # v2\n".encode() + self.checks = {HEAD: [check("Required checks"), check("Conventional PR title", identifier=2)], MERGE: []} + self.live = copy.deepcopy(SELECTED) + self.actions_policy = {"enabled_repositories": "all", "allowed_actions": "selected"} + self.repository_actions = {"enabled": True, "allowed_actions": "all"} + self.enabled_repositories = [{"id": 42, "full_name": "atrinik/example"}] + self.effective_rules = [] + self.legacy_checks = None + self.default_workflow = WORKFLOW + self.run_workflow = WORKFLOW + self.executed_workflow = WORKFLOW + self.parents = [BASE, HEAD] + self.branch_head = HEAD + self.setup = {"state": "configured", "languages": ["rust"]} + self.codeql_sources = {".github/workflows/codeql.yml": CODEQL_WORKFLOW, + "tools/ci/classify_changes.py": CLASSIFIER, + "tools/ci/require_checks.py": b"# Trusted aggregate\n"} + self.codeql_head_sources = copy.deepcopy(self.codeql_sources) + self.codeql_run = {"id": 29, "run_attempt": 1, "head_sha": HEAD, "head_branch": "main", + "event": "push", "path": ".github/workflows/codeql.yml", "check_suite_id": 777, + "status": "completed", "conclusion": "success"} + self.codeql_check = check("CodeQL validation", identifier=999) + self.codeql_check["check_suite"]["id"] = 777 + self.codeql_jobs = [] + self.analysis_reads = 0 + self.fresh_analyses = None + self.analyses = [{"id": 1, "created_at": "2026-10-11T12:00:00Z", "tool": {"name": "CodeQL"}, + "ref": "refs/heads/main", "commit_sha": HEAD, "category": "/language:rust", + "environment": '{"language":"rust"}', "error": "", "rules_count": 42}] + + def __call__(self, endpoint): + self.calls.append(endpoint) + if endpoint == "repos/atrinik/example": + return copy.deepcopy(self.metadata) + if endpoint == "repos/atrinik/example/pulls/12": + return copy.deepcopy(self.pr) + if "/contents/.github/workflows/dependency-review.yml" in endpoint: + return content(self.default_workflow if "ref=main" in endpoint else + self.executed_workflow if "ref=" + MERGE in endpoint else self.run_workflow) + if "/contents/.github/workflows/check.yml" in endpoint: + return content(self.before if "ref=" + self.pr["base"]["sha"] in endpoint else self.after) + if "/contents/" in endpoint: + path = endpoint.split("/contents/")[1].split("?")[0] + ref = parse_qs(urlparse(endpoint).query).get("ref", [None])[0] + sources = self.codeql_sources if ref == self.metadata["default_branch"] else self.codeql_head_sources + return content(sources[path]) + if "/actions/workflows/codeql.yml/runs" in endpoint: + return {"workflow_runs": [copy.deepcopy(self.codeql_run)]} + if "/actions/runs/29/attempts/1/jobs" in endpoint: + return {"total_count": len(self.codeql_jobs), "jobs": copy.deepcopy(self.codeql_jobs)} + if endpoint.endswith("/check-runs/999"): + return copy.deepcopy(self.codeql_check) + if "/check-runs/" in endpoint: + identifier = int(endpoint.rsplit("/", 1)[1]) + for checks in self.checks.values(): + for current in checks: + if current["id"] == identifier: + return copy.deepcopy(current) + raise AssertionError("Unexpected check identity: " + endpoint) + if "/actions/workflows/dependency-review.yml/runs" in endpoint: + return {"workflow_runs": [copy.deepcopy(self.run)]} + if "/attempts/2/jobs" in endpoint: + return {"total_count": len(self.jobs), "jobs": copy.deepcopy(self.jobs)} + if endpoint == "orgs/atrinik/actions/permissions": + return copy.deepcopy(self.actions_policy) + if endpoint == "repos/atrinik/example/actions/permissions": + return copy.deepcopy(self.repository_actions) + if "/actions/permissions/repositories?" in endpoint: + return {"total_count": len(self.enabled_repositories), + "repositories": copy.deepcopy(self.enabled_repositories)} + if "/rules/branches/" in endpoint: + return copy.deepcopy(self.effective_rules) + if endpoint.endswith("/protection"): + return {"required_status_checks": copy.deepcopy(self.legacy_checks)} + if "/actions/permissions/selected-actions" in endpoint: + return copy.deepcopy(self.live) + if "/pulls/12/files" in endpoint: + return copy.deepcopy(self.files) + if f"/commits/{HEAD}/pulls?" in endpoint: + return copy.deepcopy(self.associated_pulls) + if endpoint == "repos/atrinik/example/commits/" + MERGE: + return {"parents": [{"sha": value} for value in self.parents]} + if "/check-runs?" in endpoint: + values = copy.deepcopy(self.checks[endpoint.split("/commits/")[1].split("/")[0]]) + return {"total_count": len(values), "check_runs": values} + if "/actions/runs?check_suite_id=" in endpoint: + suite_id = int(parse_qs(urlparse(endpoint).query)["check_suite_id"][0]) + return {"workflow_runs": [copy.deepcopy(self.suite_runs.get(suite_id, self.run))]} + if "/actions/runs?event=pull_request&head_sha=" in endpoint: + return {"total_count": 1, "workflow_runs": [copy.deepcopy(self.run)]} + if "/check-suites/" in endpoint: + value = copy.deepcopy(self.suite) + value["id"] = int(endpoint.rsplit("/", 1)[1]) + return value + if endpoint == "repos/atrinik/example/branches/" + quote(self.metadata["default_branch"], safe=""): + return {"commit": {"sha": self.branch_head}} + if "/code-scanning/analyses?" in endpoint: + page = int(parse_qs(urlparse(endpoint).query)["page"][0]) + if page == 1: + self.analysis_reads += 1 + analyses = (self.fresh_analyses if self.analysis_reads > 1 and self.fresh_analyses is not None + else self.analyses) + return copy.deepcopy(analyses[(page - 1) * 100:page * 100]) + if endpoint.endswith("/code-scanning/default-setup"): + return copy.deepcopy(self.setup) + raise AssertionError("Unexpected read-only API endpoint: " + endpoint) + + +class ReadinessTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / ".github/workflows").mkdir(parents=True) + (self.root / ".github/workflows/dependency-review.yml").write_bytes(WORKFLOW) + (self.root / "config").mkdir() + (self.root / "config/actions-selected.json").write_text(json.dumps(SELECTED)) + (self.root / "config/repositories.json").write_text(json.dumps({ + "required_ci": {"example": ["Required checks", "Conventional PR title"]}})) + (self.root / "config/security-merge-protection.json").write_text(json.dumps({ + "codeql_required_languages": {"example": ["rust"]}, "dependency_review": []})) + (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": [], "required_partitions": {}}') + self.fixture = Fixture() + self.root_patch = patch.object(readiness, "ROOT", self.root) + self.api_patch = patch.object(readiness, "api", self.fixture) + self.root_patch.start() + self.api_patch.start() + self.addCleanup(self.root_patch.stop) + self.addCleanup(self.api_patch.stop) + + def invoke(self, mode): + stream = io.StringIO() + with patch("sys.stdout", stream): + result = readiness.main([mode, "atrinik/example/12" if mode == "sha" else "atrinik/example"]) + return result, stream.getvalue() + + def assert_deferred(self, mode, fragment): + code, text = self.invoke(mode) + self.assertEqual(code, 1, text) + self.assertIn(fragment, text) + + def test_dependency_ready_latest_attempt_exact_workflow(self): + self.assertEqual(self.invoke("dependency")[0], 0) + self.assertTrue(any("/attempts/2/jobs" in call for call in self.fixture.calls)) + + def merged_empty_fixture(self, run_empty=True, suite_empty=True): + fixture = Fixture() + fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + if run_empty: + fixture.run["pull_requests"] = [] + if suite_empty: + fixture.suite["pull_requests"] = [] + return fixture + + def test_merged_empty_associations_ready_for_dependency_and_stored_sha_canary(self): + for mode in ("dependency", "sha"): + for run_empty, suite_empty in ((True, False), (False, True), (True, True)): + for direct_head in (False, True): + with self.subTest(mode=mode, run_empty=run_empty, suite_empty=suite_empty, + direct_head=direct_head): + fixture = self.merged_empty_fixture(run_empty, suite_empty) + if direct_head: + fixture.suite["head_sha"] = HEAD + elif mode == "sha": + fixture.checks[MERGE] = [check("Required checks", MERGE), + check("Conventional PR title", MERGE, 2)] + fixture.checks[HEAD] = [] + with patch.object(readiness, "api", fixture): + result, output = self.invoke(mode) + self.assertEqual(result, 0, output) + self.assertGreaterEqual(sum(f"/commits/{HEAD}/pulls?" in endpoint + for endpoint in fixture.calls), 2) + + def test_merged_empty_association_newer_failed_or_queued_canary_still_blocks(self): + for status, conclusion in (("completed", "failure"), ("queued", None)): + with self.subTest(status=status): + fixture = self.merged_empty_fixture() + newer = check("Required checks", identifier=50, status=status, conclusion=conclusion) + newer["started_at"] = None + newer["check_suite"]["id"] = 100 + fixture.checks[HEAD].append(newer) + run = copy.deepcopy(fixture.run) + run.update(id=20, check_suite_id=100, status=status, conclusion=conclusion) + fixture.suite_runs[100] = run + with patch.object(readiness, "api", fixture): + self.assert_deferred("sha", "workflow run not successful") + + def test_merged_empty_associations_reject_ambiguous_or_missing_commit_pulls(self): + for mode in ("dependency", "sha"): + for entries in ([], [{"number": 12}, {"number": 13}]): + with self.subTest(mode=mode, entries=entries): + fixture = self.merged_empty_fixture() + fixture.associated_pulls = entries + with patch.object(readiness, "api", fixture): + self.assert_deferred(mode, "association missing or ambiguous") + + def test_merged_empty_associations_reject_open_pr(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + fixture.pr.update(state="open", merged=False) + fixture.pr["merge_commit_sha"] = MERGE + with patch.object(readiness, "api", fixture): + self.assert_deferred(mode, "association identity") + + def test_merged_empty_associations_reject_identity_mismatch(self): + for mode in ("dependency", "sha"): + for field in ("head", "head_repository", "base", "base_repository", "number"): + with self.subTest(mode=mode, field=field): + fixture = self.merged_empty_fixture() + # Change only the independently fetched association PR; retain + # the caller's initial canary so its own guards cannot mask this test. + count = 0 + def mismatch(endpoint): + nonlocal count + value = fixture(endpoint) + if endpoint.endswith("/pulls/12"): + count += 1 + if mode == "dependency" or count > 1: + if field == "head": + value["head"]["sha"] = OLD + elif field == "head_repository": + value["head"]["repo"]["full_name"] = "other/example" + elif field == "base": + value["base"]["ref"] = "other" + elif field == "base_repository": + value["base"]["repo"]["full_name"] = "other/example" + else: + value["number"] = 13 + return value + with patch.object(readiness, "api", mismatch): + self.assert_deferred(mode, "association identity") + + def test_merged_empty_associations_never_override_nonempty_conflict(self): + for mode in ("dependency", "sha"): + for conflicting in ("run", "suite"): + with self.subTest(mode=mode, conflicting=conflicting): + fixture = self.merged_empty_fixture(conflicting != "run", conflicting != "suite") + source = fixture.run if conflicting == "run" else fixture.suite + source["pull_requests"][0]["base"]["sha"] = OLD + with patch.object(readiness, "api", fixture): + self.assert_deferred(mode, "nonempty PR association conflicts") + + def test_merged_empty_associations_reject_non_pr_event_or_workflow_path(self): + for mode in ("dependency", "sha"): + for field, value in (("event", "push"), ("event", "workflow_dispatch"), + ("path", "untrusted.yml")): + with self.subTest(mode=mode, field=field, value=value): + fixture = self.merged_empty_fixture() + fixture.run[field] = value + with patch.object(readiness, "api", fixture): + result, output = self.invoke(mode) + self.assertEqual(result, 1, output) + + def test_merged_empty_associations_unreadable_is_error(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + def unreadable(endpoint): + if f"/commits/{HEAD}/pulls?" in endpoint: + raise readiness.APIError(endpoint, 403) + return fixture(endpoint) + with patch.object(readiness, "api", unreadable): + code, output = self.invoke(mode) + self.assertEqual(code, 2, output) + self.assertIn("HTTP 403", output) + + def test_merged_empty_association_collection_is_refenced(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + verify = readiness.verify_recovered_associations + def change_at_fence(coordinate, recovered): + fixture.associated_pulls.append({"number": 13}) + return verify(coordinate, recovered) + with patch.object(readiness, "api", fixture), patch.object( + readiness, "verify_recovered_associations", change_at_fence): + self.assert_deferred(mode, "association missing or ambiguous") + + def test_merged_empty_association_pr_identity_is_refenced(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + verify = readiness.verify_recovered_associations + def change_at_fence(coordinate, recovered): + fixture.pr["base"]["sha"] = OLD + return verify(coordinate, recovered) + with patch.object(readiness, "api", fixture), patch.object( + readiness, "verify_recovered_associations", change_at_fence): + self.assert_deferred(mode, "association changed during preflight") + + def test_merged_empty_association_preserves_workflow_and_merge_parent_proof(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + fixture.parents = [OLD, HEAD] + with patch.object(readiness, "api", fixture): + self.assert_deferred(mode, "merge") + fixture = self.merged_empty_fixture() + fixture.executed_workflow += b"# changed execution bytes\n" + with patch.object(readiness, "api", fixture): + self.assert_deferred("dependency", "different workflow bytes") + + def test_dependency_base_retarget_requires_edited_trigger(self): + # An otherwise green producer that ran before retargeting coverage was + # added cannot establish the complete default-branch PR contract. + old = WORKFLOW.replace(b"opened, synchronize, reopened, edited", + b"opened, synchronize, reopened") + self.fixture.default_workflow = old + self.fixture.run_workflow = old + self.fixture.executed_workflow = old + self.assert_deferred("dependency", "not landed") + + def test_dependency_base_retarget_activity_block_ready(self): + edited = WORKFLOW.replace( + b" types: [opened, synchronize, reopened, edited]", + b" types:\n - edited\n - opened\n - reopened\n - synchronize") + self.fixture.default_workflow = edited + self.fixture.run_workflow = edited + self.fixture.executed_workflow = edited + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_extra_activity_type_deferred(self): + self.fixture.default_workflow = WORKFLOW.replace(b"reopened, edited]", + b"reopened, edited, closed]") + self.assert_deferred("dependency", "not landed") + + def test_dependency_merge_queue_trigger_required(self): + self.fixture.default_workflow = WORKFLOW.replace( + b" merge_group:\n types: [checks_requested]\n", b"") + self.assert_deferred("dependency", "not landed") + + def test_dependency_merge_queue_activity_block_ready(self): + queue = WORKFLOW.replace(b" types: [checks_requested]", + b" types:\n - checks_requested") + self.fixture.default_workflow = queue + self.fixture.run_workflow = queue + self.fixture.executed_workflow = queue + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_merge_queue_wrong_activity_deferred(self): + self.fixture.default_workflow = WORKFLOW.replace(b"[checks_requested]", b"[destroyed]") + self.assert_deferred("dependency", "not landed") + + def test_dependency_merge_queue_run_does_not_replace_pr_evidence(self): + self.fixture.run["event"] = "merge_group" + self.assert_deferred("dependency", "latest dependency-review PR run not successful") + + def test_actual_component_workflow_key_order_matches_central_contract(self): + central = (ROOT / ".github/workflows/dependency-review.yml").read_bytes() + self.assertEqual(readiness.dependency_contract(COMPONENT_WORKFLOW), + readiness.dependency_contract(central)) + + def test_actual_component_key_order_preserves_branch_restriction(self): + central = (ROOT / ".github/workflows/dependency-review.yml").read_bytes() + changed = COMPONENT_WORKFLOW.replace(b" - main", b" - other") + self.assertNotEqual(readiness.dependency_contract(changed), + readiness.dependency_contract(central)) + + def test_actual_component_key_order_preserves_activity_restriction(self): + central = (ROOT / ".github/workflows/dependency-review.yml").read_bytes() + changed = COMPONENT_WORKFLOW.replace(b", edited]", b"]") + self.assertNotEqual(readiness.dependency_contract(changed), + readiness.dependency_contract(central)) + + def test_dependency_not_landed(self): + self.fixture.default_workflow = WORKFLOW.replace(b"fail-on-severity: high", b"fail-on-severity: low") + self.assert_deferred("dependency", "not landed") + + def test_dependency_producer_fresh_sha_passes_without_central_update(self): + fresh = WORKFLOW.replace(OLD.encode(), NEW.encode()).replace(b"v5.0.0", b"v5.1.0") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_mutable_action_ref_deferred(self): + self.fixture.default_workflow = WORKFLOW.replace(OLD.encode(), b"v5") + self.assert_deferred("dependency", "full-SHA") + + def test_dependency_classic_block_branch_filter_ready(self): + fresh = WORKFLOW.replace(b" branches: [main]", b" branches:\n - main") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_extra_branch_filter_deferred(self): + self.fixture.default_workflow = WORKFLOW.replace(b" branches: [main]", + b" branches:\n - main\n - extra") + self.assert_deferred("dependency", "not landed") + + def test_dependency_named_official_action_step_ready(self): + fresh = WORKFLOW.replace(b" - uses:", + b" - name: Review dependency changes\n uses:") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_labels_do_not_lock_producer_workflow(self): + fresh = WORKFLOW.replace(b"name: Dependency review\n", b"name: Dependency safety\n", 1) + fresh = fresh.replace(b" - uses:", b" - name: Validate dependencies\n uses:") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_named_step_arbitrary_run_deferred(self): + fresh = WORKFLOW.replace(b" - uses:", + b" - name: Review dependency changes\n run: arbitrary\n uses:") + self.fixture.default_workflow = fresh + self.assert_deferred("dependency", "not landed") + + def test_dependency_blank_comment_trailing_whitespace_ready(self): + fresh = WORKFLOW.replace(b"permissions:\n", b"# Explanation\n\npermissions: \n") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_failed_latest_does_not_search_older_success(self): + self.fixture.run["conclusion"] = "failure" + self.assert_deferred("dependency", "latest") + self.assertEqual(len([call for call in self.fixture.calls if "/runs?event=" in call]), 1) + + def test_dependency_successful_run_failed_job(self): + self.fixture.jobs[0]["conclusion"] = "failure" + self.assert_deferred("dependency", "job not successful") + + def test_dependency_run_different_bytes(self): + self.fixture.executed_workflow += b"# different\n" + self.assert_deferred("dependency", "different workflow bytes") + + def test_dependency_matching_head_but_executed_merge_workflow_differs(self): + self.assertEqual(self.fixture.default_workflow, self.fixture.run_workflow) + self.fixture.executed_workflow = WORKFLOW.replace(b"warn-only: false", b"warn-only: true") + self.assert_deferred("dependency", "different workflow bytes") + + def test_dependency_direct_head_execution_bound_to_pr(self): + self.fixture.suite["head_sha"] = HEAD + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_direct_head_missing_execution_pr_association(self): + self.fixture.suite.update(head_sha=HEAD, pull_requests=[]) + self.assert_deferred("dependency", "merged PR association") + + def test_dependency_executed_merge_wrong_parents_deferred(self): + self.fixture.parents = [BASE, OLD] + self.assert_deferred("dependency", "current PR head/base") + + def test_dependency_stale_execution_base_association_deferred(self): + self.fixture.suite["pull_requests"][0]["base"]["sha"] = OLD + self.assert_deferred("dependency", "execution PR association") + + def test_dependency_check_suite_drift_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if endpoint.endswith("/check-suites/99"): + calls += 1 + if calls > 1: + value["head_sha"] = OLD + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("dependency", "check suite changed") + + def test_dependency_run_association_drift_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if "/actions/workflows/dependency-review.yml/runs" in endpoint: + calls += 1 + if calls > 1: + value["workflow_runs"][0]["pull_requests"] = [] + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("dependency", "run changed") + + def test_dependency_pr_drift_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if endpoint.endswith("/pulls/12"): + calls += 1 + if calls > 1: + value["base"]["sha"] = OLD + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("dependency", "PR changed") + + def test_dependency_wrong_base(self): + self.fixture.pr["base"]["ref"] = "maintenance" + self.assert_deferred("dependency", "default branch") + + def test_dependency_stale_head(self): + self.fixture.pr["head"]["sha"] = NEW + self.assert_deferred("dependency", "head changed") + + def test_dependency_api_404_missing_is_deferred(self): + def missing(endpoint): + if "/contents/" in endpoint: + raise readiness.APIError(endpoint, 404) + return self.fixture(endpoint) + with patch.object(readiness, "api", missing): + self.assert_deferred("dependency", "missing") + + def test_api_denied_is_error_not_deferred(self): + with patch.object(readiness, "api", side_effect=readiness.APIError("repos/atrinik/example", 403)): + self.assertEqual(self.invoke("dependency")[0], 2) + + def test_sha_wildcard_accepts_fresh_rust_pin(self): + self.assertEqual(self.invoke("sha")[0], 0) + + def require_dependency_for_canary(self): + path = self.root / "config/security-merge-protection.json" + policy = json.loads(path.read_text()) + policy["dependency_review"] = ["example"] + path.write_text(json.dumps(policy)) + + def effective_check(self, context, app=15368): + return {"type": "required_status_checks", "parameters": { + "required_status_checks": [{"context": context, "integration_id": app}]}} + + def test_sha_targeted_dependency_review_missing_failed_or_queued_rejected(self): + self.require_dependency_for_canary() + self.assert_deferred("sha", "Dependency review") + self.fixture.checks[HEAD].append(check("Dependency review", identifier=30, conclusion="failure")) + self.assert_deferred("sha", "Dependency review") + self.fixture.checks[HEAD][-1].update(status="queued", conclusion=None, started_at=None) + self.assert_deferred("sha", "Dependency review") + self.fixture.checks[HEAD][-1].update(status="completed", conclusion="success") + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_effective_extra_rule_context_required_with_trusted_actions(self): + self.fixture.effective_rules = [self.effective_check("Extra validation")] + self.assert_deferred("sha", "Extra validation") + self.fixture.checks[HEAD].append(check("Extra validation", identifier=30)) + self.fixture.checks[HEAD][-1]["app"]["id"] = 999 + self.assert_deferred("sha", "Extra validation") + self.fixture.checks[HEAD][-1]["app"]["id"] = 15368 + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_effective_dependency_review_required_outside_target_inventory(self): + self.fixture.effective_rules = [self.effective_check("Dependency review")] + self.assert_deferred("sha", "Dependency review") + + def test_sha_legacy_protection_context_required(self): + self.fixture.legacy_checks = {"contexts": ["Legacy check"], "checks": []} + self.assert_deferred("sha", "Legacy check") + self.fixture.checks[HEAD].append(check("Legacy check", identifier=30)) + self.assertEqual(self.invoke("sha")[0], 0) + + def legacy_error(self, message, status="404"): + def response(endpoint): + if endpoint.endswith("/protection"): + raise readiness.APIError(endpoint, 404, {"message": message, "status": status}) + return self.fixture(endpoint) + return patch.object(readiness, "api", response) + + def disabled_legacy_response(self): + return {"data": {"repository": {"databaseId": 42, "nameWithOwner": "atrinik/example", + "isArchived": False, "ref": {"name": "main", "branchProtectionRule": None}}}} + + def test_sha_only_exact_unprotected_error_proves_absence(self): + for message, status, expected in (("Branch not protected", "404", 0), + ("Resource not found", "404", 2), ("Not Found", "404", 2), + ("Resource not accessible by integration", "404", 2), + ("Branch not protected", None, 2), ("Branch not protected", 404, 2)): + with self.subTest(message=message, status=status), self.legacy_error(message, status): + self.assertEqual(self.invoke("sha")[0], expected) + + def test_sha_missing_or_malformed_full_protection_not_absence(self): + for value in ({}, [], None, {"required_status_checks": False}, + {"required_status_checks": {}}, + {"required_status_checks": {"checks": None, "contexts": []}}, + {"required_status_checks": {"checks": [], "contexts": None}}): + def response(endpoint): + return value if endpoint.endswith("/protection") else self.fixture(endpoint) + with self.subTest(value=value), patch.object(readiness, "api", response): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_disabled_legacy_requires_exact_readonly_identity_proof(self): + result = subprocess.CompletedProcess([], 0, stdout=json.dumps(self.disabled_legacy_response()), stderr="") + with self.legacy_error("Branch protection has been disabled on this repository."), \ + patch.object(readiness.subprocess, "run", return_value=result) as mocked: + self.assertEqual(self.invoke("sha")[0], 0) + self.assertEqual(mocked.call_count, 2) + args = mocked.call_args.args[0] + self.assertEqual(args[args.index("--method") + 1], "POST") + self.assertIn("graphql", args) + self.assertIn("owner=atrinik", args) + self.assertIn("name=example", args) + self.assertIn("qualifiedName=refs/heads/main", args) + query = next(arg for arg in args if arg.startswith("query=")) + self.assertTrue(query.startswith("query=query LegacyBranchProtection(")) + self.assertNotIn("mutation", query) + + def test_sha_disabled_legacy_ambiguous_identity_or_presence_rejected(self): + mutations = [lambda r: r.update(errors=[]), + lambda r: r["data"].update(repository=None), + lambda r: r["data"]["repository"].update(databaseId=43), + lambda r: r["data"]["repository"].update(databaseId=True), + lambda r: r["data"]["repository"].update(nameWithOwner="foreign/example"), + lambda r: r["data"]["repository"].update(isArchived=True), + lambda r: r["data"]["repository"].update(ref=None), + lambda r: r["data"]["repository"]["ref"].update(name="other"), + lambda r: r["data"]["repository"]["ref"].pop("branchProtectionRule"), + lambda r: r["data"]["repository"]["ref"].update(branchProtectionRule={"id": "rule"})] + for index, mutate in enumerate(mutations): + response = self.disabled_legacy_response() + mutate(response) + result = subprocess.CompletedProcess([], 0, stdout=json.dumps(response), stderr="") + with self.subTest(index=index), self.legacy_error("Branch protection has been disabled on this repository."), \ + patch.object(readiness.subprocess, "run", return_value=result): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_disabled_legacy_unreadable_or_invalid_graphql_rejected(self): + for result in (subprocess.CompletedProcess([], 1, stdout="", stderr="denied"), + subprocess.CompletedProcess([], 0, stdout="not json", stderr="")): + with self.subTest(result=result), self.legacy_error("Branch protection has been disabled on this repository."), \ + patch.object(readiness.subprocess, "run", return_value=result): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_disabled_legacy_fresh_rest_identity_required(self): + for field, value in (("id", 43), ("full_name", "foreign/example"), + ("default_branch", "other"), ("archived", True)): + count = 0 + def response(endpoint): + nonlocal count + if endpoint.endswith("/protection"): + raise readiness.APIError(endpoint, 404, { + "message": "Branch protection has been disabled on this repository.", "status": "404"}) + current = self.fixture(endpoint) + if endpoint == "repos/atrinik/example": + count += 1 + if count > 1: + current[field] = value + return current + with self.subTest(field=field), patch.object(readiness, "api", response), \ + patch.object(readiness.subprocess, "run") as mocked: + self.assertNotEqual(self.invoke("sha")[0], 0) + mocked.assert_not_called() + + def test_sha_disabled_legacy_proof_rechecked_before_acceptance(self): + initial = self.disabled_legacy_response() + changed = copy.deepcopy(initial) + changed["data"]["repository"]["ref"]["branchProtectionRule"] = {"id": "new-rule"} + results = [subprocess.CompletedProcess([], 0, stdout=json.dumps(value), stderr="") + for value in (initial, changed)] + with self.legacy_error("Branch protection has been disabled on this repository."), \ + patch.object(readiness.subprocess, "run", side_effect=results): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_full_legacy_checks_array_requires_all_bound_contexts(self): + self.fixture.legacy_checks = {"contexts": [], "checks": [ + {"context": "Legacy check", "app_id": 15368}, + {"context": "Second legacy check", "app_id": None}]} + self.assert_deferred("sha", "Legacy check") + self.fixture.checks[HEAD].append(check("Legacy check", identifier=30)) + self.assert_deferred("sha", "Second legacy check") + self.fixture.checks[HEAD].append(check("Second legacy check", identifier=31)) + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.checks[HEAD][-1]["conclusion"] = "failure" + self.assert_deferred("sha", "Second legacy check") + + def test_sha_legacy_absence_must_remain_readable_at_final_fence(self): + count = 0 + def response(endpoint): + nonlocal count + if endpoint.endswith("/protection"): + count += 1 + if count > 1: + raise readiness.APIError(endpoint, 404, {"message": "Not Found", "status": "404"}) + return self.fixture(endpoint) + with patch.object(readiness, "api", response): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_non_actions_app_binding_never_satisfied_by_same_named_actions_check(self): + self.fixture.effective_rules = [self.effective_check("Required checks", 999)] + self.assert_deferred("sha", "non-Actions app") + self.fixture.effective_rules = [] + self.fixture.legacy_checks = {"contexts": ["Required checks"], + "checks": [{"context": "Required checks", "app_id": 999}]} + self.assert_deferred("sha", "non-Actions app") + + def test_sha_unbound_effective_context_still_requires_trusted_actions(self): + self.fixture.effective_rules = [self.effective_check("Required checks", None)] + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.checks[HEAD][0]["app"]["slug"] = "untrusted" + self.assert_deferred("sha", "Required checks") + + def test_sha_effective_rule_or_legacy_changes_during_preflight_rejected(self): + for endpoint_part in ("/rules/branches/", "/protection"): + with self.subTest(endpoint=endpoint_part): + count = 0 + def drift(endpoint): + nonlocal count + if endpoint_part in endpoint: + count += 1 + if count > 1: + if "rules/branches" in endpoint: + return [self.effective_check("New gate")] + return {"required_status_checks": {"contexts": ["New gate"], "checks": []}} + return self.fixture(endpoint) + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "effective required checks changed") + + def test_sha_effective_rules_and_legacy_unreadable_fail_closed(self): + for endpoint_part in ("/rules/branches/", "/protection"): + with self.subTest(endpoint=endpoint_part): + def unreadable(endpoint): + if endpoint_part in endpoint: + raise readiness.APIError(endpoint, 403) + return self.fixture(endpoint) + with patch.object(readiness, "api", unreadable): + code, output = self.invoke("sha") + self.assertEqual(code, 2, output) + + def test_sha_malformed_effective_checks_rejected(self): + self.fixture.effective_rules = [self.effective_check("Required checks", "15368")] + self.assertEqual(self.invoke("sha")[0], 2) + self.fixture.effective_rules = [] + self.fixture.legacy_checks = {"contexts": "Required checks", "checks": []} + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_dormant_selected_patterns_with_all_mode_rejected(self): + self.fixture.actions_policy["allowed_actions"] = "all" + self.assert_deferred("sha", "selected policy is not active") + self.assertFalse(any("/check-runs?" in endpoint for endpoint in self.fixture.calls)) + + def test_sha_selected_repository_enablement_and_identity_required(self): + self.fixture.actions_policy["enabled_repositories"] = "selected" + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.enabled_repositories[0]["id"] = 999 + self.assert_deferred("sha", "not enabled") + self.fixture.enabled_repositories[0].update(id=42, full_name="foreign/example") + self.assert_deferred("sha", "not enabled") + + def test_sha_disabled_organization_or_repository_actions_rejected(self): + self.fixture.actions_policy["enabled_repositories"] = "none" + self.assert_deferred("sha", "disabled") + self.fixture.actions_policy["enabled_repositories"] = "all" + self.fixture.repository_actions["enabled"] = False + self.assert_deferred("sha", "does not allow external") + self.fixture.repository_actions.update(enabled=True, allowed_actions="local_only") + self.assert_deferred("sha", "does not allow external") + + def test_sha_repository_all_or_selected_cannot_relax_organization_ceiling(self): + for mode in ("all", "selected"): + self.fixture.repository_actions["allowed_actions"] = mode + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.actions_policy["allowed_actions"] = "all" + self.assert_deferred("sha", "selected policy is not active") + + def test_sha_actions_mode_or_repository_changes_during_proof_rejected(self): + for owner in ("orgs/atrinik", "repos/atrinik/example"): + with self.subTest(owner=owner): + count = 0 + def drift(endpoint): + nonlocal count + value = self.fixture(endpoint) + if endpoint == owner + "/actions/permissions": + count += 1 + if count > 1: + value["allowed_actions"] = "all" if owner.startswith("orgs") else "local_only" + return value + with patch.object(readiness, "api", drift): + self.assertEqual(self.invoke("sha")[0], 1) + + def test_sha_actions_policy_unreadable_and_malformed_fail_closed(self): + for endpoint_part in ("orgs/atrinik/actions/permissions", "repos/atrinik/example/actions/permissions"): + with self.subTest(endpoint=endpoint_part): + def unreadable(endpoint): + if endpoint == endpoint_part: + raise readiness.APIError(endpoint, 403) + return self.fixture(endpoint) + with patch.object(readiness, "api", unreadable): + self.assertEqual(self.invoke("sha")[0], 2) + self.fixture.repository_actions["enabled"] = "true" + self.assertEqual(self.invoke("sha")[0], 2) + self.fixture.repository_actions["enabled"] = True + self.fixture.actions_policy["enabled_repositories"] = "unexpected" + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_selected_membership_change_refenced(self): + self.fixture.actions_policy["enabled_repositories"] = "selected" + count = 0 + def drift(endpoint): + nonlocal count + value = self.fixture(endpoint) + if "/actions/permissions/repositories?" in endpoint: + count += 1 + if count > 1: + value = {"total_count": 0, "repositories": []} + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "not enabled") + + def test_codeql_languages_schema_rejects_mapping_null_and_nonstrings(self): + for value in ({"rust": False}, None, "rust", [False], [""]): + with self.subTest(value=value): + self.fixture.setup["languages"] = value + code, output = self.invoke("codeql") + self.assertEqual(code, 2, output) + + def test_codeql_fresh_languages_schema_rejected(self): + count = 0 + def drift(endpoint): + nonlocal count + value = self.fixture(endpoint) + if endpoint.endswith("/code-scanning/default-setup"): + count += 1 + if count > 1: + value["languages"] = {"rust": False} + return value + with patch.object(readiness, "api", drift): + self.assertEqual(self.invoke("codeql")[0], 2) + + def test_sha_general_github_owned_pin_update(self): + self.fixture.before = f"steps:\n - uses: actions/checkout@{OLD} # v5\n".encode() + self.fixture.after = f"steps:\n - uses: actions/checkout@{NEW} # v6\n".encode() + self.assertEqual(self.invoke("sha")[0], 0) + + def allow_canary_action_family(self, pattern): + self.fixture.live["patterns_allowed"].append(pattern) + (self.root / "config/actions-selected.json").write_text(json.dumps(self.fixture.live)) + + def test_sha_reusable_workflow_update_is_not_action_enforcement_evidence(self): + self.allow_canary_action_family("atrinik/shared/*") + for suffix in ("yml", "yaml"): + with self.subTest(suffix=suffix): + workflow = ("name: Check\non: pull_request\njobs:\n validation:\n" + f" uses: atrinik/shared/.github/workflows/check.{suffix}@{{pin}}\n") + self.fixture.before = workflow.format(pin=OLD).encode() + self.fixture.after = workflow.format(pin=NEW).encode() + self.assert_deferred("sha", "reusable workflow reference") + # GitHub-owned allowances do not turn a reusable workflow into an action. + self.fixture.before = self.fixture.before.replace(b"atrinik/shared", b"github/shared") + self.fixture.after = self.fixture.after.replace(b"atrinik/shared", b"github/shared") + self.assert_deferred("sha", "reusable workflow reference") + + def test_sha_mixed_reusable_and_action_updates_rejected(self): + self.allow_canary_action_family("atrinik/shared/*") + workflow = ("name: Check\non: pull_request\njobs:\n validation:\n" + " uses: atrinik/shared/.github/workflows/check.yml@{workflow_pin}\n" + " local:\n runs-on: ubuntu-24.04\n steps:\n" + " - uses: actions/checkout@{action_pin}\n") + self.fixture.before = workflow.format(workflow_pin=OLD, action_pin=OLD).encode() + self.fixture.after = workflow.format(workflow_pin=NEW, action_pin=NEW).encode() + self.assert_deferred("sha", "reusable workflow reference") + # An unchanged reusable reference is ordinary context, not changed proof. + self.fixture.after = workflow.format(workflow_pin=OLD, action_pin=NEW).encode() + result, message = self.invoke("sha") + self.assertEqual(result, 0, message) + + def test_sha_real_action_subpath_update_remains_eligible(self): + self.allow_canary_action_family("atrinik/shared/*") + for action in ("atrinik/shared/actions/check", "atrinik/shared/.github/actions/check", + "github/codeql-action/init"): + with self.subTest(action=action): + workflow = ("name: Check\non: pull_request\njobs:\n validation:\n" + " runs-on: ubuntu-24.04\n steps:\n" + f" - uses: {action}@{{pin}}\n") + self.fixture.before = workflow.format(pin=OLD).encode() + self.fixture.after = workflow.format(pin=NEW).encode() + result, message = self.invoke("sha") + self.assertEqual(result, 0, message) + + def test_sha_live_old_exact_pin_deferred(self): + self.fixture.live["patterns_allowed"][-1] = "rust-lang/crates-io-auth-action@" + OLD + self.assert_deferred("sha", "live selected-actions") + + def test_sha_allowlist_file_update_rejected(self): + self.fixture.files = [{"filename": "config/actions-selected.json", "status": "modified"}] + self.assert_deferred("sha", "outside existing workflows") + + def test_sha_arbitrary_code_update_rejected_even_if_patch_hides_it(self): + self.fixture.files[0]["patch"] = "@@ fake pin-only partial diff" + self.fixture.after += b" - run: arbitrary-command\n" + self.assert_deferred("sha", "more than action pins") + + def test_sha_changed_action_identity_rejected(self): + self.fixture.after = self.fixture.after.replace(b"rust-lang/crates-io-auth-action", b"docker/setup-buildx-action") + self.assert_deferred("sha", "action structure") + + def test_sha_version_ref_rejected(self): + self.fixture.after = self.fixture.after.replace(NEW.encode(), b"v2") + self.assert_deferred("sha", "more than action pins") + + def test_sha_disallowed_action_update_rejected(self): + self.fixture.before = self.fixture.before.replace(b"rust-lang", b"untrusted") + self.fixture.after = self.fixture.after.replace(b"rust-lang", b"untrusted") + self.assert_deferred("sha", "outside desired") + + def test_sha_latest_failure_masks_old_success(self): + self.fixture.checks[HEAD].append(check("Required checks", identifier=10, conclusion="failure")) + self.assert_deferred("sha", "not successful") + + def test_sha_missing_context(self): + self.fixture.checks[HEAD].pop() + self.assert_deferred("sha", "missing") + + def test_sha_wrong_head_check_rejected(self): + self.fixture.checks[HEAD][0]["head_sha"] = OLD + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_push_checks_rejected(self): + self.fixture.run["event"] = "push" + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_workflow_dispatch_checks_rejected(self): + self.fixture.run["event"] = "workflow_dispatch" + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_unrelated_pr_checks_rejected(self): + self.fixture.run["pull_requests"][0]["number"] = 100 + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_wrong_suite_checks_rejected(self): + self.fixture.run["check_suite_id"] = 100 + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_stale_run_checks_rejected(self): + self.fixture.run["head_sha"] = OLD + self.assert_deferred("sha", "missing") + + def advance_canary_base(self): + self.fixture.pr["base"]["sha"] = OLD + self.fixture.parents = [OLD, HEAD] + + def test_sha_open_canary_base_advance_rejects_old_direct_head_success(self): + self.advance_canary_base() + self.assert_deferred("sha", "required check missing") + + def test_sha_open_canary_base_advance_requires_both_run_and_suite_current(self): + for stale in ("run", "suite"): + with self.subTest(stale=stale): + self.advance_canary_base() + self.fixture.run["pull_requests"][0]["base"]["sha"] = BASE if stale == "run" else OLD + self.fixture.suite["pull_requests"][0]["base"]["sha"] = BASE if stale == "suite" else OLD + self.assert_deferred("sha", "required check missing") + + def test_sha_open_canary_fresh_base_associations_pass_direct_and_merge_checks(self): + self.advance_canary_base() + for source in (self.fixture.run, self.fixture.suite): + source["pull_requests"][0]["base"]["sha"] = OLD + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_open_canary_old_base_associations_reject_even_with_current_merge_parents(self): + self.advance_canary_base() + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assert_deferred("sha", "required check missing") + + def test_sha_merged_canary_keeps_recorded_historical_base_proof(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.fixture.pr["base"]["sha"] = OLD + # The historical merge and association retain their recorded BASE. + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_current_base_failure_or_queued_check_cannot_hide_behind_stale_green(self): + for status, conclusion in (("completed", "failure"), ("queued", None)): + with self.subTest(status=status): + self.advance_canary_base() + self.fixture.checks[HEAD] = [check("Required checks"), check("Conventional PR title", identifier=2)] + self.fixture.suite_runs.clear() + self.add_newer_pr_check(check_status=status, check_conclusion=conclusion, + run_status=status, run_conclusion=conclusion) + self.fixture.checks[HEAD][-1]["started_at"] = None + self.fixture.suite_runs[100]["pull_requests"][0]["base"]["sha"] = OLD + def current_suite(endpoint): + value = self.fixture(endpoint) + if endpoint.endswith("/check-suites/100"): + value["pull_requests"][0]["base"]["sha"] = OLD + return value + with patch.object(readiness, "api", current_suite): + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_merged_canary_rejects_conflicting_recorded_run_and_suite_bases(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.fixture.suite["pull_requests"][0]["base"]["sha"] = OLD + self.assert_deferred("sha", "required check missing") + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assert_deferred("sha", "required check missing") + + def test_sha_merged_canary_common_base_must_match_selected_merge_parents(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + # A run lists two historical associations, but the suite corroborates + # only the base absent from this merge commit's parents. + alternate = copy.deepcopy(self.fixture.run["pull_requests"][0]) + alternate["base"]["sha"] = OLD + self.fixture.run["pull_requests"].append(alternate) + self.fixture.suite["pull_requests"] = [alternate] + self.assert_deferred("sha", "required check missing") + + def test_sha_direct_head_failed_workflow_run_rejected(self): + self.fixture.run["conclusion"] = "failure" + self.assert_deferred("sha", "workflow run not successful") + + def add_newer_pr_check(self, check_status="completed", check_conclusion="success", + run_status="completed", run_conclusion="success"): + newer = check("Required checks", identifier=50, status=check_status, conclusion=check_conclusion) + newer["started_at"] = "2026-10-11T12:01:00Z" + newer["check_suite"]["id"] = 100 + self.fixture.checks[HEAD].append(newer) + run = copy.deepcopy(self.fixture.run) + run.update(id=20, check_suite_id=100, status=run_status, conclusion=run_conclusion) + self.fixture.suite_runs[100] = run + + def test_sha_newer_failing_pr_suite_does_not_hide_behind_older_green_suite(self): + self.add_newer_pr_check(check_conclusion="failure", run_conclusion="failure") + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_newer_pending_pr_suite_does_not_hide_behind_older_green_suite(self): + self.add_newer_pr_check(check_status="queued", check_conclusion=None, + run_status="queued", run_conclusion=None) + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_newer_queued_null_start_pr_suite_blocks_older_green_suite(self): + self.add_newer_pr_check(check_status="queued", check_conclusion=None, + run_status="queued", run_conclusion=None) + self.fixture.checks[HEAD][-1]["started_at"] = None + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_newer_queued_null_start_check_blocks_older_green_in_same_suite(self): + newer = check("Required checks", identifier=50, status="queued", conclusion=None) + newer["started_at"] = None + self.fixture.checks[HEAD].append(newer) + self.assert_deferred("sha", "required check not successful") + + def test_sha_older_check_starting_later_does_not_hide_newer_failure(self): + self.add_newer_pr_check(check_conclusion="failure", run_conclusion="failure") + self.fixture.checks[HEAD][0]["started_at"] = "2026-10-11T12:02:00Z" + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_newer_green_check_in_failed_pr_run_does_not_hide_behind_older_green_suite(self): + self.add_newer_pr_check(run_conclusion="failure") + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_direct_head_run_drift_at_completion_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if "/actions/runs?check_suite_id=99" in endpoint: + calls += 1 + if calls > 1: + value["workflow_runs"][0]["run_attempt"] += 1 + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "workflow-run evidence changed") + + def test_sha_direct_head_check_drift_at_completion_deferred(self): + def drift(endpoint): + value = self.fixture(endpoint) + if endpoint.endswith("/check-runs/1"): + value["conclusion"] = "failure" + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "required check evidence changed") + + def test_sha_new_required_check_at_completion_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if f"/commits/{HEAD}/check-runs?" in endpoint: + calls += 1 + if calls > 1: + value["check_runs"].append(check("Required checks", identifier=50, conclusion="failure")) + value["total_count"] += 1 + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "required check collection changed") + + def test_sha_wrong_app_check_rejected(self): + self.fixture.checks[HEAD][0]["app"]["id"] = 1 + self.assert_deferred("sha", "missing") + + def test_sha_realistic_merge_sha_checks(self): + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_merged_canary_current_head_direct_checks(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_merged_canary_historical_merge_checks(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_closed_unmerged_canary_deferred(self): + self.fixture.pr.update(state="closed", merged=False) + self.assert_deferred("sha", "merged or open") + + def test_sha_stale_merge_run_head(self): + self.fixture.checks[MERGE] = [check("Required checks", MERGE)] + self.fixture.checks[HEAD] = [] + self.fixture.run["head_sha"] = OLD + self.assert_deferred("sha", "missing") + + def test_sha_stale_merge_commit(self): + self.fixture.parents = [BASE, OLD] + self.assert_deferred("sha", "stale") + + def test_sha_fresh_pr_head_required_at_completion(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if endpoint.endswith("/pulls/12"): + calls += 1 + if calls > 1: + value["head"]["sha"] = NEW + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "changed during") + + def test_codeql_success_current_head_rust(self): + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_large_history_stops_after_current_coverage(self): + old = copy.deepcopy(self.fixture.analyses[0]) + old.update(commit_sha=OLD, category="/partition:retired/language:rust") + self.fixture.analyses.extend([old] * 1200) + self.assertEqual(self.invoke("codeql")[0], 0) + self.assertEqual(len([call for call in self.fixture.calls if "/analyses?" in call]), 2) + + def test_codeql_retired_category_failure_does_not_gate(self): + old = copy.deepcopy(self.fixture.analyses[0]) + old.update(commit_sha=OLD, category="/partition:retired/language:rust", error="old failure") + self.fixture.analyses.insert(0, old) + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_all_live_configured_languages_required(self): + self.fixture.setup["languages"].append("python") + self.assert_deferred("codeql", "configured CodeQL language") + + def test_codeql_javascript_typescript_alias_normalized(self): + self.fixture.setup["languages"] = ["rust", "javascript/typescript"] + second = copy.deepcopy(self.fixture.analyses[0]) + second.update(category="/language:javascript-typescript", environment='{"language":"javascript-typescript"}') + self.fixture.analyses.append(second) + self.assertEqual(self.invoke("codeql")[0], 0) + + def enable_advanced(self, partitions=None): + if partitions is None: + partitions = PARTITIONS + # The policy minimum remains independent of the producer declarations. + (self.root / "config/codeql-advanced-setup.json").write_text(json.dumps({ + "repositories": ["example"], "required_partitions": {"example": [ + {"scope": item["scope"], "language": item["language"]} for item in PARTITIONS]}})) + self.set_classifier_source(classifier_source(partitions)) + (self.root / "config/security-merge-protection.json").write_text('{"codeql_required_languages": {}}') + self.fixture.analyses = [] + self.fixture.codeql_jobs = [] + for index, item in enumerate(partitions): + display, category, language = item["display"], item["category"], item["language"] + job = {"id": index + 1, "name": f"CodeQL analysis ({display})", "head_sha": HEAD, + "status": "completed", "conclusion": "success", + "started_at": "2026-10-11T11:59:00Z", "completed_at": "2026-10-11T12:01:00Z"} + self.fixture.codeql_jobs.append(job) + self.fixture.analyses.append({"id": index + 1, "tool": {"name": "CodeQL"}, + "analysis_key": ".github/workflows/codeql.yml:analyze", "category": category, + "environment": json.dumps({"display": display, "category": category, "language": language, + "scope": item["scope"], "run": ""}), + "commit_sha": HEAD, "ref": "refs/heads/main", "error": "", "rules_count": 58, + "created_at": "2026-10-11T12:00:00Z"}) + for name in ("CodeQL classification", "CodeQL validation"): + job = copy.deepcopy(self.fixture.codeql_jobs[0]) + job.update(id=999, name=name, check_run_url="https://api.github.com/repos/atrinik/example/check-runs/999") + self.fixture.codeql_jobs.append(job) + + def set_classifier_source(self, source): + self.fixture.codeql_sources["tools/ci/classify_changes.py"] = source + self.fixture.codeql_head_sources["tools/ci/classify_changes.py"] = source + + def set_codeql_workflow(self, source): + self.fixture.codeql_sources[".github/workflows/codeql.yml"] = source + self.fixture.codeql_head_sources[".github/workflows/codeql.yml"] = source + + def codeql_document(self): + import yaml + # Preserve Actions' on key when constructing modified fixtures. + return yaml.safe_load(CODEQL_WORKFLOW.replace(b"\non:", b"\n'on':")) + + def set_codeql_document(self, document): + import yaml + self.set_codeql_workflow(yaml.safe_dump(document, sort_keys=False).encode()) + + def test_codeql_yaml_dependency_missing_fails_closed_without_affecting_other_modes(self): + self.enable_advanced() + with patch.dict(sys.modules, {"yaml": None}): + result, message = self.invoke("codeql") + self.assertEqual(result, 2, message) + self.assertIn("install requirements-validation.txt", message) + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_codeql_advanced_full_run_cannot_replace_pull_request_trigger(self): + for event in ('push', 'schedule', 'workflow_dispatch'): + for triggers in (None, 'push', ['push', 'workflow_dispatch'], + {'push': {'branches': ['main']}, 'schedule': [{'cron': '0 0 * * *'}], + 'workflow_dispatch': None}, {'pull_request_target': None}): + with self.subTest(event=event, triggers=triggers): + self.enable_advanced() + self.fixture.codeql_run['event'] = event + document = self.codeql_document() + if triggers is None: + del document['on'] + else: + document['on'] = triggers + self.set_codeql_document(document) + self.assert_deferred('codeql', 'pull_request trigger missing') + + def test_codeql_advanced_pull_request_filters_cannot_skip_required_producer(self): + rejected = [False, [], 'main', {'branches': ['develop']}, + {'branches': []}, {'branches': 'main'}, {'branches': None}, + {'branches': ['main', '!main']}, {'branches': ['**']}, + {'branches': ['main', 3]}, {'branches-ignore': ['main']}, + {'branches-ignore': ['unrelated']}, {'paths': ['client/**']}, + {'paths': ['**']}, {'paths-ignore': ['docs/**']}, + {'branches': ['main'], 'paths-ignore': []}, + {'tags': ['v*']}, {'types': None}, {'types': 'opened'}, + {'types': []}, {'types': ['opened', 'reopened']}, + {'types': ['synchronize', 'reopened']}, {'types': ['opened', 'synchronize']}, + {'types': ['opened', 'synchronize', 'reopened', False]}, + {'types': ['opened', 'synchronize', 'reopened', 'invented']}, + {'types': ['opened', 'synchronize', 'reopened', 'opened']}] + for trigger in rejected: + with self.subTest(trigger=trigger): + self.enable_advanced() + document = self.codeql_document() + document['on']['pull_request'] = trigger + self.set_codeql_document(document) + self.assert_deferred('codeql', 'pull_request') + + def test_codeql_advanced_pull_request_event_forms_and_additional_types_ready(self): + for triggers in ('pull_request', ['push', 'pull_request', 'workflow_dispatch'], + {'pull_request': None}, {'pull_request': {}}, + {'pull_request': {'branches': ['main', 'maintenance']}}, + {'pull_request': {'types': ['opened', 'synchronize', 'reopened']}}, + {'pull_request': {'branches': ['main'], 'types': [ + 'ready_for_review', 'edited', 'reopened', 'opened', 'synchronize']}}): + with self.subTest(triggers=triggers): + self.enable_advanced() + document = self.codeql_document() + document['on'] = triggers + self.set_codeql_document(document) + result, message = self.invoke('codeql') + self.assertEqual(result, 0, message) + + def test_codeql_advanced_pull_request_uses_fresh_default_branch(self): + self.enable_advanced() + branch = 'release/current' + self.fixture.metadata['default_branch'] = branch + self.fixture.codeql_run['head_branch'] = branch + for analysis in self.fixture.analyses: + analysis['ref'] = 'refs/heads/' + branch + self.assert_deferred('codeql', 'pull_request does not cover the default branch') + document = self.codeql_document() + document['on']['pull_request']['branches'] = [branch] + document['on']['push']['branches'] = [branch] + self.set_codeql_document(document) + result, message = self.invoke('codeql') + self.assertEqual(result, 0, message) + self.assertIn('repos/atrinik/example/branches/release%2Fcurrent', self.fixture.calls) + + def test_codeql_advanced_default_branch_change_during_preflight_deferred(self): + self.enable_advanced() + reads = 0 + + def drift(endpoint): + nonlocal reads + result = self.fixture(endpoint) + if endpoint == 'repos/atrinik/example': + reads += 1 + if reads > 1: + result = dict(result, default_branch='release/current') + return result + + with patch.object(readiness, 'api', drift): + self.assert_deferred('codeql', 'repository default branch changed') + + def test_codeql_advanced_comments_cannot_prove_executable_wiring(self): + self.enable_advanced() + source = b"name: CodeQL\njobs:\n fake:\n runs-on: ubuntu-24.04\n steps:\n - run: true\n" + source += b"\n".join(b"# " + line for line in CODEQL_WORKFLOW.splitlines()) + self.set_codeql_workflow(source) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_disconnected_wiring_rejected_despite_green_uploads(self): + changes = [ + (("classify", "outputs", "matrix"), '${{ steps.other.outputs.codeql_matrix }}'), + (("classify", "outputs", "run"), 'true'), + (("classify", "steps", 1, "id"), 'other'), + (("classify", "steps", 1, "env", "EVENT_NAME"), 'push'), + (("classify", "steps", 1, "env", "HEAD_SHA"), '${{ github.event.pull_request.base.sha }}'), + (("analyze", "needs"), 'other'), + (("analyze", "if"), 'false'), + (("analyze", "name"), 'CodeQL analysis (C/C++ client)'), + (("analyze", "strategy", "matrix"), {"include": PARTITIONS}), + (("analyze", "strategy", "matrix"), '${{ fromJSON(needs.other.outputs.matrix) }}'), + (("analyze", "steps", 2, "with", "config-file"), 'unrelated-codeql.yml'), + (("analyze", "steps", 2, "with", "config-file"), '${{ steps.other.outputs.codeql_config }}'), + (("analyze", "steps", 2, "with", "languages"), 'c-cpp'), + (("analyze", "steps", 2, "with", "build-mode"), 'autobuild'), + (("analyze", "steps", 3, "with", "category"), '/partition:client/language:c-cpp'), + (("codeql-validation", "if"), 'success()'), + (("codeql-validation", "needs"), ['classify']), + (("codeql-validation", "strategy"), {"matrix": {"fake": [1]}}), + ] + for path, value in changes: + with self.subTest(path=path, value=value): + self.enable_advanced() + document = self.codeql_document() + target = document["jobs"] + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_classifier_and_aggregate_arguments_are_executable(self): + mutations = [ + ('classify', '--event "${EVENT_NAME}"', '--event push'), + ('classify', '--event "${EVENT_NAME}"', "--event '${EVENT_NAME}'"), + ('classify', '--event "${EVENT_NAME}"', '--event "\\${EVENT_NAME}"'), + ('classify', '--github-output "${GITHUB_OUTPUT}"', '--github-output /dev/null'), + ('analyze', '--codeql-scope "${{ matrix.scope }}"', '--codeql-scope client'), + ('analyze', 'build/codeql-config.yml', 'build/disconnected-config.yml'), + ('codeql-validation', "'${{ needs.classify.result }}'", 'success'), + ('codeql-validation', "'${{ needs.classify.outputs.run }}'", 'true'), + ('codeql-validation', "'${{ needs.analyze.result }}'", 'success'), + ] + for name, before, after in mutations: + with self.subTest(name=name, before=before): + self.enable_advanced() + document = self.codeql_document() + step = document['jobs'][name]['steps'][1] + self.assertIn(before, step['run']) + step['run'] = step['run'].replace(before, after) + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + for name in ('classify', 'analyze', 'codeql-validation'): + for mutation in ('comment-only', 'trailing-command', 'leading-command', 'substitution', 'duplicate-option'): + with self.subTest(name=name, mutation=mutation): + self.enable_advanced() + document = self.codeql_document() + step = document['jobs'][name]['steps'][1] + run = step['run'] + if mutation == 'comment-only': + step['run'] = '\n'.join('# ' + line for line in run.splitlines()) + '\ntrue\n' + elif mutation == 'trailing-command': + step['run'] = run.rstrip() + '; true\n' + elif mutation == 'leading-command': + step['run'] = 'true\n' + run + elif mutation == 'substitution': + step['run'] = run.replace('python3', '$(echo python3)', 1) + else: + step['run'] = run.rstrip() + ' --event push\n' + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_disabled_or_overridden_execution_rejected(self): + targets = [('classify', None), ('analyze', None), ('codeql-validation', None), + ('classify', 1), ('analyze', 1), ('analyze', 2), ('analyze', 3), + ('codeql-validation', 1)] + for name, index in targets: + for key, value in [('continue-on-error', True), ('if', 'false'), + ('env', {'PYTHONPATH': 'untrusted'}), + ('defaults', {'run': {'working-directory': 'elsewhere'}}), + ('shell', 'bash {0}'), ('working-directory', 'elsewhere')]: + with self.subTest(name=name, index=index, override=key): + self.enable_advanced() + document = self.codeql_document() + target = document['jobs'][name] + if index is not None: + target = target['steps'][index] + target[key] = value + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + for key, value in [('env', {'GITHUB_OUTPUT': '/dev/null'}), + ('defaults', {'run': {'shell': 'true {0}'}})]: + with self.subTest(workflow_override=key): + self.enable_advanced() + document = self.codeql_document() + document[key] = value + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + self.enable_advanced() + document = self.codeql_document() + document['jobs']['analyze'] = {'uses': 'foreign/repo/.github/workflows/codeql.yml@' + NEW} + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_current_complete_checkout_required(self): + for name in ('classify', 'analyze', 'codeql-validation'): + for key, value in [('ref', 'main'), ('ref', OLD), ('repository', 'foreign/repo'), + ('path', 'elsewhere'), ('sparse-checkout', 'only-one-file')]: + with self.subTest(name=name, key=key, value=value): + self.enable_advanced() + document = self.codeql_document() + document['jobs'][name]['steps'][0].setdefault('with', {})[key] = value + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_step_order_actions_and_duplicate_ids_rejected(self): + for mutation in ('order', 'extra', 'duplicate-id', 'mutable-init', 'foreign-analyze'): + with self.subTest(mutation=mutation): + self.enable_advanced() + document = self.codeql_document() + steps = document['jobs']['analyze']['steps'] + if mutation == 'order': + steps[2], steps[3] = steps[3], steps[2] + elif mutation == 'extra': + steps.insert(2, {'run': 'echo overwrite generated config'}) + elif mutation == 'duplicate-id': + steps[0]['id'] = 'classify' + elif mutation == 'mutable-init': + steps[2]['uses'] = 'github/codeql-action/init@v4' + else: + steps[3]['uses'] = 'foreign/codeql-action/analyze@' + NEW + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_duplicate_yaml_is_unreadable(self): + for source in (CODEQL_WORKFLOW + b'\njobs: {}\n', + CODEQL_WORKFLOW.replace(b' - id: classify', b' - id: classify\n id: other', 1), + CODEQL_WORKFLOW.replace(b' if: always()', b' if: false\n if: always()'), + b'jobs: [unclosed'): + with self.subTest(source=source[-50:]): + self.enable_advanced() + self.set_codeql_workflow(source) + result, message = self.invoke('codeql') + self.assertEqual(result, 2, message) + self.assertIn('YAML malformed or ambiguous', message) + + def test_codeql_advanced_formatting_and_owner_action_updates_ready(self): + self.enable_advanced() + document = self.codeql_document() + for job in document['jobs'].values(): + for step in job['steps']: + if 'uses' in step: + step['uses'] = step['uses'].split('@')[0] + '@' + NEW + job['steps'][0].setdefault('with', {}).update( + ref='${{github.sha}}', repository='${{ github.repository }}') + for name in ('classify', 'analyze'): + step = document['jobs'][name]['steps'][1] + step['run'] = ('# executable invocation follows\npython3 tools/ci/classify_changes.py ' + '--codeql-config build/new-config.yml --github-output "$GITHUB_OUTPUT" ' + '--head "$HEAD_SHA" --base "$BASE_SHA" --event "$EVENT_NAME"') + if name == 'analyze': + step['run'] += ' --codeql-scope "${{matrix.scope}}"' + document['jobs']['analyze']['if'] = "${{needs.classify.outputs.run=='true'}}" + document['jobs']['analyze']['strategy']['matrix'] = '${{ fromJSON ( needs.classify.outputs.matrix ) }}' + document['jobs']['analyze']['steps'][2]['with']['languages'] = '${{matrix.language}}' + document['jobs']['analyze']['steps'][3]['with']['category'] = '${{ matrix.category}}' + document['jobs']['codeql-validation']['if'] = '${{ always ( ) }}' + self.set_codeql_document(document) + result, message = self.invoke('codeql') + self.assertEqual(result, 0, message) + + def test_codeql_advanced_reduced_source_and_jobs_below_policy_deferred(self): + self.enable_advanced(PARTITIONS[:1]) + self.assert_deferred("codeql", "omits required partition coverage") + + def test_codeql_advanced_reduced_jobs_with_full_source_deferred(self): + self.enable_advanced() + self.fixture.codeql_jobs = self.fixture.codeql_jobs[:1] + self.fixture.codeql_jobs[-2:] + self.fixture.analyses = self.fixture.analyses[:1] + self.assert_deferred("codeql", "every declared partition") + + def test_codeql_advanced_additional_declared_partition_ready(self): + extra = {"display": "Extra C/C++", "scope": "extra", "language": "c-cpp", + "category": "/partition:extra/language:c-cpp", "run": True} + self.enable_advanced(PARTITIONS + [extra]) + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_advanced_additional_declaration_requires_job(self): + extra = {"display": "Extra C/C++", "scope": "extra", "language": "c-cpp", + "category": "/partition:extra/language:c-cpp", "run": True} + self.enable_advanced(PARTITIONS + [extra]) + del self.fixture.codeql_jobs[5] + self.assert_deferred("codeql", "every declared partition") + + def test_codeql_advanced_category_rename_preserves_semantic_minimum(self): + partitions = copy.deepcopy(PARTITIONS) + partitions[0]["category"] = "/partition:client-renamed/language:c-cpp" + self.enable_advanced(partitions) + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_advanced_upload_matches_declared_scope_category_and_language(self): + for field, value in (("scope", "server"), ("category", "/other/language:c-cpp"), + ("language", "python")): + with self.subTest(field=field): + self.enable_advanced() + analysis = self.fixture.analyses[0] + environment = json.loads(analysis["environment"]) + environment[field] = value + analysis["environment"] = json.dumps(environment) + if field == "category": + analysis["category"] = value + self.assert_deferred("codeql", "selected full-run job") + + def test_codeql_advanced_duplicate_declarations_rejected(self): + for field in ("display", "category", "scope"): + with self.subTest(field=field): + partitions = copy.deepcopy(PARTITIONS) + partitions[1][field] = partitions[0][field] + self.enable_advanced(partitions) + self.assert_deferred("codeql", "declarations duplicated") + + def test_codeql_advanced_nonliteral_declaration_deferred(self): + self.enable_advanced() + self.set_classifier_source(CLASSIFIER.replace(b"entries.append({", b"entries.append(dict(", 1)) + self.assertEqual(self.invoke("codeql")[0], 2) # Invalid Python is unreadable evidence. + self.set_classifier_source(CLASSIFIER + b"\n") + source = CLASSIFIER.replace(b" matrix =", b" entries.append(dynamic_partition)\n matrix =") + self.set_classifier_source(source) + self.assert_deferred("codeql", "not literal") + + def test_codeql_advanced_unsupported_matrix_mutation_deferred(self): + self.enable_advanced() + source = CLASSIFIER.replace(b" matrix =", b" entries.extend(extra)\n matrix =") + self.set_classifier_source(source) + self.assert_deferred("codeql", "shape unsupported") + + def test_codeql_advanced_missing_or_malformed_minima_unreadable(self): + for minima in (None, [], {}, [{"scope": "client"}], + [{"scope": "client", "language": "c-cpp"}] * 2): + with self.subTest(minima=minima): + self.enable_advanced() + policy = {"repositories": ["example"], "required_partitions": {"example": minima}} + (self.root / "config/codeql-advanced-setup.json").write_text(json.dumps(policy)) + self.assertEqual(self.invoke("codeql")[0], 2) + (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": ["example"]}') + self.assertEqual(self.invoke("codeql")[0], 2) + + def test_codeql_advanced_current_full_producer_all_partitions_ready(self): + self.enable_advanced() + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_advanced_large_retired_history_ready(self): + self.enable_advanced() + retired = copy.deepcopy(self.fixture.analyses[0]) + retired.update(commit_sha=OLD, category="/retired/language:c-cpp", environment='{"display":"Retired"}') + self.fixture.analyses.insert(0, retired) + self.fixture.analyses.extend([retired] * 1200) + self.assertEqual(self.invoke("codeql")[0], 0) + self.assertEqual(len([call for call in self.fixture.calls if "/analyses?" in call]), 2) + + def test_codeql_advanced_partial_uploads_deferred(self): + self.enable_advanced() + self.fixture.analyses.pop() + self.assert_deferred("codeql", "partition upload missing") + + def test_codeql_advanced_latest_failed_category_masks_success(self): + self.enable_advanced() + failed = copy.deepcopy(self.fixture.analyses[0]); failed["error"] = "failed" + self.fixture.analyses.insert(0, failed) + self.assert_deferred("codeql", "failed") + + def test_codeql_advanced_wrong_producer_deferred(self): + self.enable_advanced() + self.fixture.analyses[0]["analysis_key"] = ".github/workflows/other.yml:analyze" + self.assert_deferred("codeql", "partition upload missing") + + def test_codeql_advanced_partial_pr_event_deferred(self): + self.enable_advanced() + self.fixture.codeql_run["event"] = "pull_request" + self.assert_deferred("codeql", "full-event") + + def test_codeql_advanced_wrong_app_deferred(self): + self.enable_advanced() + self.fixture.codeql_check["app"]["id"] = 1 + self.assert_deferred("codeql", "trusted current-head producer") + + def test_codeql_advanced_stale_source_deferred(self): + self.enable_advanced() + self.fixture.codeql_head_sources["tools/ci/classify_changes.py"] += b"# different\n" + self.assert_deferred("codeql", "source is stale") + + def test_codeql_advanced_upload_outside_job_window_deferred(self): + self.enable_advanced() + self.fixture.analyses[0]["created_at"] = "2026-10-11T13:00:00Z" + self.assert_deferred("codeql", "selected full-run job") + + def prepare_analysis_recheck(self, advanced): + if advanced: + self.enable_advanced() + self.fixture.analysis_reads = 0 + self.fixture.fresh_analyses = copy.deepcopy(self.fixture.analyses) + + def test_codeql_newer_failure_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + failed = copy.deepcopy(self.fixture.analyses[0]) + failed.update(id=1000, error="new analysis failed") + self.fixture.fresh_analyses.insert(0, failed) + self.assert_deferred("codeql", "failed") + self.assertEqual(self.fixture.analysis_reads, 2) + + def test_codeql_newer_zero_rules_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + empty = copy.deepcopy(self.fixture.analyses[0]) + empty.update(id=1000, rules_count=0) + self.fixture.fresh_analyses.insert(0, empty) + self.assert_deferred("codeql", "no executed rules") + + def test_codeql_deleted_selected_analysis_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + del self.fixture.fresh_analyses[0] + self.assert_deferred("codeql", "missing within bounded evidence") + + def test_codeql_replaced_successful_coverage_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + replacement = copy.deepcopy(self.fixture.analyses[0]) + replacement["id"] = 1000 + self.fixture.fresh_analyses.insert(0, replacement) + self.assert_deferred("codeql", "analysis coverage changed during preflight") + + def test_codeql_selected_analysis_mutation_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + self.fixture.fresh_analyses[0]["rules_count"] += 1 + self.assert_deferred("codeql", "analysis coverage changed during preflight") + + def test_codeql_new_retired_analysis_during_preflight_preserves_readiness(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + retired = copy.deepcopy(self.fixture.analyses[0]) + retired.update(id=1000, commit_sha=OLD, category="/retired/language:c-cpp", + environment='{"display":"Retired"}') + self.fixture.fresh_analyses.insert(0, retired) + self.assertEqual(self.invoke("codeql")[0], 0) + self.assertEqual(self.fixture.analysis_reads, 2) + + def test_codeql_default_new_failure_with_changed_language_does_not_fallback(self): + self.prepare_analysis_recheck(False) + failed = copy.deepcopy(self.fixture.analyses[0]) + failed.update(id=1000, error="new analysis failed", environment='{"language":"python"}') + self.fixture.fresh_analyses.insert(0, failed) + self.assert_deferred("codeql", "failed") + + def test_codeql_advanced_new_failure_without_display_does_not_fallback(self): + for environment in ('{}', '{"display":"Renamed"}'): + with self.subTest(environment=environment): + self.prepare_analysis_recheck(True) + failed = copy.deepcopy(self.fixture.analyses[0]) + failed.update(id=1000, error="new analysis failed", environment=environment) + self.fixture.fresh_analyses.insert(0, failed) + self.assert_deferred("codeql", "partition upload missing") + + def test_codeql_latest_error_masks_older_success(self): + failed = copy.deepcopy(self.fixture.analyses[0]) + failed.update(id=2, error="analysis failed") + self.fixture.analyses.insert(0, failed) + self.assert_deferred("codeql", "failed") + + def test_codeql_success_old_head_deferred(self): + self.fixture.analyses[0]["commit_sha"] = OLD + self.assert_deferred("codeql", "current default head") + + def test_codeql_missing_rust_analysis_deferred(self): + self.fixture.analyses[0]["environment"] = '{"language":"python"}' + self.fixture.analyses[0]["category"] = "/language:python" + self.assert_deferred("codeql", "language") + + def test_codeql_no_executed_rules_deferred(self): + self.fixture.analyses[0]["rules_count"] = 0 + self.assert_deferred("codeql", "no executed rules") + + def test_archived_repository_deferred(self): + self.fixture.metadata["archived"] = True + self.assert_deferred("dependency", "archived") + + def test_malformed_api_evidence_error(self): + self.fixture.metadata.pop("default_branch") + self.assertEqual(self.invoke("dependency")[0], 2) + + def test_api_subprocess_uses_only_get_and_version(self): + self.api_patch.stop() + result = subprocess.CompletedProcess([], 0, stdout='{}', stderr='') + with patch.object(readiness.subprocess, "run", return_value=result) as mocked: + readiness.api("repos/atrinik/example") + args = mocked.call_args.args[0] + self.assertEqual(args[args.index("--method") + 1], "GET") + self.assertIn("X-GitHub-Api-Version: 2026-03-10", args) + + def test_api_error_retains_structured_response_without_printing_body(self): + self.api_patch.stop() + for body in ('{"message":"Branch not protected","status":"404"}', "not json", "null"): + result = subprocess.CompletedProcess([], 1, stdout=body, stderr="gh: error (HTTP 404)") + with self.subTest(body=body), patch.object(readiness.subprocess, "run", return_value=result): + with self.assertRaises(readiness.APIError) as raised: + readiness.api("repos/atrinik/example/branches/main/protection") + self.assertEqual(raised.exception.status, 404) + self.assertEqual(raised.exception.response, json.loads(body) if body != "not json" else None) + self.assertNotIn(body, str(raised.exception)) + + def test_executable_wrapper_is_bash_and_delegates_python(self): + wrapper = ROOT / "bin/security-readiness" + self.assertEqual(wrapper.read_text().splitlines()[0], "#!/usr/bin/env bash") + self.assertIn('exec python3 "${root}/lib/security_readiness.py" "$@"', wrapper.read_text()) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/security-readiness.sh b/tests/security-readiness.sh new file mode 100755 index 0000000..900b90c --- /dev/null +++ b/tests/security-readiness.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +exec python3 "${root}/tests/security-readiness.py" diff --git a/tests/validate-codeql-advanced.sh b/tests/validate-codeql-advanced.sh index a6e92b1..3fd7ddb 100755 --- a/tests/validate-codeql-advanced.sh +++ b/tests/validate-codeql-advanced.sh @@ -10,6 +10,8 @@ mkdir -p "${temporary}/bin" cp "${root}/bin/validate" "${temporary}/bin/validate" cp -R "${root}/config" "${temporary}/config" cp -R "${root}/community-health" "${temporary}/community-health" +cp -R "${root}/.github" "${temporary}/.github" +"${temporary}/bin/validate" >/dev/null assert_invalid() { local description=$1 @@ -42,6 +44,27 @@ assert_invalid 'an archived advanced CodeQL inventory repository' cp "${root}/config/codeql-advanced-setup.json" \ "${temporary}/config/codeql-advanced-setup.json" +for filter in \ + 'del(.required_partitions)' \ + '.required_partitions = {}' \ + '.required_partitions.classic = []' \ + '.required_partitions.classic = {}' \ + '.required_partitions.classic[0].scope = null' \ + 'del(.required_partitions.classic[0].language)' \ + '.required_partitions.classic += [.required_partitions.classic[0]]' \ + '.required_partitions.other = .required_partitions.classic'; do + rewrite_json "${filter}" "${temporary}/config/codeql-advanced-setup.json" + assert_invalid "malformed or mismatched advanced partition policy: ${filter}" + cp "${root}/config/codeql-advanced-setup.json" \ + "${temporary}/config/codeql-advanced-setup.json" +done + +rewrite_json '.repositories = [] | .required_partitions = {}' \ + "${temporary}/config/codeql-advanced-setup.json" +"${temporary}/bin/validate" >/dev/null +cp "${root}/config/codeql-advanced-setup.json" \ + "${temporary}/config/codeql-advanced-setup.json" + rewrite_json '.secret_scanning = "disabled"' \ "${temporary}/config/code-security-advanced.json" assert_invalid 'advanced security configuration drift from the baseline' diff --git a/tests/validate-crate-trusted-publishing.sh b/tests/validate-crate-trusted-publishing.sh index 0cd963e..bae75c2 100755 --- a/tests/validate-crate-trusted-publishing.sh +++ b/tests/validate-crate-trusted-publishing.sh @@ -80,7 +80,7 @@ reset_policy jq '.patterns_allowed += ["rust-lang/*"]' "${root}/config/actions-selected.json" \ >"${temporary}/config/actions-selected.json" if "${temporary}/bin/validate" >/dev/null 2>&1; then - echo "error: accepted broader Rust action access beside the exact pin" >&2 + echo "error: accepted broader Rust action access beside the exact action" >&2 exit 1 fi reset_policy diff --git a/tests/validate-retired-required-ci.sh b/tests/validate-retired-required-ci.sh new file mode 100755 index 0000000..8bfcc2e --- /dev/null +++ b/tests/validate-retired-required-ci.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +mkdir -p "${temporary}/bin" +cp "${root}/bin/validate" "${temporary}/bin/" +cp -R "${root}/config" "${root}/community-health" "${root}/.github" "${temporary}/" +"${temporary}/bin/validate" >/dev/null +printf '{"retirements":[{"repository":"web-platform","repository_id":123,"ruleset_id":3,"required_ci":["Platform validation","Conventional PR title"]}]}\n' >"${temporary}/valid.json" +cp "${temporary}/valid.json" "${temporary}/config/retired-required-ci.json" +"${temporary}/bin/validate" >/dev/null +for filter in \ + '.retirements[0].repository = "client"' '.retirements[0].repository = "legacy-client"' \ + '.retirements[0].repository = "unknown"' '.retirements[0].required_ci = []' \ + '.retirements[0].required_ci = [""]' '.retirements[0].required_ci = [" "]' \ + '.retirements[0].required_ci = ["Duplicate", "Duplicate"]' \ + '.retirements[0].required_ci = [null]' '.retirements[0].repository_id = 0' \ + '.retirements[0].ruleset_id = -1' '.retirements[0].ruleset_id = "3"' \ + '.retirements[0].repository_id = 1.5' '.retirements += [.retirements[0]]' \ + '.retirements[0].unknown_field = true' 'del(.retirements[0].ruleset_id)'; do + jq "${filter}" "${temporary}/valid.json" >"${temporary}/config/retired-required-ci.json" + if "${temporary}/bin/validate" >/dev/null 2>&1; then + echo "error: accepted invalid required-CI retirement: ${filter}" >&2 + exit 1 + fi +done +echo 'Required-CI retirement schema tests passed.' diff --git a/tests/validate-security-merge.sh b/tests/validate-security-merge.sh new file mode 100755 index 0000000..5efadea --- /dev/null +++ b/tests/validate-security-merge.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +root=${ATRINIK_PUBLISH_TEST_ROOT:-${root}} +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +mkdir "${temporary}/bin" +cp "${root}/bin/validate" "${temporary}/bin/" +cp -R "${root}/config" "${root}/community-health" "${root}/.github" "${temporary}/" + +reject() { + local file=$1 filter=$2 + jq "${filter}" "${root}/config/${file}" >"${temporary}/config/${file}" + if "${temporary}/bin/validate" >/dev/null 2>&1; then + echo "error: accepted invalid security policy ${file}: ${filter}" >&2 + exit 1 + fi + cp "${root}/config/${file}" "${temporary}/config/${file}" +} + +"${temporary}/bin/validate" >/dev/null +reject security-merge-protection.json '.dependency_review += ["legacy-client"]' +reject security-merge-protection.json '.code_scanning -= ["service-updater"]' +reject security-merge-protection.json '.dependency_review += ["web-platform"]' +reject security-merge-protection.json '.codeql_required_languages["content-toolkit"] = []' +reject security-merge-protection.json '.codeql_required_languages.classic = ["rust"]' +reject security-merge-protection.json '.codeql_required_languages.server = ["invented"]' +for filter in 'del(.activation)' '.activation = null' '.activation = []' \ + '.activation.unexpected = {}' 'del(.activation.code_scanning)' \ + '.activation.code_scanning = []' '.activation.code_scanning = null' \ + 'del(.activation.code_scanning.classic)' '.activation.code_scanning.classic = null' \ + '.activation.code_scanning.classic = "unknown"' '.activation.code_scanning.classic = true' \ + '.activation.dependency_review.classic = {}' '.activation.dependency_review.classic = "ready"' \ + '.activation.dependency_review.nawerhals = "enforced"'; do + reject security-merge-protection.json "${filter}" +done +reject rulesets/code-scanning.json '.bypass_actors[0].bypass_mode = "always"' +reject rulesets/code-scanning.json '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' +reject rulesets/code-scanning.json '.rules[0].parameters.code_scanning_tools[0].alerts_threshold = "errors"' +reject rulesets/dependency-review.json '.rules[0].parameters.required_status_checks[0].integration_id = null' +reject actions-permissions.json '.sha_pinning_required = false' +reject actions-sha-pinning.json '.phase = "pending" | .canary = "atrinik/github-settings/99"' +reject actions-sha-pinning.json '.phase = "enforced" | .canary = null' +reject actions-sha-pinning.json '.phase = "enforced" | .canary = "atrinik/unregistered/99"' +reject actions-sha-pinning.json '.phase = "enforced" | .canary = "foreign/github-settings/99"' +reject actions-sha-pinning.json '.phase = "enforced" | .canary = "atrinik/github-settings/0"' +reject actions-sha-pinning.json '.phase = "unknown"' +reject actions-sha-pinning.json '.phase = null' +reject actions-sha-pinning.json '.unexpected = true' + +# Reviewed enforced source binds its canary to a repository with required CI. +jq '.phase = "enforced" | .canary = "atrinik/github-settings/99"' \ + "${root}/config/actions-sha-pinning.json" >"${temporary}/config/actions-sha-pinning.json" +"${temporary}/bin/validate" >/dev/null +cp "${root}/config/actions-sha-pinning.json" "${temporary}/config/actions-sha-pinning.json" + +# Source review can promote gates independently without a version catalog. +jq '.activation.code_scanning.classic = "enforced" | .activation.dependency_review.atrinik = "enforced"' \ + "${root}/config/security-merge-protection.json" >"${temporary}/config/security-merge-protection.json" +"${temporary}/bin/validate" >/dev/null +cp "${root}/config/security-merge-protection.json" "${temporary}/config/security-merge-protection.json" + +# A coordinated target retirement is valid; unrelated governed policies stay. +jq 'del(.required_ci["service-updater"])' "${root}/config/repositories.json" \ + >"${temporary}/config/repositories.json" +jq '.code_scanning -= ["service-updater"] | .dependency_review -= ["service-updater"] | del(.activation.code_scanning["service-updater"], .activation.dependency_review["service-updater"])' \ + "${root}/config/security-merge-protection.json" >"${temporary}/config/security-merge-protection.json" +"${temporary}/bin/validate" >/dev/null + +echo 'Security merge policy validation tests passed.'