From 76d56bae75c47da158f4031bb9cb0f107583a9e6 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 11:20:40 -0500 Subject: [PATCH 01/22] feat: stage security merge protection and audit effective policy --- .github/workflows/dependency-review.yml | 24 + AGENTS.md | 15 + CONTRIBUTING.md | 12 + README.md | 85 ++- bin/audit | 349 ++++++++++-- bin/publish | 165 +++++- bin/security-readiness | 5 + bin/validate | 47 +- config/actions-permissions.json | 3 +- config/actions-selected.json | 2 +- config/rulesets/code-scanning.json | 34 ++ config/rulesets/dependency-review.json | 34 ++ config/security-merge-protection.json | 53 ++ lib/security_readiness.py | 604 +++++++++++++++++++++ tests/audit-fixtures.py | 251 +++++++++ tests/audit.sh | 352 +++++------- tests/publish-maintenance-branch.sh | 12 +- tests/publish-security-merge.sh | 251 +++++++++ tests/security-readiness.py | 573 +++++++++++++++++++ tests/security-readiness.sh | 5 + tests/validate-crate-trusted-publishing.sh | 11 +- tests/validate-security-merge.sh | 32 ++ 22 files changed, 2630 insertions(+), 289 deletions(-) create mode 100644 .github/workflows/dependency-review.yml create mode 100755 bin/security-readiness create mode 100644 config/rulesets/code-scanning.json create mode 100644 config/rulesets/dependency-review.json create mode 100644 config/security-merge-protection.json create mode 100644 lib/security_readiness.py create mode 100755 tests/audit-fixtures.py create mode 100755 tests/publish-security-merge.sh create mode 100644 tests/security-readiness.py create mode 100755 tests/security-readiness.sh create mode 100755 tests/validate-security-merge.sh diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 0000000..df3e793 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,24 @@ +name: Dependency review + +on: + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + dependency-review: + name: Dependency review + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: high + fail-on-scopes: runtime,development,unknown + warn-only: false + comment-summary-in-pr: never + show-openssf-scorecard: false + retry-on-snapshot-warnings: true + retry-on-snapshot-warnings-timeout: 120 diff --git a/AGENTS.md b/AGENTS.md index 10ed425..d7dbd9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -167,3 +167,18 @@ `release_policy: none` and are omitted from `release_tags`. - Update this `AGENTS.md` in the same change when major rework alters governance ownership, policy scope, publisher behavior, required checks, or validation. + +- `config/security-merge-protection.json` owns the dependency-review and CodeQL + target sets and default-setup required language minima. Keep both sets equal + to the required-CI inventory. Union required languages with live default-setup + languages, fail on unreadable language state, and patch only real drift. +- Keep `06 - CodeQL security` and `07 - Dependency review` repository-owned in + both policy scopes; Team has no organization code-scanning rule support. + New gates require read-only `bin/security-readiness` evidence for landed and + successful producers; existing active gates survive later PR failures. Keep + aggregate checks, zero approvals, and owner pull-request-only bypass intact. +- Desired Actions SHA enforcement is staged behind `--sha-pinning-canary` proof + of an ordinary action-update PR passing current-head CI without an allowlist + edit. Missing evidence defers only that field; never disable an existing true + setting. Allow reviewed action names with `@*` so pin updates do not require + central allowlist edits. Keep full SHAs in producer workflows. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b203d15..c340bb5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,3 +125,15 @@ organization-owner authorization for `--apply`. Re-run the complete preflight after apply and again before separately authorized exact branch deletion; never combine either operation with tag, release, asset, default-branch, or unrelated policy changes. + +Security merge controls use repository-level rules in both publisher scopes. +Keep `config/security-merge-protection.json` aligned with `required_ci` and +preserve ordinary aggregates. New dependency gates require the reviewed +workflow on the default branch and a successful corresponding PR run; CodeQL +gates require successful current-head analyses, including required languages. +Keep existing gates active on later failures. Tests must cover missing/failed +producers, unreadable evidence, archived targets, language unions, both scopes, +and zero security mutations on a converged rerun. SHA enforcement must remain +pending until the ordinary action-update canary proves successful current-head +CI without an allowlist edit. Action allowances use owner/action wildcards; +full commit pins belong in workflows, not redundant version assertions. diff --git a/README.md b/README.md index 8402a95..7cf86e6 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,8 @@ read-only and are skipped on later runs. remain outside GitHub and this repository. - GitHub Actions defaults to read-only, cannot approve pull requests, and may use only Atrinik, GitHub, Codecov coverage, explicitly allowed Docker - actions, and the exact reviewed crates.io authentication action commit. + actions, and `rust-lang/crates-io-auth-action@*`. Workflow references remain + full commit SHAs; the action allowlist permits routine pin updates. - Historical repositories listed in `config/repositories.json` are archived. The five former standalone classic component repositories are already archived read-only after their history, active work, issues, and release @@ -147,9 +148,82 @@ separate checked procedure below. Codecov must be installed for the listed repositories so their OIDC-authenticated coverage uploads and badges remain available. +## Staged security merge protection + +`config/security-merge-protection.json` targets the 21 repositories in +`required_ci`, including `service-updater`; `.github` and `web-platform` are +excluded. Existing aggregate checks and zero required approvals remain intact. +The existing publisher restores `service-updater`'s already inventoried PR, +`Updater validation`/title CI, and release-tag protections. + +Both policy scopes install repository-owned `06 - CodeQL security` rules: +GitHub Team does not support organization code-scanning rules. CodeQL blocks +high and critical security findings (`high_or_higher`), with ordinary quality +alerts disabled (`none`). A separate repository-owned `07 - Dependency review` +requires the GitHub Actions `Dependency review` job. Both permit only +organization owners to bypass through a pull request; bypass can merge findings +and must be a deliberate owner decision. Direct pushes gain no exception. + +The dependency workflow runs on every pull request to `main`, with read-only +contents permission and no checkout. It rejects new high/critical vulnerable +runtime, development, or unknown-scope dependencies and fails on review errors. +It relies on GitHub's dependency graph and submitted snapshots; unsupported or +missing manifests cannot provide comprehensive dependency coverage. This is +separate from Dependabot's scheduled update and alert capabilities. Default +CodeQL setup similarly analyzes supported languages and its detected build +coverage; enabling it alone does not prove a successful analysis. Rust in +`content-toolkit` is an explicit minimum, unioned with its existing languages. +Legacy `javascript`/`typescript` aliases use `javascript-typescript` in the +PATCH payload, preserving that coverage and other live language entries. +Read failures never replace that live language set with an incomplete set. + +GitHub's [code-scanning merge-protection limitations](https://docs.github.com/en/code-security/concepts/code-scanning/merge-protection) +exclude Dependabot pull requests analyzed by default setup and merge queue +groups. Dependency review remains a required check for those Dependabot PRs; +it does not inherit that CodeQL exception. CodeQL merge protection also depends +on the alert's affected lines appearing in the PR diff. Ordinary quality alerts +remain excluded by `alerts_threshold: none`. + +Rollout is owned by an organization administrator: + +1. Merge the dependency workflow PRs and let ordinary pull requests prove their + `Dependency review` jobs. Merge this policy and review `bin/publish`'s plan. +2. Apply only with separate live-settings authorization. Existing governance, + the wildcard action allowance, and the Rust language correction can converge + while security gates remain `PENDING`. The publisher creates each gate only + after `bin/security-readiness` verifies its producer. Dependency readiness + requires the reviewed workflow contract on the default branch, a successful + latest PR run/job, and the same workflow at the check suite's executed + revision. Synthetic merge revisions must bind the associated PR's base and + head. CodeQL requires current-head coverage for every configured default-setup + language plus policy minima. Classic instead requires its successful full + advanced producer and all current source-derived analysis partitions; + changed-path PR subsets cannot establish readiness. The bounded newest-first + query stops once current coverage is proven; retired categories and old + analysis history do not require manual cleanup. +3. Rerun the plan after the pending analyses complete. Existing active gates + remain enforced even when a later PR fails a security check. Scope migrations + preserve these repository-owned gates. A converged security gate or language + configuration causes no mutation or security-configuration reattachment. +4. Desired organization `sha_pinning_required` is true, but the publisher omits + this field until an ordinary action-pin update PR passes CI without allowlist + edits. Verify it using `bin/security-readiness sha OWNER/REPOSITORY/PR`, then + pass `--sha-pinning-canary OWNER/REPOSITORY/PR` to the reviewed plan/apply. + The verifier checks live allowance, exact PR changes, and successful current + PR-head CI. Missing or failed evidence defers SHA enforcement while the other + controls converge; a live true setting is always preserved. Full SHA pinning + is not a claim that an allowed action itself is trustworthy. + +Read-only preflight failures and unfinished producers are explicit pending +controls, never successful rollout evidence. The audit reports pending controls +separately and rejects unreadable evidence. No paid subscription is introduced. +Rollback requires reviewed desired-state changes and separately authorized +apply; reverting a producer while its required gate remains active can prevent +merges. Do not remove a gate merely because a vulnerable PR fails it. + ## Usage -Requirements: `git`, `gh`, `jq`, and an authenticated organization-owner +Requirements: `git`, `gh`, `jq`, `python3`, and an authenticated organization-owner account. ```sh @@ -1093,8 +1167,8 @@ The existing `crates-io-bootstrap` environment and published 0.1 crate are separate historical state and are not changed by this contract. The Actions allowlist adds only -`rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18`, -verified against upstream `v1.0.5`. The +`rust-lang/crates-io-auth-action@*`, +allowing full-SHA updates of that action. The currently inventoried [pinned official action](https://github.com/rust-lang/crates-io-auth-action/blob/c6f97d42243bad5fab37ca0427f495c86d5b1a18/README.md) exchanges GitHub OIDC for a temporary crates.io token and revokes it in its post step. Only the owning workflow's upload job may request `id-token: write`, @@ -1117,7 +1191,8 @@ Activation is a later, separately authorized operation: No future release SHA or crate digest belongs in this planning record. 3. Review `bin/publish` and apply only the merged, authorized desired-state policy. The organization allowlist applies to all governed repositories, but - the one added authentication-action pattern matches only that exact commit. + the authentication-action pattern matches only that action name and permits + routine full-SHA updates. Check the complete plan for unrelated drift before requesting apply. 4. With separate owner authorization, provision `crates-io-release` with the exact desired reviewer/branch/bypass settings and register a crates.io diff --git a/bin/audit b/bin/audit index ebcb69b..c778ee0 100755 --- a/bin/audit +++ b/bin/audit @@ -23,10 +23,12 @@ trap 'rm -rf "${temporary}"' EXIT "${root}/bin/validate" >/dev/null github_api() { - gh api -H "X-GitHub-Api-Version: ${api_version}" "$@" + gh api --method GET -H "X-GitHub-Api-Version: ${api_version}" "$@" } declare -a findings=() +declare -a pending=() +declare -A ruleset_cache=() record_finding() { findings+=("$1") @@ -61,6 +63,14 @@ is_temporary_security_advisory_repository() { "${repositories_config}" >/dev/null } +# Capture before parsing: process substitutions otherwise hide API failures. +if ! github_api --paginate --slurp "orgs/${organization}/repos?per_page=100&type=all" | + jq -e 'add | if type == "array" then . else error("invalid inventory") end' \ + >"${temporary}/inventory.json"; then + echo "error: organization repository inventory could not be read" >&2 + exit 1 +fi + declare -A live_archived=() while IFS=$'\t' read -r \ repository archived private visibility disabled fork has_issues has_projects has_wiki; do @@ -71,24 +81,11 @@ while IFS=$'\t' read -r \ continue fi live_archived["${repository}"]=${archived} -done < <( - github_api --paginate "orgs/${organization}/repos?per_page=100&type=all" \ - --jq ' - .[] | - [ - .name, - (.archived | tostring), - (.private | tostring), - .visibility, - (.disabled | tostring), - (.fork | tostring), - (.has_issues | tostring), - (.has_projects | tostring), - (.has_wiki | tostring) - ] | - @tsv - ' -) +done < <(jq -r ' + .[] | [.name, (.archived | tostring), (.private | tostring), .visibility, + (.disabled | tostring), (.fork | tostring), (.has_issues | tostring), + (.has_projects | tostring), (.has_wiki | tostring)] | @tsv +' "${temporary}/inventory.json") declare -A expected_active=() declare -A expected_archived=() @@ -140,19 +137,6 @@ for repository in "${!expected_archived[@]}"; do fi done -expected_rule_name() { - local repository=$1 - local base=$2 - - if jq -e --arg repository "${repository}" \ - '.repositories | index($repository) != null' "${advisory_windows}" \ - >/dev/null; then - printf '%s - classic\n' "${base}" - else - printf '%s\n' "${base}" - fi -} - has_configured_repository() { local property=$1 local repository=$2 @@ -175,6 +159,159 @@ has_release_tags() { >/dev/null } +audit_actions() { + local current + for policy in permissions selected-actions workflow; do + local endpoint="orgs/${organization}/actions/permissions" + local config="${root}/config/actions-permissions.json" + case ${policy} in + selected-actions) endpoint+=/selected-actions; config="${root}/config/actions-selected.json" ;; + workflow) endpoint+=/workflow; config="${root}/config/actions-workflow.json" ;; + esac + if ! current=$(github_api "${endpoint}"); then + record_finding "organization Actions ${policy} could not be read: ${organization}" + continue + fi + if ! jq -e --slurpfile desired "${config}" ' + . as $current | $desired[0] | to_entries | all(.[]; + if .key == "sha_pinning_required" then true + elif .key == "patterns_allowed" then + (.value | sort) == ($current.patterns_allowed | sort) + else $current[.key] == .value end) + ' <<<"${current}" >/dev/null; then + record_finding "organization Actions ${policy} drift: ${organization}" + fi + if [[ ${policy} == permissions ]] && + jq -e '.sha_pinning_required == true' "${config}" >/dev/null; then + case $(jq -r '.sha_pinning_required | tostring' <<<"${current}") in + true) ;; + false) pending+=("Actions SHA pinning deferred pending canary proof: ${organization}") ;; + *) record_finding "Actions SHA pinning state could not be read: ${organization}" ;; + esac + fi + done +} + +audit_actions + +# The branch endpoint proves effective applicability; inherited names alone do not. +# Details additionally prove bypass and ref/repository scope, including tag policy. +ruleset_targets_repository() { + local detail=$1 repository=$2 id=$3 pattern matched=false + if jq -e '.conditions.repository_id != null' <<<"${detail}" >/dev/null; then + jq -e --argjson id "${id}" '.conditions.repository_id.repository_ids | index($id) != null' \ + <<<"${detail}" >/dev/null || return 1 + fi + if jq -e '.conditions.repository_name != null' <<<"${detail}" >/dev/null; then + while IFS= read -r pattern; do + # Intentional GitHub repository-name wildcard matching. + # shellcheck disable=SC2053 + if [[ ${pattern} == '~ALL' || ${repository} == ${pattern} ]]; then matched=true; fi + done < <(jq -r '.conditions.repository_name.include[]' <<<"${detail}") + ${matched} || return 1 + while IFS= read -r pattern; do + # Intentional GitHub repository-name wildcard matching. + # shellcheck disable=SC2053 + if [[ ${pattern} == '~ALL' || ${repository} == ${pattern} ]]; then return 1; fi + done < <(jq -r '.conditions.repository_name.exclude[]' <<<"${detail}") + fi +} + +check_ruleset() { + local repository=$1 name=$2 template=$3 list=$4 metadata=$5 + local entry id source endpoint detail wanted + entry=$(jq -c --arg name "${name}" '[.[] | select(.name == $name)]' <<<"${list}") + if [[ $(jq length <<<"${entry}") != 1 ]]; then + record_finding "missing or duplicate inherited ruleset '${name}': ${organization}/${repository}" + return + fi + id=$(jq -r '.[0].id' <<<"${entry}") + source=$(jq -r '.[0].source_type' <<<"${entry}") + case ${source} in + Organization) endpoint="orgs/${organization}/rulesets/${id}" ;; + Repository) endpoint="repos/${organization}/${repository}/rulesets/${id}" ;; + *) record_finding "ruleset source could not be read '${name}': ${organization}/${repository}"; return ;; + esac + if [[ ! -v ruleset_cache["${endpoint}"] ]]; then + if ! detail=$(github_api "${endpoint}"); then + record_finding "ruleset contents could not be read '${name}': ${organization}/${repository}" + return + fi + ruleset_cache["${endpoint}"]=${detail} + fi + detail=${ruleset_cache["${endpoint}"]} + wanted=$(cat "${template}") + if [[ ${name} == '03 - Required CI - '* ]]; then + wanted=$(jq --argjson checks "${expected_checks}" \ + '.rules[0].parameters.required_status_checks = $checks' <<<"${wanted}") + # Dependency review is governed by its separate rollout ruleset. + wanted=$(jq '.rules[0].parameters.required_status_checks |= map(select(.context != "Dependency review"))' <<<"${wanted}") + fi + if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ + "${advisory_windows}" >/dev/null && [[ ${name} == '03 - Required CI - '* ]]; then + wanted=$(jq '.bypass_actors[].bypass_mode = "always"' <<<"${wanted}") + fi + if ! ruleset_targets_repository "${detail}" "${repository}" "$(jq -r .id "${metadata}")" || + ! jq -e --argjson wanted "${wanted}" ' + # GitHub returns null for OrganizationAdmin, whose desired actor ID is 1. + def normalize_bypass: + map(if .actor_type == "OrganizationAdmin" and .actor_id == null then + .actor_id = 1 else . end) | sort_by(.actor_type, .actor_id, .bypass_mode); + .enforcement == $wanted.enforcement and .target == $wanted.target and + (.bypass_actors | normalize_bypass) == ($wanted.bypass_actors | normalize_bypass) and + .conditions.ref_name == $wanted.conditions.ref_name and + (.rules | map(if .type == "required_status_checks" then + .parameters.required_status_checks |= sort_by(.context, .integration_id) else . end) | sort_by(.type)) == + ($wanted.rules | map(if .type == "required_status_checks" then + .parameters.required_status_checks |= sort_by(.context, .integration_id) else . end) | sort_by(.type)) + ' <<<"${detail}" >/dev/null; then + record_finding "ruleset scope or policy drift '${name}': ${organization}/${repository}" + fi +} + +audit_security() { + local repository=$1 desired="${root}/config/code-security.json" setup="${root}/config/codeql-default-setup.json" + local current languages + if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ + "${root}/config/codeql-advanced-setup.json" >/dev/null; then + desired="${root}/config/code-security-advanced.json" + setup="${root}/config/codeql-default-setup-advanced.json" + fi + if current=$(github_api "repos/${organization}/${repository}/code-security-configuration"); then + if ! jq -e --slurpfile desired "${desired}" ' + .status == "enforced" and (.configuration | . as $config | + . + {code_security: (.code_security // + (if .advanced_security == "enabled" or .advanced_security == "code_security" then "enabled" else "disabled" end)), + secret_protection: (.secret_protection // + (if .advanced_security == "enabled" or .advanced_security == "secret_protection" then "enabled" else "disabled" end))} | + . as $normalized | $desired[0] | to_entries | all(.[]; $normalized[.key] == .value)) + ' <<<"${current}" >/dev/null; then + record_finding "security configuration attachment or baseline drift: ${organization}/${repository}" + fi + else + record_finding "security configuration could not be read: ${organization}/${repository}" + fi + if current=$(github_api "repos/${organization}/${repository}/code-scanning/default-setup"); then + languages=$(jq -c --arg repository "${repository}" '.codeql_required_languages[$repository] // []' \ + "${root}/config/security-merge-protection.json") + if ! jq -e --slurpfile desired "${setup}" --argjson languages "${languages}" ' + . as $current | ($desired[0] | to_entries | all(.[]; $current[.key] == .value)) and + (($languages - ($current.languages // [])) | length == 0) + ' <<<"${current}" >/dev/null; then + record_finding "CodeQL setup or required language drift: ${organization}/${repository}" + fi + else + record_finding "CodeQL setup could not be read: ${organization}/${repository}" + fi + if ! github_api "repos/${organization}/${repository}/vulnerability-alerts" >/dev/null; then + record_finding "Dependabot alerts disabled or unreadable: ${organization}/${repository}" + fi + if ! current=$(github_api "repos/${organization}/${repository}/private-vulnerability-reporting") || + ! jq -e '.enabled == true' <<<"${current}" >/dev/null; then + record_finding "private vulnerability reporting disabled or unreadable: ${organization}/${repository}" + fi +} + for repository in "${!expected_active[@]}"; do [[ -v live_archived["${repository}"] ]] || continue [[ ${live_archived["${repository}"]} == false ]] || continue @@ -185,6 +322,16 @@ for repository in "${!expected_active[@]}"; do continue fi + if current=$(github_api "repos/${organization}/${repository}/actions/permissions/workflow"); then + if ! jq -e --slurpfile desired "${root}/config/actions-workflow.json" ' + . as $current | $desired[0] | to_entries | all(.[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + record_finding "repository Actions workflow permissions drift: ${organization}/${repository}" + fi + else + record_finding "repository Actions workflow permissions could not be read: ${organization}/${repository}" + fi + if ! jq -e --slurpfile desired "${repository_defaults}" ' .archived == false and .default_branch == $desired[0].default_branch and @@ -214,32 +361,118 @@ for repository in "${!expected_active[@]}"; do record_finding "repository custom properties could not be read: ${organization}/${repository}" fi - if rulesets=$(github_api \ - "repos/${organization}/${repository}/rulesets?includes_parents=true" | \ - jq -r '.[].name' | sort); then - required_rulesets=( - "$(expected_rule_name "${repository}" "01 - Default branch integrity")" - "$(expected_rule_name "${repository}" "01 - Default branch linear history")" - ) - if has_configured_repository '.pull_request_gate' "${repository}"; then - required_rulesets+=( - "$(expected_rule_name "${repository}" "02 - Changes through pull requests")" - ) + rulesets='[]' + if ! rulesets=$(github_api --paginate --slurp \ + "repos/${organization}/${repository}/rulesets?includes_parents=true&per_page=100" | jq -e 'add'); then + record_finding "inherited rulesets could not be read: ${organization}/${repository}" + rulesets='[]' + fi + branch=$(jq -r '.default_branch' "${repository_defaults}") + effective='[]' + if ! effective=$(github_api --paginate --slurp \ + "repos/${organization}/${repository}/rules/branches/${branch}?per_page=100" | jq -e 'add'); then + record_finding "effective default-branch rules could not be read: ${organization}/${repository}" + effective='[]' + fi + expected_checks=$(jq -c --arg repository "${repository}" \ + '(.required_ci[$repository] // []) | map({context: ., integration_id: 15368})' "${repositories_config}") + dependency_required=false + if jq -e --arg repository "${repository}" '.dependency_review | index($repository) != null' \ + "${root}/config/security-merge-protection.json" >/dev/null; then + if jq -e 'any(.[]; .type == "required_status_checks" and + any(.parameters.required_status_checks[]; .context == "Dependency review"))' \ + <<<"${effective}" >/dev/null || + jq -e 'any(.[]; .name == "07 - Dependency review")' <<<"${rulesets}" >/dev/null; then + dependency_required=true + elif "${root}/bin/security-readiness" dependency "${organization}/${repository}" \ + >"${temporary}/${repository}.readiness" 2>&1; then + dependency_required=true + else + readiness_status=$? + if ((readiness_status == 1)); then + pending+=("Dependency review merge protection awaits landed successful producer: ${organization}/${repository}") + else + record_finding "Dependency review readiness could not be read: ${organization}/${repository}" + fi fi - if has_required_ci "${repository}"; then - required_rulesets+=("03 - Required CI - ${repository}") + if ${dependency_required}; then + expected_checks=$(jq '. + [{context: "Dependency review", integration_id: 15368}]' <<<"${expected_checks}") + check_ruleset "${repository}" '07 - Dependency review' \ + "${root}/config/rulesets/dependency-review.json" "${rulesets}" "${metadata_file}" fi - if has_release_tags "${repository}"; then - required_rulesets+=("04 - Immutable release tags") + fi + if ! jq -e --argjson checks "${expected_checks}" ' + ([.[] | select(.type == "required_status_checks") | .parameters.required_status_checks[]] | + unique_by(.context, .integration_id) | sort_by(.context, .integration_id)) == + ($checks | sort_by(.context, .integration_id)) and + all(.[] | select(.type == "required_status_checks"); .parameters.strict_required_status_checks_policy == true) + ' <<<"${effective}" >/dev/null; then + record_finding "effective required CI contexts or app drift: ${organization}/${repository}" + fi + for policy in default-branch-integrity default-branch-linear-history; do + template="${root}/config/rulesets/${policy}.json" + if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ + "${advisory_windows}" >/dev/null; then + template="${root}/config/rulesets/${policy}-classic.json" + fi + check_ruleset "${repository}" "$(jq -r .name "${template}")" "${template}" "${rulesets}" "${metadata_file}" + if ! jq -e --slurpfile desired "${template}" \ + '. as $effective | all($desired[0].rules[]; . as $rule | any($effective[]; .type == $rule.type))' \ + <<<"${effective}" >/dev/null; then + record_finding "effective default-branch integrity drift: ${organization}/${repository}" + fi + done + if has_configured_repository '.pull_request_gate' "${repository}"; then + template="${root}/config/rulesets/pull-request-gate.json" + if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ + "${advisory_windows}" >/dev/null; then template="${root}/config/rulesets/pull-request-gate-classic.json"; fi + check_ruleset "${repository}" "$(jq -r .name "${template}")" "${template}" "${rulesets}" "${metadata_file}" + if ! jq -e --slurpfile desired "${template}" \ + '[.[] | select(.type == "pull_request") | .parameters] == [$desired[0].rules[0].parameters]' \ + <<<"${effective}" >/dev/null; then + record_finding "effective pull-request policy drift: ${organization}/${repository}" fi - for ruleset in "${required_rulesets[@]}"; do - if ! grep -Fxq -- "${ruleset}" <<<"${rulesets}"; then - record_finding "missing inherited ruleset '${ruleset}': ${organization}/${repository}" + elif jq -e 'any(.[]; .type == "pull_request")' <<<"${effective}" >/dev/null; then + record_finding "unexpected effective pull-request gate: ${organization}/${repository}" + fi + if has_required_ci "${repository}"; then + check_ruleset "${repository}" "03 - Required CI - ${repository}" \ + "${root}/config/rulesets/required-ci.json" "${rulesets}" "${metadata_file}" + fi + if has_release_tags "${repository}"; then + check_ruleset "${repository}" '04 - Immutable release tags' \ + "${root}/config/rulesets/immutable-release-tags.json" "${rulesets}" "${metadata_file}" + fi + if jq -e --arg repository "${repository}" '.code_scanning | index($repository) != null' \ + "${root}/config/security-merge-protection.json" >/dev/null; then + codeql_required=false + if jq -e 'any(.[]; .type == "code_scanning")' <<<"${effective}" >/dev/null || + jq -e 'any(.[]; .name == "06 - CodeQL security")' <<<"${rulesets}" >/dev/null; then + codeql_required=true + elif "${root}/bin/security-readiness" codeql "${organization}/${repository}" \ + >"${temporary}/${repository}.codeql-readiness" 2>&1; then + codeql_required=true + else + readiness_status=$? + if ((readiness_status == 1)); then + pending+=("CodeQL merge protection awaits successful analysis: ${organization}/${repository}") + else + record_finding "CodeQL readiness could not be read: ${organization}/${repository}" fi - done - else - record_finding "inherited rulesets could not be read: ${organization}/${repository}" + fi + if ${codeql_required}; then + check_ruleset "${repository}" '06 - CodeQL security' \ + "${root}/config/rulesets/code-scanning.json" "${rulesets}" "${metadata_file}" + if ! jq -e --slurpfile desired "${root}/config/rulesets/code-scanning.json" \ + '[.[] | select(.type == "code_scanning") | {type, parameters}] == $desired[0].rules' \ + <<<"${effective}" >/dev/null; then + record_finding "effective CodeQL merge protection drift: ${organization}/${repository}" + fi + fi + elif jq -e 'any(.[]; .type == "code_scanning")' <<<"${effective}" >/dev/null; then + record_finding "unexpected effective CodeQL merge protection: ${organization}/${repository}" fi + audit_security "${repository}" if ! jq -e ' .security_and_analysis.secret_scanning.status == "enabled" and @@ -251,10 +484,18 @@ for repository in "${!expected_active[@]}"; do fi done +if ((${#pending[@]})); then + printf 'PENDING %s\n' "${pending[@]}" | sort -u +fi + if ((${#findings[@]})); then printf 'DRIFT %s\n' "${findings[@]}" | sort -u echo "Governance audit failed: ${#findings[@]} finding(s)." >&2 exit 1 fi -echo "Governance audit passed: ${#expected_active[@]} active repositories checked." +if ((${#pending[@]})); then + echo "Governance audit baseline passed with ${#pending[@]} pending rollout(s): ${#expected_active[@]} active repositories checked." +else + echo "Governance audit passed: ${#expected_active[@]} active repositories checked." +fi diff --git a/bin/publish b/bin/publish index 4052acf..f888d23 100755 --- a/bin/publish +++ b/bin/publish @@ -7,6 +7,7 @@ policy_scope=${ATRINIK_POLICY_SCOPE:-auto} api_version=2026-03-10 apply=false retire_maintenance= +sha_pinning_canary= while (($#)); do case $1 in @@ -14,6 +15,14 @@ while (($#)); do apply=true shift ;; + --sha-pinning-canary) + if (($# < 2)) || [[ -z $2 ]]; then + echo "error: --sha-pinning-canary requires OWNER/REPOSITORY/PR" >&2 + exit 2 + fi + sha_pinning_canary=$2 + shift 2 + ;; --retire-maintenance) if (($# < 2)) || [[ -z $2 ]]; then echo "error: --retire-maintenance requires REPOSITORY/BRANCH" >&2 @@ -23,7 +32,7 @@ while (($#)); do shift 2 ;; *) - echo "usage: $0 [--apply] [--retire-maintenance REPOSITORY/BRANCH]" >&2 + echo "usage: $0 [--apply] [--retire-maintenance REPOSITORY/BRANCH] [--sha-pinning-canary OWNER/REPOSITORY/PR]" >&2 exit 2 ;; esac @@ -37,7 +46,7 @@ auto | organization | repository) ;; ;; esac -for command in gh git jq sort; do +for command in gh git jq sort python3; do if ! command -v "${command}" >/dev/null 2>&1; then echo "error: ${command} is required" >&2 exit 1 @@ -51,6 +60,11 @@ advisory_merge_windows_config=${root}/config/advisory-merge-windows.json codeql_advanced_config=${root}/config/codeql-advanced-setup.json immutable_releases_config=${root}/config/immutable-releases.json retired_maintenance_config=${root}/config/retired-maintenance-branches.json +security_merge_config=${root}/config/security-merge-protection.json +if [[ -n ${sha_pinning_canary} && ${sha_pinning_canary} != "${organization}/"* ]]; then + echo "error: SHA canary must belong to ${organization}" >&2 + exit 2 +fi temporary_files=() trap 'rm -f "${temporary_files[@]}"' EXIT @@ -1000,35 +1014,167 @@ wait_for_code_security_configuration() { } converge_codeql_default_setup() { - local repository input desired_state desired_query current - local current_state current_query + local repository input desired_state desired_query current required_languages + local current_state current_query language_match for repository in "${active_repositories[@]}"; do input=${root}/config/codeql-default-setup.json if repository_uses_advanced_codeql "${repository}"; then input=${root}/config/codeql-default-setup-advanced.json fi - + required_languages=$(jq -c --arg repository "${repository}" \ + '.codeql_required_languages[$repository] // []' "${security_merge_config}") desired_state=$(jq -r '.state' "${input}") desired_query=$(jq -r '.query_suite // empty' "${input}") + current='{}' if current=$(github_api \ "repos/${organization}/${repository}/code-scanning/default-setup" \ 2>/dev/null); then current_state=$(jq -r '.state // empty' <<<"${current}") + if [[ ${required_languages} != '[]' ]] && ! jq -e ' + (.state == "not-configured" and .languages == null) or + (.languages | type == "array" and all(.[]; type == "string")) + ' <<<"${current}" >/dev/null; then + echo "error: invalid existing CodeQL languages for ${repository}" >&2 + return 1 + fi current_query=$(jq -r '.query_suite // empty' <<<"${current}") - if [[ ${current_state} == "${desired_state}" ]] && + language_match=$(jq --argjson required "${required_languages}" \ + '($required - (.languages // [])) == []' <<<"${current}") + if [[ ${current_state} == "${desired_state}" && ${language_match} == true ]] && { [[ -z ${desired_query} ]] || [[ ${current_query} == "${desired_query}" ]]; }; then echo "KEEP /repos/${organization}/${repository}/code-scanning/default-setup is ${desired_state}" continue fi + elif [[ ${required_languages} != '[]' ]]; then + # Never replace an unreadable live language set with an incomplete list. + echo "error: cannot read existing CodeQL languages for ${repository}" >&2 + return 1 + fi + if [[ ${required_languages} != '[]' ]]; then + local language_payload + language_payload=$(mktemp) + temporary_files+=("${language_payload}") + jq --argjson current "${current}" --argjson required "${required_languages}" \ + '.languages = (($current.languages // []) + $required | + map(if . == "javascript" or . == "typescript" then + "javascript-typescript" else . end) | unique)' \ + "${input}" >"${language_payload}" + input=${language_payload} fi - run_api PATCH \ "repos/${organization}/${repository}/code-scanning/default-setup" \ "${input}" done } +# These repository rules remain repository-owned even in organization scope: +# GitHub Team does not support organization-level code-scanning rules. +publish_security_merge_rulesets() { + local repository kind template name existing id current reason status readiness_mode + for kind in code_scanning dependency_review; do + if [[ ${kind} == code_scanning ]]; then + template=${root}/config/rulesets/code-scanning.json + readiness_mode=codeql + else + template=${root}/config/rulesets/dependency-review.json + readiness_mode=dependency + fi + name=$(jq -r '.name' "${template}") + while IFS= read -r repository; do + if [[ ! -v active_repository_ids["${repository}"] ]]; then + echo "SKIP ${organization}/${repository} is not active; ${name}" + continue + fi + existing=$(github_api \ + "repos/${organization}/${repository}/rulesets?includes_parents=false") + if ! jq -e --arg name "${name}" \ + '[.[] | select(.name == $name)] | length <= 1' <<<"${existing}" >/dev/null; then + echo "error: ambiguous managed ruleset ${name} on ${repository}" >&2 + return 1 + fi + id=$(jq -r --arg name "${name}" \ + '.[] | select(.name == $name) | .id' <<<"${existing}") + current='{}' + if [[ -n ${id} ]]; then + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") + if jq -e --slurpfile desired "${template}" ' + # GitHub returns the organization-owner sentinel with a null ID. + def normalize: + .bypass_actors |= (map( + if .actor_type == "OrganizationAdmin" and .actor_id == null then + .actor_id = 1 + else . end) | sort_by(.actor_type, .actor_id, .bypass_mode)); + normalize as $current | + all($desired[0] | normalize | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + echo "KEEP /repos/${organization}/${repository}/rulesets/${id} ${name}" + continue + fi + fi + # Preserve an already active gate when a later pull request fails it. + if ! jq -e --arg kind "${kind}" ' + .enforcement == "active" and + if $kind == "code_scanning" then + any(.rules[]?; .type == "code_scanning" and + any(.parameters.code_scanning_tools[]?; .tool == "CodeQL")) + else + any(.rules[]?; .type == "required_status_checks" and + any(.parameters.required_status_checks[]?; + .context == "Dependency review" and .integration_id == 15368)) + end + ' <<<"${current}" >/dev/null; then + status=0 + reason=$("${root}/bin/security-readiness" "${readiness_mode}" \ + "${organization}/${repository}") || status=$? + if ((status != 0)); then + echo "PENDING ${organization}/${repository} ${name}: ${reason}" + continue + fi + fi + if [[ -n ${id} ]]; then + run_api PUT "repos/${organization}/${repository}/rulesets/${id}" "${template}" + else + run_api POST "repos/${organization}/${repository}/rulesets" "${template}" + fi + done < <(jq -r --arg kind "${kind}" '.[$kind][]' "${security_merge_config}") + done +} + +converge_actions_permissions() { + local current input reason + current=$(github_api "orgs/${organization}/actions/permissions") + if ! jq -e 'type == "object" and + (.enabled_repositories | type == "string") and + (.allowed_actions | type == "string") and + (.sha_pinning_required == null or (.sha_pinning_required | type == "boolean")) + ' <<<"${current}" >/dev/null; then + echo "error: cannot read current organization Actions permissions" >&2 + return 1 + fi + input=$(mktemp) + temporary_files+=("${input}") + cp "${root}/config/actions-permissions.json" "${input}" + if ! jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then + if [[ -n ${sha_pinning_canary} ]] && + reason=$("${root}/bin/security-readiness" sha "${sha_pinning_canary}"); then + echo "READY SHA pinning: ${reason}" + else + echo "PENDING SHA pinning: ${reason:-supply --sha-pinning-canary OWNER/REPOSITORY/PR after an ordinary action update passes CI}" + # Omit the field: pending evidence never enables or disables this setting. + jq 'del(.sha_pinning_required)' "${input}" >"${input}.pending" + mv "${input}.pending" "${input}" + fi + fi + if jq -e --slurpfile desired "${input}" ' + . as $current | all($desired[0] | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + echo "KEEP /orgs/${organization}/actions/permissions" + else + run_api PUT "orgs/${organization}/actions/permissions" "${input}" + fi +} + publish_organization_security() { local advanced_id baseline_id defaults_payload defaults local advanced_repository_count baseline_repository_count @@ -1331,8 +1477,7 @@ done < <( converge_immutable_release_policy converge_organization_settings "${organization_metadata}" -run_api PUT "orgs/${organization}/actions/permissions" \ - "${root}/config/actions-permissions.json" +converge_actions_permissions run_api PUT "orgs/${organization}/actions/permissions/selected-actions" \ "${root}/config/actions-selected.json" run_api PUT "orgs/${organization}/actions/permissions/workflow" \ @@ -1351,6 +1496,8 @@ else publish_repository_security fi +publish_security_merge_rulesets + while IFS= read -r repository; do if ! repository_archived=$( github_api "repos/${organization}/${repository}" --jq '.archived' 2>/dev/null diff --git a/bin/security-readiness b/bin/security-readiness new file mode 100755 index 0000000..a00982e --- /dev/null +++ b/bin/security-readiness @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +exec python3 "${root}/lib/security_readiness.py" "$@" diff --git a/bin/validate b/bin/validate index db70f03..798aaa3 100755 --- a/bin/validate +++ b/bin/validate @@ -51,6 +51,47 @@ jq -e ' } ' "${organization_config}" >/dev/null +jq -e --slurpfile repositories "${repositories_config}" \ + --slurpfile advanced "${advanced_codeql_config}" ' + (keys == ["code_scanning", "codeql_required_languages", "dependency_review"]) and + .dependency_review == ($repositories[0].required_ci | keys) and + .code_scanning == .dependency_review and + (.codeql_required_languages | type == "object") and + all(.codeql_required_languages | to_entries[]; + .key as $repository | + ($repositories[0].required_ci | has($repository)) and + ($advanced[0].repositories | index($repository)) == null and + (.value | type == "array" and length > 0 and . == (sort | unique)) and + all(.value[]; IN("c-cpp", "csharp", "go", "java-kotlin", + "javascript-typescript", "python", "ruby", "rust", "swift"))) and + .codeql_required_languages["content-toolkit"] == ["rust"] +' "${root}/config/security-merge-protection.json" >/dev/null + +jq -e '.sha_pinning_required == true' \ + "${root}/config/actions-permissions.json" >/dev/null + +for security_ruleset in code-scanning dependency-review; do + jq -e ' + .target == "branch" and .enforcement == "active" and + .conditions == {ref_name: {include: ["~DEFAULT_BRANCH"], exclude: []}} and + .bypass_actors == [{actor_id: 1, actor_type: "OrganizationAdmin", + bypass_mode: "pull_request"}] + ' "${root}/config/rulesets/${security_ruleset}.json" >/dev/null +done +jq -e ' + .name == "06 - CodeQL security" and + .rules == [{type: "code_scanning", parameters: {code_scanning_tools: [{ + tool: "CodeQL", security_alerts_threshold: "high_or_higher", alerts_threshold: "none" + }]}}] +' "${root}/config/rulesets/code-scanning.json" >/dev/null +jq -e ' + .name == "07 - Dependency review" and + .rules == [{type: "required_status_checks", parameters: { + required_status_checks: [{context: "Dependency review", integration_id: 15368}], + strict_required_status_checks_policy: true + }}] +' "${root}/config/rulesets/dependency-review.json" >/dev/null + jq -e ' .bypass_actors == [] and [.rules[].type] == ["deletion", "non_fast_forward"] @@ -780,9 +821,9 @@ jq -e \ "provider", "repository", "repository_id", "runbook", "workflow_filename", "workflow_ref", "workflow_trigger"]) and .activation_state == "pending-source-release" and - .authentication_action == "rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18" and - (.authentication_action as $action | - ($actions[0].patterns_allowed | map(select(startswith("rust-lang/")))) == [$action]) and + (.authentication_action | test("^rust-lang/crates-io-auth-action@[0-9a-f]{40}$")) and + (($actions[0].patterns_allowed | map(select(startswith("rust-lang/")))) == + ["rust-lang/crates-io-auth-action@*"]) and .crate == "atrinik-protocol" and .provider == "github" and .repository == "atrinik/protocol" and .repository_id == 1327106950 and ($repositories[0].pull_request_gate | index("protocol")) != null and diff --git a/config/actions-permissions.json b/config/actions-permissions.json index 9f8c2fe..8ed7a80 100644 --- a/config/actions-permissions.json +++ b/config/actions-permissions.json @@ -1,4 +1,5 @@ { "enabled_repositories": "all", - "allowed_actions": "selected" + "allowed_actions": "selected", + "sha_pinning_required": true } diff --git a/config/actions-selected.json b/config/actions-selected.json index 47b5dd9..33aac03 100644 --- a/config/actions-selected.json +++ b/config/actions-selected.json @@ -7,6 +7,6 @@ "docker/login-action@*", "docker/setup-buildx-action@*", "docker/build-push-action@*", - "rust-lang/crates-io-auth-action@c6f97d42243bad5fab37ca0427f495c86d5b1a18" + "rust-lang/crates-io-auth-action@*" ] } diff --git a/config/rulesets/code-scanning.json b/config/rulesets/code-scanning.json new file mode 100644 index 0000000..a3a0f14 --- /dev/null +++ b/config/rulesets/code-scanning.json @@ -0,0 +1,34 @@ +{ + "name": "06 - CodeQL security", + "target": "branch", + "enforcement": "active", + "bypass_actors": [ + { + "actor_id": 1, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "rules": [ + { + "type": "code_scanning", + "parameters": { + "code_scanning_tools": [ + { + "tool": "CodeQL", + "security_alerts_threshold": "high_or_higher", + "alerts_threshold": "none" + } + ] + } + } + ] +} diff --git a/config/rulesets/dependency-review.json b/config/rulesets/dependency-review.json new file mode 100644 index 0000000..2fe155b --- /dev/null +++ b/config/rulesets/dependency-review.json @@ -0,0 +1,34 @@ +{ + "name": "07 - Dependency review", + "target": "branch", + "enforcement": "active", + "bypass_actors": [ + { + "actor_id": 1, + "actor_type": "OrganizationAdmin", + "bypass_mode": "pull_request" + } + ], + "conditions": { + "ref_name": { + "exclude": [], + "include": [ + "~DEFAULT_BRANCH" + ] + } + }, + "rules": [ + { + "type": "required_status_checks", + "parameters": { + "required_status_checks": [ + { + "context": "Dependency review", + "integration_id": 15368 + } + ], + "strict_required_status_checks_policy": true + } + } + ] +} diff --git a/config/security-merge-protection.json b/config/security-merge-protection.json new file mode 100644 index 0000000..56ea14b --- /dev/null +++ b/config/security-merge-protection.json @@ -0,0 +1,53 @@ +{ + "dependency_review": [ + "agent-integrations", + "atrinik", + "classic", + "client", + "content", + "content-toolkit", + "deploy-control", + "devcontainer", + "editor", + "github-settings", + "metaserver-worker", + "observatory", + "playtester", + "protocol", + "renderer", + "resources", + "server", + "service-updater", + "sound", + "tools", + "website" + ], + "code_scanning": [ + "agent-integrations", + "atrinik", + "classic", + "client", + "content", + "content-toolkit", + "deploy-control", + "devcontainer", + "editor", + "github-settings", + "metaserver-worker", + "observatory", + "playtester", + "protocol", + "renderer", + "resources", + "server", + "service-updater", + "sound", + "tools", + "website" + ], + "codeql_required_languages": { + "content-toolkit": [ + "rust" + ] + } +} diff --git a/lib/security_readiness.py b/lib/security_readiness.py new file mode 100644 index 0000000..29d09c5 --- /dev/null +++ b/lib/security_readiness.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python3 +"""Read-only preflight for security controls; 0 ready, 1 deferred, 2 unreadable.""" + +import argparse +import ast +import base64 +import binascii +import difflib +import fnmatch +import json +from pathlib import Path +import re +import subprocess +import sys +from urllib.parse import quote, urlencode + + +ROOT = Path(__file__).resolve().parent.parent +VERSION = "2026-03-10" +ACTION_APP = 15368 +SHA = re.compile(r"[0-9a-fA-F]{40}\Z") +COORDINATE = re.compile(r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+\Z") +SIMPLE_NAME = r"(?:[A-Za-z][A-Za-z0-9 ._/-]*|'[A-Za-z][A-Za-z0-9 ._/-]*'|\"[A-Za-z][A-Za-z0-9 ._/-]*\")" +USES = re.compile( + r"(?P[ \t]*(?:-[ \t]+)?uses:[ \t]+)" + r"(?P[A-Za-z0-9_.-]+/[A-Za-z0-9_./-]+)@" + r"(?P[0-9a-fA-F]{40})(?P[ \t]*)" + r"(?P#[ \t]*v?[0-9]+(?:\.[0-9A-Za-z_-]+)*(?:[ \t]*)?)?" + r"(?P\r?\n)?\Z" +) + + +class Deferred(Exception): + pass + + +class EvidenceError(Exception): + pass + + +class APIError(EvidenceError): + def __init__(self, endpoint, status=None): + self.status = status + super().__init__(f"API unreadable: {endpoint.split('?')[0]}" + + (f" (HTTP {status})" if status else "")) + + +def api(endpoint): + """Use GET explicitly: query parameters must never select gh's POST default.""" + try: + result = subprocess.run( + ["gh", "api", "--method", "GET", "-H", f"X-GitHub-Api-Version: {VERSION}", + "-H", "Accept: application/vnd.github+json", endpoint], + capture_output=True, text=True, timeout=30, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise APIError(endpoint) from exc + if result.returncode: + status = re.search(r"HTTP (\d{3})", result.stderr) + raise APIError(endpoint, int(status[1]) if status else None) + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise EvidenceError("API returned invalid JSON") from exc + + +def config(name): + return json.loads((ROOT / "config" / name).read_text()) + + +def require(condition, reason): + if not condition: + raise Deferred(reason) + + +def repository(coordinate): + metadata = api(f"repos/{coordinate}") + if metadata["full_name"].lower() != coordinate.lower(): + raise EvidenceError("repository identity changed") + require(metadata["archived"] is False, "repository archived") + if not isinstance(metadata["default_branch"], str) or not metadata["default_branch"]: + raise EvidenceError("default branch missing") + return metadata + + +def contents(coordinate, path, ref): + endpoint = f"repos/{coordinate}/contents/{path}?{urlencode({'ref': ref})}" + try: + value = api(endpoint) + except APIError as exc: + if exc.status == 404: + raise Deferred(f"{path} missing at required revision") from exc + raise + if value.get("type") != "file" or value.get("encoding") != "base64": + raise EvidenceError("workflow content is not an encoded file") + try: + # GitHub wraps content with newlines; reject other non-base64 bytes. + return base64.b64decode(value["content"].replace("\n", ""), validate=True) + except (binascii.Error, ValueError) as exc: + raise EvidenceError("workflow content encoding invalid") from exc + + +def pages(endpoint, key=None): + """Bound enumeration; missing or oversized evidence cannot imply readiness.""" + results = [] + for page in range(1, 11): + value = api(endpoint + ("&" if "?" in endpoint else "?") + + urlencode({"per_page": 100, "page": page})) + entries = value[key] if key else value + if not isinstance(entries, list): + raise EvidenceError("API collection malformed") + results.extend(entries) + if len(entries) < 100: + if key and "total_count" in value and len(results) != value["total_count"]: + raise EvidenceError("API collection changed during enumeration") + return results + raise Deferred("evidence exceeds bounded enumeration") + + +def latest_dependency_run(coordinate): + try: + value = api(f"repos/{coordinate}/actions/workflows/dependency-review.yml/runs" + "?event=pull_request&per_page=1") + except APIError as exc: + if exc.status == 404: + raise Deferred("dependency-review workflow not landed") from exc + raise + require(bool(value["workflow_runs"]), "dependency-review PR run missing") + return value["workflow_runs"][0] + + +def dependency_contract(data): + """Allow producer-owned immutable pin bumps without weakening the contract. + + Keep indentation and every policy value intact. Ignore blank lines, standalone + comments and trailing whitespace; normalize only this official action's SHA + and its optional same-line version annotation. + """ + lines = [] + pins = 0 + for line in data.decode("utf-8").splitlines(): + if not line.strip() or line.lstrip().startswith("#"): + continue + match = USES.fullmatch(line) + if match and match["action"] == "actions/dependency-review-action": + pins += 1 + line = match["prefix"] + "actions/dependency-review-action@" + if re.fullmatch("name: " + SIMPLE_NAME, line.rstrip()): + line = "name: Dependency review" + lines.append(line.rstrip()) + require(pins == 1, "dependency-review workflow requires one official full-SHA action") + normalized = [] + index = 0 + while index < len(lines): + line = lines[index] + step_name = re.fullmatch(r"( *)- name: " + SIMPLE_NAME, line) + if step_name and index + 1 < len(lines): + following = lines[index + 1] + if following == step_name[1] + " uses: actions/dependency-review-action@": + line = step_name[1] + "- uses: actions/dependency-review-action@" + index += 1 + # Producers may spell the same one-element branch filter as a block + # list or a flow list. Preserve key indentation and reject extra items. + flow = re.fullmatch(r"( *)branches: *\[ *(?:main|'main'|\"main\") *\]", line) + block = re.fullmatch(r"( *)branches:", line) + if flow: + line = flow[1] + "branches: [main]" + elif block and index + 1 < len(lines): + item = re.fullmatch(r"( *)- +(?:main|'main'|\"main\")", lines[index + 1]) + if item and len(item[1]) > len(block[1]): + line = block[1] + "branches: [main]" + index += 1 + normalized.append(line) + index += 1 + return normalized + + +def dependency(coordinate): + metadata = repository(coordinate) + path = ".github/workflows/dependency-review.yml" + desired = dependency_contract((ROOT / path).read_bytes()) + landed = contents(coordinate, path, metadata["default_branch"]) + require(dependency_contract(landed) == desired, + "dependency-review workflow not landed with desired enforcement contract") + run = latest_dependency_run(coordinate) + require(run["event"] == "pull_request" and run["status"] == "completed" + and run["conclusion"] == "success", "latest dependency-review PR run not successful") + require(bool(run["pull_requests"]), "dependency-review PR association missing") + suite_endpoint = f"repos/{coordinate}/check-suites/{run['check_suite_id']}" + suite = api(suite_endpoint) + require(suite["id"] == run["check_suite_id"] and suite["app"]["id"] == ACTION_APP + and suite["status"] == "completed" and suite["conclusion"] == "success", + "dependency-review execution check suite not successful") + executed = suite["head_sha"] + require(bool(SHA.fullmatch(executed)), "dependency-review executed SHA missing") + merge_parents = None + if executed != run["head_sha"]: + merge_commit = api(f"repos/{coordinate}/commits/{executed}") + merge_parents = [parent["sha"] for parent in merge_commit["parents"]] + pulls = [] + for reference in run["pull_requests"]: + pr = api(f"repos/{coordinate}/pulls/{reference['number']}") + require(pr["base"]["repo"]["full_name"].lower() == coordinate.lower() + and pr["base"]["ref"] == metadata["default_branch"], + "dependency-review PR does not target default branch") + require(pr["head"]["sha"] == run["head_sha"], "dependency-review PR head changed") + require(reference["head"]["sha"] == pr["head"]["sha"] + and reference["base"]["sha"] == pr["base"]["sha"] + and reference["base"]["ref"] == pr["base"]["ref"], + "dependency-review run PR association is stale") + require(any(item["number"] == pr["number"] + and item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["sha"] == pr["base"]["sha"] + and item["base"]["ref"] == pr["base"]["ref"] + for item in suite["pull_requests"]), + "dependency-review execution PR association missing or stale") + if merge_parents is not None: + require(len(merge_parents) == 2 and set(merge_parents) + == {pr["head"]["sha"], pr["base"]["sha"]}, + "dependency-review executed merge does not match current PR head/base") + pulls.append(pr) + require(contents(coordinate, path, executed) == landed, + "dependency-review run used different workflow bytes") + jobs = pages(f"repos/{coordinate}/actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs", "jobs") + matches = [job for job in jobs if job["name"] == "Dependency review"] + require(len(matches) == 1 and matches[0]["status"] == "completed" + and matches[0]["conclusion"] == "success", "Dependency review job not successful") + fresh = latest_dependency_run(coordinate) + run_keys = ("id", "run_attempt", "status", "conclusion", "event", "head_sha", + "check_suite_id", "pull_requests") + require(all(fresh[key] == run[key] for key in run_keys), + "dependency-review run changed during preflight") + fresh_suite = api(suite_endpoint) + suite_keys = ("id", "head_sha", "app", "status", "conclusion", "pull_requests") + require(all(fresh_suite[key] == suite[key] for key in suite_keys), + "dependency-review execution check suite changed during preflight") + for pr in pulls: + fresh_pr = api(f"repos/{coordinate}/pulls/{pr['number']}") + require(fresh_pr["head"]["sha"] == pr["head"]["sha"] + and fresh_pr["base"]["sha"] == pr["base"]["sha"] + and fresh_pr["base"]["ref"] == pr["base"]["ref"] + and fresh_pr["base"]["repo"]["full_name"] == pr["base"]["repo"]["full_name"] + and all(fresh_pr[key] == pr[key] for key in ("state", "draft", "merged")), + "dependency-review PR changed during preflight") + require(contents(coordinate, path, metadata["default_branch"]) == landed, + "default-branch dependency-review workflow changed during preflight") + require(repository(coordinate)["default_branch"] == metadata["default_branch"], + "repository default branch changed during preflight") + return "workflow enforcement contract landed; latest PR job successful with same workflow bytes" + + +def pin_changes(before, after, selected): + """Compare complete files, so API's potentially truncated patches grant nothing.""" + old = before.decode("utf-8").splitlines(keepends=True) + new = after.decode("utf-8").splitlines(keepends=True) + changed = 0 + for tag, old_start, old_end, new_start, new_end in difflib.SequenceMatcher( + None, old, new, autojunk=False).get_opcodes(): + if tag == "equal": + continue + require(tag == "replace" and old_end - old_start == new_end - new_start, + "canary changes more than action pins") + for left, right in zip(old[old_start:old_end], new[new_start:new_end]): + left_match, right_match = USES.fullmatch(left), USES.fullmatch(right) + require(bool(left_match and right_match), "canary changes more than action pins") + for key in ("prefix", "action", "space", "newline"): + require(left_match[key] == right_match[key], "canary changes action structure") + require(left_match["sha"] != right_match["sha"], "canary contains unrelated comment changes") + action = right_match["action"] + identity = f"{action}@{right_match['sha']}" + require((selected["github_owned_allowed"] and action.split("/")[0].lower() + in {"actions", "github"}) + or any(fnmatch.fnmatchcase(identity, pattern) + for pattern in selected["patterns_allowed"]), + "new action pin is outside desired selected-actions patterns") + changed += 1 + require(changed > 0, "canary has no full-SHA action update") + return changed + + +def merge_check_run(coordinate, check, pr, merges): + if check["head_sha"] == pr["head"]["sha"]: + return True + if check["head_sha"] not in merges: + return False + value = api(f"repos/{coordinate}/actions/runs?check_suite_id={check['check_suite']['id']}&per_page=1") + runs = value["workflow_runs"] + return bool(len(runs) == 1 and runs[0]["event"] == "pull_request" + and runs[0]["head_sha"] == pr["head"]["sha"] + and any(item["number"] == pr["number"] for item in runs[0]["pull_requests"])) + + +def sha(coordinate, number): + metadata = repository(coordinate) + pr = api(f"repos/{coordinate}/pulls/{number}") + require(pr["merged"] is True or (pr["state"] == "open" and not pr["draft"]), + "action-update canary must be merged or open and review-ready") + require(pr["base"]["repo"]["full_name"].lower() == coordinate.lower() + and pr["base"]["ref"] == metadata["default_branch"], + "canary does not target default branch") + require(bool(SHA.fullmatch(pr["head"]["sha"])), "canary head SHA missing") + selected = config("actions-selected.json") + live = api(f"orgs/{coordinate.split('/')[0]}/actions/permissions/selected-actions") + require(selected["github_owned_allowed"] is True and selected["verified_allowed"] is False, + "desired selected-actions policy is not conservative") + require(live["github_owned_allowed"] is True and live["verified_allowed"] is False + and sorted(live["patterns_allowed"]) == sorted(selected["patterns_allowed"]), + "live selected-actions policy differs from desired policy") + required = config("repositories.json")["required_ci"].get(coordinate.split('/')[1]) + require(bool(required), "canary has no configured required contexts") + files = pages(f"repos/{coordinate}/pulls/{number}/files") + require(bool(files) and len(files) == pr["changed_files"], "canary file evidence incomplete") + updates = 0 + for item in files: + path = item["filename"] + require(item["status"] == "modified" and + re.fullmatch(r"\.github/workflows/[^/]+\.ya?ml", path) is not None, + "canary contains changes outside existing workflows") + updates += pin_changes(contents(coordinate, path, pr["base"]["sha"]), + contents(pr["head"]["repo"]["full_name"], path, pr["head"]["sha"]), selected) + merge = pr.get("merge_commit_sha") if not pr["merged"] else None + merges = set() + if merge: + require(bool(SHA.fullmatch(merge)), "canary merge SHA malformed") + commit = api(f"repos/{coordinate}/commits/{merge}") + require({parent["sha"] for parent in commit["parents"]} + == {pr["base"]["sha"], pr["head"]["sha"]}, "canary merge SHA is stale") + merges.add(merge) + if pr["merged"]: + # After squash/rebase merge, merge_commit_sha identifies publication, + # not the temporary merge commit that PR Actions checks evaluated. + runs = pages(f"repos/{coordinate}/actions/runs?event=pull_request&head_sha={pr['head']['sha']}", + "workflow_runs") + for run in runs: + references = [item for item in run["pull_requests"] if item["number"] == number] + if run["event"] != "pull_request" or run["head_sha"] != pr["head"]["sha"] or not references: + continue + suite = api(f"repos/{coordinate}/check-suites/{run['check_suite_id']}") + candidate = suite["head_sha"] + if candidate == pr["head"]["sha"]: + continue + require(bool(SHA.fullmatch(candidate)), "historical canary merge SHA malformed") + commit = api(f"repos/{coordinate}/commits/{candidate}") + require(any(item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["ref"] == metadata["default_branch"] + and {parent["sha"] for parent in commit["parents"]} + == {item["base"]["sha"], pr["head"]["sha"]} for item in references), + "historical canary merge does not match PR head/base") + merges.add(candidate) + checks = [] + for ref in [pr["head"]["sha"], *sorted(merges)]: + checks.extend(pages(f"repos/{coordinate}/commits/{ref}/check-runs?filter=all&app_id={ACTION_APP}", + "check_runs")) + for context in required: + candidates = [check for check in checks if check["name"] == context + and check["app"]["id"] == ACTION_APP + and check["app"]["slug"] == "github-actions" + and merge_check_run(coordinate, check, pr, merges)] + require(bool(candidates), f"required check missing for canary head: {context}") + latest = max(candidates, key=lambda check: (check.get("started_at") or "", check["id"])) + require(latest["status"] == "completed" and latest["conclusion"] == "success", + f"required check not successful for canary head: {context}") + fresh = api(f"repos/{coordinate}/pulls/{number}") + require(fresh["head"]["sha"] == pr["head"]["sha"] + and fresh["base"]["sha"] == pr["base"]["sha"] + and fresh.get("merge_commit_sha") == pr.get("merge_commit_sha") + and fresh["state"] == pr["state"] and fresh["merged"] == pr["merged"] + and (fresh["merged"] or not fresh["draft"]), "canary changed during preflight") + fresh_live = api(f"orgs/{coordinate.split('/')[0]}/actions/permissions/selected-actions") + require(fresh_live["github_owned_allowed"] is True and fresh_live["verified_allowed"] is False + and sorted(fresh_live["patterns_allowed"]) == sorted(selected["patterns_allowed"]), + "live selected-actions policy changed during preflight") + require(repository(coordinate)["default_branch"] == metadata["default_branch"], + "repository default branch changed during preflight") + return f"ordinary action-update PR #{number}: {updates} allowed full-SHA updates; required checks passed" + + +def canonical_language(value): + return {"javascript/typescript": "javascript-typescript", "javascript": "javascript-typescript", + "typescript": "javascript-typescript"}.get(value, value) + + +def analysis_language(analysis): + environment = json.loads(analysis.get("environment") or "{}") + language = environment.get("language") + if not language: + match = re.search(r"(?:^|/)language:([a-z0-9+-]+)(?:/|$)", analysis["category"]) + language = match[1] if match else None + return canonical_language(language), environment + + +def successful_analysis(analysis, head): + require(analysis["commit_sha"] == head, "latest CodeQL analysis does not cover current default head") + require(analysis["error"] in ("", None) and analysis["rules_count"] > 0, + "latest CodeQL analysis failed or contains no executed rules") + + +def newest_analysis_pages(coordinate, branch): + endpoint = f"repos/{coordinate}/code-scanning/analyses?" + urlencode( + {"ref": f"refs/heads/{branch}", "tool_name": "CodeQL"}) + for page in range(1, 11): + entries = api(endpoint + f"&per_page=100&page={page}") + if not isinstance(entries, list): + raise EvidenceError("CodeQL analysis collection malformed") + yield entries + if len(entries) < 100: + return + + +def default_codeql(coordinate, branch, head, minima): + endpoint = f"repos/{coordinate}/code-scanning/default-setup" + setup = api(endpoint) + require(setup["state"] == "configured", "CodeQL default setup not configured") + languages = {canonical_language(value) for value in setup["languages"]} + require(bool(languages), "CodeQL default setup has no configured languages") + require(set(minima) <= languages, "required CodeQL languages not configured") + found = set() + for entries in newest_analysis_pages(coordinate, branch): + for analysis in entries: + if analysis["tool"]["name"] != "CodeQL" or analysis["ref"] != f"refs/heads/{branch}": + continue + language, _ = analysis_language(analysis) + # Default setup owns one canonical category per configured language. + # Retired advanced partitions, even in that language, are historical. + category_language = analysis["category"].removeprefix("/language:") + if (not analysis["category"].startswith("/language:") + or canonical_language(category_language) != language + or language not in languages or language in found): + continue + successful_analysis(analysis, head) + found.add(language) + if found == languages: + fresh = api(endpoint) + require(fresh["state"] == setup["state"] + and {canonical_language(value) for value in fresh["languages"]} == languages, + "CodeQL default setup changed during preflight") + return "latest canonical default-setup analyses cover every configured language on current head" + raise Deferred("required configured CodeQL language analysis missing within bounded evidence") + + +def classifier_full_events(source): + """Read the owner's full-event contract without executing fetched Python.""" + tree = ast.parse(source) + assignments = [node for node in tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "FULL_EVENTS" for target in node.targets)] + require(len(assignments) == 1, "advanced CodeQL classifier full-event contract missing") + events = ast.literal_eval(assignments[0].value) + mains = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "main"] + require(len(mains) == 1, "advanced CodeQL classifier main contract missing") + full = False + for node in ast.walk(mains[0]): + if (isinstance(node, ast.If) and isinstance(node.test, ast.Compare) + and ast.unparse(node.test) == "arguments.event in FULL_EVENTS"): + full = any(isinstance(item, ast.Assign) and isinstance(item.value, ast.Constant) + and item.value.value is True and any(isinstance(target, ast.Name) + and target.id == "full" for target in item.targets) for item in node.body) + require(full and {"push", "schedule", "workflow_dispatch"} <= set(events), + "advanced CodeQL classifier does not select full analysis for full events") + return set(events) & {"push", "schedule", "workflow_dispatch"} + + +def latest_full_codeql_run(coordinate, branch, head, events): + value = api(f"repos/{coordinate}/actions/workflows/codeql.yml/runs?head_sha={head}&per_page=100") + candidates = [run for run in value["workflow_runs"] if run["event"] in events + and run["head_sha"] == head and run["head_branch"] == branch] + require(bool(candidates), "current-head full-event CodeQL workflow run missing") + run = candidates[0] + require(run["path"] == ".github/workflows/codeql.yml" and run["status"] == "completed" + and run["conclusion"] == "success", "latest full-event CodeQL workflow run not successful") + return run + + +def advanced_codeql(coordinate, branch, head, minima): + sources = {} + for path in (".github/workflows/codeql.yml", "tools/ci/classify_changes.py", "tools/ci/require_checks.py"): + sources[path] = contents(coordinate, path, branch) + require(contents(coordinate, path, head) == sources[path], "advanced CodeQL producer source is stale") + workflow = sources[".github/workflows/codeql.yml"].decode() + for marker in ("name: CodeQL analysis (${{ matrix.display }})", "languages: ${{ matrix.language }}", + "category: ${{ matrix.category }}", "python3 tools/ci/classify_changes.py", + "python3 tools/ci/require_checks.py optional", "${{ needs.classify.outputs.run }}", + "${{ needs.analyze.result }}"): + require(marker in workflow, "advanced CodeQL workflow producer contract unsupported") + events = classifier_full_events(sources["tools/ci/classify_changes.py"]) + run = latest_full_codeql_run(coordinate, branch, head, events) + jobs_endpoint = f"repos/{coordinate}/actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs" + jobs = pages(jobs_endpoint, "jobs") + require(bool(jobs) and all(job["head_sha"] == head and job["status"] == "completed" + and job["conclusion"] == "success" for job in jobs), + "advanced CodeQL full-run jobs not all successful on current head") + require(len([job for job in jobs if job["name"] == "CodeQL classification"]) == 1, + "advanced CodeQL classification job missing") + validation = [job for job in jobs if job["name"] == "CodeQL validation"] + require(len(validation) == 1, "advanced CodeQL validation job missing") + expected_url = f"https://api.github.com/repos/{coordinate}/check-runs/" + check_url = validation[0]["check_run_url"] + require(check_url.startswith(expected_url) and check_url[len(expected_url):].isdigit(), + "advanced CodeQL validation check identity missing") + check_endpoint = f"repos/{coordinate}/check-runs/{check_url[len(expected_url):]}" + check = api(check_endpoint) + require(check["name"] == "CodeQL validation" and check["head_sha"] == head + and check["app"]["id"] == ACTION_APP and check["app"]["slug"] == "github-actions" + and check["check_suite"]["id"] == run["check_suite_id"] + and check["status"] == "completed" and check["conclusion"] == "success", + "advanced CodeQL validation check is not the trusted current-head producer") + selected = {} + for job in jobs: + match = re.fullmatch(r"CodeQL analysis \((.+)\)", job["name"]) + if match: + require(match[1] not in selected, "advanced CodeQL analysis jobs duplicated") + selected[match[1]] = job + require(bool(selected), "advanced CodeQL full run selected no analysis jobs") + found, categories, languages = set(), set(), set() + for entries in newest_analysis_pages(coordinate, branch): + for analysis in entries: + if (analysis["tool"]["name"] != "CodeQL" or analysis["ref"] != f"refs/heads/{branch}" + or analysis.get("analysis_key") != ".github/workflows/codeql.yml:analyze"): + continue + category = analysis["category"] + if category in categories: + continue + categories.add(category) + language, environment = analysis_language(analysis) + display = environment.get("display") + if display not in selected: + continue + job = selected[display] + if analysis["created_at"] < job["started_at"]: + continue + require(display not in found, "advanced CodeQL selected display has ambiguous upload categories") + successful_analysis(analysis, head) + category_language = re.search(r"(?:^|/)language:([a-z0-9+-]+)$", category) + require(environment.get("category") == category and category_language is not None + and canonical_language(category_language[1]) == language + and job["started_at"] <= analysis["created_at"] <= job["completed_at"], + "advanced CodeQL analysis upload does not belong to selected full-run job") + found.add(display) + languages.add(language) + if found == set(selected): + break + if found == set(selected): + require({canonical_language(value) for value in minima} <= languages, + "required CodeQL language analysis missing") + fresh = latest_full_codeql_run(coordinate, branch, head, events) + require(all(fresh[key] == run[key] for key in + ("id", "run_attempt", "head_sha", "head_branch", "event", "path", "check_suite_id")), + "advanced CodeQL full run changed during preflight") + require(api(check_endpoint) == check, "advanced CodeQL validation check changed during preflight") + require(pages(jobs_endpoint, "jobs") == jobs, "advanced CodeQL full-run jobs changed during preflight") + for path, source in sources.items(): + require(contents(coordinate, path, branch) == source, + "advanced CodeQL producer source changed during preflight") + return "current full-event CodeQL producer and every selected partition upload successful" + raise Deferred("required advanced CodeQL partition upload missing within bounded evidence") + + +def codeql(coordinate): + metadata = repository(coordinate) + branch = metadata["default_branch"] + branch_endpoint = f"repos/{coordinate}/branches/{quote(branch, safe='')}" + head = api(branch_endpoint)["commit"]["sha"] + require(bool(SHA.fullmatch(head)), "default-branch head missing") + required = config("security-merge-protection.json")["codeql_required_languages"].get( + coordinate.split('/')[1], []) + advanced = config("codeql-advanced-setup.json")["repositories"] + if coordinate.split('/')[1] in advanced: + reason = advanced_codeql(coordinate, branch, head, required) + else: + reason = default_codeql(coordinate, branch, head, [canonical_language(value) for value in required]) + require(api(branch_endpoint)["commit"]["sha"] == head, "default branch changed during CodeQL preflight") + require(repository(coordinate)["default_branch"] == branch, + "repository default branch changed during preflight") + return reason + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=("dependency", "sha", "codeql")) + parser.add_argument("target", help="OWNER/REPO, or OWNER/REPO/PR for sha") + args = parser.parse_args(argv) + try: + parts = args.target.split("/") + coordinate = "/".join(parts[:2]) + if not COORDINATE.fullmatch(coordinate) or len(parts) != (3 if args.mode == "sha" else 2): + raise EvidenceError("invalid repository or PR coordinate") + if args.mode == "sha": + if not parts[2].isdigit() or int(parts[2]) < 1: + raise EvidenceError("invalid PR number") + reason = sha(coordinate, int(parts[2])) + else: + reason = {"dependency": dependency, "codeql": codeql}[args.mode](coordinate) + print(f"ready: {reason}") + return 0 + except Deferred as exc: + print(f"deferred: {exc}") + return 1 + except (EvidenceError, OSError, ValueError, KeyError, TypeError, AttributeError, SyntaxError) as exc: + reason = str(exc) if isinstance(exc, EvidenceError) else "local or API evidence malformed/unreadable" + print(f"error: {reason}") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py new file mode 100755 index 0000000..5c04c3c --- /dev/null +++ b/tests/audit-fixtures.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Read-only GitHub API double for the effective-governance audit tests.""" +import copy +import json +import os +import sys +from pathlib import Path + +root = Path(os.environ['AUDIT_CONFIG_ROOT']) +scenario = os.environ.get('GH_TEST_SCENARIO', 'pass') + + +def config(name): + return json.loads((root / 'config' / name).read_text()) + + +def reject(message): + print(message, file=sys.stderr) + sys.exit(2) + + +args = sys.argv[1:] +if not args or args.pop(0) != 'api': + reject('API only') +slurp = False +endpoint = None +while args: + arg = args.pop(0) + if arg == '--method': + if args.pop(0) != 'GET': + reject('audit attempted API mutation') + elif arg == '-H': + args.pop(0) + elif arg == '--paginate': + pass + elif arg == '--slurp': + slurp = True + elif arg.startswith('-'): + reject(f'unexpected or mutating API flag: {arg}') + else: + endpoint = arg +if endpoint is None: + reject('endpoint missing') +with open(os.environ['AUDIT_API_LOG'], 'a', encoding='utf-8') as log: + log.write(endpoint + '\n') + +properties = config('repository-properties.json') +repositories = config('repositories.json') +policy = config('security-merge-protection.json') +active = sorted(name for name, value in properties['repositories'].items() + if value['lifecycle'] != 'archived') + + +def rulesets(repository): + specs = [('default-branch-integrity', None), ('default-branch-linear-history', None)] + if repository in repositories['pull_request_gate']: + specs.append(('pull-request-gate', None)) + if repository in repositories['required_ci']: + specs.append(('required-ci', '03 - Required CI - ' + repository)) + if repository in repositories['release_tags']: + specs.append(('immutable-release-tags', None)) + if repository in policy['code_scanning'] and scenario not in ('pending-codeql', 'ready-missing-codeql', 'codeql-readiness-unreadable'): + specs.append(('code-scanning', None)) + if repository in policy['dependency_review'] and scenario not in ('pending', 'ready-missing', 'dependency-readiness-unreadable'): + specs.append(('dependency-review', None)) + result = [] + for index, (template, name) in enumerate(specs, 1): + item = config('rulesets/' + template + '.json') + if name: + item['name'] = name + item['rules'][0]['parameters']['required_status_checks'] = [ + {'context': context, 'integration_id': 15368} + for context in repositories['required_ci'][repository]] + item.update(id=active.index(repository) * 10 + index, + source_type='Repository' if scenario == 'repository-rulesets' else 'Organization') + # Match GitHub's actual owner representation, including repository rulesets. + for actor in item['bypass_actors']: + if actor['actor_type'] == 'OrganizationAdmin': + actor['actor_id'] = None + if scenario == 'owner-id-drift' and repository == 'observatory' and template == 'pull-request-gate': + item['bypass_actors'][0]['actor_id'] = 2 + item['conditions']['repository_name'] = { + 'include': [repository], 'exclude': [], 'protected': False} + # Keep inherited names while making actual targeting ineffective. + if scenario == 'wrong-pr-scope' and repository == '.github' and template == 'pull-request-gate': + item['conditions']['repository_name']['include'] = ['client'] + if scenario == 'pending-wrong-dependency-scope' and repository == 'observatory' and template == 'dependency-review': + item['conditions']['repository_name']['exclude'] = ['observatory'] + if scenario == 'missing-tag-scope' and repository == 'service-updater' and template == 'immutable-release-tags': + item['conditions']['repository_name']['exclude'] = ['service-updater'] + if scenario == 'tag-ref-drift' and repository == 'service-updater' and template == 'immutable-release-tags': + item['conditions']['ref_name']['include'] = ['refs/tags/unrelated*'] + if scenario == 'integrity-bypass' and repository == 'observatory' and template == 'default-branch-integrity': + item['bypass_actors'] = [{'actor_type': 'OrganizationAdmin', 'actor_id': 1, 'bypass_mode': 'always'}] + if scenario == 'pr-bypass' and repository == 'observatory' and template == 'pull-request-gate': + item['bypass_actors'][0]['bypass_mode'] = 'always' + if scenario == 'codeql-threshold' and repository == 'observatory' and template == 'code-scanning': + item['rules'][0]['parameters']['code_scanning_tools'][0]['security_alerts_threshold'] = 'critical' + if scenario == 'codeql-tool' and repository == 'observatory' and template == 'code-scanning': + item['rules'][0]['parameters']['code_scanning_tools'][0]['tool'] = 'other-scanner' + if scenario == 'codeql-quality' and repository == 'observatory' and template == 'code-scanning': + item['rules'][0]['parameters']['code_scanning_tools'][0]['alerts_threshold'] = 'errors' + result.append(item) + return result + + +payload = None +if endpoint.startswith('orgs/atrinik/repos?'): + if scenario == 'inventory-unreadable': + reject('inventory read denied') + payload = [dict(name=name, archived=value['lifecycle'] == 'archived', private=False, + visibility='public', disabled=False, fork=False, has_issues=True, + has_projects=True, has_wiki=True) + for name, value in properties['repositories'].items()] + if scenario == 'unmanaged': + payload.append(dict(name='unmanaged', archived=False, private=False, visibility='public', + disabled=False, fork=False, has_issues=True, has_projects=True, has_wiki=True)) + if scenario == 'temporary': + payload.append(dict(name='classic-ghsa-8533-3vg8-r287', archived=False, private=True, + visibility='private', disabled=False, fork=False, + has_issues=False, has_projects=False, has_wiki=False)) +elif endpoint == 'orgs/atrinik/properties/schema': + payload = properties['definitions'] + if scenario == 'schema-drift': + payload[0]['description'] = 'drift' +elif endpoint.startswith('orgs/atrinik/actions/permissions'): + suffix = endpoint.removeprefix('orgs/atrinik/actions/permissions') + filename = {'': 'actions-permissions.json', '/workflow': 'actions-workflow.json', + '/selected-actions': 'actions-selected.json'}[suffix] + if scenario == 'actions-unreadable': + reject('Actions read denied') + payload = config(filename) + if filename == 'actions-permissions.json': + payload['sha_pinning_required'] = scenario != 'pending' + if scenario == 'sha-unreadable': + del payload['sha_pinning_required'] + if filename == 'actions-workflow.json': + if scenario == 'actions-write-token': + payload['default_workflow_permissions'] = 'write' + if scenario == 'actions-approval': + payload['can_approve_pull_request_reviews'] = True + if filename == 'actions-selected.json' and scenario == 'actions-allowlist': + payload['patterns_allowed'].append('*') +elif endpoint.startswith('orgs/atrinik/rulesets/'): + identifier = int(endpoint.rsplit('/', 1)[1]) + payload = next((item for repository in active for item in rulesets(repository) + if item['id'] == identifier), None) + if payload is None or scenario == 'ruleset-unreadable': + reject('ruleset read denied') +elif endpoint.startswith('repos/atrinik/'): + repository, _, path = endpoint.removeprefix('repos/atrinik/').partition('/') + if repository not in active: + reject('audit attempted to inspect archived or foreign repository') + target = repository == 'observatory' + if path == 'actions/permissions/workflow': + if scenario == 'repository-actions-unreadable' and target: + reject('repository workflow permissions read denied') + payload = config('actions-workflow.json') + if scenario == 'repository-write-token' and target: + payload['default_workflow_permissions'] = 'write' + if scenario == 'repository-approval' and target: + payload['can_approve_pull_request_reviews'] = True + elif path == 'properties/values': + payload = [{'property_name': key, 'value': value} + for key, value in properties['repositories'][repository].items()] + if scenario == 'properties-drift' and target: + payload[0]['value'] = 'drift' + elif path.startswith('rulesets?'): + payload = [{'name': item['name'], 'id': item['id'], 'source_type': item['source_type']} + for item in rulesets(repository)] + elif path.startswith('rulesets/'): + identifier = int(path.rsplit('/', 1)[1]) + payload = next((item for item in rulesets(repository) if item['id'] == identifier), None) + if payload is None: + reject('repository ruleset missing') + elif path.startswith('rules/branches/'): + if scenario == 'effective-unreadable' and target: + reject('effective branch read denied') + payload = [] + for item in rulesets(repository): + scope = item['conditions']['repository_name'] + if item['target'] == 'branch' and repository in scope['include'] and repository not in scope['exclude']: + payload.extend(copy.deepcopy(item['rules'])) + for rule in payload: + if target and rule['type'] == 'required_status_checks': + if scenario == 'wrong-app': + rule['parameters']['required_status_checks'][0]['integration_id'] = 9 + if scenario == 'wrong-context': + rule['parameters']['required_status_checks'][0]['context'] += ' drift' + if scenario == 'strict-ci': + rule['parameters']['strict_required_status_checks_policy'] = False + if target and rule['type'] == 'pull_request': + if scenario == 'approval-count': + rule['parameters']['required_approving_review_count'] = 1 + if scenario == 'thread-resolution': + rule['parameters']['required_review_thread_resolution'] = False + if scenario == 'effective-integrity' and target: + payload = [item for item in payload if item['type'] != 'deletion'] + if scenario == 'unexpected-codeql' and repository == '.github': + payload.extend(config('rulesets/code-scanning.json')['rules']) + if scenario == 'unexpected-pr' and repository == 'web-platform': + payload.append(config('rulesets/pull-request-gate.json')['rules'][0]) + elif path == 'code-security-configuration': + advanced = repository in config('codeql-advanced-setup.json')['repositories'] + if scenario == 'configuration-unreadable' and target: + reject('configuration read denied') + payload = {'status': 'enforced', 'configuration': config( + 'code-security-advanced.json' if advanced else 'code-security.json')} + if scenario == 'legacy-security': + payload['configuration'].pop('code_security') + payload['configuration'].pop('secret_protection') + payload['configuration']['advanced_security'] = 'enabled' + if scenario == 'configuration-detached' and target: + payload['status'] = 'attached' + if scenario == 'dependency-graph' and target: + payload['configuration']['dependency_graph'] = 'disabled' + if scenario == 'advanced-config' and repository == 'classic': + payload['configuration'] = config('code-security.json') + elif path == 'code-scanning/default-setup': + advanced = repository in config('codeql-advanced-setup.json')['repositories'] + if scenario == 'setup-unreadable' and target: + reject('setup read denied') + payload = config('codeql-default-setup-advanced.json' if advanced else 'codeql-default-setup.json') + payload['languages'] = policy['codeql_required_languages'].get(repository, []) + if scenario == 'missing-rust' and repository == 'content-toolkit': + payload['languages'] = ['javascript-typescript'] + if scenario == 'advanced-setup' and repository == 'classic': + payload['state'] = 'configured' + if scenario == 'default-setup' and target: + payload['state'] = 'not-configured' + elif path == 'vulnerability-alerts': + if scenario == 'dependabot-alerts' and target: + reject('alerts disabled') + sys.exit(0) # GitHub's enabled response is HTTP 204, without JSON. + elif path == 'private-vulnerability-reporting': + payload = {'enabled': not (scenario == 'private-reporting' and target)} + elif path == '': + payload = dict(config('repository-defaults.json'), id=active.index(repository) + 1, + name=repository, archived=False, security_and_analysis={ + name: {'status': 'enabled'} for name in ( + 'secret_scanning', 'secret_scanning_push_protection', + 'secret_scanning_validity_checks', 'dependabot_security_updates')}) + if scenario == 'merge-drift' and target: + payload['allow_merge_commit'] = True + if scenario == 'secret-baseline' and target: + payload['security_and_analysis']['secret_scanning']['status'] = 'disabled' + else: + reject('unexpected repository endpoint: ' + endpoint) +else: + reject('unexpected endpoint: ' + endpoint) +print(json.dumps([payload] if slurp else payload)) diff --git a/tests/audit.sh b/tests/audit.sh index d34373c..390da1d 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -5,228 +5,160 @@ set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) temporary=$(mktemp -d) trap 'rm -rf "${temporary}"' EXIT -mkdir "${temporary}/bin" - -cat >"${temporary}/bin/gh" <<'EOF' +mkdir -p "${temporary}/mock" "${temporary}/full/bin" +cp -R "${root}/config" "${temporary}/full/config" +cp "${root}/bin/audit" "${temporary}/full/bin/audit" +# Schema validation has its own policy tests; these isolate live API evidence. +printf '#!/usr/bin/env bash\nexit 0\n' >"${temporary}/full/bin/validate" +cat >"${temporary}/full/bin/security-readiness" <<'EOF' #!/usr/bin/env bash - -set -euo pipefail - -[[ ${1:-} == api ]] || exit 1 -shift -jq_filter= -endpoint= -while (($#)); do - case $1 in - -H) - shift 2 - ;; - --paginate) - shift - ;; - --jq) - jq_filter=$2 - shift 2 - ;; - *) - endpoint=$1 - shift - ;; - esac -done - -emit() { - local payload=$1 - if [[ -n ${jq_filter} ]]; then - jq -r "${jq_filter}" <<<"${payload}" - else - printf '%s\n' "${payload}" - fi -} - -config_root=${AUDIT_CONFIG_ROOT:?} -scenario=${GH_TEST_SCENARIO:-pass} - -case ${endpoint} in -orgs/atrinik/properties/schema) - if [[ ${scenario} == schema-drift ]]; then - jq -n --slurpfile config \ - "${config_root}/config/repository-properties.json" \ - '$config[0].definitions | map(. + {description: "drift"})' - else - jq -n --slurpfile config \ - "${config_root}/config/repository-properties.json" \ - '$config[0].definitions' - fi - ;; -orgs/atrinik/repos\?*) - payload=$(jq -n --slurpfile config "${config_root}/config/repository-properties.json" ' - [ - $config[0].repositories | - to_entries[] | - { - name: .key, - archived: (.value.lifecycle == "archived"), - private: false, - visibility: "public", - disabled: false, - fork: false, - has_issues: true, - has_projects: true, - has_wiki: true - } - ] | - if $ENV.GH_TEST_SCENARIO == "unmanaged" then - . + [{ - name: "unmanaged", - archived: false, - private: false, - visibility: "public", - disabled: false, - fork: false, - has_issues: true, - has_projects: true, - has_wiki: true - }] - elif $ENV.GH_TEST_SCENARIO == "temporary" then - . + [{ - name: "classic-ghsa-8533-3vg8-r287", - archived: false, - private: true, - visibility: "private", - disabled: false, - fork: false, - has_issues: false, - has_projects: false, - has_wiki: false - }] - else - . - end - ') - emit "${payload}" - ;; -repos/atrinik/*) - path=${endpoint#repos/atrinik/} - case ${path} in - */properties/values) - repository=${path%/properties/values} - if [[ ${scenario} == properties-drift && ${repository} == observatory ]]; then - jq -n ' - [ - {property_name: "component_role", value: ["unclassified"]}, - {property_name: "lifecycle", value: "seed"}, - {property_name: "provider_set", value: ["unclassified"]}, - {property_name: "release_policy", value: "none"} - ] - ' - else - jq -n --slurpfile config "${config_root}/config/repository-properties.json" \ - --arg repository "${repository}" ' - $config[0].repositories[$repository] | - to_entries | - map({property_name: .key, value: .value}) - ' - fi - ;; - */rulesets\?includes_parents=true) - repository=${path%/rulesets?includes_parents=true} - jq -n --slurpfile repositories "${config_root}/config/repositories.json" \ - --arg repository "${repository}" ' - [ - "01 - Default branch integrity", - "01 - Default branch linear history" - ] + - (if ($repositories[0].pull_request_gate | index($repository)) != null then - ["02 - Changes through pull requests"] - else [] end) + - (if ($repositories[0].required_ci | has($repository)) then - ["03 - Required CI - " + $repository] - else [] end) + - (if ($repositories[0].release_tags | index($repository)) != null then - ["04 - Immutable release tags"] - else [] end) | - map({name: .}) - ' - ;; - *) - repository=${path} - allow_merge_commit=false - if [[ ${scenario} == merge-drift && ${repository} == observatory ]]; then - allow_merge_commit=true - fi - jq -n --slurpfile defaults "${config_root}/config/repository-defaults.json" \ - --arg repository "${repository}" --argjson allow_merge_commit ${allow_merge_commit} ' - { - id: 1, - name: $repository, - archived: false, - default_branch: $defaults[0].default_branch, - allow_merge_commit: $allow_merge_commit, - allow_rebase_merge: $defaults[0].allow_rebase_merge, - allow_squash_merge: $defaults[0].allow_squash_merge, - delete_branch_on_merge: $defaults[0].delete_branch_on_merge, - squash_merge_commit_title: $defaults[0].squash_merge_commit_title, - squash_merge_commit_message: $defaults[0].squash_merge_commit_message, - security_and_analysis: { - secret_scanning: {status: "enabled"}, - secret_scanning_push_protection: {status: "enabled"}, - secret_scanning_validity_checks: {status: "enabled"}, - dependabot_security_updates: {status: "enabled"} - } - } - ' - ;; - esac - ;; -*) - echo "unexpected endpoint: ${endpoint}" >&2 - exit 1 - ;; +case ${GH_TEST_SCENARIO:-pass} in + pending|pending-codeql|pending-wrong-dependency-scope) exit 1 ;; + readiness-unreadable|dependency-readiness-unreadable|codeql-readiness-unreadable) exit 2 ;; + *) exit 0 ;; esac EOF -chmod +x "${temporary}/bin/gh" +chmod +x "${temporary}/full/bin/validate" "${temporary}/full/bin/security-readiness" +cp "${root}/tests/audit-fixtures.py" "${temporary}/mock/gh" +chmod +x "${temporary}/mock/gh" +# Keep standalone tests runnable while publisher-owned schema is being delivered. +# Once integrated these files are copied directly from authoritative policy. +if [[ ! -f ${temporary}/full/config/security-merge-protection.json ]]; then + jq '{dependency_review: (.required_ci | keys), code_scanning: (.required_ci | keys), + codeql_required_languages: {"content-toolkit": ["rust"]}}' \ + "${temporary}/full/config/repositories.json" >"${temporary}/full/config/security-merge-protection.json" + jq '.name = "06 - CodeQL security" | .rules = [{type: "code_scanning", parameters: { + code_scanning_tools: [{tool: "CodeQL", security_alerts_threshold: "high_or_higher", alerts_threshold: "none"}]}}]' \ + "${temporary}/full/config/rulesets/required-ci.json" >"${temporary}/full/config/rulesets/code-scanning.json" + jq '.name = "07 - Dependency review" | .rules[0].parameters.required_status_checks = + [{context: "Dependency review", integration_id: 15368}]' \ + "${temporary}/full/config/rulesets/required-ci.json" >"${temporary}/full/config/rulesets/dependency-review.json" +fi +jq '.sha_pinning_required = true' "${temporary}/full/config/actions-permissions.json" \ + >"${temporary}/actions.json" +mv "${temporary}/actions.json" "${temporary}/full/config/actions-permissions.json" +jq '.pull_request_gate |= ((. + [".github"]) | unique)' "${temporary}/full/config/repositories.json" \ + >"${temporary}/repositories.json" +mv "${temporary}/repositories.json" "${temporary}/full/config/repositories.json" +for repository in .github classic observatory content-toolkit service-updater web-platform; do + fixture="${temporary}/fixture-${repository}" + cp -R "${temporary}/full" "${fixture}" + jq --arg repository "${repository}" \ + '.repositories |= with_entries(select(.key == $repository))' \ + "${temporary}/full/config/repository-properties.json" >"${fixture}/config/repository-properties.json" +done run_audit() { - local scenario=$1 + local scenario=$1 fixture=${2:-focused} repository=observatory + if [[ ${fixture} == focused ]]; then + case ${scenario} in + wrong-pr-scope|unexpected-codeql) repository=.github ;; + advanced-config|advanced-setup) repository=classic ;; + missing-rust) repository=content-toolkit ;; + missing-tag-scope|tag-ref-drift) repository=service-updater ;; + unexpected-pr) repository=web-platform ;; + esac + fixture="fixture-${repository}" + fi GH_TEST_SCENARIO=${scenario} \ - PATH="${temporary}/bin:${PATH}" \ - AUDIT_CONFIG_ROOT="${root}" \ - "${root}/bin/audit" + PATH="${temporary}/mock:${PATH}" \ + AUDIT_CONFIG_ROOT="${temporary}/${fixture}" \ + AUDIT_API_LOG="${temporary}/api.log" \ + "${temporary}/${fixture}/bin/audit" } -output=$(run_audit pass) +output=$(run_audit pass full) grep -Fxq 'Governance audit passed: 23 active repositories checked.' <<<"${output}" - -temporary_output=$(run_audit temporary) -grep -Fxq 'Governance audit passed: 23 active repositories checked.' <<<"${temporary_output}" - -if run_audit unmanaged >"${temporary}/unmanaged.out" 2>"${temporary}/unmanaged.err"; then - echo "expected unmanaged repository audit to fail" >&2 - exit 1 -fi -grep -Fq 'active repository is not in the desired-state inventory: atrinik/unmanaged' \ - "${temporary}/unmanaged.out" - -if run_audit merge-drift >"${temporary}/merge.out" 2>"${temporary}/merge.err"; then - echo "expected merge drift audit to fail" >&2 +output=$(run_audit temporary full) +grep -Fxq 'Governance audit passed: 23 active repositories checked.' <<<"${output}" +output=$(run_audit legacy-security) +grep -Fq 'Governance audit passed:' <<<"${output}" +output=$(run_audit repository-rulesets) +grep -Fq 'Governance audit passed:' <<<"${output}" +output=$(run_audit pending) +grep -Fq 'PENDING Actions SHA pinning deferred pending canary proof' <<<"${output}" +grep -Fq 'PENDING Dependency review merge protection awaits landed successful producer' <<<"${output}" +grep -Fq 'Governance audit baseline passed with' <<<"${output}" +if grep -Fq 'Governance audit passed:' <<<"${output}"; then + echo 'pending policy must not report full governance enforcement' >&2 exit 1 fi -grep -Fq 'repository merge policy drift: atrinik/observatory' \ - "${temporary}/merge.out" -if run_audit properties-drift >"${temporary}/properties.out" 2>"${temporary}/properties.err"; then - echo "expected property drift audit to fail" >&2 - exit 1 -fi -grep -Fq 'repository custom-property drift: atrinik/observatory' \ - "${temporary}/properties.out" +assert_drift() { + local scenario=$1 message=$2 + if run_audit "${scenario}" >"${temporary}/${scenario}.out" 2>"${temporary}/${scenario}.err"; then + echo "expected ${scenario} audit to fail" >&2 + exit 1 + fi + if ! grep -Fq "${message}" "${temporary}/${scenario}.out" "${temporary}/${scenario}.err"; then + cat "${temporary}/${scenario}.out" "${temporary}/${scenario}.err" >&2 + echo "missing expected ${scenario} diagnostic: ${message}" >&2 + exit 1 + fi +} -if run_audit schema-drift >"${temporary}/schema.out" 2>"${temporary}/schema.err"; then - echo "expected property schema drift audit to fail" >&2 +assert_drift unmanaged 'active repository is not in the desired-state inventory' +assert_drift merge-drift 'repository merge policy drift' +assert_drift properties-drift 'repository custom-property drift' +assert_drift schema-drift 'organization custom-property schema drift' +assert_drift inventory-unreadable 'organization repository inventory could not be read' +assert_drift wrong-pr-scope 'effective pull-request policy drift: atrinik/.github' +assert_drift missing-tag-scope "ruleset scope or policy drift '04 - Immutable release tags': atrinik/service-updater" +assert_drift tag-ref-drift "ruleset scope or policy drift '04 - Immutable release tags'" +assert_drift integrity-bypass "ruleset scope or policy drift '01 - Default branch integrity'" +assert_drift pr-bypass "ruleset scope or policy drift '02 - Changes through pull requests'" +assert_drift owner-id-drift "ruleset scope or policy drift '02 - Changes through pull requests'" +assert_drift pending-wrong-dependency-scope "ruleset scope or policy drift '07 - Dependency review'" +for scenario in wrong-app wrong-context strict-ci; do + assert_drift "${scenario}" 'effective required CI contexts or app drift' +done +for scenario in approval-count thread-resolution; do + assert_drift "${scenario}" 'effective pull-request policy drift' +done +assert_drift effective-integrity 'effective default-branch integrity drift' +assert_drift unexpected-pr 'effective pull-request policy drift: atrinik/web-platform' +assert_drift unexpected-codeql 'unexpected effective CodeQL merge protection: atrinik/.github' +for scenario in codeql-threshold codeql-tool codeql-quality; do + assert_drift "${scenario}" 'effective CodeQL merge protection drift' +done +for scenario in actions-write-token actions-approval; do + assert_drift "${scenario}" 'organization Actions workflow drift' +done +for scenario in repository-write-token repository-approval; do + assert_drift "${scenario}" 'repository Actions workflow permissions drift' +done +assert_drift repository-actions-unreadable 'repository Actions workflow permissions could not be read' +assert_drift actions-allowlist 'organization Actions selected-actions drift' +assert_drift actions-unreadable 'organization Actions permissions could not be read' +assert_drift sha-unreadable 'Actions SHA pinning state could not be read' +assert_drift missing-rust 'CodeQL setup or required language drift: atrinik/content-toolkit' +for scenario in advanced-setup default-setup; do + assert_drift "${scenario}" 'CodeQL setup or required language drift' +done +for scenario in configuration-detached dependency-graph advanced-config; do + assert_drift "${scenario}" 'security configuration attachment or baseline drift' +done +assert_drift configuration-unreadable 'security configuration could not be read' +assert_drift setup-unreadable 'CodeQL setup could not be read' +assert_drift dependabot-alerts 'Dependabot alerts disabled or unreadable' +assert_drift private-reporting 'private vulnerability reporting disabled or unreadable' +assert_drift secret-baseline 'exposed security baseline drift' +assert_drift effective-unreadable 'effective default-branch rules could not be read' +assert_drift ruleset-unreadable 'ruleset contents could not be read' +assert_drift ready-missing 'effective required CI contexts or app drift' +assert_drift ready-missing-codeql 'effective CodeQL merge protection drift' +assert_drift dependency-readiness-unreadable 'Dependency review readiness could not be read' +assert_drift codeql-readiness-unreadable 'CodeQL readiness could not be read' +output=$(run_audit pending-codeql) +grep -Fq 'PENDING CodeQL merge protection awaits successful analysis' <<<"${output}" +grep -Fq 'Governance audit baseline passed with' <<<"${output}" +# An existing dependency gate survives a temporarily unreadable producer readiness +# API; read readiness is required only when first installing the context. +output=$(run_audit readiness-unreadable) +grep -Fq 'Governance audit passed:' <<<"${output}" + +if grep -Eq '^repos/atrinik/(legacy-|nawerhals)' "${temporary}/api.log"; then + echo 'audit must not inspect archived repositories' >&2 exit 1 fi -grep -Fq 'organization custom-property schema drift: atrinik' \ - "${temporary}/schema.out" - -echo "Governance drift audit tests passed." +echo 'Governance effective-policy audit tests passed.' diff --git a/tests/publish-maintenance-branch.sh b/tests/publish-maintenance-branch.sh index fda3923..f616ed9 100755 --- a/tests/publish-maintenance-branch.sh +++ b/tests/publish-maintenance-branch.sh @@ -226,6 +226,9 @@ if [[ ${method} != GET ]]; then fi case "${endpoint}|${jq_filter}" in +"orgs/atrinik/actions/permissions|") + printf '{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":false}\n' + ;; "orgs/atrinik|") if [[ ${GH_ORGANIZATION_SCENARIO:-converged} == drifted ]]; then jq -n '{ @@ -1085,8 +1088,9 @@ assert_idempotent_organization_security() { advisory_active_root=${temporary}/advisory-window-active mkdir -p "${advisory_active_root}/bin" -cp "${root}/bin/publish" "${root}/bin/validate" \ +cp "${root}/bin/publish" "${root}/bin/validate" "${root}/bin/security-readiness" \ "${advisory_active_root}/bin/" +cp -R "${root}/lib" "${advisory_active_root}/lib" cp -R "${root}/config" "${advisory_active_root}/config" cp -R "${root}/community-health" \ "${advisory_active_root}/community-health" @@ -1239,7 +1243,8 @@ assert_codeql_default_setup "${creation_log}" rollback_root=${temporary}/empty-inventory mkdir -p "${rollback_root}/bin" -cp "${root}/bin/publish" "${root}/bin/validate" "${rollback_root}/bin/" +cp "${root}/bin/publish" "${root}/bin/validate" "${root}/bin/security-readiness" "${rollback_root}/bin/" +cp -R "${root}/lib" "${rollback_root}/lib" cp -R "${root}/config" "${rollback_root}/config" cp -R "${root}/community-health" "${rollback_root}/community-health" mkdir -p "${rollback_root}/.github" @@ -1328,8 +1333,9 @@ jq -s -e ' advisory_rollback_root=${temporary}/advisory-window-rollback mkdir -p "${advisory_rollback_root}/bin" -cp "${root}/bin/publish" "${root}/bin/validate" \ +cp "${root}/bin/publish" "${root}/bin/validate" "${root}/bin/security-readiness" \ "${advisory_rollback_root}/bin/" +cp -R "${root}/lib" "${advisory_rollback_root}/lib" cp -R "${root}/config" "${advisory_rollback_root}/config" cp -R "${root}/community-health" \ "${advisory_rollback_root}/community-health" diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh new file mode 100755 index 0000000..da40f40 --- /dev/null +++ b/tests/publish-security-merge.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash + +# Sourced production functions consume the globals below. +# shellcheck disable=SC2034 +set -euo pipefail + +source_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +# An override permits testing the integrated candidate from an isolated worktree. +source_root=${ATRINIK_PUBLISH_TEST_ROOT:-${source_root}} +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +root=${temporary}/fixture +mkdir -p "${root}/bin" "${root}/config/rulesets" +cp "${source_root}/config/"*.json "${root}/config/" +cp "${source_root}/config/rulesets/"*.json "${root}/config/rulesets/" + +# Source production functions without executing validation, authentication or +# live publication. Every API call is replaced below by a fail-closed mock. +sed -n '/^converge_organization_settings() {/,/^organization_metadata=/p' \ + "${source_root}/bin/publish" | sed '$d' >"${temporary}/functions.sh" +# shellcheck source=/dev/null +source "${temporary}/functions.sh" +# Exercise the production scope dispatch, including its following gate call. +awk ' + /^if \[\[ \$\{policy_scope\} == organization \]\]; then$/ { capture = 1 } + capture { print } + capture && /^publish_security_merge_rulesets$/ { exit } +' "${source_root}/bin/publish" >"${temporary}/dispatch.sh" +[[ -s ${temporary}/dispatch.sh ]] + +organization=atrinik +apply=true +temporary_files=() +security_merge_config=${root}/config/security-merge-protection.json +codeql_advanced_config=${root}/config/codeql-advanced-setup.json +active_repositories=(content-toolkit) +declare -A active_repository_ids=([content-toolkit]=123) +api_log=${temporary}/api.jsonl +readiness_log=${temporary}/readiness.jsonl +export READINESS_LOG=${readiness_log} +cat >"${root}/bin/security-readiness" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +jq -cn --arg mode "$1" --arg repository "$2" \ + '{mode: $mode, repository: $repository}' >>"${READINESS_LOG}" +case ${READINESS_STATUS:-ready} in +ready) echo 'producer evidence ready'; exit 0 ;; +missing) echo 'producer is missing'; exit 1 ;; +failed) echo 'producer failed'; exit 1 ;; +unreadable) echo 'producer evidence unreadable'; exit 2 ;; +*) exit 99 ;; +esac +MOCK +chmod +x "${root}/bin/security-readiness" + +# Minimal live response fixtures preserve the API shapes used by the publisher. +github_api() { + local endpoint=$1 + jq -cn --arg endpoint "${endpoint}" \ + '{method: "GET", endpoint: $endpoint}' >>"${api_log}" + case ${endpoint} in + repos/atrinik/content-toolkit/code-scanning/default-setup) + [[ ${codeql_state} != unreadable ]] || return 1 + printf '%s\n' "${codeql_state}" + ;; + 'repos/atrinik/content-toolkit/rulesets?includes_parents=false') + if [[ ${ruleset_state} == missing ]]; then + echo '[]' + else + jq -n --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [{id: 6, name: $codeql[0].name}, {id: 7, name: $dependency[0].name}]' + fi + ;; + repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) + local template=code-scanning + [[ ${endpoint##*/} != 7 ]] || template=dependency-review + if [[ ${ruleset_state} == exact ]]; then + jq '. + {id: 100, source_type: "Repository"}' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == owner-null ]]; then + jq '.bypass_actors[0].actor_id = null' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == active-drift ]]; then + jq '.bypass_actors = []' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == active-unrelated-gate ]]; then + if [[ ${template} == code-scanning ]]; then + jq '.rules[0].parameters.code_scanning_tools[0].tool = "Another scanner"' \ + "${root}/config/rulesets/${template}.json" + else + jq '.rules[0].parameters.required_status_checks[0].integration_id = 999' \ + "${root}/config/rulesets/${template}.json" + fi + elif [[ ${ruleset_state} == active-without-gate ]]; then + jq '.rules = []' "${root}/config/rulesets/${template}.json" + else + jq '.enforcement = "disabled"' "${root}/config/rulesets/${template}.json" + fi + ;; + orgs/atrinik/actions/permissions) printf '%s\n' "${actions_state}" ;; + *) echo "unexpected mock API: ${endpoint}" >&2; return 99 ;; + esac +} + +run_api() { + jq -cn --arg method "$1" --arg endpoint "$2" --slurpfile payload "$3" \ + '{method: $method, endpoint: $endpoint, payload: $payload[0]}' >>"${api_log}" +} +reset_case() { + : >"${api_log}" + : >"${readiness_log}" + codeql_state='{"state":"configured","query_suite":"default","languages":["python","javascript-typescript"]}' + ruleset_state=missing + actions_state='{"enabled_repositories":"selected","allowed_actions":"all","sha_pinning_required":false}' + sha_pinning_canary= + export READINESS_STATUS=ready +} +assert_no_writes() { + jq -se 'all(.[]; .method == "GET")' "${api_log}" >/dev/null +} + +reset_case +converge_codeql_default_setup >"${temporary}/languages.txt" +jq -se ' + [.[] | select(.method == "PATCH")] == [{ + method: "PATCH", + endpoint: "repos/atrinik/content-toolkit/code-scanning/default-setup", + payload: {state: "configured", query_suite: "default", + languages: ["javascript-typescript", "python", "rust"]} + }]' "${api_log}" >/dev/null +reset_case +codeql_state='{"state":"configured","query_suite":"default","languages":["actions","javascript","javascript-typescript","python","typescript"]}' +converge_codeql_default_setup >"${temporary}/languages-aliases.txt" +jq -se '[.[] | select(.method == "PATCH")][0].payload.languages == + ["actions", "javascript-typescript", "python", "rust"]' "${api_log}" >/dev/null +reset_case +codeql_state='{"state":"configured","query_suite":"default","languages":["rust","python"]}' +converge_codeql_default_setup >"${temporary}/languages-keep.txt" +assert_no_writes +reset_case +codeql_state=unreadable +if converge_codeql_default_setup >"${temporary}/languages-error.txt" 2>&1; then + echo 'expected unreadable language inventory to fail closed' >&2 + exit 1 +fi +assert_no_writes +reset_case +codeql_state='{"state":"configured","query_suite":"default","languages":null}' +if converge_codeql_default_setup >"${temporary}/languages-malformed.txt" 2>&1; then + echo 'expected malformed language inventory to fail closed' >&2 + exit 1 +fi +assert_no_writes + +# Use the actual dispatch: security producer readiness follows base convergence +# for both scopes, and these Team-compatible gates always target repositories. +publish_organization_rulesets() { echo organization-rules >>"${temporary}/order"; } +publish_organization_security() { echo organization-security >>"${temporary}/order"; } +publish_repository_rulesets() { echo repository-rules >>"${temporary}/order"; } +publish_repository_security() { echo repository-security >>"${temporary}/order"; } +for policy_scope in organization repository; do + for producer in missing failed unreadable ready; do + reset_case + export READINESS_STATUS=${producer} + : >"${temporary}/order" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/${policy_scope}-${producer}.txt" + [[ $(cat "${temporary}/order") == "${policy_scope}-rules"$'\n'"${policy_scope}-security" ]] + if [[ ${producer} == ready ]]; then + jq -se --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [.[] | select(.method != "GET")] == [ + {method: "POST", endpoint: "repos/atrinik/content-toolkit/rulesets", payload: $codeql[0]}, + {method: "POST", endpoint: "repos/atrinik/content-toolkit/rulesets", payload: $dependency[0]} + ]' "${api_log}" >/dev/null + else + assert_no_writes + fi + jq -se '[.[].mode] == ["codeql", "dependency"] and + all(.[]; .repository == "atrinik/content-toolkit")' "${readiness_log}" >/dev/null + jq -se 'all(.[]; .endpoint | startswith("repos/atrinik/content-toolkit/"))' \ + "${api_log}" >/dev/null + done +done + +reset_case +ruleset_state=exact +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/exact.txt" +assert_no_writes +[[ ! -s ${readiness_log} ]] +reset_case +ruleset_state=owner-null +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/owner-null.txt" +assert_no_writes +[[ ! -s ${readiness_log} ]] +reset_case +ruleset_state=active-drift +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/active-drift.txt" +jq -se '[.[] | select(.method != "GET")] | length == 2 and + all(.[]; .method == "PUT" and .payload.enforcement == "active")' \ + "${api_log}" >/dev/null +[[ ! -s ${readiness_log} ]] +for ruleset_state in disabled active-without-gate active-unrelated-gate; do + : >"${api_log}" + : >"${readiness_log}" + export READINESS_STATUS=failed + publish_security_merge_rulesets >"${temporary}/${ruleset_state}.txt" + assert_no_writes + jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null +done + +# An archived or missing repository is absent from the active identity map. +reset_case +active_repository_ids=() +publish_security_merge_rulesets >"${temporary}/inactive.txt" +[[ ! -s ${api_log} && ! -s ${readiness_log} ]] +active_repository_ids=([content-toolkit]=123) + +reset_case +converge_actions_permissions >"${temporary}/sha-deferred.txt" +jq -se '[.[] | select(.method == "PUT")][0].payload == + {enabled_repositories: "all", allowed_actions: "selected"}' "${api_log}" >/dev/null +[[ ! -s ${readiness_log} ]] +reset_case +actions_state='{"enabled_repositories":"selected","allowed_actions":"all","sha_pinning_required":true}' +converge_actions_permissions >"${temporary}/sha-retained.txt" +jq -se '[.[] | select(.method == "PUT")][0].payload.sha_pinning_required == true' \ + "${api_log}" >/dev/null +[[ ! -s ${readiness_log} ]] +reset_case +sha_pinning_canary=atrinik/github-settings/99 +export READINESS_STATUS=failed +converge_actions_permissions >"${temporary}/sha-failed.txt" +jq -se '[.[] | select(.method == "PUT")][0].payload | has("sha_pinning_required") | not' \ + "${api_log}" >/dev/null +reset_case +sha_pinning_canary=atrinik/github-settings/99 +converge_actions_permissions >"${temporary}/sha-ready.txt" +jq -se --slurpfile desired "${root}/config/actions-permissions.json" ' + [.[] | select(.method == "PUT")][0].payload == $desired[0]' "${api_log}" >/dev/null +jq -se '. == [{mode: "sha", repository: "atrinik/github-settings/99"}]' \ + "${readiness_log}" >/dev/null +reset_case +actions_state=$(cat "${root}/config/actions-permissions.json") +converge_actions_permissions >"${temporary}/sha-keep.txt" +assert_no_writes + +rm -f "${temporary_files[@]}" +echo 'Security merge publisher tests passed.' diff --git a/tests/security-readiness.py b/tests/security-readiness.py new file mode 100644 index 0000000..64ca596 --- /dev/null +++ b/tests/security-readiness.py @@ -0,0 +1,573 @@ +#!/usr/bin/env python3 +"""Deterministic, read-only security readiness contract tests.""" + +import base64 +import copy +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch +from urllib.parse import parse_qs, urlparse + + +ROOT = Path(__file__).resolve().parent.parent +sys.dont_write_bytecode = True +SPEC = importlib.util.spec_from_file_location("security_readiness", ROOT / "lib/security_readiness.py") +readiness = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(readiness) +HEAD, BASE, MERGE = "a" * 40, "b" * 40, "c" * 40 +OLD, NEW = "d" * 40, "e" * 40 +WORKFLOW = ("name: Dependency review\n\non:\n pull_request:\n branches: [main]\n" + "permissions:\n contents: read\njobs:\n dependency-review:\n" + " name: Dependency review\n runs-on: ubuntu-24.04\n steps:\n" + f" - uses: actions/dependency-review-action@{OLD} # v5.0.0\n" + " with:\n fail-on-severity: high\n warn-only: false\n").encode() +SELECTED = {"github_owned_allowed": True, "verified_allowed": False, + "patterns_allowed": ["rust-lang/crates-io-auth-action@*", "docker/setup-buildx-action@*"]} +CODEQL_WORKFLOW = b"""name: CodeQL +jobs: + analyze: + name: CodeQL analysis (${{ matrix.display }}) + run: python3 tools/ci/classify_changes.py + languages: ${{ matrix.language }} + category: ${{ matrix.category }} + validation: + name: CodeQL validation + run: python3 tools/ci/require_checks.py optional '${{ needs.classify.outputs.run }}' '${{ needs.analyze.result }}' +""" +CLASSIFIER = b"""FULL_EVENTS = {'push', 'schedule', 'workflow_dispatch', 'merge_group'} +def main(): + if arguments.event in FULL_EVENTS: + full = True + paths = [] +""" + + +def content(data): + return {"type": "file", "encoding": "base64", "content": base64.b64encode(data).decode()} + + +def check(name, ref=HEAD, identifier=1, conclusion="success", status="completed"): + return {"name": name, "head_sha": ref, "id": identifier, "status": status, + "conclusion": conclusion, "started_at": "2026-10-11T12:00:00Z", + "app": {"id": 15368, "slug": "github-actions"}, "check_suite": {"id": 99}} + + +class Fixture: + def __init__(self): + self.calls = [] + self.metadata = {"full_name": "atrinik/example", "default_branch": "main", "archived": False} + self.pr = {"number": 12, "state": "open", "draft": False, "merged": False, + "changed_files": 1, "merge_commit_sha": MERGE, + "base": {"ref": "main", "sha": BASE, "repo": {"full_name": "atrinik/example"}}, + "head": {"sha": HEAD, "repo": {"full_name": "atrinik/example"}}} + self.run = {"id": 9, "run_attempt": 2, "event": "pull_request", "status": "completed", + "conclusion": "success", "head_sha": HEAD, "check_suite_id": 99, + "pull_requests": [{"number": 12, "head": {"sha": HEAD}, + "base": {"sha": BASE, "ref": "main"}}]} + self.suite = {"id": 99, "head_sha": MERGE, "status": "completed", "conclusion": "success", + "app": {"id": 15368}, "pull_requests": copy.deepcopy(self.run["pull_requests"])} + self.jobs = [{"name": "Dependency review", "status": "completed", "conclusion": "success"}] + self.files = [{"filename": ".github/workflows/check.yml", "status": "modified"}] + self.before = f"name: Check\nsteps:\n - uses: rust-lang/crates-io-auth-action@{OLD} # v1\n".encode() + self.after = f"name: Check\nsteps:\n - uses: rust-lang/crates-io-auth-action@{NEW} # v2\n".encode() + self.checks = {HEAD: [check("Required checks"), check("Conventional PR title", identifier=2)], MERGE: []} + self.live = copy.deepcopy(SELECTED) + self.default_workflow = WORKFLOW + self.run_workflow = WORKFLOW + self.executed_workflow = WORKFLOW + self.parents = [BASE, HEAD] + self.branch_head = HEAD + self.setup = {"state": "configured", "languages": ["rust"]} + self.codeql_sources = {".github/workflows/codeql.yml": CODEQL_WORKFLOW, + "tools/ci/classify_changes.py": CLASSIFIER, + "tools/ci/require_checks.py": b"# Trusted aggregate\n"} + self.codeql_head_sources = copy.deepcopy(self.codeql_sources) + self.codeql_run = {"id": 29, "run_attempt": 1, "head_sha": HEAD, "head_branch": "main", + "event": "push", "path": ".github/workflows/codeql.yml", "check_suite_id": 777, + "status": "completed", "conclusion": "success"} + self.codeql_check = check("CodeQL validation", identifier=999) + self.codeql_check["check_suite"]["id"] = 777 + self.codeql_jobs = [] + self.analyses = [{"id": 1, "created_at": "2026-10-11T12:00:00Z", "tool": {"name": "CodeQL"}, + "ref": "refs/heads/main", "commit_sha": HEAD, "category": "/language:rust", + "environment": '{"language":"rust"}', "error": "", "rules_count": 42}] + + def __call__(self, endpoint): + self.calls.append(endpoint) + if endpoint == "repos/atrinik/example": + return copy.deepcopy(self.metadata) + if endpoint == "repos/atrinik/example/pulls/12": + return copy.deepcopy(self.pr) + if "/contents/.github/workflows/dependency-review.yml" in endpoint: + return content(self.default_workflow if "ref=main" in endpoint else + self.executed_workflow if "ref=" + MERGE in endpoint else self.run_workflow) + if "/contents/.github/workflows/check.yml" in endpoint: + return content(self.before if "ref=" + BASE in endpoint else self.after) + if "/contents/" in endpoint: + path = endpoint.split("/contents/")[1].split("?")[0] + sources = self.codeql_sources if "ref=main" in endpoint else self.codeql_head_sources + return content(sources[path]) + if "/actions/workflows/codeql.yml/runs" in endpoint: + return {"workflow_runs": [copy.deepcopy(self.codeql_run)]} + if "/actions/runs/29/attempts/1/jobs" in endpoint: + return {"total_count": len(self.codeql_jobs), "jobs": copy.deepcopy(self.codeql_jobs)} + if endpoint.endswith("/check-runs/999"): + return copy.deepcopy(self.codeql_check) + if "/actions/workflows/dependency-review.yml/runs" in endpoint: + return {"workflow_runs": [copy.deepcopy(self.run)]} + if "/attempts/2/jobs" in endpoint: + return {"total_count": len(self.jobs), "jobs": copy.deepcopy(self.jobs)} + if "/actions/permissions/selected-actions" in endpoint: + return copy.deepcopy(self.live) + if "/pulls/12/files" in endpoint: + return copy.deepcopy(self.files) + if endpoint == "repos/atrinik/example/commits/" + MERGE: + return {"parents": [{"sha": value} for value in self.parents]} + if "/check-runs?" in endpoint: + values = copy.deepcopy(self.checks[endpoint.split("/commits/")[1].split("/")[0]]) + return {"total_count": len(values), "check_runs": values} + if "/actions/runs?check_suite_id=99" in endpoint: + return {"workflow_runs": [copy.deepcopy(self.run)]} + if "/actions/runs?event=pull_request&head_sha=" in endpoint: + return {"total_count": 1, "workflow_runs": [copy.deepcopy(self.run)]} + if "/check-suites/99" in endpoint: + return copy.deepcopy(self.suite) + if "/branches/main" in endpoint: + return {"commit": {"sha": self.branch_head}} + if "/code-scanning/analyses?" in endpoint: + page = int(parse_qs(urlparse(endpoint).query)["page"][0]) + return copy.deepcopy(self.analyses[(page - 1) * 100:page * 100]) + if endpoint.endswith("/code-scanning/default-setup"): + return copy.deepcopy(self.setup) + raise AssertionError("Unexpected read-only API endpoint: " + endpoint) + + +class ReadinessTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + (self.root / ".github/workflows").mkdir(parents=True) + (self.root / ".github/workflows/dependency-review.yml").write_bytes(WORKFLOW) + (self.root / "config").mkdir() + (self.root / "config/actions-selected.json").write_text(json.dumps(SELECTED)) + (self.root / "config/repositories.json").write_text(json.dumps({ + "required_ci": {"example": ["Required checks", "Conventional PR title"]}})) + (self.root / "config/security-merge-protection.json").write_text(json.dumps({ + "codeql_required_languages": {"example": ["rust"]}})) + (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": []}') + self.fixture = Fixture() + self.root_patch = patch.object(readiness, "ROOT", self.root) + self.api_patch = patch.object(readiness, "api", self.fixture) + self.root_patch.start() + self.api_patch.start() + self.addCleanup(self.root_patch.stop) + self.addCleanup(self.api_patch.stop) + + def invoke(self, mode): + stream = io.StringIO() + with patch("sys.stdout", stream): + result = readiness.main([mode, "atrinik/example/12" if mode == "sha" else "atrinik/example"]) + return result, stream.getvalue() + + def assert_deferred(self, mode, fragment): + code, text = self.invoke(mode) + self.assertEqual(code, 1, text) + self.assertIn(fragment, text) + + def test_dependency_ready_latest_attempt_exact_workflow(self): + self.assertEqual(self.invoke("dependency")[0], 0) + self.assertTrue(any("/attempts/2/jobs" in call for call in self.fixture.calls)) + + def test_dependency_not_landed(self): + self.fixture.default_workflow = WORKFLOW.replace(b"fail-on-severity: high", b"fail-on-severity: low") + self.assert_deferred("dependency", "not landed") + + def test_dependency_producer_fresh_sha_passes_without_central_update(self): + fresh = WORKFLOW.replace(OLD.encode(), NEW.encode()).replace(b"v5.0.0", b"v5.1.0") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_mutable_action_ref_deferred(self): + self.fixture.default_workflow = WORKFLOW.replace(OLD.encode(), b"v5") + self.assert_deferred("dependency", "full-SHA") + + def test_dependency_classic_block_branch_filter_ready(self): + fresh = WORKFLOW.replace(b" branches: [main]", b" branches:\n - main") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_extra_branch_filter_deferred(self): + self.fixture.default_workflow = WORKFLOW.replace(b" branches: [main]", + b" branches:\n - main\n - extra") + self.assert_deferred("dependency", "not landed") + + def test_dependency_named_official_action_step_ready(self): + fresh = WORKFLOW.replace(b" - uses:", + b" - name: Review dependency changes\n uses:") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_labels_do_not_lock_producer_workflow(self): + fresh = WORKFLOW.replace(b"name: Dependency review\n", b"name: Dependency safety\n", 1) + fresh = fresh.replace(b" - uses:", b" - name: Validate dependencies\n uses:") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_named_step_arbitrary_run_deferred(self): + fresh = WORKFLOW.replace(b" - uses:", + b" - name: Review dependency changes\n run: arbitrary\n uses:") + self.fixture.default_workflow = fresh + self.assert_deferred("dependency", "not landed") + + def test_dependency_blank_comment_trailing_whitespace_ready(self): + fresh = WORKFLOW.replace(b"permissions:\n", b"# Explanation\n\npermissions: \n") + self.fixture.default_workflow = fresh + self.fixture.run_workflow = fresh + self.fixture.executed_workflow = fresh + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_failed_latest_does_not_search_older_success(self): + self.fixture.run["conclusion"] = "failure" + self.assert_deferred("dependency", "latest") + self.assertEqual(len([call for call in self.fixture.calls if "/runs?event=" in call]), 1) + + def test_dependency_successful_run_failed_job(self): + self.fixture.jobs[0]["conclusion"] = "failure" + self.assert_deferred("dependency", "job not successful") + + def test_dependency_run_different_bytes(self): + self.fixture.executed_workflow += b"# different\n" + self.assert_deferred("dependency", "different workflow bytes") + + def test_dependency_matching_head_but_executed_merge_workflow_differs(self): + self.assertEqual(self.fixture.default_workflow, self.fixture.run_workflow) + self.fixture.executed_workflow = WORKFLOW.replace(b"warn-only: false", b"warn-only: true") + self.assert_deferred("dependency", "different workflow bytes") + + def test_dependency_direct_head_execution_bound_to_pr(self): + self.fixture.suite["head_sha"] = HEAD + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_direct_head_missing_execution_pr_association(self): + self.fixture.suite.update(head_sha=HEAD, pull_requests=[]) + self.assert_deferred("dependency", "execution PR association") + + def test_dependency_executed_merge_wrong_parents_deferred(self): + self.fixture.parents = [BASE, OLD] + self.assert_deferred("dependency", "current PR head/base") + + def test_dependency_stale_execution_base_association_deferred(self): + self.fixture.suite["pull_requests"][0]["base"]["sha"] = OLD + self.assert_deferred("dependency", "execution PR association") + + def test_dependency_check_suite_drift_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if endpoint.endswith("/check-suites/99"): + calls += 1 + if calls > 1: + value["head_sha"] = OLD + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("dependency", "check suite changed") + + def test_dependency_run_association_drift_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if "/actions/workflows/dependency-review.yml/runs" in endpoint: + calls += 1 + if calls > 1: + value["workflow_runs"][0]["pull_requests"] = [] + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("dependency", "run changed") + + def test_dependency_pr_drift_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if endpoint.endswith("/pulls/12"): + calls += 1 + if calls > 1: + value["base"]["sha"] = OLD + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("dependency", "PR changed") + + def test_dependency_wrong_base(self): + self.fixture.pr["base"]["ref"] = "maintenance" + self.assert_deferred("dependency", "default branch") + + def test_dependency_stale_head(self): + self.fixture.pr["head"]["sha"] = NEW + self.assert_deferred("dependency", "head changed") + + def test_dependency_api_404_missing_is_deferred(self): + def missing(endpoint): + if "/contents/" in endpoint: + raise readiness.APIError(endpoint, 404) + return self.fixture(endpoint) + with patch.object(readiness, "api", missing): + self.assert_deferred("dependency", "missing") + + def test_api_denied_is_error_not_deferred(self): + with patch.object(readiness, "api", side_effect=readiness.APIError("repos/atrinik/example", 403)): + self.assertEqual(self.invoke("dependency")[0], 2) + + def test_sha_wildcard_accepts_fresh_rust_pin(self): + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_general_github_owned_pin_update(self): + self.fixture.before = f"steps:\n - uses: actions/checkout@{OLD} # v5\n".encode() + self.fixture.after = f"steps:\n - uses: actions/checkout@{NEW} # v6\n".encode() + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_live_old_exact_pin_deferred(self): + self.fixture.live["patterns_allowed"][-1] = "rust-lang/crates-io-auth-action@" + OLD + self.assert_deferred("sha", "live selected-actions") + + def test_sha_allowlist_file_update_rejected(self): + self.fixture.files = [{"filename": "config/actions-selected.json", "status": "modified"}] + self.assert_deferred("sha", "outside existing workflows") + + def test_sha_arbitrary_code_update_rejected_even_if_patch_hides_it(self): + self.fixture.files[0]["patch"] = "@@ fake pin-only partial diff" + self.fixture.after += b" - run: arbitrary-command\n" + self.assert_deferred("sha", "more than action pins") + + def test_sha_changed_action_identity_rejected(self): + self.fixture.after = self.fixture.after.replace(b"rust-lang/crates-io-auth-action", b"docker/setup-buildx-action") + self.assert_deferred("sha", "action structure") + + def test_sha_version_ref_rejected(self): + self.fixture.after = self.fixture.after.replace(NEW.encode(), b"v2") + self.assert_deferred("sha", "more than action pins") + + def test_sha_disallowed_action_update_rejected(self): + self.fixture.before = self.fixture.before.replace(b"rust-lang", b"untrusted") + self.fixture.after = self.fixture.after.replace(b"rust-lang", b"untrusted") + self.assert_deferred("sha", "outside desired") + + def test_sha_latest_failure_masks_old_success(self): + self.fixture.checks[HEAD].append(check("Required checks", identifier=10, conclusion="failure")) + self.assert_deferred("sha", "not successful") + + def test_sha_missing_context(self): + self.fixture.checks[HEAD].pop() + self.assert_deferred("sha", "missing") + + def test_sha_wrong_head_check_rejected(self): + self.fixture.checks[HEAD][0]["head_sha"] = OLD + self.assert_deferred("sha", "missing") + + def test_sha_wrong_app_check_rejected(self): + self.fixture.checks[HEAD][0]["app"]["id"] = 1 + self.assert_deferred("sha", "missing") + + def test_sha_realistic_merge_sha_checks(self): + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_merged_canary_current_head_direct_checks(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_merged_canary_historical_merge_checks(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_closed_unmerged_canary_deferred(self): + self.fixture.pr.update(state="closed", merged=False) + self.assert_deferred("sha", "merged or open") + + def test_sha_stale_merge_run_head(self): + self.fixture.checks[MERGE] = [check("Required checks", MERGE)] + self.fixture.checks[HEAD] = [] + self.fixture.run["head_sha"] = OLD + self.assert_deferred("sha", "missing") + + def test_sha_stale_merge_commit(self): + self.fixture.parents = [BASE, OLD] + self.assert_deferred("sha", "stale") + + def test_sha_fresh_pr_head_required_at_completion(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if endpoint.endswith("/pulls/12"): + calls += 1 + if calls > 1: + value["head"]["sha"] = NEW + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "changed during") + + def test_codeql_success_current_head_rust(self): + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_large_history_stops_after_current_coverage(self): + old = copy.deepcopy(self.fixture.analyses[0]) + old.update(commit_sha=OLD, category="/partition:retired/language:rust") + self.fixture.analyses.extend([old] * 1200) + self.assertEqual(self.invoke("codeql")[0], 0) + self.assertEqual(len([call for call in self.fixture.calls if "/analyses?" in call]), 1) + + def test_codeql_retired_category_failure_does_not_gate(self): + old = copy.deepcopy(self.fixture.analyses[0]) + old.update(commit_sha=OLD, category="/partition:retired/language:rust", error="old failure") + self.fixture.analyses.insert(0, old) + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_all_live_configured_languages_required(self): + self.fixture.setup["languages"].append("python") + self.assert_deferred("codeql", "configured CodeQL language") + + def test_codeql_javascript_typescript_alias_normalized(self): + self.fixture.setup["languages"] = ["rust", "javascript/typescript"] + second = copy.deepcopy(self.fixture.analyses[0]) + second.update(category="/language:javascript-typescript", environment='{"language":"javascript-typescript"}') + self.fixture.analyses.append(second) + self.assertEqual(self.invoke("codeql")[0], 0) + + def enable_advanced(self): + (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": ["example"]}') + (self.root / "config/security-merge-protection.json").write_text('{"codeql_required_languages": {}}') + displays = [("C/C++ client", "client", "c-cpp"), ("C/C++ server", "server", "c-cpp"), + ("C/C++ core", "core", "c-cpp"), ("Python", "core", "python"), + ("GitHub Actions", "core", "actions")] + self.fixture.analyses = [] + self.fixture.codeql_jobs = [] + for index, (display, partition, language) in enumerate(displays): + category = f"/partition:{partition}/language:{language}" + job = {"id": index + 1, "name": f"CodeQL analysis ({display})", "head_sha": HEAD, + "status": "completed", "conclusion": "success", + "started_at": "2026-10-11T11:59:00Z", "completed_at": "2026-10-11T12:01:00Z"} + self.fixture.codeql_jobs.append(job) + self.fixture.analyses.append({"id": index + 1, "tool": {"name": "CodeQL"}, + "analysis_key": ".github/workflows/codeql.yml:analyze", "category": category, + "environment": json.dumps({"display": display, "category": category, "language": language, + "scope": partition, "run": ""}), + "commit_sha": HEAD, "ref": "refs/heads/main", "error": "", "rules_count": 58, + "created_at": "2026-10-11T12:00:00Z"}) + for name in ("CodeQL classification", "CodeQL validation"): + job = copy.deepcopy(self.fixture.codeql_jobs[0]) + job.update(id=999, name=name, check_run_url="https://api.github.com/repos/atrinik/example/check-runs/999") + self.fixture.codeql_jobs.append(job) + + def test_codeql_advanced_current_full_producer_all_partitions_ready(self): + self.enable_advanced() + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_advanced_large_retired_history_ready(self): + self.enable_advanced() + retired = copy.deepcopy(self.fixture.analyses[0]) + retired.update(commit_sha=OLD, category="/retired/language:c-cpp", environment='{"display":"Retired"}') + self.fixture.analyses.insert(0, retired) + self.fixture.analyses.extend([retired] * 1200) + self.assertEqual(self.invoke("codeql")[0], 0) + self.assertEqual(len([call for call in self.fixture.calls if "/analyses?" in call]), 1) + + def test_codeql_advanced_partial_uploads_deferred(self): + self.enable_advanced() + self.fixture.analyses.pop() + self.assert_deferred("codeql", "partition upload missing") + + def test_codeql_advanced_latest_failed_category_masks_success(self): + self.enable_advanced() + failed = copy.deepcopy(self.fixture.analyses[0]); failed["error"] = "failed" + self.fixture.analyses.insert(0, failed) + self.assert_deferred("codeql", "failed") + + def test_codeql_advanced_wrong_producer_deferred(self): + self.enable_advanced() + self.fixture.analyses[0]["analysis_key"] = ".github/workflows/other.yml:analyze" + self.assert_deferred("codeql", "partition upload missing") + + def test_codeql_advanced_partial_pr_event_deferred(self): + self.enable_advanced() + self.fixture.codeql_run["event"] = "pull_request" + self.assert_deferred("codeql", "full-event") + + def test_codeql_advanced_wrong_app_deferred(self): + self.enable_advanced() + self.fixture.codeql_check["app"]["id"] = 1 + self.assert_deferred("codeql", "trusted current-head producer") + + def test_codeql_advanced_stale_source_deferred(self): + self.enable_advanced() + self.fixture.codeql_head_sources["tools/ci/classify_changes.py"] += b"# different\n" + self.assert_deferred("codeql", "source is stale") + + def test_codeql_advanced_upload_outside_job_window_deferred(self): + self.enable_advanced() + self.fixture.analyses[0]["created_at"] = "2026-10-11T13:00:00Z" + self.assert_deferred("codeql", "selected full-run job") + + def test_codeql_latest_error_masks_older_success(self): + failed = copy.deepcopy(self.fixture.analyses[0]) + failed.update(id=2, error="analysis failed") + self.fixture.analyses.insert(0, failed) + self.assert_deferred("codeql", "failed") + + def test_codeql_success_old_head_deferred(self): + self.fixture.analyses[0]["commit_sha"] = OLD + self.assert_deferred("codeql", "current default head") + + def test_codeql_missing_rust_analysis_deferred(self): + self.fixture.analyses[0]["environment"] = '{"language":"python"}' + self.fixture.analyses[0]["category"] = "/language:python" + self.assert_deferred("codeql", "language") + + def test_codeql_no_executed_rules_deferred(self): + self.fixture.analyses[0]["rules_count"] = 0 + self.assert_deferred("codeql", "no executed rules") + + def test_archived_repository_deferred(self): + self.fixture.metadata["archived"] = True + self.assert_deferred("dependency", "archived") + + def test_malformed_api_evidence_error(self): + self.fixture.metadata.pop("default_branch") + self.assertEqual(self.invoke("dependency")[0], 2) + + def test_api_subprocess_uses_only_get_and_version(self): + self.api_patch.stop() + result = subprocess.CompletedProcess([], 0, stdout='{}', stderr='') + with patch.object(readiness.subprocess, "run", return_value=result) as mocked: + readiness.api("repos/atrinik/example") + args = mocked.call_args.args[0] + self.assertEqual(args[args.index("--method") + 1], "GET") + self.assertIn("X-GitHub-Api-Version: 2026-03-10", args) + + def test_executable_wrapper_is_bash_and_delegates_python(self): + wrapper = ROOT / "bin/security-readiness" + self.assertEqual(wrapper.read_text().splitlines()[0], "#!/usr/bin/env bash") + self.assertIn('exec python3 "${root}/lib/security_readiness.py" "$@"', wrapper.read_text()) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/security-readiness.sh b/tests/security-readiness.sh new file mode 100755 index 0000000..900b90c --- /dev/null +++ b/tests/security-readiness.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +exec python3 "${root}/tests/security-readiness.py" diff --git a/tests/validate-crate-trusted-publishing.sh b/tests/validate-crate-trusted-publishing.sh index c2c8c50..eae64e6 100755 --- a/tests/validate-crate-trusted-publishing.sh +++ b/tests/validate-crate-trusted-publishing.sh @@ -57,24 +57,29 @@ done jq '.patterns_allowed |= map(select(startswith("rust-lang/crates-io-auth-action@") | not))' \ "${root}/config/actions-selected.json" >"${temporary}/config/actions-selected.json" if "${temporary}/bin/validate" >/dev/null 2>&1; then - echo "error: accepted planned publisher without exact action allowance" >&2 + echo "error: accepted planned publisher without action allowance" >&2 exit 1 fi reset_policy jq '.patterns_allowed |= map(if startswith("rust-lang/crates-io-auth-action@") then "rust-lang/*" else . end)' \ "${root}/config/actions-selected.json" >"${temporary}/config/actions-selected.json" if "${temporary}/bin/validate" >/dev/null 2>&1; then - echo "error: accepted wildcard action allowance instead of exact pin" >&2 + echo "error: accepted organization-wide Rust allowance" >&2 exit 1 fi reset_policy jq '.patterns_allowed += ["rust-lang/*"]' "${root}/config/actions-selected.json" \ >"${temporary}/config/actions-selected.json" if "${temporary}/bin/validate" >/dev/null 2>&1; then - echo "error: accepted broader Rust action access beside the exact pin" >&2 + echo "error: accepted broader Rust action access beside the exact action" >&2 exit 1 fi reset_policy +# A new reviewed immutable release must not require an allowlist edit. +jq '.crates_io_trusted_publishing[0].authentication_action = "rust-lang/crates-io-auth-action@1111111111111111111111111111111111111111"' \ + "${root}/config/manual-settings.json" >"${temporary}/config/manual-settings.json" +"${temporary}/bin/validate" >/dev/null +reset_policy # Removing unactivated manual intent is a valid reviewed rollback. jq '.crates_io_trusted_publishing = []' "${root}/config/manual-settings.json" \ >"${temporary}/config/manual-settings.json" diff --git a/tests/validate-security-merge.sh b/tests/validate-security-merge.sh new file mode 100755 index 0000000..ac0558f --- /dev/null +++ b/tests/validate-security-merge.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +mkdir "${temporary}/bin" +cp "${root}/bin/validate" "${temporary}/bin/" +cp -R "${root}/config" "${root}/community-health" "${root}/.github" "${temporary}/" + +reject() { + local file=$1 filter=$2 + jq "${filter}" "${root}/config/${file}" >"${temporary}/config/${file}" + if "${temporary}/bin/validate" >/dev/null 2>&1; then + echo "error: accepted invalid security policy ${file}: ${filter}" >&2 + exit 1 + fi + cp "${root}/config/${file}" "${temporary}/config/${file}" +} + +"${temporary}/bin/validate" >/dev/null +reject security-merge-protection.json '.dependency_review += ["legacy-client"]' +reject security-merge-protection.json '.code_scanning -= ["service-updater"]' +reject security-merge-protection.json '.dependency_review += ["web-platform"]' +reject security-merge-protection.json '.codeql_required_languages["content-toolkit"] = []' +reject security-merge-protection.json '.codeql_required_languages.classic = ["rust"]' +reject security-merge-protection.json '.codeql_required_languages.server = ["invented"]' +reject rulesets/code-scanning.json '.bypass_actors[0].bypass_mode = "always"' +reject rulesets/code-scanning.json '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' +reject rulesets/code-scanning.json '.rules[0].parameters.code_scanning_tools[0].alerts_threshold = "errors"' +reject rulesets/dependency-review.json '.rules[0].parameters.required_status_checks[0].integration_id = null' +reject actions-permissions.json '.sha_pinning_required = false' +echo 'Security merge policy validation tests passed.' From 235e05c66e0dcfcb0807c087aa05308fe3268b02 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 11:56:12 -0500 Subject: [PATCH 02/22] fix: verify dependency lifecycle producers semantically --- .github/workflows/dependency-review.yml | 3 + README.md | 26 +++++-- lib/security_readiness.py | 33 +++++++++ tests/security-readiness.py | 97 +++++++++++++++++++++++++ 4 files changed, 152 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index df3e793..e75109f 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -3,6 +3,9 @@ name: Dependency review on: pull_request: branches: [main] + types: [opened, synchronize, reopened, edited] + merge_group: + types: [checks_requested] permissions: contents: read diff --git a/README.md b/README.md index 7cf86e6..20ab112 100644 --- a/README.md +++ b/README.md @@ -164,13 +164,25 @@ requires the GitHub Actions `Dependency review` job. Both permit only organization owners to bypass through a pull request; bypass can merge findings and must be a deliberate owner decision. Direct pushes gain no exception. -The dependency workflow runs on every pull request to `main`, with read-only -contents permission and no checkout. It rejects new high/critical vulnerable -runtime, development, or unknown-scope dependencies and fails on review errors. +The dependency workflow runs on pull requests to `main` when opened, updated, +reopened, or edited, including a base-branch retarget to `main`. It also runs +for merge queue groups when checks are requested. The action reads each event's +base and head revisions directly, with read-only contents permission and no +checkout. It rejects new high/critical +vulnerable runtime, development, or unknown-scope dependencies and fails on +review errors. It relies on GitHub's dependency graph and submitted snapshots; unsupported or missing manifests cannot provide comprehensive dependency coverage. This is -separate from Dependabot's scheduled update and alert capabilities. Default -CodeQL setup similarly analyzes supported languages and its detected build +separate from Dependabot's scheduled update and alert capabilities. + +GitHub's [Dependabot alert limitations](https://docs.github.com/en/code-security/concepts/supply-chain-security/dependabot-alerts#limitations) +exclude advisory alerts for Actions referenced by commit SHA. SHA pinning and +a passing dependency review therefore do not prove that an Action commit is +free of known vulnerabilities; review upstream advisories when updating pins. +The high/critical dependency gate remains enabled for vulnerabilities GitHub +reports through its supported dependency data. + +Default CodeQL setup similarly analyzes supported languages and its detected build coverage; enabling it alone does not prove a successful analysis. Rust in `content-toolkit` is an explicit minimum, unioned with its existing languages. Legacy `javascript`/`typescript` aliases use `javascript-typescript` in the @@ -179,8 +191,8 @@ Read failures never replace that live language set with an incomplete set. GitHub's [code-scanning merge-protection limitations](https://docs.github.com/en/code-security/concepts/code-scanning/merge-protection) exclude Dependabot pull requests analyzed by default setup and merge queue -groups. Dependency review remains a required check for those Dependabot PRs; -it does not inherit that CodeQL exception. CodeQL merge protection also depends +groups. Dependency review runs for queue groups and remains a required check +for those Dependabot PRs; it does not inherit those CodeQL exceptions. CodeQL merge protection also depends on the alert's affected lines appearing in the PR diff. Ordinary quality alerts remain excluded by `alerts_threshold: none`. diff --git a/lib/security_readiness.py b/lib/security_readiness.py index 29d09c5..0646de7 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -170,8 +170,41 @@ def dependency_contract(data): if item and len(item[1]) > len(block[1]): line = block[1] + "branches: [main]" index += 1 + # Activity order and flow/block spelling do not change which PR or + # merge-group events trigger review. Keep duplicates and unknown events visible to the + # contract comparison rather than silently broadening the trigger. + activity_flow = re.fullmatch(r"( *)types: *\[(.*)\]", line) + activity_block = re.fullmatch(r"( *)types:", line) + activities = None + if activity_flow: + activities = [item.strip() for item in activity_flow[2].split(",")] + indentation = activity_flow[1] + elif activity_block: + activities = [] + indentation = activity_block[1] + while index + 1 < len(lines): + item = re.fullmatch(r"( *)- +(.+)", lines[index + 1]) + if not item or len(item[1]) <= len(indentation): + break + activities.append(item[2]) + index += 1 + if activities is not None: + values = [] + for activity in activities: + value = re.fullmatch(r"(?:([a-z_]+)|'([a-z_]+)'|\"([a-z_]+)\")", activity) + require(value is not None, "dependency-review activity types unsupported") + values.append(next(part for part in value.groups() if part is not None)) + line = indentation + "types: [" + ", ".join(sorted(values)) + "]" normalized.append(line) index += 1 + # YAML mapping key order does not change the PR trigger. Reorder only the + # two recognized, already-normalized keys; retain every value and any + # additional or duplicated key for the exact contract comparison below. + for index in range(len(normalized) - 2): + if (normalized[index] == " pull_request:" + and normalized[index + 1].startswith(" types: [") + and normalized[index + 2] == " branches: [main]"): + normalized[index + 1:index + 3] = [normalized[index + 2], normalized[index + 1]] return normalized diff --git a/tests/security-readiness.py b/tests/security-readiness.py index 64ca596..e066710 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -23,10 +23,45 @@ HEAD, BASE, MERGE = "a" * 40, "b" * 40, "c" * 40 OLD, NEW = "d" * 40, "e" * 40 WORKFLOW = ("name: Dependency review\n\non:\n pull_request:\n branches: [main]\n" + " types: [opened, synchronize, reopened, edited]\n" + " merge_group:\n types: [checks_requested]\n" "permissions:\n contents: read\njobs:\n dependency-review:\n" " name: Dependency review\n runs-on: ubuntu-24.04\n steps:\n" f" - uses: actions/dependency-review-action@{OLD} # v5.0.0\n" " with:\n fail-on-severity: high\n warn-only: false\n").encode() +# Snapshot of the actual content-toolkit producer, including its types-first +# mapping, block branch filter, named step, and complete enforcement settings. +# Immutable action versions remain producer-owned and are normalized by policy. +COMPONENT_WORKFLOW = b"""name: Dependency review + +on: + pull_request: + types: [opened, synchronize, reopened, edited] + branches: + - main + merge_group: + types: [checks_requested] + +permissions: + contents: read + +jobs: + dependency-review: + name: Dependency review + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Review dependency changes + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: high + fail-on-scopes: runtime,development,unknown + warn-only: false + comment-summary-in-pr: never + show-openssf-scorecard: false + retry-on-snapshot-warnings: true + retry-on-snapshot-warnings-timeout: 120 +""" SELECTED = {"github_owned_allowed": True, "verified_allowed": False, "patterns_allowed": ["rust-lang/crates-io-auth-action@*", "docker/setup-buildx-action@*"]} CODEQL_WORKFLOW = b"""name: CodeQL @@ -185,6 +220,68 @@ def test_dependency_ready_latest_attempt_exact_workflow(self): self.assertEqual(self.invoke("dependency")[0], 0) self.assertTrue(any("/attempts/2/jobs" in call for call in self.fixture.calls)) + def test_dependency_base_retarget_requires_edited_trigger(self): + # An otherwise green producer that ran before retargeting coverage was + # added cannot establish the complete default-branch PR contract. + old = WORKFLOW.replace(b"opened, synchronize, reopened, edited", + b"opened, synchronize, reopened") + self.fixture.default_workflow = old + self.fixture.run_workflow = old + self.fixture.executed_workflow = old + self.assert_deferred("dependency", "not landed") + + def test_dependency_base_retarget_activity_block_ready(self): + edited = WORKFLOW.replace( + b" types: [opened, synchronize, reopened, edited]", + b" types:\n - edited\n - opened\n - reopened\n - synchronize") + self.fixture.default_workflow = edited + self.fixture.run_workflow = edited + self.fixture.executed_workflow = edited + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_extra_activity_type_deferred(self): + self.fixture.default_workflow = WORKFLOW.replace(b"reopened, edited]", + b"reopened, edited, closed]") + self.assert_deferred("dependency", "not landed") + + def test_dependency_merge_queue_trigger_required(self): + self.fixture.default_workflow = WORKFLOW.replace( + b" merge_group:\n types: [checks_requested]\n", b"") + self.assert_deferred("dependency", "not landed") + + def test_dependency_merge_queue_activity_block_ready(self): + queue = WORKFLOW.replace(b" types: [checks_requested]", + b" types:\n - checks_requested") + self.fixture.default_workflow = queue + self.fixture.run_workflow = queue + self.fixture.executed_workflow = queue + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_dependency_merge_queue_wrong_activity_deferred(self): + self.fixture.default_workflow = WORKFLOW.replace(b"[checks_requested]", b"[destroyed]") + self.assert_deferred("dependency", "not landed") + + def test_dependency_merge_queue_run_does_not_replace_pr_evidence(self): + self.fixture.run["event"] = "merge_group" + self.assert_deferred("dependency", "latest dependency-review PR run not successful") + + def test_actual_component_workflow_key_order_matches_central_contract(self): + central = (ROOT / ".github/workflows/dependency-review.yml").read_bytes() + self.assertEqual(readiness.dependency_contract(COMPONENT_WORKFLOW), + readiness.dependency_contract(central)) + + def test_actual_component_key_order_preserves_branch_restriction(self): + central = (ROOT / ".github/workflows/dependency-review.yml").read_bytes() + changed = COMPONENT_WORKFLOW.replace(b" - main", b" - other") + self.assertNotEqual(readiness.dependency_contract(changed), + readiness.dependency_contract(central)) + + def test_actual_component_key_order_preserves_activity_restriction(self): + central = (ROOT / ".github/workflows/dependency-review.yml").read_bytes() + changed = COMPONENT_WORKFLOW.replace(b", edited]", b"]") + self.assertNotEqual(readiness.dependency_contract(changed), + readiness.dependency_contract(central)) + def test_dependency_not_landed(self): self.fixture.default_workflow = WORKFLOW.replace(b"fail-on-severity: high", b"fail-on-severity: low") self.assert_deferred("dependency", "not landed") From 18cb52d6db7dea15d9e97cb93421f7a268ff6adb Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 12:29:10 -0500 Subject: [PATCH 03/22] fix: fence security rollout and converge effective policy --- AGENTS.md | 18 ++- CONTRIBUTING.md | 8 +- README.md | 34 +++-- bin/audit | 84 ++++++++---- bin/publish | 171 +++++++++++++++++++---- bin/validate | 10 ++ config/actions-sha-pinning.json | 4 + lib/security_readiness.py | 59 ++++++-- tests/audit-fixtures.py | 55 ++++++-- tests/audit.sh | 47 +++++-- tests/publish-maintenance-branch.sh | 3 + tests/publish-security-merge.sh | 205 ++++++++++++++++++++++++++-- tests/security-readiness.py | 113 ++++++++++++++- tests/validate-security-merge.sh | 23 ++++ 14 files changed, 716 insertions(+), 118 deletions(-) create mode 100644 config/actions-sha-pinning.json diff --git a/AGENTS.md b/AGENTS.md index d7dbd9b..2580ffc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -177,8 +177,16 @@ New gates require read-only `bin/security-readiness` evidence for landed and successful producers; existing active gates survive later PR failures. Keep aggregate checks, zero approvals, and owner pull-request-only bypass intact. -- Desired Actions SHA enforcement is staged behind `--sha-pinning-canary` proof - of an ordinary action-update PR passing current-head CI without an allowlist - edit. Missing evidence defers only that field; never disable an existing true - setting. Allow reviewed action names with `@*` so pin updates do not require - central allowlist edits. Keep full SHAs in producer workflows. +- `config/actions-sha-pinning.json` persists SHA rollout as `pending` with no + canary or `enforced` with an accepted PR coordinate. Promote it through normal + source review only after ordinary PR-bound action-update evidence passes. + Pending command-line canaries only probe readiness; they never enable SHA + enforcement. Enforced policy uses its stored proof to restore drift and fails + if proof cannot be revalidated; audit reports disabled enforced policy as + drift. Preserve existing true settings. Allow action names with `@*`; keep + full SHAs in producer workflows instead of duplicate pin catalogs. +- Existing security gates may waive initial producer readiness only when the + effective default-branch API proves the same ruleset and desired tool/context. + Retired targets lose only exact repository-owned managed security rules; + preserve archived, foreign, and modified same-name rules. Keep repository + workflow overrides read-only and unable to approve PRs, with delta-aware writes. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c340bb5..2fe2958 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -134,6 +134,10 @@ gates require successful current-head analyses, including required languages. Keep existing gates active on later failures. Tests must cover missing/failed producers, unreadable evidence, archived targets, language unions, both scopes, and zero security mutations on a converged rerun. SHA enforcement must remain -pending until the ordinary action-update canary proves successful current-head -CI without an allowlist edit. Action allowances use owner/action wildcards; +pending until an ordinary action-update canary proves successful PR-bound +current-head CI without an allowlist edit and the persistent rollout source is +promoted to `enforced` with that accepted PR. Later disabled enforcement is drift, +not pending rollout; ordinary publication revalidates the recorded proof before +restoring it. Test rule retirement, effective default-branch applicability, and +repository workflow permission overrides. Action allowances use owner/action wildcards; full commit pins belong in workflows, not redundant version assertions. diff --git a/README.md b/README.md index 20ab112..1c606db 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,9 @@ read-only and are skipped on later runs. - GitHub Actions defaults to read-only, cannot approve pull requests, and may use only Atrinik, GitHub, Codecov coverage, explicitly allowed Docker actions, and `rust-lang/crates-io-auth-action@*`. Workflow references remain - full commit SHAs; the action allowlist permits routine pin updates. + full commit SHAs; the action allowlist permits routine pin updates. The + publisher also restores per-repository workflow permission overrides to + read-only with PR approvals disabled, applying only real drift. - Historical repositories listed in `config/repositories.json` are archived. The five former standalone classic component repositories are already archived read-only after their history, active work, issues, and release @@ -217,14 +219,28 @@ Rollout is owned by an organization administrator: remain enforced even when a later PR fails a security check. Scope migrations preserve these repository-owned gates. A converged security gate or language configuration causes no mutation or security-configuration reattachment. -4. Desired organization `sha_pinning_required` is true, but the publisher omits - this field until an ordinary action-pin update PR passes CI without allowlist - edits. Verify it using `bin/security-readiness sha OWNER/REPOSITORY/PR`, then - pass `--sha-pinning-canary OWNER/REPOSITORY/PR` to the reviewed plan/apply. - The verifier checks live allowance, exact PR changes, and successful current - PR-head CI. Missing or failed evidence defers SHA enforcement while the other - controls converge; a live true setting is always preserved. Full SHA pinning - is not a claim that an allowed action itself is trustworthy. + Moving a rule from another branch onto the default branch requires fresh + producer proof; only an already effective gate with the same tool or + dependency context can retain protection after a later producer failure. + Retiring a target removes its required-CI and security inventory entries + together. The publisher removes only matching repository-owned security + rules from active retired targets, preserving foreign or modified same-name + rules for explicit review and leaving archived repositories untouched. +4. Desired organization `sha_pinning_required` is true. Its durable rollout + contract is `config/actions-sha-pinning.json`: the initial `pending` phase + has no accepted canary and never enables the field. Verify an ordinary + action-pin update PR with `bin/security-readiness sha OWNER/REPOSITORY/PR`. + The optional publisher `--sha-pinning-canary` argument only reports readiness + while pending. After successful PR evidence, promote the source to + `{"phase":"enforced","canary":"OWNER/REPOSITORY/PR"}` through the normal + review and merge process before the first separately authorized live apply. + This records the accepted PR, with no duplicate action-version inventory. + The verifier checks live allowance, exact PR changes, and successful PR-bound + current-head CI. Ordinary subsequent publication uses that stored canary, + revalidates it and restores a disabled setting. Unreadable or failed evidence + blocks restoration with an error; the audit reports disabled enforced policy + as drift, never as initial rollout pending. A live true setting is always + preserved. Full SHA pinning does not prove an allowed action is trustworthy. Read-only preflight failures and unfinished producers are explicit pending controls, never successful rollout evidence. The audit reports pending controls diff --git a/bin/audit b/bin/audit index c778ee0..5c81ab8 100755 --- a/bin/audit +++ b/bin/audit @@ -183,9 +183,15 @@ audit_actions() { fi if [[ ${policy} == permissions ]] && jq -e '.sha_pinning_required == true' "${config}" >/dev/null; then - case $(jq -r '.sha_pinning_required | tostring' <<<"${current}") in + case $(jq -r '.sha_pinning_required | if type == "boolean" then tostring else "unreadable" end' <<<"${current}") in true) ;; - false) pending+=("Actions SHA pinning deferred pending canary proof: ${organization}") ;; + false) + case $(jq -r '.phase' "${root}/config/actions-sha-pinning.json") in + pending) pending+=("Actions SHA pinning deferred pending canary proof: ${organization}") ;; + enforced) record_finding "Actions SHA pinning enforcement drift: ${organization}" ;; + *) record_finding "Actions SHA pinning rollout phase could not be read: ${organization}" ;; + esac + ;; *) record_finding "Actions SHA pinning state could not be read: ${organization}" ;; esac fi @@ -196,30 +202,39 @@ audit_actions # The branch endpoint proves effective applicability; inherited names alone do not. # Details additionally prove bypass and ref/repository scope, including tag policy. -ruleset_targets_repository() { - local detail=$1 repository=$2 id=$3 pattern matched=false - if jq -e '.conditions.repository_id != null' <<<"${detail}" >/dev/null; then - jq -e --argjson id "${id}" '.conditions.repository_id.repository_ids | index($id) != null' \ - <<<"${detail}" >/dev/null || return 1 - fi - if jq -e '.conditions.repository_name != null' <<<"${detail}" >/dev/null; then - while IFS= read -r pattern; do - # Intentional GitHub repository-name wildcard matching. - # shellcheck disable=SC2053 - if [[ ${pattern} == '~ALL' || ${repository} == ${pattern} ]]; then matched=true; fi - done < <(jq -r '.conditions.repository_name.include[]' <<<"${detail}") - ${matched} || return 1 - while IFS= read -r pattern; do - # Intentional GitHub repository-name wildcard matching. - # shellcheck disable=SC2053 - if [[ ${pattern} == '~ALL' || ${repository} == ${pattern} ]]; then return 1; fi - done < <(jq -r '.conditions.repository_name.exclude[]' <<<"${detail}") - fi +# Match the entire publisher-owned repository scope. Being included in a wider +# scope does not prove convergence: a widened tag rule can constrain services +# with no release contract. Repository-level rulesets have only ref conditions. +organization_ruleset_scope() { + local name=$1 repository=$2 + jq -cn --arg name "${name}" --arg repository "${repository}" \ + --slurpfile repositories "${repositories_config}" \ + --slurpfile windows "${advisory_windows}" \ + --slurpfile security "${root}/config/security-merge-protection.json" ' + {include: [], exclude: [], protected: false} | + if $name == "01 - Default branch integrity" or + $name == "01 - Default branch linear history" then + .include = ["~ALL"] | .exclude = $windows[0].repositories + elif $name == "01 - Default branch integrity - classic" or + $name == "01 - Default branch linear history - classic" or + $name == "02 - Changes through pull requests - classic" then + .include = ["classic"] + elif $name == "02 - Changes through pull requests" then + .include = ($repositories[0].pull_request_gate - $windows[0].repositories) + elif ($name | startswith("03 - Required CI - ")) then + .include = [$repository] + elif $name == "04 - Immutable release tags" then + .include = $repositories[0].release_tags + elif $name == "06 - CodeQL security" then + .include = $security[0].code_scanning + elif $name == "07 - Dependency review" then + .include = $security[0].dependency_review + else error("unknown managed ruleset scope") end' } check_ruleset() { - local repository=$1 name=$2 template=$3 list=$4 metadata=$5 - local entry id source endpoint detail wanted + local repository=$1 name=$2 template=$3 list=$4 + local entry id source endpoint detail wanted scope entry=$(jq -c --arg name "${name}" '[.[] | select(.name == $name)]' <<<"${list}") if [[ $(jq length <<<"${entry}") != 1 ]]; then record_finding "missing or duplicate inherited ruleset '${name}': ${organization}/${repository}" @@ -232,6 +247,11 @@ check_ruleset() { Repository) endpoint="repos/${organization}/${repository}/rulesets/${id}" ;; *) record_finding "ruleset source could not be read '${name}': ${organization}/${repository}"; return ;; esac + if [[ ${name} == '04 - Immutable release tags' && ${source} == Repository ]] && + ! has_release_tags "${repository}"; then + record_finding "unexpected managed release-tag protection: ${organization}/${repository}" + return + fi if [[ ! -v ruleset_cache["${endpoint}"] ]]; then if ! detail=$(github_api "${endpoint}"); then record_finding "ruleset contents could not be read '${name}': ${organization}/${repository}" @@ -251,15 +271,21 @@ check_ruleset() { "${advisory_windows}" >/dev/null && [[ ${name} == '03 - Required CI - '* ]]; then wanted=$(jq '.bypass_actors[].bypass_mode = "always"' <<<"${wanted}") fi - if ! ruleset_targets_repository "${detail}" "${repository}" "$(jq -r .id "${metadata}")" || - ! jq -e --argjson wanted "${wanted}" ' + if [[ ${source} == Organization ]]; then + scope=$(organization_ruleset_scope "${name}" "${repository}") + wanted=$(jq --argjson scope "${scope}" '.conditions.repository_name = $scope' <<<"${wanted}") + fi + if ! jq -e --argjson wanted "${wanted}" ' # GitHub returns null for OrganizationAdmin, whose desired actor ID is 1. def normalize_bypass: map(if .actor_type == "OrganizationAdmin" and .actor_id == null then .actor_id = 1 else . end) | sort_by(.actor_type, .actor_id, .bypass_mode); + def normalize_conditions: + with_entries(if .key == "ref_name" or .key == "repository_name" then + .value.include |= sort | .value.exclude |= sort else . end); .enforcement == $wanted.enforcement and .target == $wanted.target and (.bypass_actors | normalize_bypass) == ($wanted.bypass_actors | normalize_bypass) and - .conditions.ref_name == $wanted.conditions.ref_name and + (.conditions | normalize_conditions) == ($wanted.conditions | normalize_conditions) and (.rules | map(if .type == "required_status_checks" then .parameters.required_status_checks |= sort_by(.context, .integration_id) else . end) | sort_by(.type)) == ($wanted.rules | map(if .type == "required_status_checks" then @@ -439,7 +465,11 @@ for repository in "${!expected_active[@]}"; do check_ruleset "${repository}" "03 - Required CI - ${repository}" \ "${root}/config/rulesets/required-ci.json" "${rulesets}" "${metadata_file}" fi - if has_release_tags "${repository}"; then + # Parent listings can contain correctly scoped but nonapplicable rulesets. + # Inspect the exact managed name on nonrelease repositories too, so widening + # its complete scope fails while foreign, unmanaged tag rules remain unowned. + if has_release_tags "${repository}" || + jq -e 'any(.[]; .name == "04 - Immutable release tags")' <<<"${rulesets}" >/dev/null; then check_ruleset "${repository}" '04 - Immutable release tags' \ "${root}/config/rulesets/immutable-release-tags.json" "${rulesets}" "${metadata_file}" fi diff --git a/bin/publish b/bin/publish index f888d23..259176f 100755 --- a/bin/publish +++ b/bin/publish @@ -1068,10 +1068,87 @@ converge_codeql_default_setup() { done } +security_ruleset_matches() { + local template=$1 current=$2 + jq -e --slurpfile desired "${template}" ' + # GitHub returns the organization-owner sentinel with a null ID. + def normalize: + .bypass_actors |= (map( + if .actor_type == "OrganizationAdmin" and .actor_id == null then + .actor_id = 1 + else . end) | sort_by(.actor_type, .actor_id, .bypass_mode)); + normalize as $current | + all($desired[0] | normalize | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null +} + +security_gate_applies_to_default_branch() { + local repository=$1 kind=$2 id=$3 metadata branch effective + metadata=$(github_api "repos/${organization}/${repository}") || return 1 + jq -e --argjson id "${active_repository_ids["${repository}"]}" ' + .id == $id and .archived == false and + (.default_branch | type == "string" and length > 0) + ' <<<"${metadata}" >/dev/null || return 1 + branch=$(jq -r '.default_branch | @uri' <<<"${metadata}") + effective=$(github_api "repos/${organization}/${repository}/rules/branches/${branch}") || return 1 + jq -e --argjson id "${id}" --arg kind "${kind}" ' + any(.[]; .ruleset_id == $id and + if $kind == "code_scanning" then + .type == "code_scanning" and + any(.parameters.code_scanning_tools[]?; .tool == "CodeQL") + else + .type == "required_status_checks" and + any(.parameters.required_status_checks[]?; + .context == "Dependency review" and .integration_id == 15368) + end) + ' <<<"${effective}" >/dev/null +} + +delete_removed_security_merge_rulesets() { + local repository kind template name existing rule id current metadata + for repository in "${active_repositories[@]}"; do + [[ -v active_repository_ids["${repository}"] ]] || continue + # The complete live inventory was checked against governed repositories + # before this function; archived repositories never enter this collection. + for kind in code_scanning dependency_review; do + if jq -e --arg kind "${kind}" --arg repository "${repository}" \ + '.[$kind] | index($repository) != null' "${security_merge_config}" >/dev/null; then + continue + fi + if [[ ${kind} == code_scanning ]]; then + template=${root}/config/rulesets/code-scanning.json + else + template=${root}/config/rulesets/dependency-review.json + fi + name=$(jq -r '.name' "${template}") + existing=$(github_api "repos/${organization}/${repository}/rulesets?includes_parents=false") + while IFS= read -r rule; do + id=$(jq -r '.id' <<<"${rule}") + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") + if jq -e --argjson id "${id}" --arg source "${organization}/${repository}" ' + .id == $id and .source_type == "Repository" and .source == $source + ' <<<"${current}" >/dev/null && security_ruleset_matches "${template}" "${current}"; then + metadata=$(github_api "repos/${organization}/${repository}") + if jq -e --argjson id "${active_repository_ids["${repository}"]}" ' + .id == $id and .archived == false + ' <<<"${metadata}" >/dev/null; then + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" + else + echo "SKIP ${organization}/${repository} ${name}: repository identity or archive state changed; retained" + fi + else + echo "SKIP ${organization}/${repository} ${name}: unrecognized ownership or payload; retained" + fi + done < <(jq -c --arg name "${name}" '.[] | select(.name == $name)' <<<"${existing}") + done + done +} + # These repository rules remain repository-owned even in organization scope: # GitHub Team does not support organization-level code-scanning rules. publish_security_merge_rulesets() { local repository kind template name existing id current reason status readiness_mode + delete_removed_security_merge_rulesets for kind in code_scanning dependency_review; do if [[ ${kind} == code_scanning ]]; then template=${root}/config/rulesets/code-scanning.json @@ -1098,32 +1175,13 @@ publish_security_merge_rulesets() { current='{}' if [[ -n ${id} ]]; then current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") - if jq -e --slurpfile desired "${template}" ' - # GitHub returns the organization-owner sentinel with a null ID. - def normalize: - .bypass_actors |= (map( - if .actor_type == "OrganizationAdmin" and .actor_id == null then - .actor_id = 1 - else . end) | sort_by(.actor_type, .actor_id, .bypass_mode)); - normalize as $current | - all($desired[0] | normalize | to_entries[]; $current[.key] == .value) - ' <<<"${current}" >/dev/null; then + if security_ruleset_matches "${template}" "${current}"; then echo "KEEP /repos/${organization}/${repository}/rulesets/${id} ${name}" continue fi fi # Preserve an already active gate when a later pull request fails it. - if ! jq -e --arg kind "${kind}" ' - .enforcement == "active" and - if $kind == "code_scanning" then - any(.rules[]?; .type == "code_scanning" and - any(.parameters.code_scanning_tools[]?; .tool == "CodeQL")) - else - any(.rules[]?; .type == "required_status_checks" and - any(.parameters.required_status_checks[]?; - .context == "Dependency review" and .integration_id == 15368)) - end - ' <<<"${current}" >/dev/null; then + if [[ -z ${id} ]] || ! security_gate_applies_to_default_branch "${repository}" "${kind}" "${id}"; then status=0 reason=$("${root}/bin/security-readiness" "${readiness_mode}" \ "${organization}/${repository}") || status=$? @@ -1141,8 +1199,27 @@ publish_security_merge_rulesets() { done } +validate_sha_canary_target() { + local canary=$1 owner repository number metadata + IFS=/ read -r owner repository number <<<"${canary}" + if [[ ${owner} != "${organization}" || ! ${number} =~ ^[1-9][0-9]*$ || + ! ${repository} =~ ^[A-Za-z0-9_.-]+$ ]] || + [[ ! -v active_repository_ids["${repository}"] ]]; then + echo "error: SHA canary must identify an active governed repository in ${organization}" >&2 + return 1 + fi + metadata=$(github_api "repos/${organization}/${repository}") + if ! jq -e --argjson id "${active_repository_ids["${repository}"]}" \ + --arg name "${organization}/${repository}" ' + .id == $id and .full_name == $name and .archived == false + ' <<<"${metadata}" >/dev/null; then + echo "error: SHA canary repository identity or archive state changed" >&2 + return 1 + fi +} + converge_actions_permissions() { - local current input reason + local current input reason phase canary current=$(github_api "orgs/${organization}/actions/permissions") if ! jq -e 'type == "object" and (.enabled_repositories | type == "string") and @@ -1155,13 +1232,28 @@ converge_actions_permissions() { input=$(mktemp) temporary_files+=("${input}") cp "${root}/config/actions-permissions.json" "${input}" + phase=$(jq -r '.phase' "${root}/config/actions-sha-pinning.json") + canary=$(jq -r '.canary // empty' "${root}/config/actions-sha-pinning.json") if ! jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then - if [[ -n ${sha_pinning_canary} ]] && - reason=$("${root}/bin/security-readiness" sha "${sha_pinning_canary}"); then - echo "READY SHA pinning: ${reason}" + if [[ ${phase} == enforced ]]; then + validate_sha_canary_target "${canary}" || return 1 + if ! reason=$("${root}/bin/security-readiness" sha "${canary}"); then + echo "error: enforced SHA pinning cannot be restored: ${reason}" >&2 + return 1 + fi + echo "READY enforced SHA pinning: ${reason}" else - echo "PENDING SHA pinning: ${reason:-supply --sha-pinning-canary OWNER/REPOSITORY/PR after an ordinary action update passes CI}" - # Omit the field: pending evidence never enables or disables this setting. + if [[ -n ${sha_pinning_canary} ]]; then + validate_sha_canary_target "${sha_pinning_canary}" || return 1 + if reason=$("${root}/bin/security-readiness" sha "${sha_pinning_canary}"); then + echo "READY canary evidence: ${reason}; source rollout phase remains pending" + else + echo "PENDING canary evidence: ${reason}" + fi + fi + echo "PENDING SHA pinning: promote the reviewed source rollout phase and accepted canary after readiness" + # Pending source never enables or disables SHA pinning, including when + # an optional command-line probe succeeds. Enforced source retains proof. jq 'del(.sha_pinning_required)' "${input}" >"${input}.pending" mv "${input}.pending" "${input}" fi @@ -1175,6 +1267,29 @@ converge_actions_permissions() { fi } +converge_repository_workflow_permissions() { + local repository current + for repository in "${active_repositories[@]}"; do + [[ -v active_repository_ids["${repository}"] ]] || continue + current=$(github_api "repos/${organization}/${repository}/actions/permissions/workflow") + if ! jq -e 'type == "object" and + (.default_workflow_permissions | IN("read", "write")) and + (.can_approve_pull_request_reviews | type == "boolean") + ' <<<"${current}" >/dev/null; then + echo "error: cannot read repository workflow permissions: ${repository}" >&2 + return 1 + fi + if jq -e --slurpfile desired "${root}/config/actions-workflow.json" ' + . as $current | all($desired[0] | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + echo "KEEP /repos/${organization}/${repository}/actions/permissions/workflow" + else + run_api PUT "repos/${organization}/${repository}/actions/permissions/workflow" \ + "${root}/config/actions-workflow.json" + fi + done +} + publish_organization_security() { local advanced_id baseline_id defaults_payload defaults local advanced_repository_count baseline_repository_count @@ -1488,6 +1603,8 @@ for repository in "${active_repositories[@]}"; do "${root}/config/repository-defaults.json" done +converge_repository_workflow_permissions + if [[ ${policy_scope} == organization ]]; then publish_organization_rulesets publish_organization_security diff --git a/bin/validate b/bin/validate index 798aaa3..c4dcca6 100755 --- a/bin/validate +++ b/bin/validate @@ -70,6 +70,16 @@ jq -e --slurpfile repositories "${repositories_config}" \ jq -e '.sha_pinning_required == true' \ "${root}/config/actions-permissions.json" >/dev/null +jq -e --slurpfile repositories "${repositories_config}" ' + (keys == ["canary", "phase"]) and + if .phase == "pending" then .canary == null + elif .phase == "enforced" then + (.canary | type == "string" and test("^atrinik/[A-Za-z0-9_.-]+/[1-9][0-9]*$")) and + ((.canary | split("/")[1]) as $repository | + ($repositories[0].required_ci | has($repository))) + else false end +' "${root}/config/actions-sha-pinning.json" >/dev/null + for security_ruleset in code-scanning dependency-review; do jq -e ' .target == "branch" and .enforcement == "active" and diff --git a/config/actions-sha-pinning.json b/config/actions-sha-pinning.json new file mode 100644 index 0000000..8e5861e --- /dev/null +++ b/config/actions-sha-pinning.json @@ -0,0 +1,4 @@ +{ + "phase": "pending", + "canary": null +} diff --git a/lib/security_readiness.py b/lib/security_readiness.py index 0646de7..75db0fc 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -311,16 +311,24 @@ def pin_changes(before, after, selected): return changed -def merge_check_run(coordinate, check, pr, merges): - if check["head_sha"] == pr["head"]["sha"]: - return True - if check["head_sha"] not in merges: +def merge_check_run(coordinate, check, pr, merges, proofs): + """Establish PR association before selecting the latest check, even if failing.""" + if check["head_sha"] != pr["head"]["sha"] and check["head_sha"] not in merges: return False - value = api(f"repos/{coordinate}/actions/runs?check_suite_id={check['check_suite']['id']}&per_page=1") - runs = value["workflow_runs"] - return bool(len(runs) == 1 and runs[0]["event"] == "pull_request" - and runs[0]["head_sha"] == pr["head"]["sha"] - and any(item["number"] == pr["number"] for item in runs[0]["pull_requests"])) + suite_id = check["check_suite"]["id"] + if suite_id not in proofs: + endpoint = f"repos/{coordinate}/actions/runs?check_suite_id={suite_id}&per_page=1" + value = api(endpoint) + runs = value["workflow_runs"] + valid = bool(len(runs) == 1 and runs[0]["event"] == "pull_request" + and runs[0]["head_sha"] == pr["head"]["sha"] + and runs[0]["check_suite_id"] == suite_id + and any(item["number"] == pr["number"] + and item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["ref"] == pr["base"]["ref"] + for item in runs[0]["pull_requests"])) + proofs[suite_id] = {"endpoint": endpoint, "run": runs[0] if valid else None} + return proofs[suite_id]["run"] is not None def sha(coordinate, number): @@ -384,15 +392,44 @@ def sha(coordinate, number): for ref in [pr["head"]["sha"], *sorted(merges)]: checks.extend(pages(f"repos/{coordinate}/commits/{ref}/check-runs?filter=all&app_id={ACTION_APP}", "check_runs")) + proofs, selected_checks = {}, [] for context in required: candidates = [check for check in checks if check["name"] == context and check["app"]["id"] == ACTION_APP and check["app"]["slug"] == "github-actions" - and merge_check_run(coordinate, check, pr, merges)] + and merge_check_run(coordinate, check, pr, merges, proofs)] require(bool(candidates), f"required check missing for canary head: {context}") - latest = max(candidates, key=lambda check: (check.get("started_at") or "", check["id"])) + # Check IDs order creation, including queued checks with no started_at. + # Scheduling/start time must not let an older green check hide a newer + # PR-associated check or suite that has not completed successfully. + latest = max(candidates, key=lambda check: check["id"]) + run = proofs[latest["check_suite"]["id"]]["run"] + require(run["status"] == "completed" and run["conclusion"] == "success", + f"required PR workflow run not successful for canary head: {context}") require(latest["status"] == "completed" and latest["conclusion"] == "success", f"required check not successful for canary head: {context}") + selected_checks.append(latest) + run_keys = ("id", "run_attempt", "event", "head_sha", "check_suite_id", "status", "conclusion", + "pull_requests") + check_keys = ("id", "name", "head_sha", "status", "conclusion", "app", "check_suite", "started_at") + fresh_checks = [] + for ref in [pr["head"]["sha"], *sorted(merges)]: + fresh_checks.extend(pages(f"repos/{coordinate}/commits/{ref}/check-runs?filter=all&app_id={ACTION_APP}", + "check_runs")) + def check_snapshot(values): + return sorted(json.dumps({key: check[key] for key in check_keys}, sort_keys=True) + for check in values if check["name"] in required) + require(check_snapshot(fresh_checks) == check_snapshot(checks), + "canary required check collection changed during preflight") + for suite_id in {check["check_suite"]["id"] for check in selected_checks}: + proof = proofs[suite_id] + fresh_runs = api(proof["endpoint"])["workflow_runs"] + require(len(fresh_runs) == 1 and all(fresh_runs[0][key] == proof["run"][key] for key in run_keys), + "canary PR workflow-run evidence changed during preflight") + for check in selected_checks: + fresh_check = api(f"repos/{coordinate}/check-runs/{check['id']}") + require(all(fresh_check[key] == check[key] for key in check_keys), + "canary required check evidence changed during preflight") fresh = api(f"repos/{coordinate}/pulls/{number}") require(fresh["head"]["sha"] == pr["head"]["sha"] and fresh["base"]["sha"] == pr["base"]["sha"] diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index 5c04c3c..4102ecc 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Read-only GitHub API double for the effective-governance audit tests.""" import copy +import fnmatch import json import os import sys @@ -57,35 +58,69 @@ def rulesets(repository): specs.append(('pull-request-gate', None)) if repository in repositories['required_ci']: specs.append(('required-ci', '03 - Required CI - ' + repository)) - if repository in repositories['release_tags']: + if repository in repositories['release_tags'] or scenario in ( + 'inherited-tag-unaffected', 'tag-extra-nonrelease', 'tag-all-nonrelease', 'tag-repository-nonrelease'): specs.append(('immutable-release-tags', None)) if repository in policy['code_scanning'] and scenario not in ('pending-codeql', 'ready-missing-codeql', 'codeql-readiness-unreadable'): specs.append(('code-scanning', None)) if repository in policy['dependency_review'] and scenario not in ('pending', 'ready-missing', 'dependency-readiness-unreadable'): specs.append(('dependency-review', None)) + if scenario == 'foreign-tag': + specs.append(('immutable-release-tags', 'Foreign tag retention')) result = [] for index, (template, name) in enumerate(specs, 1): item = config('rulesets/' + template + '.json') if name: item['name'] = name + if template == 'required-ci': item['rules'][0]['parameters']['required_status_checks'] = [ {'context': context, 'integration_id': 15368} for context in repositories['required_ci'][repository]] item.update(id=active.index(repository) * 10 + index, - source_type='Repository' if scenario == 'repository-rulesets' else 'Organization') + source_type='Repository' if scenario == 'repository-rulesets' or + template in ('code-scanning', 'dependency-review') else 'Organization') + if scenario == 'tag-repository-nonrelease' and template == 'immutable-release-tags': + item['source_type'] = 'Repository' # Match GitHub's actual owner representation, including repository rulesets. for actor in item['bypass_actors']: if actor['actor_type'] == 'OrganizationAdmin': actor['actor_id'] = None if scenario == 'owner-id-drift' and repository == 'observatory' and template == 'pull-request-gate': item['bypass_actors'][0]['actor_id'] = 2 - item['conditions']['repository_name'] = { - 'include': [repository], 'exclude': [], 'protected': False} + if item['source_type'] == 'Organization': + if template in ('default-branch-integrity', 'default-branch-linear-history'): + included = ['~ALL'] + excluded = config('advisory-merge-windows.json')['repositories'] + elif template == 'pull-request-gate': + windows = config('advisory-merge-windows.json')['repositories'] + included = [name for name in repositories['pull_request_gate'] if name not in windows] + excluded = [] + elif template == 'immutable-release-tags': + included, excluded = repositories['release_tags'], [] + else: + included, excluded = [repository], [] + item['conditions']['repository_name'] = { + 'include': list(included), 'exclude': list(excluded), 'protected': False} + if scenario == 'scope-order' and 'repository_name' in item['conditions']: + item['conditions']['repository_name']['include'].reverse() + item['conditions']['repository_name']['exclude'].reverse() + if scenario == 'repository-scope-extra' and template == 'dependency-review': + item['conditions']['repository_name'] = {'include': ['~ALL'], 'exclude': [], 'protected': False} + if scenario in ('tag-all-scope', 'tag-all-nonrelease') and template == 'immutable-release-tags': + item['conditions']['repository_name']['include'] = ['~ALL'] + if scenario in ('tag-extra-repo', 'tag-extra-nonrelease') and template == 'immutable-release-tags': + item['conditions']['repository_name']['include'].append('.github') + if scenario == 'tag-extra-exclude' and template == 'immutable-release-tags': + item['conditions']['repository_name']['exclude'] = ['foreign-repository'] + if scenario == 'pr-all-scope' and template == 'pull-request-gate': + item['conditions']['repository_name']['include'] = ['~ALL'] + if scenario == 'default-extra-exclude' and template == 'default-branch-integrity': + item['conditions']['repository_name']['exclude'].append('foreign-repository') # Keep inherited names while making actual targeting ineffective. if scenario == 'wrong-pr-scope' and repository == '.github' and template == 'pull-request-gate': item['conditions']['repository_name']['include'] = ['client'] if scenario == 'pending-wrong-dependency-scope' and repository == 'observatory' and template == 'dependency-review': - item['conditions']['repository_name']['exclude'] = ['observatory'] + item['conditions']['repository_name'] = {'include': ['observatory'], 'exclude': ['observatory'], 'protected': False} if scenario == 'missing-tag-scope' and repository == 'service-updater' and template == 'immutable-release-tags': item['conditions']['repository_name']['exclude'] = ['service-updater'] if scenario == 'tag-ref-drift' and repository == 'service-updater' and template == 'immutable-release-tags': @@ -131,7 +166,9 @@ def rulesets(repository): reject('Actions read denied') payload = config(filename) if filename == 'actions-permissions.json': - payload['sha_pinning_required'] = scenario != 'pending' + payload['sha_pinning_required'] = scenario not in ('pending', 'sha-enforced-off', 'phase-unreadable') + if scenario == 'sha-invalid-type': + payload['sha_pinning_required'] = 'false' if scenario == 'sha-unreadable': del payload['sha_pinning_required'] if filename == 'actions-workflow.json': @@ -178,8 +215,10 @@ def rulesets(repository): reject('effective branch read denied') payload = [] for item in rulesets(repository): - scope = item['conditions']['repository_name'] - if item['target'] == 'branch' and repository in scope['include'] and repository not in scope['exclude']: + scope = item['conditions'].get('repository_name', {'include': [repository], 'exclude': []}) + included = any(pattern == '~ALL' or fnmatch.fnmatchcase(repository, pattern) for pattern in scope['include']) + excluded = any(pattern == '~ALL' or fnmatch.fnmatchcase(repository, pattern) for pattern in scope['exclude']) + if item['target'] == 'branch' and included and not excluded: payload.extend(copy.deepcopy(item['rules'])) for rule in payload: if target and rule['type'] == 'required_status_checks': diff --git a/tests/audit.sh b/tests/audit.sh index 390da1d..1c6fe5b 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -21,19 +21,9 @@ EOF chmod +x "${temporary}/full/bin/validate" "${temporary}/full/bin/security-readiness" cp "${root}/tests/audit-fixtures.py" "${temporary}/mock/gh" chmod +x "${temporary}/mock/gh" -# Keep standalone tests runnable while publisher-owned schema is being delivered. -# Once integrated these files are copied directly from authoritative policy. -if [[ ! -f ${temporary}/full/config/security-merge-protection.json ]]; then - jq '{dependency_review: (.required_ci | keys), code_scanning: (.required_ci | keys), - codeql_required_languages: {"content-toolkit": ["rust"]}}' \ - "${temporary}/full/config/repositories.json" >"${temporary}/full/config/security-merge-protection.json" - jq '.name = "06 - CodeQL security" | .rules = [{type: "code_scanning", parameters: { - code_scanning_tools: [{tool: "CodeQL", security_alerts_threshold: "high_or_higher", alerts_threshold: "none"}]}}]' \ - "${temporary}/full/config/rulesets/required-ci.json" >"${temporary}/full/config/rulesets/code-scanning.json" - jq '.name = "07 - Dependency review" | .rules[0].parameters.required_status_checks = - [{context: "Dependency review", integration_id: 15368}]' \ - "${temporary}/full/config/rulesets/required-ci.json" >"${temporary}/full/config/rulesets/dependency-review.json" -fi +# Exercise both reviewed rollout phases using test-owned desired state. +printf '%s\n' '{"phase":"pending","canary":null}' \ + >"${temporary}/full/config/actions-sha-pinning.json" jq '.sha_pinning_required = true' "${temporary}/full/config/actions-permissions.json" \ >"${temporary}/actions.json" mv "${temporary}/actions.json" "${temporary}/full/config/actions-permissions.json" @@ -52,14 +42,27 @@ run_audit() { local scenario=$1 fixture=${2:-focused} repository=observatory if [[ ${fixture} == focused ]]; then case ${scenario} in - wrong-pr-scope|unexpected-codeql) repository=.github ;; + wrong-pr-scope|unexpected-codeql|tag-extra-nonrelease|tag-all-nonrelease|tag-repository-nonrelease|inherited-tag-unaffected|foreign-tag) repository=.github ;; advanced-config|advanced-setup) repository=classic ;; missing-rust) repository=content-toolkit ;; - missing-tag-scope|tag-ref-drift) repository=service-updater ;; + missing-tag-scope|tag-ref-drift|tag-all-scope|tag-extra-repo|tag-extra-exclude) repository=service-updater ;; unexpected-pr) repository=web-platform ;; esac fixture="fixture-${repository}" fi + case ${scenario} in + sha-enforced-off|sha-enforced-on) + printf '%s\n' '{"phase":"enforced","canary":"atrinik/observatory/1"}' \ + >"${temporary}/${fixture}/config/actions-sha-pinning.json" + ;; + phase-unreadable) + printf '%s\n' '{}' >"${temporary}/${fixture}/config/actions-sha-pinning.json" + ;; + *) + printf '%s\n' '{"phase":"pending","canary":null}' \ + >"${temporary}/${fixture}/config/actions-sha-pinning.json" + ;; + esac GH_TEST_SCENARIO=${scenario} \ PATH="${temporary}/mock:${PATH}" \ AUDIT_CONFIG_ROOT="${temporary}/${fixture}" \ @@ -75,6 +78,10 @@ output=$(run_audit legacy-security) grep -Fq 'Governance audit passed:' <<<"${output}" output=$(run_audit repository-rulesets) grep -Fq 'Governance audit passed:' <<<"${output}" +for scenario in scope-order inherited-tag-unaffected foreign-tag sha-enforced-on; do + output=$(run_audit "${scenario}") + grep -Fq 'Governance audit passed:' <<<"${output}" +done output=$(run_audit pending) grep -Fq 'PENDING Actions SHA pinning deferred pending canary proof' <<<"${output}" grep -Fq 'PENDING Dependency review merge protection awaits landed successful producer' <<<"${output}" @@ -105,6 +112,13 @@ assert_drift inventory-unreadable 'organization repository inventory could not b assert_drift wrong-pr-scope 'effective pull-request policy drift: atrinik/.github' assert_drift missing-tag-scope "ruleset scope or policy drift '04 - Immutable release tags': atrinik/service-updater" assert_drift tag-ref-drift "ruleset scope or policy drift '04 - Immutable release tags'" +for scenario in tag-all-scope tag-extra-repo tag-extra-exclude tag-all-nonrelease tag-extra-nonrelease; do + assert_drift "${scenario}" "ruleset scope or policy drift '04 - Immutable release tags'" +done +assert_drift tag-repository-nonrelease 'unexpected managed release-tag protection: atrinik/.github' +assert_drift pr-all-scope "ruleset scope or policy drift '02 - Changes through pull requests'" +assert_drift default-extra-exclude "ruleset scope or policy drift '01 - Default branch integrity'" +assert_drift repository-scope-extra "ruleset scope or policy drift '07 - Dependency review'" assert_drift integrity-bypass "ruleset scope or policy drift '01 - Default branch integrity'" assert_drift pr-bypass "ruleset scope or policy drift '02 - Changes through pull requests'" assert_drift owner-id-drift "ruleset scope or policy drift '02 - Changes through pull requests'" @@ -131,6 +145,9 @@ assert_drift repository-actions-unreadable 'repository Actions workflow permissi assert_drift actions-allowlist 'organization Actions selected-actions drift' assert_drift actions-unreadable 'organization Actions permissions could not be read' assert_drift sha-unreadable 'Actions SHA pinning state could not be read' +assert_drift sha-invalid-type 'Actions SHA pinning state could not be read' +assert_drift sha-enforced-off 'Actions SHA pinning enforcement drift' +assert_drift phase-unreadable 'Actions SHA pinning rollout phase could not be read' assert_drift missing-rust 'CodeQL setup or required language drift: atrinik/content-toolkit' for scenario in advanced-setup default-setup; do assert_drift "${scenario}" 'CodeQL setup or required language drift' diff --git a/tests/publish-maintenance-branch.sh b/tests/publish-maintenance-branch.sh index f616ed9..5fd7467 100755 --- a/tests/publish-maintenance-branch.sh +++ b/tests/publish-maintenance-branch.sh @@ -226,6 +226,9 @@ if [[ ${method} != GET ]]; then fi case "${endpoint}|${jq_filter}" in +repos/atrinik/*/actions/permissions/workflow\|*) + printf '{"default_workflow_permissions":"read","can_approve_pull_request_reviews":false}\n' + ;; "orgs/atrinik/actions/permissions|") printf '{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":false}\n' ;; diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index da40f40..83d736f 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -33,6 +33,7 @@ apply=true temporary_files=() security_merge_config=${root}/config/security-merge-protection.json codeql_advanced_config=${root}/config/codeql-advanced-setup.json +actions_sha_pinning_config=${root}/config/actions-sha-pinning.json active_repositories=(content-toolkit) declare -A active_repository_ids=([content-toolkit]=123) api_log=${temporary}/api.jsonl @@ -59,6 +60,28 @@ github_api() { jq -cn --arg endpoint "${endpoint}" \ '{method: "GET", endpoint: $endpoint}' >>"${api_log}" case ${endpoint} in + repos/atrinik/content-toolkit) + printf '%s\n' "${repository_state}" + ;; + repos/atrinik/content-toolkit/rules/branches/main | repos/atrinik/content-toolkit/rules/branches/release%2Fcurrent) + if [[ ${effective_state} == covering ]]; then + jq -n --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [$codeql[0].rules[] + {ruleset_id: 6}, + $dependency[0].rules[] + {ruleset_id: 7}]' + elif [[ ${effective_state} == different-id ]]; then + jq -n --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [$codeql[0].rules[] + {ruleset_id: 600}, + $dependency[0].rules[] + {ruleset_id: 700}]' + else + echo '[]' + fi + ;; + repos/atrinik/content-toolkit/actions/permissions/workflow) + [[ ${workflow_state} != unreadable ]] || return 1 + printf '%s\n' "${workflow_state}" + ;; repos/atrinik/content-toolkit/code-scanning/default-setup) [[ ${codeql_state} != unreadable ]] || return 1 printf '%s\n' "${codeql_state}" @@ -69,17 +92,39 @@ github_api() { else jq -n --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' - [{id: 6, name: $codeql[0].name}, {id: 7, name: $dependency[0].name}]' + [{id: 1, name: "01 - Default branch integrity"}, + {id: 3, name: "03 - Required CI"}, + {id: 6, name: $codeql[0].name}, {id: 7, name: $dependency[0].name}]' fi ;; repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) local template=code-scanning [[ ${endpoint##*/} != 7 ]] || template=dependency-review - if [[ ${ruleset_state} == exact ]]; then - jq '. + {id: 100, source_type: "Repository"}' \ + if [[ ${ruleset_state} == exact || ${ruleset_state} == removal-owned ]]; then + jq --argjson id "${endpoint##*/}" \ + '. + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"}' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == removal-owner-null ]]; then + jq --argjson id "${endpoint##*/}" \ + '. + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + .bypass_actors[0].actor_id = null' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == removal-foreign ]]; then + jq --argjson id "${endpoint##*/}" \ + '. + {id: $id, source_type: "Organization", source: "atrinik"}' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == removal-id-drift ]]; then + jq '. + {id: 999, source_type: "Repository", source: "atrinik/content-toolkit"}' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == removal-name-collision ]]; then + jq --argjson id "${endpoint##*/}" \ + '. + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + .rules = [{type: "deletion"}, {type: "non_fast_forward"}]' \ "${root}/config/rulesets/${template}.json" elif [[ ${ruleset_state} == owner-null ]]; then jq '.bypass_actors[0].actor_id = null' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == active-wrong-ref ]]; then + jq '.conditions.ref_name.include = ["refs/heads/feature"]' \ + "${root}/config/rulesets/${template}.json" elif [[ ${ruleset_state} == active-drift ]]; then jq '.bypass_actors = []' "${root}/config/rulesets/${template}.json" elif [[ ${ruleset_state} == active-unrelated-gate ]]; then @@ -102,6 +147,10 @@ github_api() { } run_api() { + if [[ $1 == DELETE ]]; then + jq -cn --arg endpoint "$2" '{method: "DELETE", endpoint: $endpoint}' >>"${api_log}" + return + fi jq -cn --arg method "$1" --arg endpoint "$2" --slurpfile payload "$3" \ '{method: $method, endpoint: $endpoint, payload: $payload[0]}' >>"${api_log}" } @@ -110,6 +159,11 @@ reset_case() { : >"${readiness_log}" codeql_state='{"state":"configured","query_suite":"default","languages":["python","javascript-typescript"]}' ruleset_state=missing + effective_state=covering + repository_state='{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' + workflow_state='{"default_workflow_permissions":"write","can_approve_pull_request_reviews":true}' + cp "${source_root}/config/security-merge-protection.json" "${security_merge_config}" + echo '{"phase":"pending","canary":null}' >"${actions_sha_pinning_config}" actions_state='{"enabled_repositories":"selected","allowed_actions":"all","sha_pinning_required":false}' sha_pinning_canary= export READINESS_STATUS=ready @@ -203,6 +257,7 @@ jq -se '[.[] | select(.method != "GET")] | length == 2 and "${api_log}" >/dev/null [[ ! -s ${readiness_log} ]] for ruleset_state in disabled active-without-gate active-unrelated-gate; do + effective_state=absent : >"${api_log}" : >"${readiness_log}" export READINESS_STATUS=failed @@ -211,6 +266,70 @@ for ruleset_state in disabled active-without-gate active-unrelated-gate; do jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null done +# An active rule on another ref does not grandfather a new default-branch gate. +for effective_state in absent different-id; do + : >"${api_log}" + : >"${readiness_log}" + ruleset_state=active-wrong-ref + export READINESS_STATUS=missing + publish_security_merge_rulesets >"${temporary}/wrong-ref-${effective_state}.txt" + assert_no_writes + jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null +done +reset_case +ruleset_state=active-drift +repository_state='{"id":123,"archived":false,"default_branch":"release/current"}' +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/encoded-ref.txt" +jq -se 'any(.[]; .endpoint == "repos/atrinik/content-toolkit/rules/branches/release%2Fcurrent") and + ([.[] | select(.method == "PUT")] | length == 2)' "${api_log}" >/dev/null +[[ ! -s ${readiness_log} ]] + +# Stale identities and archived metadata cannot grandfather a gate. +for repository_state in '{"id":999,"archived":false,"default_branch":"main"}' '{"id":123,"archived":true,"default_branch":"main"}'; do + : >"${api_log}" + : >"${readiness_log}" + ruleset_state=active-drift + effective_state=covering + export READINESS_STATUS=missing + publish_security_merge_rulesets >"${temporary}/metadata-drift.txt" + assert_no_writes + jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null +done + +# Exact repository-owned retired targets are removed; foreign collisions stay. +for removal in removal-owned removal-owner-null removal-foreign removal-id-drift removal-name-collision; do + reset_case + ruleset_state=${removal} + jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"]' \ + "${security_merge_config}" >"${temporary}/removed.json" + cp "${temporary}/removed.json" "${security_merge_config}" + publish_security_merge_rulesets >"${temporary}/${removal}.txt" + if [[ ${removal} == removal-owned || ${removal} == removal-owner-null ]]; then + jq -se '[.[] | select(.method != "GET")] == [ + {method: "DELETE", endpoint: "repos/atrinik/content-toolkit/rulesets/6"}, + {method: "DELETE", endpoint: "repos/atrinik/content-toolkit/rulesets/7"} + ]' "${api_log}" >/dev/null + else + assert_no_writes + rg -q '^SKIP ' "${temporary}/${removal}.txt" + fi + [[ ! -s ${readiness_log} ]] +done + +# Retirements also fence a repository archived or replaced after enumeration. +for retired_state in '{"id":123,"archived":true}' '{"id":999,"archived":false}'; do + reset_case + repository_state=${retired_state} + ruleset_state="removal-owned" + jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"]' \ + "${security_merge_config}" >"${temporary}/removed.json" + cp "${temporary}/removed.json" "${security_merge_config}" + publish_security_merge_rulesets >"${temporary}/removed-state-changed.txt" + assert_no_writes + [[ ! -s ${readiness_log} ]] +done + # An archived or missing repository is absent from the active identity map. reset_case active_repository_ids=() @@ -229,23 +348,85 @@ converge_actions_permissions >"${temporary}/sha-retained.txt" jq -se '[.[] | select(.method == "PUT")][0].payload.sha_pinning_required == true' \ "${api_log}" >/dev/null [[ ! -s ${readiness_log} ]] +for producer in failed ready; do + reset_case + sha_pinning_canary=atrinik/content-toolkit/99 + export READINESS_STATUS=${producer} + converge_actions_permissions >"${temporary}/sha-pending-${producer}.txt" + jq -se '[.[] | select(.method == "PUT")][0].payload | has("sha_pinning_required") | not' \ + "${api_log}" >/dev/null +done reset_case -sha_pinning_canary=atrinik/github-settings/99 -export READINESS_STATUS=failed -converge_actions_permissions >"${temporary}/sha-failed.txt" -jq -se '[.[] | select(.method == "PUT")][0].payload | has("sha_pinning_required") | not' \ - "${api_log}" >/dev/null -reset_case -sha_pinning_canary=atrinik/github-settings/99 -converge_actions_permissions >"${temporary}/sha-ready.txt" +echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" +converge_actions_permissions >"${temporary}/sha-enforced-ready.txt" jq -se --slurpfile desired "${root}/config/actions-permissions.json" ' [.[] | select(.method == "PUT")][0].payload == $desired[0]' "${api_log}" >/dev/null -jq -se '. == [{mode: "sha", repository: "atrinik/github-settings/99"}]' \ +jq -se '. == [{mode: "sha", repository: "atrinik/content-toolkit/99"}]' \ "${readiness_log}" >/dev/null +for producer in failed unreadable; do + reset_case + echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" + export READINESS_STATUS=${producer} + if converge_actions_permissions >"${temporary}/sha-enforced-${producer}.txt" 2>&1; then + echo "expected enforced SHA evidence failure to fail closed: ${producer}" >&2 + exit 1 + fi + assert_no_writes +done reset_case +echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" actions_state=$(cat "${root}/config/actions-permissions.json") +export READINESS_STATUS=failed converge_actions_permissions >"${temporary}/sha-keep.txt" assert_no_writes +[[ ! -s ${readiness_log} ]] + +# Neither a foreign owner nor an unregistered repository can lend canary proof. +for phase in pending enforced; do + for rejected_canary in foreign/content-toolkit/99 atrinik/unregistered/99; do + reset_case + if [[ ${phase} == enforced ]]; then + jq -n --arg canary "${rejected_canary}" '{phase:"enforced", canary:$canary}' \ + >"${actions_sha_pinning_config}" + else + sha_pinning_canary=${rejected_canary} + fi + if converge_actions_permissions >"${temporary}/sha-identity-rejected.txt" 2>&1; then + echo 'expected foreign or unregistered canary to be rejected' >&2 + exit 1 + fi + assert_no_writes + [[ ! -s ${readiness_log} ]] + done +done +reset_case +repository_state='{"id":999,"full_name":"atrinik/content-toolkit","archived":false}' +sha_pinning_canary=atrinik/content-toolkit/99 +if converge_actions_permissions >"${temporary}/sha-replaced-repository.txt" 2>&1; then + echo 'expected a replaced canary repository to be rejected' >&2 + exit 1 +fi +assert_no_writes +[[ ! -s ${readiness_log} ]] + +reset_case +converge_repository_workflow_permissions >"${temporary}/workflow-drift.txt" +jq -se --slurpfile desired "${root}/config/actions-workflow.json" ' + [.[] | select(.method != "GET")] == [{method: "PUT", + endpoint: "repos/atrinik/content-toolkit/actions/permissions/workflow", payload: $desired[0]}] +' "${api_log}" >/dev/null +reset_case +workflow_state=$(cat "${root}/config/actions-workflow.json") +converge_repository_workflow_permissions >"${temporary}/workflow-noop.txt" +assert_no_writes +for workflow_state in unreadable '{}' '{"default_workflow_permissions":"read","can_approve_pull_request_reviews":"false"}'; do + : >"${api_log}" + if converge_repository_workflow_permissions >"${temporary}/workflow-error.txt" 2>&1; then + echo 'expected unreadable or malformed workflow permissions to fail closed' >&2 + exit 1 + fi + assert_no_writes +done rm -f "${temporary_files[@]}" echo 'Security merge publisher tests passed.' diff --git a/tests/security-readiness.py b/tests/security-readiness.py index e066710..37ca5e1 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -105,6 +105,7 @@ def __init__(self): "conclusion": "success", "head_sha": HEAD, "check_suite_id": 99, "pull_requests": [{"number": 12, "head": {"sha": HEAD}, "base": {"sha": BASE, "ref": "main"}}]} + self.suite_runs = {} self.suite = {"id": 99, "head_sha": MERGE, "status": "completed", "conclusion": "success", "app": {"id": 15368}, "pull_requests": copy.deepcopy(self.run["pull_requests"])} self.jobs = [{"name": "Dependency review", "status": "completed", "conclusion": "success"}] @@ -154,6 +155,13 @@ def __call__(self, endpoint): return {"total_count": len(self.codeql_jobs), "jobs": copy.deepcopy(self.codeql_jobs)} if endpoint.endswith("/check-runs/999"): return copy.deepcopy(self.codeql_check) + if "/check-runs/" in endpoint: + identifier = int(endpoint.rsplit("/", 1)[1]) + for checks in self.checks.values(): + for current in checks: + if current["id"] == identifier: + return copy.deepcopy(current) + raise AssertionError("Unexpected check identity: " + endpoint) if "/actions/workflows/dependency-review.yml/runs" in endpoint: return {"workflow_runs": [copy.deepcopy(self.run)]} if "/attempts/2/jobs" in endpoint: @@ -167,8 +175,9 @@ def __call__(self, endpoint): if "/check-runs?" in endpoint: values = copy.deepcopy(self.checks[endpoint.split("/commits/")[1].split("/")[0]]) return {"total_count": len(values), "check_runs": values} - if "/actions/runs?check_suite_id=99" in endpoint: - return {"workflow_runs": [copy.deepcopy(self.run)]} + if "/actions/runs?check_suite_id=" in endpoint: + suite_id = int(parse_qs(urlparse(endpoint).query)["check_suite_id"][0]) + return {"workflow_runs": [copy.deepcopy(self.suite_runs.get(suite_id, self.run))]} if "/actions/runs?event=pull_request&head_sha=" in endpoint: return {"total_count": 1, "workflow_runs": [copy.deepcopy(self.run)]} if "/check-suites/99" in endpoint: @@ -477,6 +486,106 @@ def test_sha_wrong_head_check_rejected(self): self.fixture.checks[HEAD][0]["head_sha"] = OLD self.assert_deferred("sha", "missing") + def test_sha_direct_head_push_checks_rejected(self): + self.fixture.run["event"] = "push" + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_workflow_dispatch_checks_rejected(self): + self.fixture.run["event"] = "workflow_dispatch" + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_unrelated_pr_checks_rejected(self): + self.fixture.run["pull_requests"][0]["number"] = 100 + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_wrong_suite_checks_rejected(self): + self.fixture.run["check_suite_id"] = 100 + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_stale_run_checks_rejected(self): + self.fixture.run["head_sha"] = OLD + self.assert_deferred("sha", "missing") + + def test_sha_direct_head_failed_workflow_run_rejected(self): + self.fixture.run["conclusion"] = "failure" + self.assert_deferred("sha", "workflow run not successful") + + def add_newer_pr_check(self, check_status="completed", check_conclusion="success", + run_status="completed", run_conclusion="success"): + newer = check("Required checks", identifier=50, status=check_status, conclusion=check_conclusion) + newer["started_at"] = "2026-10-11T12:01:00Z" + newer["check_suite"]["id"] = 100 + self.fixture.checks[HEAD].append(newer) + run = copy.deepcopy(self.fixture.run) + run.update(id=20, check_suite_id=100, status=run_status, conclusion=run_conclusion) + self.fixture.suite_runs[100] = run + + def test_sha_newer_failing_pr_suite_does_not_hide_behind_older_green_suite(self): + self.add_newer_pr_check(check_conclusion="failure", run_conclusion="failure") + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_newer_pending_pr_suite_does_not_hide_behind_older_green_suite(self): + self.add_newer_pr_check(check_status="queued", check_conclusion=None, + run_status="queued", run_conclusion=None) + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_newer_queued_null_start_pr_suite_blocks_older_green_suite(self): + self.add_newer_pr_check(check_status="queued", check_conclusion=None, + run_status="queued", run_conclusion=None) + self.fixture.checks[HEAD][-1]["started_at"] = None + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_newer_queued_null_start_check_blocks_older_green_in_same_suite(self): + newer = check("Required checks", identifier=50, status="queued", conclusion=None) + newer["started_at"] = None + self.fixture.checks[HEAD].append(newer) + self.assert_deferred("sha", "required check not successful") + + def test_sha_older_check_starting_later_does_not_hide_newer_failure(self): + self.add_newer_pr_check(check_conclusion="failure", run_conclusion="failure") + self.fixture.checks[HEAD][0]["started_at"] = "2026-10-11T12:02:00Z" + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_newer_green_check_in_failed_pr_run_does_not_hide_behind_older_green_suite(self): + self.add_newer_pr_check(run_conclusion="failure") + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_direct_head_run_drift_at_completion_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if "/actions/runs?check_suite_id=99" in endpoint: + calls += 1 + if calls > 1: + value["workflow_runs"][0]["run_attempt"] += 1 + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "workflow-run evidence changed") + + def test_sha_direct_head_check_drift_at_completion_deferred(self): + def drift(endpoint): + value = self.fixture(endpoint) + if endpoint.endswith("/check-runs/1"): + value["conclusion"] = "failure" + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "required check evidence changed") + + def test_sha_new_required_check_at_completion_deferred(self): + calls = 0 + def drift(endpoint): + nonlocal calls + value = self.fixture(endpoint) + if f"/commits/{HEAD}/check-runs?" in endpoint: + calls += 1 + if calls > 1: + value["check_runs"].append(check("Required checks", identifier=50, conclusion="failure")) + value["total_count"] += 1 + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "required check collection changed") + def test_sha_wrong_app_check_rejected(self): self.fixture.checks[HEAD][0]["app"]["id"] = 1 self.assert_deferred("sha", "missing") diff --git a/tests/validate-security-merge.sh b/tests/validate-security-merge.sh index ac0558f..561f815 100755 --- a/tests/validate-security-merge.sh +++ b/tests/validate-security-merge.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -euo pipefail root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +root=${ATRINIK_PUBLISH_TEST_ROOT:-${root}} temporary=$(mktemp -d) trap 'rm -rf "${temporary}"' EXIT mkdir "${temporary}/bin" @@ -29,4 +30,26 @@ reject rulesets/code-scanning.json '.rules[0].parameters.code_scanning_tools[0]. reject rulesets/code-scanning.json '.rules[0].parameters.code_scanning_tools[0].alerts_threshold = "errors"' reject rulesets/dependency-review.json '.rules[0].parameters.required_status_checks[0].integration_id = null' reject actions-permissions.json '.sha_pinning_required = false' +reject actions-sha-pinning.json '.phase = "pending" | .canary = "atrinik/github-settings/99"' +reject actions-sha-pinning.json '.phase = "enforced" | .canary = null' +reject actions-sha-pinning.json '.phase = "enforced" | .canary = "atrinik/unregistered/99"' +reject actions-sha-pinning.json '.phase = "enforced" | .canary = "foreign/github-settings/99"' +reject actions-sha-pinning.json '.phase = "enforced" | .canary = "atrinik/github-settings/0"' +reject actions-sha-pinning.json '.phase = "unknown"' +reject actions-sha-pinning.json '.phase = null' +reject actions-sha-pinning.json '.unexpected = true' + +# Reviewed enforced source binds its canary to a repository with required CI. +jq '.phase = "enforced" | .canary = "atrinik/github-settings/99"' \ + "${root}/config/actions-sha-pinning.json" >"${temporary}/config/actions-sha-pinning.json" +"${temporary}/bin/validate" >/dev/null +cp "${root}/config/actions-sha-pinning.json" "${temporary}/config/actions-sha-pinning.json" + +# A coordinated target retirement is valid; unrelated governed policies stay. +jq 'del(.required_ci["service-updater"])' "${root}/config/repositories.json" \ + >"${temporary}/config/repositories.json" +jq '.code_scanning -= ["service-updater"] | .dependency_review -= ["service-updater"]' \ + "${root}/config/security-merge-protection.json" >"${temporary}/config/security-merge-protection.json" +"${temporary}/bin/validate" >/dev/null + echo 'Security merge policy validation tests passed.' From 295e6ec2dcd89aebf00b3f27554cbdc0e49d09cf Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 12:39:11 -0500 Subject: [PATCH 04/22] fix: preserve merged producer proof and portable policy tests --- lib/security_readiness.py | 96 ++++++++++++++---- tests/check-project-health.sh | 2 +- tests/publish-security-merge.sh | 2 +- tests/security-readiness.py | 171 +++++++++++++++++++++++++++++++- 4 files changed, 249 insertions(+), 22 deletions(-) diff --git a/lib/security_readiness.py b/lib/security_readiness.py index 75db0fc..278b932 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -208,6 +208,50 @@ def dependency_contract(data): return normalized +def pull_identity(pr): + return {"number": pr["number"], "state": pr["state"], "merged": pr["merged"], "draft": pr["draft"], + "head": (pr["head"]["sha"], pr["head"]["repo"]["full_name"]), + "base": (pr["base"]["sha"], pr["base"]["ref"], pr["base"]["repo"]["full_name"])} + + +def merged_run_pull(coordinate, run, default_branch): + """Recover only an unambiguous, fully fetched merged PR after refs disappear.""" + require(run["event"] == "pull_request" + and re.fullmatch(r"\.github/workflows/[^/]+\.ya?ml", run["path"]) is not None, + "merged PR association requires a PR workflow") + require(bool(SHA.fullmatch(run["head_sha"])), "merged PR association head missing") + associated = pages(f"repos/{coordinate}/commits/{run['head_sha']}/pulls") + require(len(associated) == 1, "merged PR association missing or ambiguous") + pr = api(f"repos/{coordinate}/pulls/{associated[0]['number']}") + require(pr["number"] == associated[0]["number"] and pr["merged"] is True + and pr["state"] == "closed" and pr["head"]["sha"] == run["head_sha"] + and pr["head"]["repo"]["full_name"].lower() == run["head_repository"]["full_name"].lower() + and pr["base"]["repo"]["full_name"].lower() == coordinate.lower() + and pr["base"]["ref"] == default_branch, + "merged PR association identity does not match run/default branch") + return pr + + +def association_references(coordinate, run, suite, default_branch, recovered): + if run["pull_requests"] and suite["pull_requests"]: + return run["pull_requests"], suite["pull_requests"] + pr = merged_run_pull(coordinate, run, default_branch) + for references in (run["pull_requests"], suite["pull_requests"]): + require(all(item["number"] == pr["number"] + and item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["sha"] == pr["base"]["sha"] + and item["base"]["ref"] == pr["base"]["ref"] for item in references), + "nonempty PR association conflicts with merged PR") + recovered.append((run, default_branch, pull_identity(pr))) + return [pr], [pr] + + +def verify_recovered_associations(coordinate, recovered): + for run, default_branch, identity in recovered: + require(pull_identity(merged_run_pull(coordinate, run, default_branch)) == identity, + "merged PR association changed during preflight") + + def dependency(coordinate): metadata = repository(coordinate) path = ".github/workflows/dependency-review.yml" @@ -218,7 +262,7 @@ def dependency(coordinate): run = latest_dependency_run(coordinate) require(run["event"] == "pull_request" and run["status"] == "completed" and run["conclusion"] == "success", "latest dependency-review PR run not successful") - require(bool(run["pull_requests"]), "dependency-review PR association missing") + require(run["path"] == path, "dependency-review run workflow path changed") suite_endpoint = f"repos/{coordinate}/check-suites/{run['check_suite_id']}" suite = api(suite_endpoint) require(suite["id"] == run["check_suite_id"] and suite["app"]["id"] == ACTION_APP @@ -230,8 +274,10 @@ def dependency(coordinate): if executed != run["head_sha"]: merge_commit = api(f"repos/{coordinate}/commits/{executed}") merge_parents = [parent["sha"] for parent in merge_commit["parents"]] - pulls = [] - for reference in run["pull_requests"]: + recovered, pulls = [], [] + references, suite_references = association_references( + coordinate, run, suite, metadata["default_branch"], recovered) + for reference in references: pr = api(f"repos/{coordinate}/pulls/{reference['number']}") require(pr["base"]["repo"]["full_name"].lower() == coordinate.lower() and pr["base"]["ref"] == metadata["default_branch"], @@ -245,7 +291,7 @@ def dependency(coordinate): and item["head"]["sha"] == pr["head"]["sha"] and item["base"]["sha"] == pr["base"]["sha"] and item["base"]["ref"] == pr["base"]["ref"] - for item in suite["pull_requests"]), + for item in suite_references), "dependency-review execution PR association missing or stale") if merge_parents is not None: require(len(merge_parents) == 2 and set(merge_parents) @@ -260,7 +306,7 @@ def dependency(coordinate): and matches[0]["conclusion"] == "success", "Dependency review job not successful") fresh = latest_dependency_run(coordinate) run_keys = ("id", "run_attempt", "status", "conclusion", "event", "head_sha", - "check_suite_id", "pull_requests") + "check_suite_id", "pull_requests", "path", "head_repository") require(all(fresh[key] == run[key] for key in run_keys), "dependency-review run changed during preflight") fresh_suite = api(suite_endpoint) @@ -275,6 +321,7 @@ def dependency(coordinate): and fresh_pr["base"]["repo"]["full_name"] == pr["base"]["repo"]["full_name"] and all(fresh_pr[key] == pr[key] for key in ("state", "draft", "merged")), "dependency-review PR changed during preflight") + verify_recovered_associations(coordinate, recovered) require(contents(coordinate, path, metadata["default_branch"]) == landed, "default-branch dependency-review workflow changed during preflight") require(repository(coordinate)["default_branch"] == metadata["default_branch"], @@ -311,7 +358,7 @@ def pin_changes(before, after, selected): return changed -def merge_check_run(coordinate, check, pr, merges, proofs): +def merge_check_run(coordinate, check, pr, merges, proofs, recovered): """Establish PR association before selecting the latest check, even if failing.""" if check["head_sha"] != pr["head"]["sha"] and check["head_sha"] not in merges: return False @@ -322,12 +369,18 @@ def merge_check_run(coordinate, check, pr, merges, proofs): runs = value["workflow_runs"] valid = bool(len(runs) == 1 and runs[0]["event"] == "pull_request" and runs[0]["head_sha"] == pr["head"]["sha"] - and runs[0]["check_suite_id"] == suite_id - and any(item["number"] == pr["number"] - and item["head"]["sha"] == pr["head"]["sha"] - and item["base"]["ref"] == pr["base"]["ref"] - for item in runs[0]["pull_requests"])) - proofs[suite_id] = {"endpoint": endpoint, "run": runs[0] if valid else None} + and runs[0]["check_suite_id"] == suite_id) + suite = None + if valid: + suite = api(f"repos/{coordinate}/check-suites/{suite_id}") + references, suite_references = association_references( + coordinate, runs[0], suite, pr["base"]["ref"], recovered) + valid = suite["id"] == suite_id and suite["app"]["id"] == ACTION_APP and all( + any(item["number"] == pr["number"] + and item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["ref"] == pr["base"]["ref"] for item in refs) + for refs in (references, suite_references)) + proofs[suite_id] = {"endpoint": endpoint, "run": runs[0] if valid else None, "suite": suite} return proofs[suite_id]["run"] is not None @@ -360,7 +413,7 @@ def sha(coordinate, number): updates += pin_changes(contents(coordinate, path, pr["base"]["sha"]), contents(pr["head"]["repo"]["full_name"], path, pr["head"]["sha"]), selected) merge = pr.get("merge_commit_sha") if not pr["merged"] else None - merges = set() + merges, recovered = set(), [] if merge: require(bool(SHA.fullmatch(merge)), "canary merge SHA malformed") commit = api(f"repos/{coordinate}/commits/{merge}") @@ -373,10 +426,14 @@ def sha(coordinate, number): runs = pages(f"repos/{coordinate}/actions/runs?event=pull_request&head_sha={pr['head']['sha']}", "workflow_runs") for run in runs: - references = [item for item in run["pull_requests"] if item["number"] == number] - if run["event"] != "pull_request" or run["head_sha"] != pr["head"]["sha"] or not references: + if run["event"] != "pull_request" or run["head_sha"] != pr["head"]["sha"]: continue suite = api(f"repos/{coordinate}/check-suites/{run['check_suite_id']}") + references, _ = association_references( + coordinate, run, suite, metadata["default_branch"], recovered) + references = [item for item in references if item["number"] == number] + if not references: + continue candidate = suite["head_sha"] if candidate == pr["head"]["sha"]: continue @@ -397,7 +454,7 @@ def sha(coordinate, number): candidates = [check for check in checks if check["name"] == context and check["app"]["id"] == ACTION_APP and check["app"]["slug"] == "github-actions" - and merge_check_run(coordinate, check, pr, merges, proofs)] + and merge_check_run(coordinate, check, pr, merges, proofs, recovered)] require(bool(candidates), f"required check missing for canary head: {context}") # Check IDs order creation, including queued checks with no started_at. # Scheduling/start time must not let an older green check hide a newer @@ -410,7 +467,7 @@ def sha(coordinate, number): f"required check not successful for canary head: {context}") selected_checks.append(latest) run_keys = ("id", "run_attempt", "event", "head_sha", "check_suite_id", "status", "conclusion", - "pull_requests") + "pull_requests", "path", "head_repository") check_keys = ("id", "name", "head_sha", "status", "conclusion", "app", "check_suite", "started_at") fresh_checks = [] for ref in [pr["head"]["sha"], *sorted(merges)]: @@ -426,6 +483,11 @@ def check_snapshot(values): fresh_runs = api(proof["endpoint"])["workflow_runs"] require(len(fresh_runs) == 1 and all(fresh_runs[0][key] == proof["run"][key] for key in run_keys), "canary PR workflow-run evidence changed during preflight") + fresh_suite = api(f"repos/{coordinate}/check-suites/{suite_id}") + suite_keys = ("id", "head_sha", "app", "status", "conclusion", "pull_requests") + require(all(fresh_suite[key] == proof["suite"][key] for key in suite_keys), + "canary check-suite evidence changed during preflight") + verify_recovered_associations(coordinate, recovered) for check in selected_checks: fresh_check = api(f"repos/{coordinate}/check-runs/{check['id']}") require(all(fresh_check[key] == check[key] for key in check_keys), diff --git a/tests/check-project-health.sh b/tests/check-project-health.sh index b9c7709..ba86f6b 100755 --- a/tests/check-project-health.sh +++ b/tests/check-project-health.sh @@ -801,7 +801,7 @@ fi grep -Fq '2026-08-07T01:00:00Z' "${temporary}/old-success.out" grep -Fq 'actions/runs/1' "${temporary}/old-success.out" -if rg -n 'health-token|settings-token' "${temporary}"/*.out \ +if grep -En 'health-token|settings-token' "${temporary}"/*.out \ "${temporary}"/*.err "${temporary}/step-summary" >/dev/null; then echo "error: health output exposed credential material" >&2 exit 1 diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index 83d736f..df8e0ae 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -312,7 +312,7 @@ for removal in removal-owned removal-owner-null removal-foreign removal-id-drift ]' "${api_log}" >/dev/null else assert_no_writes - rg -q '^SKIP ' "${temporary}/${removal}.txt" + grep -q '^SKIP ' "${temporary}/${removal}.txt" fi [[ ! -s ${readiness_log} ]] done diff --git a/tests/security-readiness.py b/tests/security-readiness.py index 37ca5e1..4500d02 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -103,9 +103,12 @@ def __init__(self): "head": {"sha": HEAD, "repo": {"full_name": "atrinik/example"}}} self.run = {"id": 9, "run_attempt": 2, "event": "pull_request", "status": "completed", "conclusion": "success", "head_sha": HEAD, "check_suite_id": 99, + "path": ".github/workflows/dependency-review.yml", + "head_repository": {"full_name": "atrinik/example"}, "pull_requests": [{"number": 12, "head": {"sha": HEAD}, "base": {"sha": BASE, "ref": "main"}}]} self.suite_runs = {} + self.associated_pulls = [{"number": 12}] self.suite = {"id": 99, "head_sha": MERGE, "status": "completed", "conclusion": "success", "app": {"id": 15368}, "pull_requests": copy.deepcopy(self.run["pull_requests"])} self.jobs = [{"name": "Dependency review", "status": "completed", "conclusion": "success"}] @@ -170,6 +173,8 @@ def __call__(self, endpoint): return copy.deepcopy(self.live) if "/pulls/12/files" in endpoint: return copy.deepcopy(self.files) + if f"/commits/{HEAD}/pulls?" in endpoint: + return copy.deepcopy(self.associated_pulls) if endpoint == "repos/atrinik/example/commits/" + MERGE: return {"parents": [{"sha": value} for value in self.parents]} if "/check-runs?" in endpoint: @@ -180,8 +185,10 @@ def __call__(self, endpoint): return {"workflow_runs": [copy.deepcopy(self.suite_runs.get(suite_id, self.run))]} if "/actions/runs?event=pull_request&head_sha=" in endpoint: return {"total_count": 1, "workflow_runs": [copy.deepcopy(self.run)]} - if "/check-suites/99" in endpoint: - return copy.deepcopy(self.suite) + if "/check-suites/" in endpoint: + value = copy.deepcopy(self.suite) + value["id"] = int(endpoint.rsplit("/", 1)[1]) + return value if "/branches/main" in endpoint: return {"commit": {"sha": self.branch_head}} if "/code-scanning/analyses?" in endpoint: @@ -229,6 +236,164 @@ def test_dependency_ready_latest_attempt_exact_workflow(self): self.assertEqual(self.invoke("dependency")[0], 0) self.assertTrue(any("/attempts/2/jobs" in call for call in self.fixture.calls)) + def merged_empty_fixture(self, run_empty=True, suite_empty=True): + fixture = Fixture() + fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + if run_empty: + fixture.run["pull_requests"] = [] + if suite_empty: + fixture.suite["pull_requests"] = [] + return fixture + + def test_merged_empty_associations_ready_for_dependency_and_stored_sha_canary(self): + for mode in ("dependency", "sha"): + for run_empty, suite_empty in ((True, False), (False, True), (True, True)): + for direct_head in (False, True): + with self.subTest(mode=mode, run_empty=run_empty, suite_empty=suite_empty, + direct_head=direct_head): + fixture = self.merged_empty_fixture(run_empty, suite_empty) + if direct_head: + fixture.suite["head_sha"] = HEAD + elif mode == "sha": + fixture.checks[MERGE] = [check("Required checks", MERGE), + check("Conventional PR title", MERGE, 2)] + fixture.checks[HEAD] = [] + with patch.object(readiness, "api", fixture): + result, output = self.invoke(mode) + self.assertEqual(result, 0, output) + self.assertGreaterEqual(sum(f"/commits/{HEAD}/pulls?" in endpoint + for endpoint in fixture.calls), 2) + + def test_merged_empty_association_newer_failed_or_queued_canary_still_blocks(self): + for status, conclusion in (("completed", "failure"), ("queued", None)): + with self.subTest(status=status): + fixture = self.merged_empty_fixture() + newer = check("Required checks", identifier=50, status=status, conclusion=conclusion) + newer["started_at"] = None + newer["check_suite"]["id"] = 100 + fixture.checks[HEAD].append(newer) + run = copy.deepcopy(fixture.run) + run.update(id=20, check_suite_id=100, status=status, conclusion=conclusion) + fixture.suite_runs[100] = run + with patch.object(readiness, "api", fixture): + self.assert_deferred("sha", "workflow run not successful") + + def test_merged_empty_associations_reject_ambiguous_or_missing_commit_pulls(self): + for mode in ("dependency", "sha"): + for entries in ([], [{"number": 12}, {"number": 13}]): + with self.subTest(mode=mode, entries=entries): + fixture = self.merged_empty_fixture() + fixture.associated_pulls = entries + with patch.object(readiness, "api", fixture): + self.assert_deferred(mode, "association missing or ambiguous") + + def test_merged_empty_associations_reject_open_pr(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + fixture.pr.update(state="open", merged=False) + fixture.pr["merge_commit_sha"] = MERGE + with patch.object(readiness, "api", fixture): + self.assert_deferred(mode, "association identity") + + def test_merged_empty_associations_reject_identity_mismatch(self): + for mode in ("dependency", "sha"): + for field in ("head", "head_repository", "base", "base_repository", "number"): + with self.subTest(mode=mode, field=field): + fixture = self.merged_empty_fixture() + # Change only the independently fetched association PR; retain + # the caller's initial canary so its own guards cannot mask this test. + count = 0 + def mismatch(endpoint): + nonlocal count + value = fixture(endpoint) + if endpoint.endswith("/pulls/12"): + count += 1 + if mode == "dependency" or count > 1: + if field == "head": + value["head"]["sha"] = OLD + elif field == "head_repository": + value["head"]["repo"]["full_name"] = "other/example" + elif field == "base": + value["base"]["ref"] = "other" + elif field == "base_repository": + value["base"]["repo"]["full_name"] = "other/example" + else: + value["number"] = 13 + return value + with patch.object(readiness, "api", mismatch): + self.assert_deferred(mode, "association identity") + + def test_merged_empty_associations_never_override_nonempty_conflict(self): + for mode in ("dependency", "sha"): + for conflicting in ("run", "suite"): + with self.subTest(mode=mode, conflicting=conflicting): + fixture = self.merged_empty_fixture(conflicting != "run", conflicting != "suite") + source = fixture.run if conflicting == "run" else fixture.suite + source["pull_requests"][0]["base"]["sha"] = OLD + with patch.object(readiness, "api", fixture): + self.assert_deferred(mode, "nonempty PR association conflicts") + + def test_merged_empty_associations_reject_non_pr_event_or_workflow_path(self): + for mode in ("dependency", "sha"): + for field, value in (("event", "push"), ("event", "workflow_dispatch"), + ("path", "untrusted.yml")): + with self.subTest(mode=mode, field=field, value=value): + fixture = self.merged_empty_fixture() + fixture.run[field] = value + with patch.object(readiness, "api", fixture): + result, output = self.invoke(mode) + self.assertEqual(result, 1, output) + + def test_merged_empty_associations_unreadable_is_error(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + def unreadable(endpoint): + if f"/commits/{HEAD}/pulls?" in endpoint: + raise readiness.APIError(endpoint, 403) + return fixture(endpoint) + with patch.object(readiness, "api", unreadable): + code, output = self.invoke(mode) + self.assertEqual(code, 2, output) + self.assertIn("HTTP 403", output) + + def test_merged_empty_association_collection_is_refenced(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + verify = readiness.verify_recovered_associations + def change_at_fence(coordinate, recovered): + fixture.associated_pulls.append({"number": 13}) + return verify(coordinate, recovered) + with patch.object(readiness, "api", fixture), patch.object( + readiness, "verify_recovered_associations", change_at_fence): + self.assert_deferred(mode, "association missing or ambiguous") + + def test_merged_empty_association_pr_identity_is_refenced(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + verify = readiness.verify_recovered_associations + def change_at_fence(coordinate, recovered): + fixture.pr["base"]["sha"] = OLD + return verify(coordinate, recovered) + with patch.object(readiness, "api", fixture), patch.object( + readiness, "verify_recovered_associations", change_at_fence): + self.assert_deferred(mode, "association changed during preflight") + + def test_merged_empty_association_preserves_workflow_and_merge_parent_proof(self): + for mode in ("dependency", "sha"): + with self.subTest(mode=mode): + fixture = self.merged_empty_fixture() + fixture.parents = [OLD, HEAD] + with patch.object(readiness, "api", fixture): + self.assert_deferred(mode, "merge") + fixture = self.merged_empty_fixture() + fixture.executed_workflow += b"# changed execution bytes\n" + with patch.object(readiness, "api", fixture): + self.assert_deferred("dependency", "different workflow bytes") + def test_dependency_base_retarget_requires_edited_trigger(self): # An otherwise green producer that ran before retargeting coverage was # added cannot establish the complete default-branch PR contract. @@ -371,7 +536,7 @@ def test_dependency_direct_head_execution_bound_to_pr(self): def test_dependency_direct_head_missing_execution_pr_association(self): self.fixture.suite.update(head_sha=HEAD, pull_requests=[]) - self.assert_deferred("dependency", "execution PR association") + self.assert_deferred("dependency", "merged PR association") def test_dependency_executed_merge_wrong_parents_deferred(self): self.fixture.parents = [BASE, OLD] From 2a5a1a9409e74066c0280e1b6cf3c676b54d4942 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 12:56:00 -0500 Subject: [PATCH 05/22] fix: bind canary base proof and security gate ownership --- bin/audit | 12 +++--- lib/security_readiness.py | 33 ++++++++++++---- tests/audit-fixtures.py | 7 ++++ tests/audit.sh | 2 + tests/security-readiness.py | 79 ++++++++++++++++++++++++++++++++++++- 5 files changed, 118 insertions(+), 15 deletions(-) diff --git a/bin/audit b/bin/audit index 5c81ab8..441ad36 100755 --- a/bin/audit +++ b/bin/audit @@ -209,8 +209,7 @@ organization_ruleset_scope() { local name=$1 repository=$2 jq -cn --arg name "${name}" --arg repository "${repository}" \ --slurpfile repositories "${repositories_config}" \ - --slurpfile windows "${advisory_windows}" \ - --slurpfile security "${root}/config/security-merge-protection.json" ' + --slurpfile windows "${advisory_windows}" ' {include: [], exclude: [], protected: false} | if $name == "01 - Default branch integrity" or $name == "01 - Default branch linear history" then @@ -225,10 +224,6 @@ organization_ruleset_scope() { .include = [$repository] elif $name == "04 - Immutable release tags" then .include = $repositories[0].release_tags - elif $name == "06 - CodeQL security" then - .include = $security[0].code_scanning - elif $name == "07 - Dependency review" then - .include = $security[0].dependency_review else error("unknown managed ruleset scope") end' } @@ -242,6 +237,11 @@ check_ruleset() { fi id=$(jq -r '.[0].id' <<<"${entry}") source=$(jq -r '.[0].source_type' <<<"${entry}") + if [[ ${name} == '06 - CodeQL security' || ${name} == '07 - Dependency review' ]] && + [[ ${source} != Repository ]]; then + record_finding "managed security ruleset must be repository-owned '${name}': ${organization}/${repository}" + return + fi case ${source} in Organization) endpoint="orgs/${organization}/rulesets/${id}" ;; Repository) endpoint="repos/${organization}/${repository}/rulesets/${id}" ;; diff --git a/lib/security_readiness.py b/lib/security_readiness.py index 278b932..0033a87 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -370,18 +370,35 @@ def merge_check_run(coordinate, check, pr, merges, proofs, recovered): valid = bool(len(runs) == 1 and runs[0]["event"] == "pull_request" and runs[0]["head_sha"] == pr["head"]["sha"] and runs[0]["check_suite_id"] == suite_id) - suite = None + suite, bases = None, set() if valid: suite = api(f"repos/{coordinate}/check-suites/{suite_id}") references, suite_references = association_references( coordinate, runs[0], suite, pr["base"]["ref"], recovered) - valid = suite["id"] == suite_id and suite["app"]["id"] == ACTION_APP and all( - any(item["number"] == pr["number"] - and item["head"]["sha"] == pr["head"]["sha"] - and item["base"]["ref"] == pr["base"]["ref"] for item in refs) - for refs in (references, suite_references)) - proofs[suite_id] = {"endpoint": endpoint, "run": runs[0] if valid else None, "suite": suite} - return proofs[suite_id]["run"] is not None + association_bases = [ + {item["base"]["sha"] for item in refs if item["number"] == pr["number"] + and item["head"]["sha"] == pr["head"]["sha"] + and item["base"]["ref"] == pr["base"]["ref"]} + for refs in (references, suite_references)] + bases = association_bases[0] & association_bases[1] + # Open PRs require the current base. Merged PRs retain the common + # recorded base from their run and suite, even after main advances. + if not pr["merged"]: + bases &= {pr["base"]["sha"]} + valid = suite["id"] == suite_id and suite["app"]["id"] == ACTION_APP and bool(bases) + proofs[suite_id] = {"endpoint": endpoint, "run": runs[0] if valid else None, + "suite": suite, "bases": bases, "parents": {}} + proof = proofs[suite_id] + if proof["run"] is None: + return False + if check["head_sha"] != pr["head"]["sha"]: + if check["head_sha"] not in proof["parents"]: + commit = api(f"repos/{coordinate}/commits/{check['head_sha']}") + proof["parents"][check["head_sha"]] = [item["sha"] for item in commit["parents"]] + parents = proof["parents"][check["head_sha"]] + return len(parents) == 2 and any(set(parents) == {base, pr["head"]["sha"]} + for base in proof["bases"]) + return True def sha(coordinate, number): diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index 4102ecc..4816078 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -79,6 +79,9 @@ def rulesets(repository): item.update(id=active.index(repository) * 10 + index, source_type='Repository' if scenario == 'repository-rulesets' or template in ('code-scanning', 'dependency-review') else 'Organization') + if ((scenario == 'organization-codeql-owner' and template == 'code-scanning') or + (scenario == 'organization-dependency-owner' and template == 'dependency-review')): + item['source_type'] = 'Organization' if scenario == 'tag-repository-nonrelease' and template == 'immutable-release-tags': item['source_type'] = 'Repository' # Match GitHub's actual owner representation, including repository rulesets. @@ -97,6 +100,10 @@ def rulesets(repository): excluded = [] elif template == 'immutable-release-tags': included, excluded = repositories['release_tags'], [] + elif template == 'code-scanning': + included, excluded = policy['code_scanning'], [] + elif template == 'dependency-review': + included, excluded = policy['dependency_review'], [] else: included, excluded = [repository], [] item['conditions']['repository_name'] = { diff --git a/tests/audit.sh b/tests/audit.sh index 1c6fe5b..d002df6 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -119,6 +119,8 @@ assert_drift tag-repository-nonrelease 'unexpected managed release-tag protectio assert_drift pr-all-scope "ruleset scope or policy drift '02 - Changes through pull requests'" assert_drift default-extra-exclude "ruleset scope or policy drift '01 - Default branch integrity'" assert_drift repository-scope-extra "ruleset scope or policy drift '07 - Dependency review'" +assert_drift organization-codeql-owner "managed security ruleset must be repository-owned '06 - CodeQL security'" +assert_drift organization-dependency-owner "managed security ruleset must be repository-owned '07 - Dependency review'" assert_drift integrity-bypass "ruleset scope or policy drift '01 - Default branch integrity'" assert_drift pr-bypass "ruleset scope or policy drift '02 - Changes through pull requests'" assert_drift owner-id-drift "ruleset scope or policy drift '02 - Changes through pull requests'" diff --git a/tests/security-readiness.py b/tests/security-readiness.py index 4500d02..fa83b02 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -147,7 +147,7 @@ def __call__(self, endpoint): return content(self.default_workflow if "ref=main" in endpoint else self.executed_workflow if "ref=" + MERGE in endpoint else self.run_workflow) if "/contents/.github/workflows/check.yml" in endpoint: - return content(self.before if "ref=" + BASE in endpoint else self.after) + return content(self.before if "ref=" + self.pr["base"]["sha"] in endpoint else self.after) if "/contents/" in endpoint: path = endpoint.split("/contents/")[1].split("?")[0] sources = self.codeql_sources if "ref=main" in endpoint else self.codeql_head_sources @@ -671,6 +671,83 @@ def test_sha_direct_head_stale_run_checks_rejected(self): self.fixture.run["head_sha"] = OLD self.assert_deferred("sha", "missing") + def advance_canary_base(self): + self.fixture.pr["base"]["sha"] = OLD + self.fixture.parents = [OLD, HEAD] + + def test_sha_open_canary_base_advance_rejects_old_direct_head_success(self): + self.advance_canary_base() + self.assert_deferred("sha", "required check missing") + + def test_sha_open_canary_base_advance_requires_both_run_and_suite_current(self): + for stale in ("run", "suite"): + with self.subTest(stale=stale): + self.advance_canary_base() + self.fixture.run["pull_requests"][0]["base"]["sha"] = BASE if stale == "run" else OLD + self.fixture.suite["pull_requests"][0]["base"]["sha"] = BASE if stale == "suite" else OLD + self.assert_deferred("sha", "required check missing") + + def test_sha_open_canary_fresh_base_associations_pass_direct_and_merge_checks(self): + self.advance_canary_base() + for source in (self.fixture.run, self.fixture.suite): + source["pull_requests"][0]["base"]["sha"] = OLD + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_open_canary_old_base_associations_reject_even_with_current_merge_parents(self): + self.advance_canary_base() + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assert_deferred("sha", "required check missing") + + def test_sha_merged_canary_keeps_recorded_historical_base_proof(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.fixture.pr["base"]["sha"] = OLD + # The historical merge and association retain their recorded BASE. + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_current_base_failure_or_queued_check_cannot_hide_behind_stale_green(self): + for status, conclusion in (("completed", "failure"), ("queued", None)): + with self.subTest(status=status): + self.advance_canary_base() + self.fixture.checks[HEAD] = [check("Required checks"), check("Conventional PR title", identifier=2)] + self.fixture.suite_runs.clear() + self.add_newer_pr_check(check_status=status, check_conclusion=conclusion, + run_status=status, run_conclusion=conclusion) + self.fixture.checks[HEAD][-1]["started_at"] = None + self.fixture.suite_runs[100]["pull_requests"][0]["base"]["sha"] = OLD + def current_suite(endpoint): + value = self.fixture(endpoint) + if endpoint.endswith("/check-suites/100"): + value["pull_requests"][0]["base"]["sha"] = OLD + return value + with patch.object(readiness, "api", current_suite): + self.assert_deferred("sha", "workflow run not successful") + + def test_sha_merged_canary_rejects_conflicting_recorded_run_and_suite_bases(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.fixture.suite["pull_requests"][0]["base"]["sha"] = OLD + self.assert_deferred("sha", "required check missing") + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + self.assert_deferred("sha", "required check missing") + + def test_sha_merged_canary_common_base_must_match_selected_merge_parents(self): + self.fixture.pr.update(state="closed", merged=True, merge_commit_sha=NEW) + self.fixture.checks[MERGE] = [check("Required checks", MERGE), check("Conventional PR title", MERGE, 2)] + self.fixture.checks[HEAD] = [] + # A run lists two historical associations, but the suite corroborates + # only the base absent from this merge commit's parents. + alternate = copy.deepcopy(self.fixture.run["pull_requests"][0]) + alternate["base"]["sha"] = OLD + self.fixture.run["pull_requests"].append(alternate) + self.fixture.suite["pull_requests"] = [alternate] + self.assert_deferred("sha", "required check missing") + def test_sha_direct_head_failed_workflow_run_rejected(self): self.fixture.run["conclusion"] = "failure" self.assert_deferred("sha", "workflow run not successful") From 7d4f137a9789f7d3ba20b1bf72c99ffbe3d86e7a Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 13:36:23 -0500 Subject: [PATCH 06/22] fix: verify active security policy and stage SHA restoration --- README.md | 16 ++- bin/audit | 3 + bin/publish | 120 +++++++++++----- lib/security_readiness.py | 110 ++++++++++++--- tests/audit-fixtures.py | 18 +++ tests/audit.sh | 7 +- tests/publish-maintenance-branch.sh | 12 +- tests/publish-security-merge.sh | 203 +++++++++++++++++++++++++++- tests/security-readiness.py | 202 ++++++++++++++++++++++++++- 9 files changed, 626 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 1c606db..13a393a 100644 --- a/README.md +++ b/README.md @@ -235,10 +235,18 @@ Rollout is owned by an organization administrator: `{"phase":"enforced","canary":"OWNER/REPOSITORY/PR"}` through the normal review and merge process before the first separately authorized live apply. This records the accepted PR, with no duplicate action-version inventory. - The verifier checks live allowance, exact PR changes, and successful PR-bound - current-head CI. Ordinary subsequent publication uses that stored canary, - revalidates it and restores a disabled setting. Unreadable or failed evidence - blocks restoration with an error; the audit reports disabled enforced policy + The verifier checks the active conservative organization Actions mode, + repository enablement, exact PR changes, and successful PR-bound current-head + checks. These include configured CI, desired Dependency review for targeted + repositories, and all effective ruleset and legacy branch-protection status + contexts, with trusted app identities. This proves current policy and PR-run + evidence; it does not attest historical organization settings. + Ordinary subsequent publication uses that stored canary. It first converges + selected mode, then allowed-action patterns, then revalidates the canary + before restoring a disabled SHA setting. A dry run reports the required + staging without claiming that planned settings are active. Unreadable or + failed proof blocks SHA restoration while retaining conservative policy + repairs; the audit reports disabled enforced policy as drift, never as initial rollout pending. A live true setting is always preserved. Full SHA pinning does not prove an allowed action is trustworthy. diff --git a/bin/audit b/bin/audit index 441ad36..1f92f58 100755 --- a/bin/audit +++ b/bin/audit @@ -322,6 +322,9 @@ audit_security() { "${root}/config/security-merge-protection.json") if ! jq -e --slurpfile desired "${setup}" --argjson languages "${languages}" ' . as $current | ($desired[0] | to_entries | all(.[]; $current[.key] == .value)) and + (if $desired[0].state == "configured" then + $current.languages | if type == "array" then all(.[]; type == "string") else false end + else true end) and (($languages - ($current.languages // [])) | length == 0) ' <<<"${current}" >/dev/null; then record_finding "CodeQL setup or required language drift: ${organization}/${repository}" diff --git a/bin/publish b/bin/publish index 259176f..95fe152 100755 --- a/bin/publish +++ b/bin/publish @@ -1031,9 +1031,9 @@ converge_codeql_default_setup() { "repos/${organization}/${repository}/code-scanning/default-setup" \ 2>/dev/null); then current_state=$(jq -r '.state // empty' <<<"${current}") - if [[ ${required_languages} != '[]' ]] && ! jq -e ' + if ! jq -e ' (.state == "not-configured" and .languages == null) or - (.languages | type == "array" and all(.[]; type == "string")) + (.languages | type == "array" and all(.[]; type == "string" and length > 0)) ' <<<"${current}" >/dev/null; then echo "error: invalid existing CodeQL languages for ${repository}" >&2 return 1 @@ -1219,51 +1219,105 @@ validate_sha_canary_target() { } converge_actions_permissions() { - local current input reason phase canary - current=$(github_api "orgs/${organization}/actions/permissions") - if ! jq -e 'type == "object" and - (.enabled_repositories | type == "string") and - (.allowed_actions | type == "string") and - (.sha_pinning_required == null or (.sha_pinning_required | type == "boolean")) - ' <<<"${current}" >/dev/null; then + local current selected input reason phase canary staging_needed=false + local permissions_filter='type == "object" and + (.enabled_repositories | IN("all", "none", "selected")) and + (.allowed_actions | IN("all", "local_only", "selected")) and + (.sha_pinning_required == null or (.sha_pinning_required | type == "boolean"))' + current=$(github_api "orgs/${organization}/actions/permissions") || return 1 + if ! jq -e "${permissions_filter}" <<<"${current}" >/dev/null; then echo "error: cannot read current organization Actions permissions" >&2 return 1 fi - input=$(mktemp) - temporary_files+=("${input}") - cp "${root}/config/actions-permissions.json" "${input}" phase=$(jq -r '.phase' "${root}/config/actions-sha-pinning.json") canary=$(jq -r '.canary // empty' "${root}/config/actions-sha-pinning.json") if ! jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then if [[ ${phase} == enforced ]]; then validate_sha_canary_target "${canary}" || return 1 - if ! reason=$("${root}/bin/security-readiness" sha "${canary}"); then - echo "error: enforced SHA pinning cannot be restored: ${reason}" >&2 - return 1 - fi - echo "READY enforced SHA pinning: ${reason}" - else - if [[ -n ${sha_pinning_canary} ]]; then - validate_sha_canary_target "${sha_pinning_canary}" || return 1 - if reason=$("${root}/bin/security-readiness" sha "${sha_pinning_canary}"); then - echo "READY canary evidence: ${reason}; source rollout phase remains pending" - else - echo "PENDING canary evidence: ${reason}" - fi - fi - echo "PENDING SHA pinning: promote the reviewed source rollout phase and accepted canary after readiness" - # Pending source never enables or disables SHA pinning, including when - # an optional command-line probe succeeds. Enforced source retains proof. - jq 'del(.sha_pinning_required)' "${input}" >"${input}.pending" - mv "${input}.pending" "${input}" + elif [[ -n ${sha_pinning_canary} ]]; then + validate_sha_canary_target "${sha_pinning_canary}" || return 1 fi fi + + # GitHub requires selected mode before its selected-actions endpoint can be + # updated. Stage both conservative controls before proving SHA compatibility. + # Omit SHA entirely: this preserves an existing value without enabling it + # from a stale read if another actor changes the field during staging. + input=$(mktemp) + temporary_files+=("${input}") + jq 'del(.sha_pinning_required)' "${root}/config/actions-permissions.json" >"${input}" if jq -e --slurpfile desired "${input}" ' . as $current | all($desired[0] | to_entries[]; $current[.key] == .value) ' <<<"${current}" >/dev/null; then echo "KEEP /orgs/${organization}/actions/permissions" else - run_api PUT "orgs/${organization}/actions/permissions" "${input}" + staging_needed=true + run_api PUT "orgs/${organization}/actions/permissions" "${input}" || return 1 + fi + # This endpoint may require selected mode. A dry run with a mode change + # can plan its update, but cannot claim to have read the staged live policy. + if ! ${apply} && [[ $(jq -r '.allowed_actions' <<<"${current}") != selected ]]; then + selected=null + else + selected=$(github_api "orgs/${organization}/actions/permissions/selected-actions") || return 1 + if ! jq -e ' + (.github_owned_allowed | type == "boolean") and + (.verified_allowed | type == "boolean") and + (.patterns_allowed | type == "array" and all(.[]; type == "string")) + ' <<<"${selected}" >/dev/null; then + echo "error: cannot read current selected Actions policy" >&2 + return 1 + fi + fi + if jq -e --slurpfile desired "${root}/config/actions-selected.json" ' + .github_owned_allowed == $desired[0].github_owned_allowed and + .verified_allowed == $desired[0].verified_allowed and + (.patterns_allowed | sort) == ($desired[0].patterns_allowed | sort) + ' <<<"${selected}" >/dev/null; then + echo "KEEP /orgs/${organization}/actions/permissions/selected-actions" + else + staging_needed=true + run_api PUT "orgs/${organization}/actions/permissions/selected-actions" \ + "${root}/config/actions-selected.json" || return 1 + fi + if ${apply}; then + # Base the SHA decision on live post-staging state: another writer may + # disable it while conservative mode or pattern updates are in flight. + current=$(github_api "orgs/${organization}/actions/permissions") || return 1 + if ! jq -e "${permissions_filter}" <<<"${current}" >/dev/null || + ! jq -e --slurpfile desired "${input}" ' + . as $current | all($desired[0] | to_entries[]; $current[.key] == .value) + ' <<<"${current}" >/dev/null; then + echo "error: organization Actions permissions unreadable or changed during staging" >&2 + return 1 + fi + fi + if jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then + return 0 + fi + if [[ ${phase} == enforced ]]; then + if ! ${apply} && ${staging_needed}; then + echo "PLAN SHA pinning proof and restoration after Actions policy staging is applied" + return 0 + fi + validate_sha_canary_target "${canary}" || return 1 + if ! reason=$("${root}/bin/security-readiness" sha "${canary}"); then + echo "error: enforced SHA pinning cannot be restored: ${reason}" >&2 + return 1 + fi + echo "READY enforced SHA pinning: ${reason}" + run_api PUT "orgs/${organization}/actions/permissions" \ + "${root}/config/actions-permissions.json" || return 1 + else + if [[ -n ${sha_pinning_canary} ]]; then + validate_sha_canary_target "${sha_pinning_canary}" || return 1 + if reason=$("${root}/bin/security-readiness" sha "${sha_pinning_canary}"); then + echo "READY canary evidence: ${reason}; source rollout phase remains pending" + else + echo "PENDING canary evidence: ${reason}" + fi + fi + echo "PENDING SHA pinning: promote the reviewed source rollout phase and accepted canary after readiness" fi } @@ -1593,8 +1647,6 @@ converge_immutable_release_policy converge_organization_settings "${organization_metadata}" converge_actions_permissions -run_api PUT "orgs/${organization}/actions/permissions/selected-actions" \ - "${root}/config/actions-selected.json" run_api PUT "orgs/${organization}/actions/permissions/workflow" \ "${root}/config/actions-workflow.json" diff --git a/lib/security_readiness.py b/lib/security_readiness.py index 0033a87..cd83429 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -401,6 +401,82 @@ def merge_check_run(coordinate, check, pr, merges, proofs, recovered): return True +def canary_actions_policy(coordinate, metadata): + organization = coordinate.split('/')[0] + if type(metadata["id"]) is not int or metadata["id"] < 1: + raise EvidenceError("canary repository identity malformed") + current = api(f"orgs/{organization}/actions/permissions") + if (current["allowed_actions"] not in {"all", "local_only", "selected"} + or current["enabled_repositories"] not in {"all", "none", "selected"}): + raise EvidenceError("organization Actions permissions malformed") + require(current["allowed_actions"] == "selected", "organization Actions selected policy is not active") + require(current["enabled_repositories"] != "none", "organization Actions are disabled") + membership = [] + if current["enabled_repositories"] == "selected": + membership = pages(f"orgs/{organization}/actions/permissions/repositories", "repositories") + require(any(item["id"] == metadata["id"] and item["full_name"].lower() == coordinate.lower() + for item in membership), "canary repository is not enabled for organization Actions") + repository_policy = api(f"repos/{coordinate}/actions/permissions") + if (not isinstance(repository_policy["enabled"], bool) + or repository_policy["allowed_actions"] not in {"all", "local_only", "selected"}): + raise EvidenceError("repository Actions permissions malformed") + require(repository_policy["enabled"] and repository_policy["allowed_actions"] != "local_only", + "canary repository does not allow external Actions") + # Repository 'all' cannot relax the organization's selected-actions ceiling. + selected = api(f"orgs/{organization}/actions/permissions/selected-actions") + desired = config("actions-selected.json") + require(desired["github_owned_allowed"] is True and desired["verified_allowed"] is False, + "desired selected-actions policy is not conservative") + for value in (selected, desired): + if (not isinstance(value["github_owned_allowed"], bool) + or not isinstance(value["verified_allowed"], bool) + or not isinstance(value["patterns_allowed"], list) + or not all(isinstance(item, str) for item in value["patterns_allowed"])): + raise EvidenceError("selected-actions policy malformed") + require(selected["github_owned_allowed"] is True and selected["verified_allowed"] is False + and sorted(selected["patterns_allowed"]) == sorted(desired["patterns_allowed"]), + "live selected-actions policy differs from desired policy") + return {"mode": current["allowed_actions"], "enabled": current["enabled_repositories"], + "membership": sorted((item["id"], item["full_name"]) for item in membership), + "repository": repository_policy, "selected": selected} + + +def canary_required_checks(coordinate, branch): + rules = pages(f"repos/{coordinate}/rules/branches/{quote(branch, safe='')}") + entries = [] + for rule in rules: + if rule["type"] == "required_status_checks": + entries.extend((item["context"], item["integration_id"]) + for item in rule["parameters"]["required_status_checks"]) + try: + legacy = api(f"repos/{coordinate}/branches/{quote(branch, safe='')}/protection/required_status_checks") + except APIError as exc: + if exc.status != 404: + raise + legacy = None + if legacy is not None: + if (not isinstance(legacy["checks"], list) or not isinstance(legacy["contexts"], list) + or not all(isinstance(value, str) and value for value in legacy["contexts"])): + raise EvidenceError("legacy required status checks malformed") + entries.extend((item["context"], item["app_id"]) for item in legacy["checks"]) + entries.extend((context, None) for context in legacy["contexts"]) + for context, app in entries: + if not isinstance(context, str) or not context or (app is not None and type(app) is not int): + raise EvidenceError("effective required status check malformed") + require(app in (None, -1, ACTION_APP), + f"effective required check is bound to a non-Actions app: {context}") + repository_name = coordinate.split('/')[1] + configured = config("repositories.json")["required_ci"].get(repository_name) + require(bool(configured), "canary has no configured required contexts") + if not isinstance(configured, list) or not all(isinstance(value, str) and value for value in configured): + raise EvidenceError("configured required checks malformed") + required = set(configured) | {context for context, _ in entries} + if repository_name in config("security-merge-protection.json")["dependency_review"]: + required.add("Dependency review") + return (list(dict.fromkeys(configured)) + sorted(required - set(configured)), + sorted(json.dumps(item) for item in entries)) + + def sha(coordinate, number): metadata = repository(coordinate) pr = api(f"repos/{coordinate}/pulls/{number}") @@ -410,15 +486,9 @@ def sha(coordinate, number): and pr["base"]["ref"] == metadata["default_branch"], "canary does not target default branch") require(bool(SHA.fullmatch(pr["head"]["sha"])), "canary head SHA missing") + actions_policy = canary_actions_policy(coordinate, metadata) selected = config("actions-selected.json") - live = api(f"orgs/{coordinate.split('/')[0]}/actions/permissions/selected-actions") - require(selected["github_owned_allowed"] is True and selected["verified_allowed"] is False, - "desired selected-actions policy is not conservative") - require(live["github_owned_allowed"] is True and live["verified_allowed"] is False - and sorted(live["patterns_allowed"]) == sorted(selected["patterns_allowed"]), - "live selected-actions policy differs from desired policy") - required = config("repositories.json")["required_ci"].get(coordinate.split('/')[1]) - require(bool(required), "canary has no configured required contexts") + required, required_evidence = canary_required_checks(coordinate, metadata["default_branch"]) files = pages(f"repos/{coordinate}/pulls/{number}/files") require(bool(files) and len(files) == pr["changed_files"], "canary file evidence incomplete") updates = 0 @@ -515,12 +585,13 @@ def check_snapshot(values): and fresh.get("merge_commit_sha") == pr.get("merge_commit_sha") and fresh["state"] == pr["state"] and fresh["merged"] == pr["merged"] and (fresh["merged"] or not fresh["draft"]), "canary changed during preflight") - fresh_live = api(f"orgs/{coordinate.split('/')[0]}/actions/permissions/selected-actions") - require(fresh_live["github_owned_allowed"] is True and fresh_live["verified_allowed"] is False - and sorted(fresh_live["patterns_allowed"]) == sorted(selected["patterns_allowed"]), - "live selected-actions policy changed during preflight") - require(repository(coordinate)["default_branch"] == metadata["default_branch"], - "repository default branch changed during preflight") + require(canary_actions_policy(coordinate, metadata) == actions_policy, + "live Actions permissions changed during preflight") + require(canary_required_checks(coordinate, metadata["default_branch"]) + == (required, required_evidence), "effective required checks changed during preflight") + fresh_metadata = repository(coordinate) + require(fresh_metadata["default_branch"] == metadata["default_branch"] + and fresh_metadata["id"] == metadata["id"], "repository identity/default branch changed during preflight") return f"ordinary action-update PR #{number}: {updates} allowed full-SHA updates; required checks passed" @@ -556,11 +627,18 @@ def newest_analysis_pages(coordinate, branch): return +def configured_codeql_languages(setup): + values = setup["languages"] + if not isinstance(values, list) or not all(isinstance(item, str) and item for item in values): + raise EvidenceError("CodeQL default-setup languages malformed") + return {canonical_language(value) for value in values} + + def default_codeql(coordinate, branch, head, minima): endpoint = f"repos/{coordinate}/code-scanning/default-setup" setup = api(endpoint) require(setup["state"] == "configured", "CodeQL default setup not configured") - languages = {canonical_language(value) for value in setup["languages"]} + languages = configured_codeql_languages(setup) require(bool(languages), "CodeQL default setup has no configured languages") require(set(minima) <= languages, "required CodeQL languages not configured") found = set() @@ -581,7 +659,7 @@ def default_codeql(coordinate, branch, head, minima): if found == languages: fresh = api(endpoint) require(fresh["state"] == setup["state"] - and {canonical_language(value) for value in fresh["languages"]} == languages, + and configured_codeql_languages(fresh) == languages, "CodeQL default setup changed during preflight") return "latest canonical default-setup analyses cover every configured language on current head" raise Deferred("required configured CodeQL language analysis missing within bounded evidence") diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index 4816078..c8e2613 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -268,6 +268,24 @@ def rulesets(repository): reject('setup read denied') payload = config('codeql-default-setup-advanced.json' if advanced else 'codeql-default-setup.json') payload['languages'] = policy['codeql_required_languages'].get(repository, []) + if target: + malformed_languages = { + 'languages-null': None, + 'languages-scalar': 'javascript-typescript', + 'languages-object': {'language': 'javascript-typescript'}, + 'languages-number': [7], + 'languages-mixed': ['javascript-typescript', None], + } + if scenario in malformed_languages: + payload['languages'] = malformed_languages[scenario] + elif scenario == 'languages-missing': + del payload['languages'] + elif scenario == 'languages-readable': + payload['languages'] = ['javascript-typescript', 'future-language'] + elif scenario == 'languages-empty': + payload['languages'] = [] + if scenario == 'advanced-null-languages' and repository == 'classic': + payload['languages'] = None if scenario == 'missing-rust' and repository == 'content-toolkit': payload['languages'] = ['javascript-typescript'] if scenario == 'advanced-setup' and repository == 'classic': diff --git a/tests/audit.sh b/tests/audit.sh index d002df6..36d5b9d 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -43,7 +43,7 @@ run_audit() { if [[ ${fixture} == focused ]]; then case ${scenario} in wrong-pr-scope|unexpected-codeql|tag-extra-nonrelease|tag-all-nonrelease|tag-repository-nonrelease|inherited-tag-unaffected|foreign-tag) repository=.github ;; - advanced-config|advanced-setup) repository=classic ;; + advanced-config|advanced-setup|advanced-null-languages) repository=classic ;; missing-rust) repository=content-toolkit ;; missing-tag-scope|tag-ref-drift|tag-all-scope|tag-extra-repo|tag-extra-exclude) repository=service-updater ;; unexpected-pr) repository=web-platform ;; @@ -78,7 +78,7 @@ output=$(run_audit legacy-security) grep -Fq 'Governance audit passed:' <<<"${output}" output=$(run_audit repository-rulesets) grep -Fq 'Governance audit passed:' <<<"${output}" -for scenario in scope-order inherited-tag-unaffected foreign-tag sha-enforced-on; do +for scenario in scope-order inherited-tag-unaffected foreign-tag sha-enforced-on languages-readable languages-empty advanced-null-languages; do output=$(run_audit "${scenario}") grep -Fq 'Governance audit passed:' <<<"${output}" done @@ -151,6 +151,9 @@ assert_drift sha-invalid-type 'Actions SHA pinning state could not be read' assert_drift sha-enforced-off 'Actions SHA pinning enforcement drift' assert_drift phase-unreadable 'Actions SHA pinning rollout phase could not be read' assert_drift missing-rust 'CodeQL setup or required language drift: atrinik/content-toolkit' +for scenario in languages-missing languages-null languages-scalar languages-object languages-number languages-mixed; do + assert_drift "${scenario}" 'CodeQL setup or required language drift: atrinik/observatory' +done for scenario in advanced-setup default-setup; do assert_drift "${scenario}" 'CodeQL setup or required language drift' done diff --git a/tests/publish-maintenance-branch.sh b/tests/publish-maintenance-branch.sh index 5fd7467..aa9468b 100755 --- a/tests/publish-maintenance-branch.sh +++ b/tests/publish-maintenance-branch.sh @@ -6,6 +6,7 @@ root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) temporary=$(mktemp -d) trap 'rm -rf "${temporary}"' EXIT mkdir "${temporary}/bin" +export GH_SELECTED_ACTIONS_POLICY=${root}/config/actions-selected.json cat >"${temporary}/bin/gh" <<'EOF' #!/usr/bin/env bash @@ -229,6 +230,9 @@ case "${endpoint}|${jq_filter}" in repos/atrinik/*/actions/permissions/workflow\|*) printf '{"default_workflow_permissions":"read","can_approve_pull_request_reviews":false}\n' ;; +"orgs/atrinik/actions/permissions/selected-actions|") + cat "${GH_SELECTED_ACTIONS_POLICY}" + ;; "orgs/atrinik/actions/permissions|") printf '{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":false}\n' ;; @@ -351,17 +355,17 @@ repos/atrinik/*/actions/permissions/workflow\|*) if [[ ${security_scenario} == drifted || ${security_scenario} == create || ${GH_EMPTY_ADVANCED_INVENTORY:-false} == true ]]; then - printf '{"state":"configured","query_suite":"default"}\n' + printf '{"state":"configured","query_suite":"default","languages":["c-cpp"]}\n' else - printf '{"state":"not-configured","query_suite":"default"}\n' + printf '{"state":"not-configured","query_suite":"default","languages":null}\n' fi ;; "repos/atrinik/content/code-scanning/default-setup|"*) if [[ ${security_scenario} == drifted || ${security_scenario} == create ]]; then - printf '{"state":"not-configured"}\n' + printf '{"state":"not-configured","languages":null}\n' else - printf '{"state":"configured","query_suite":"default"}\n' + printf '{"state":"configured","query_suite":"default","languages":["python"]}\n' fi ;; "repos/atrinik/nawerhals|.archived") diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index df8e0ae..0f8b279 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -39,11 +39,24 @@ declare -A active_repository_ids=([content-toolkit]=123) api_log=${temporary}/api.jsonl readiness_log=${temporary}/readiness.jsonl export READINESS_LOG=${readiness_log} +export MOCK_API_LOG=${api_log} +export MOCK_ACTIONS_POLICY=${temporary}/actions-policy.json +export MOCK_ACTIONS_SELECTED=${root}/config/actions-selected.json cat >"${root}/bin/security-readiness" <<'MOCK' #!/usr/bin/env bash set -euo pipefail jq -cn --arg mode "$1" --arg repository "$2" \ '{mode: $mode, repository: $repository}' >>"${READINESS_LOG}" +if [[ $1 == sha ]]; then + jq -cn '{method: "PROOF", endpoint: "sha"}' >>"${MOCK_API_LOG}" + if [[ ${CHECK_POLICY_STAGED:-false} == true ]]; then + jq -e --slurpfile selected "${MOCK_ACTIONS_SELECTED}" ' + .permissions.allowed_actions == "selected" and + .permissions.enabled_repositories == "all" and + .selected == $selected[0] + ' "${MOCK_ACTIONS_POLICY}" >/dev/null || exit 98 + fi +fi case ${READINESS_STATUS:-ready} in ready) echo 'producer evidence ready'; exit 0 ;; missing) echo 'producer is missing'; exit 1 ;; @@ -142,6 +155,13 @@ github_api() { fi ;; orgs/atrinik/actions/permissions) printf '%s\n' "${actions_state}" ;; + orgs/atrinik/actions/permissions/selected-actions) + # Model the API prerequisite: the dormant list cannot be read or changed + # until the publisher has staged conservative selected mode. + [[ $(jq -r '.allowed_actions' <<<"${actions_state}") == selected ]] || return 1 + [[ ${selected_state} != unreadable ]] || return 1 + printf '%s\n' "${selected_state}" + ;; *) echo "unexpected mock API: ${endpoint}" >&2; return 99 ;; esac } @@ -153,6 +173,32 @@ run_api() { fi jq -cn --arg method "$1" --arg endpoint "$2" --slurpfile payload "$3" \ '{method: $method, endpoint: $endpoint, payload: $payload[0]}' >>"${api_log}" + if ${apply}; then + case $2 in + orgs/atrinik/actions/permissions) + actions_state=$(jq --slurpfile payload "$3" '. * $payload[0]' <<<"${actions_state}") + ;; + orgs/atrinik/actions/permissions/selected-actions) + [[ $(jq -r '.allowed_actions' <<<"${actions_state}") == selected ]] || return 1 + selected_state=$(cat "$3") + case ${staging_race} in + disable | replace-repository) + actions_state=$(jq '.sha_pinning_required = false' <<<"${actions_state}") + if [[ ${staging_race} == replace-repository ]]; then + repository_state=$(jq '.id = 999' <<<"${repository_state}") + fi + ;; + malformed) + actions_state=$(jq '.sha_pinning_required = "false"' <<<"${actions_state}") + ;; + esac + ;; + esac + if [[ ${selected_state} != unreadable ]]; then + jq -n --argjson permissions "${actions_state}" --argjson selected "${selected_state}" \ + '{permissions:$permissions, selected:$selected}' >"${MOCK_ACTIONS_POLICY}" + fi + fi } reset_case() { : >"${api_log}" @@ -165,7 +211,11 @@ reset_case() { cp "${source_root}/config/security-merge-protection.json" "${security_merge_config}" echo '{"phase":"pending","canary":null}' >"${actions_sha_pinning_config}" actions_state='{"enabled_repositories":"selected","allowed_actions":"all","sha_pinning_required":false}' + selected_state=$(cat "${root}/config/actions-selected.json") sha_pinning_canary= + staging_race= + apply=true + export CHECK_POLICY_STAGED=false export READINESS_STATUS=ready } assert_no_writes() { @@ -345,8 +395,9 @@ jq -se '[.[] | select(.method == "PUT")][0].payload == reset_case actions_state='{"enabled_repositories":"selected","allowed_actions":"all","sha_pinning_required":true}' converge_actions_permissions >"${temporary}/sha-retained.txt" -jq -se '[.[] | select(.method == "PUT")][0].payload.sha_pinning_required == true' \ +jq -se '[.[] | select(.method == "PUT")][0].payload | has("sha_pinning_required") | not' \ "${api_log}" >/dev/null +[[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == true ]] [[ ! -s ${readiness_log} ]] for producer in failed ready; do reset_case @@ -360,7 +411,7 @@ reset_case echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" converge_actions_permissions >"${temporary}/sha-enforced-ready.txt" jq -se --slurpfile desired "${root}/config/actions-permissions.json" ' - [.[] | select(.method == "PUT")][0].payload == $desired[0]' "${api_log}" >/dev/null + [.[] | select(.method == "PUT")][-1].payload == $desired[0]' "${api_log}" >/dev/null jq -se '. == [{mode: "sha", repository: "atrinik/content-toolkit/99"}]' \ "${readiness_log}" >/dev/null for producer in failed unreadable; do @@ -371,7 +422,7 @@ for producer in failed unreadable; do echo "expected enforced SHA evidence failure to fail closed: ${producer}" >&2 exit 1 fi - assert_no_writes + jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null done reset_case echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" @@ -381,6 +432,152 @@ converge_actions_permissions >"${temporary}/sha-keep.txt" assert_no_writes [[ ! -s ${readiness_log} ]] +# Simultaneous mode, patterns and SHA drift repairs conservative policy first. +for producer in ready failed unreadable; do + reset_case + echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" + selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":["old/action@*"]}' + export CHECK_POLICY_STAGED=true + export READINESS_STATUS=${producer} + if converge_actions_permissions >"${temporary}/sha-staged-${producer}.txt" 2>&1; then + [[ ${producer} == ready ]] + else + [[ ${producer} != ready ]] + fi + jq -se ' + [.[] | select(.method == "PUT" or .method == "PROOF")] as $events | + $events[0].endpoint == "orgs/atrinik/actions/permissions" and + ($events[0].payload | has("sha_pinning_required") | not) and + $events[1].endpoint == "orgs/atrinik/actions/permissions/selected-actions" and + $events[2].method == "PROOF" + ' "${api_log}" >/dev/null + if [[ ${producer} == ready ]]; then + jq -se '[.[] | select(.method == "PUT" or .method == "PROOF")] | + length == 4 and .[3].payload.sha_pinning_required == true' "${api_log}" >/dev/null + # A second converged publication neither mutates nor re-proves retained SHA. + : >"${api_log}" + : >"${readiness_log}" + converge_actions_permissions >"${temporary}/sha-staged-converged.txt" + assert_no_writes + [[ ! -s ${readiness_log} ]] + else + jq -se '[.[] | select(.method == "PUT" or .method == "PROOF")] | length == 3' "${api_log}" >/dev/null + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == false ]] + fi +done + +# A disable concurrent with pattern staging must not reuse the initial true. +for producer in ready failed unreadable; do + reset_case + echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" + actions_state='{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":true}' + selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":[]}' + staging_race=disable + export READINESS_STATUS=${producer} + if converge_actions_permissions >"${temporary}/sha-concurrent-disable-${producer}.txt" 2>&1; then + [[ ${producer} == ready ]] + else + [[ ${producer} != ready ]] + fi + jq -se ' + to_entries as $events | + ([$events[] | select(.value.method == "PUT" and + .value.endpoint == "orgs/atrinik/actions/permissions/selected-actions")][0].key) as $staged | + ([$events[] | select(.key > $staged and .value.method == "GET" and + .value.endpoint == "orgs/atrinik/actions/permissions")][0].key) as $read | + ([$events[] | select(.key > $read and .value.method == "GET" and + .value.endpoint == "repos/atrinik/content-toolkit")][0].key) as $identity | + ([$events[] | select(.value.method == "PROOF")][0].key) as $proof | + $staged < $read and $read < $identity and $identity < $proof and + all($events[] | select(.key < $proof and .value.method == "PUT"); + .value.payload.sha_pinning_required != true) + ' "${api_log}" >/dev/null + if [[ ${producer} == ready ]]; then + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == true ]] + jq -se '.[-1].method == "PUT" and .[-1].payload.sha_pinning_required == true' "${api_log}" >/dev/null + else + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == false ]] + jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null + fi +done +for race in malformed replace-repository; do + reset_case + echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" + actions_state='{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":true}' + selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":[]}' + staging_race=${race} + if converge_actions_permissions >"${temporary}/sha-concurrent-${race}.txt" 2>&1; then + echo 'expected changed post-staging evidence to fail closed' >&2 + exit 1 + fi + [[ ! -s ${readiness_log} ]] + jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null +done +reset_case +actions_state='{"enabled_repositories":"all","allowed_actions":"selected","sha_pinning_required":true}' +selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":[]}' +staging_race=disable +converge_actions_permissions >"${temporary}/sha-concurrent-disable-pending.txt" +[[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == false ]] +[[ ! -s ${readiness_log} ]] +jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null + +reset_case +echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" +apply=false +converge_actions_permissions >"${temporary}/sha-stage-plan.txt" 2>&1 +grep -q 'PLAN SHA pinning proof' "${temporary}/sha-stage-plan.txt" +if grep -Eq 'jq:|error:' "${temporary}/sha-stage-plan.txt"; then + echo 'unexpected error in SHA staging dry run' >&2 + exit 1 +fi +[[ ! -s ${readiness_log} ]] +jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null +[[ $(jq -r '.allowed_actions' <<<"${actions_state}") == all ]] + +# An unreadable staged allowlist never reaches the canary or writes SHA. +reset_case +echo '{"phase":"enforced","canary":"atrinik/content-toolkit/99"}' >"${actions_sha_pinning_config}" +selected_state=unreadable +if converge_actions_permissions >"${temporary}/sha-staged-unreadable-policy.txt" 2>&1; then + echo 'expected unreadable staged Actions policy to fail closed' >&2 + exit 1 +fi +[[ ! -s ${readiness_log} ]] +jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null + +# Pending rollout may repair policy and probe, but never enables SHA pinning. +reset_case +selected_state='{"github_owned_allowed":true,"verified_allowed":false,"patterns_allowed":[]}' +sha_pinning_canary=atrinik/content-toolkit/99 +export CHECK_POLICY_STAGED=true +converge_actions_permissions >"${temporary}/sha-pending-staged.txt" +jq -se 'all(.[] | select(.method == "PUT"); .payload.sha_pinning_required != true)' "${api_log}" >/dev/null + +# Malformed configured languages are unreadable for ordinary repositories too. +for languages in null '{}' '[false]' '[""]' '"rust"'; do + reset_case + jq '.codeql_required_languages = {}' "${security_merge_config}" >"${temporary}/no-minima.json" + cp "${temporary}/no-minima.json" "${security_merge_config}" + codeql_state=$(jq -cn --argjson languages "${languages}" \ + '{state:"configured", query_suite:"default", languages:$languages}') + if converge_codeql_default_setup >"${temporary}/malformed-languages.txt" 2>&1; then + echo 'expected malformed configured CodeQL languages to fail closed without minima' >&2 + exit 1 + fi + assert_no_writes +done +reset_case +jq '.codeql_required_languages = {}' "${security_merge_config}" >"${temporary}/no-minima.json" +cp "${temporary}/no-minima.json" "${security_merge_config}" +codeql_state='{"state":"configured","query_suite":"default","languages":[]}' +converge_codeql_default_setup >"${temporary}/readable-empty-languages.txt" +assert_no_writes +reset_case +codeql_state='{"state":"not-configured","languages":null}' +converge_codeql_default_setup >"${temporary}/unconfigured-languages.txt" +jq -se '[.[] | select(.method == "PATCH")][0].payload.languages == ["rust"]' "${api_log}" >/dev/null + # Neither a foreign owner nor an unregistered repository can lend canary proof. for phase in pending enforced; do for rejected_canary in foreign/content-toolkit/99 atrinik/unregistered/99; do diff --git a/tests/security-readiness.py b/tests/security-readiness.py index fa83b02..63f76e3 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -96,7 +96,7 @@ def check(name, ref=HEAD, identifier=1, conclusion="success", status="completed" class Fixture: def __init__(self): self.calls = [] - self.metadata = {"full_name": "atrinik/example", "default_branch": "main", "archived": False} + self.metadata = {"id": 42, "full_name": "atrinik/example", "default_branch": "main", "archived": False} self.pr = {"number": 12, "state": "open", "draft": False, "merged": False, "changed_files": 1, "merge_commit_sha": MERGE, "base": {"ref": "main", "sha": BASE, "repo": {"full_name": "atrinik/example"}}, @@ -117,6 +117,11 @@ def __init__(self): self.after = f"name: Check\nsteps:\n - uses: rust-lang/crates-io-auth-action@{NEW} # v2\n".encode() self.checks = {HEAD: [check("Required checks"), check("Conventional PR title", identifier=2)], MERGE: []} self.live = copy.deepcopy(SELECTED) + self.actions_policy = {"enabled_repositories": "all", "allowed_actions": "selected"} + self.repository_actions = {"enabled": True, "allowed_actions": "all"} + self.enabled_repositories = [{"id": 42, "full_name": "atrinik/example"}] + self.effective_rules = [] + self.legacy_checks = None self.default_workflow = WORKFLOW self.run_workflow = WORKFLOW self.executed_workflow = WORKFLOW @@ -169,6 +174,19 @@ def __call__(self, endpoint): return {"workflow_runs": [copy.deepcopy(self.run)]} if "/attempts/2/jobs" in endpoint: return {"total_count": len(self.jobs), "jobs": copy.deepcopy(self.jobs)} + if endpoint == "orgs/atrinik/actions/permissions": + return copy.deepcopy(self.actions_policy) + if endpoint == "repos/atrinik/example/actions/permissions": + return copy.deepcopy(self.repository_actions) + if "/actions/permissions/repositories?" in endpoint: + return {"total_count": len(self.enabled_repositories), + "repositories": copy.deepcopy(self.enabled_repositories)} + if "/rules/branches/" in endpoint: + return copy.deepcopy(self.effective_rules) + if endpoint.endswith("/protection/required_status_checks"): + if self.legacy_checks is None: + raise readiness.APIError(endpoint, 404) + return copy.deepcopy(self.legacy_checks) if "/actions/permissions/selected-actions" in endpoint: return copy.deepcopy(self.live) if "/pulls/12/files" in endpoint: @@ -211,7 +229,7 @@ def setUp(self): (self.root / "config/repositories.json").write_text(json.dumps({ "required_ci": {"example": ["Required checks", "Conventional PR title"]}})) (self.root / "config/security-merge-protection.json").write_text(json.dumps({ - "codeql_required_languages": {"example": ["rust"]}})) + "codeql_required_languages": {"example": ["rust"]}, "dependency_review": []})) (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": []}') self.fixture = Fixture() self.root_patch = patch.object(readiness, "ROOT", self.root) @@ -608,6 +626,186 @@ def test_api_denied_is_error_not_deferred(self): def test_sha_wildcard_accepts_fresh_rust_pin(self): self.assertEqual(self.invoke("sha")[0], 0) + def require_dependency_for_canary(self): + path = self.root / "config/security-merge-protection.json" + policy = json.loads(path.read_text()) + policy["dependency_review"] = ["example"] + path.write_text(json.dumps(policy)) + + def effective_check(self, context, app=15368): + return {"type": "required_status_checks", "parameters": { + "required_status_checks": [{"context": context, "integration_id": app}]}} + + def test_sha_targeted_dependency_review_missing_failed_or_queued_rejected(self): + self.require_dependency_for_canary() + self.assert_deferred("sha", "Dependency review") + self.fixture.checks[HEAD].append(check("Dependency review", identifier=30, conclusion="failure")) + self.assert_deferred("sha", "Dependency review") + self.fixture.checks[HEAD][-1].update(status="queued", conclusion=None, started_at=None) + self.assert_deferred("sha", "Dependency review") + self.fixture.checks[HEAD][-1].update(status="completed", conclusion="success") + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_effective_extra_rule_context_required_with_trusted_actions(self): + self.fixture.effective_rules = [self.effective_check("Extra validation")] + self.assert_deferred("sha", "Extra validation") + self.fixture.checks[HEAD].append(check("Extra validation", identifier=30)) + self.fixture.checks[HEAD][-1]["app"]["id"] = 999 + self.assert_deferred("sha", "Extra validation") + self.fixture.checks[HEAD][-1]["app"]["id"] = 15368 + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_effective_dependency_review_required_outside_target_inventory(self): + self.fixture.effective_rules = [self.effective_check("Dependency review")] + self.assert_deferred("sha", "Dependency review") + + def test_sha_legacy_protection_context_required(self): + self.fixture.legacy_checks = {"contexts": ["Legacy check"], "checks": []} + self.assert_deferred("sha", "Legacy check") + self.fixture.checks[HEAD].append(check("Legacy check", identifier=30)) + self.assertEqual(self.invoke("sha")[0], 0) + + def test_sha_non_actions_app_binding_never_satisfied_by_same_named_actions_check(self): + self.fixture.effective_rules = [self.effective_check("Required checks", 999)] + self.assert_deferred("sha", "non-Actions app") + self.fixture.effective_rules = [] + self.fixture.legacy_checks = {"contexts": ["Required checks"], + "checks": [{"context": "Required checks", "app_id": 999}]} + self.assert_deferred("sha", "non-Actions app") + + def test_sha_unbound_effective_context_still_requires_trusted_actions(self): + self.fixture.effective_rules = [self.effective_check("Required checks", None)] + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.checks[HEAD][0]["app"]["slug"] = "untrusted" + self.assert_deferred("sha", "Required checks") + + def test_sha_effective_rule_or_legacy_changes_during_preflight_rejected(self): + for endpoint_part in ("/rules/branches/", "/protection/required_status_checks"): + with self.subTest(endpoint=endpoint_part): + count = 0 + def drift(endpoint): + nonlocal count + if endpoint_part in endpoint: + count += 1 + if count > 1: + if "rules/branches" in endpoint: + return [self.effective_check("New gate")] + return {"contexts": ["New gate"], "checks": []} + return self.fixture(endpoint) + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "effective required checks changed") + + def test_sha_effective_rules_and_legacy_unreadable_fail_closed(self): + for endpoint_part in ("/rules/branches/", "/protection/required_status_checks"): + with self.subTest(endpoint=endpoint_part): + def unreadable(endpoint): + if endpoint_part in endpoint: + raise readiness.APIError(endpoint, 403) + return self.fixture(endpoint) + with patch.object(readiness, "api", unreadable): + code, output = self.invoke("sha") + self.assertEqual(code, 2, output) + + def test_sha_malformed_effective_checks_rejected(self): + self.fixture.effective_rules = [self.effective_check("Required checks", "15368")] + self.assertEqual(self.invoke("sha")[0], 2) + self.fixture.effective_rules = [] + self.fixture.legacy_checks = {"contexts": "Required checks", "checks": []} + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_dormant_selected_patterns_with_all_mode_rejected(self): + self.fixture.actions_policy["allowed_actions"] = "all" + self.assert_deferred("sha", "selected policy is not active") + self.assertFalse(any("/check-runs?" in endpoint for endpoint in self.fixture.calls)) + + def test_sha_selected_repository_enablement_and_identity_required(self): + self.fixture.actions_policy["enabled_repositories"] = "selected" + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.enabled_repositories[0]["id"] = 999 + self.assert_deferred("sha", "not enabled") + self.fixture.enabled_repositories[0].update(id=42, full_name="foreign/example") + self.assert_deferred("sha", "not enabled") + + def test_sha_disabled_organization_or_repository_actions_rejected(self): + self.fixture.actions_policy["enabled_repositories"] = "none" + self.assert_deferred("sha", "disabled") + self.fixture.actions_policy["enabled_repositories"] = "all" + self.fixture.repository_actions["enabled"] = False + self.assert_deferred("sha", "does not allow external") + self.fixture.repository_actions.update(enabled=True, allowed_actions="local_only") + self.assert_deferred("sha", "does not allow external") + + def test_sha_repository_all_or_selected_cannot_relax_organization_ceiling(self): + for mode in ("all", "selected"): + self.fixture.repository_actions["allowed_actions"] = mode + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.actions_policy["allowed_actions"] = "all" + self.assert_deferred("sha", "selected policy is not active") + + def test_sha_actions_mode_or_repository_changes_during_proof_rejected(self): + for owner in ("orgs/atrinik", "repos/atrinik/example"): + with self.subTest(owner=owner): + count = 0 + def drift(endpoint): + nonlocal count + value = self.fixture(endpoint) + if endpoint == owner + "/actions/permissions": + count += 1 + if count > 1: + value["allowed_actions"] = "all" if owner.startswith("orgs") else "local_only" + return value + with patch.object(readiness, "api", drift): + self.assertEqual(self.invoke("sha")[0], 1) + + def test_sha_actions_policy_unreadable_and_malformed_fail_closed(self): + for endpoint_part in ("orgs/atrinik/actions/permissions", "repos/atrinik/example/actions/permissions"): + with self.subTest(endpoint=endpoint_part): + def unreadable(endpoint): + if endpoint == endpoint_part: + raise readiness.APIError(endpoint, 403) + return self.fixture(endpoint) + with patch.object(readiness, "api", unreadable): + self.assertEqual(self.invoke("sha")[0], 2) + self.fixture.repository_actions["enabled"] = "true" + self.assertEqual(self.invoke("sha")[0], 2) + self.fixture.repository_actions["enabled"] = True + self.fixture.actions_policy["enabled_repositories"] = "unexpected" + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_selected_membership_change_refenced(self): + self.fixture.actions_policy["enabled_repositories"] = "selected" + count = 0 + def drift(endpoint): + nonlocal count + value = self.fixture(endpoint) + if "/actions/permissions/repositories?" in endpoint: + count += 1 + if count > 1: + value = {"total_count": 0, "repositories": []} + return value + with patch.object(readiness, "api", drift): + self.assert_deferred("sha", "not enabled") + + def test_codeql_languages_schema_rejects_mapping_null_and_nonstrings(self): + for value in ({"rust": False}, None, "rust", [False], [""]): + with self.subTest(value=value): + self.fixture.setup["languages"] = value + code, output = self.invoke("codeql") + self.assertEqual(code, 2, output) + + def test_codeql_fresh_languages_schema_rejected(self): + count = 0 + def drift(endpoint): + nonlocal count + value = self.fixture(endpoint) + if endpoint.endswith("/code-scanning/default-setup"): + count += 1 + if count > 1: + value["languages"] = {"rust": False} + return value + with patch.object(readiness, "api", drift): + self.assertEqual(self.invoke("codeql")[0], 2) + def test_sha_general_github_owned_pin_update(self): self.fixture.before = f"steps:\n - uses: actions/checkout@{OLD} # v5\n".encode() self.fixture.after = f"steps:\n - uses: actions/checkout@{NEW} # v6\n".encode() From 14ce873d2151a5d5c5d7dcf7aaae0838e5ff8365 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 13:58:08 -0500 Subject: [PATCH 07/22] fix: fail closed on CodeQL reads and run requested canary probes --- bin/publish | 34 +++++++++-------- tests/publish-security-merge.sh | 68 +++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 16 deletions(-) diff --git a/bin/publish b/bin/publish index 95fe152..0a843ef 100755 --- a/bin/publish +++ b/bin/publish @@ -1046,8 +1046,9 @@ converge_codeql_default_setup() { echo "KEEP /repos/${organization}/${repository}/code-scanning/default-setup is ${desired_state}" continue fi - elif [[ ${required_languages} != '[]' ]]; then - # Never replace an unreadable live language set with an incomplete list. + else + # A failed read is unknown state, including repositories with no policy + # language minimum or with advanced setup. Never publish from that state. echo "error: cannot read existing CodeQL languages for ${repository}" >&2 return 1 fi @@ -1231,12 +1232,11 @@ converge_actions_permissions() { fi phase=$(jq -r '.phase' "${root}/config/actions-sha-pinning.json") canary=$(jq -r '.canary // empty' "${root}/config/actions-sha-pinning.json") - if ! jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then - if [[ ${phase} == enforced ]]; then - validate_sha_canary_target "${canary}" || return 1 - elif [[ -n ${sha_pinning_canary} ]]; then - validate_sha_canary_target "${sha_pinning_canary}" || return 1 - fi + if [[ ${phase} == pending && -n ${sha_pinning_canary} ]]; then + validate_sha_canary_target "${sha_pinning_canary}" || return 1 + elif [[ ${phase} == enforced ]] && + ! jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then + validate_sha_canary_target "${canary}" || return 1 fi # GitHub requires selected mode before its selected-actions endpoint can be @@ -1292,6 +1292,16 @@ converge_actions_permissions() { return 1 fi fi + # An explicitly requested pending-phase probe is read-only and independent + # of whether SHA pinning is already enabled. Do not let preservation skip it. + if [[ ${phase} == pending && -n ${sha_pinning_canary} ]]; then + validate_sha_canary_target "${sha_pinning_canary}" || return 1 + if reason=$("${root}/bin/security-readiness" sha "${sha_pinning_canary}"); then + echo "READY canary evidence: ${reason}; source rollout phase remains pending" + else + echo "PENDING canary evidence: ${reason}" + fi + fi if jq -e '.sha_pinning_required == true' <<<"${current}" >/dev/null; then return 0 fi @@ -1309,14 +1319,6 @@ converge_actions_permissions() { run_api PUT "orgs/${organization}/actions/permissions" \ "${root}/config/actions-permissions.json" || return 1 else - if [[ -n ${sha_pinning_canary} ]]; then - validate_sha_canary_target "${sha_pinning_canary}" || return 1 - if reason=$("${root}/bin/security-readiness" sha "${sha_pinning_canary}"); then - echo "READY canary evidence: ${reason}; source rollout phase remains pending" - else - echo "PENDING canary evidence: ${reason}" - fi - fi echo "PENDING SHA pinning: promote the reviewed source rollout phase and accepted canary after readiness" fi } diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index 0f8b279..185ffbe 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -61,6 +61,7 @@ case ${READINESS_STATUS:-ready} in ready) echo 'producer evidence ready'; exit 0 ;; missing) echo 'producer is missing'; exit 1 ;; failed) echo 'producer failed'; exit 1 ;; +stale) echo 'canary base changed during preflight'; exit 1 ;; unreadable) echo 'producer evidence unreadable'; exit 2 ;; *) exit 99 ;; esac @@ -96,6 +97,10 @@ github_api() { printf '%s\n' "${workflow_state}" ;; repos/atrinik/content-toolkit/code-scanning/default-setup) + if [[ ${codeql_state} == http-* ]]; then + echo "gh: default-setup read failed (HTTP ${codeql_state#http-})" >&2 + return 1 + fi [[ ${codeql_state} != unreadable ]] || return 1 printf '%s\n' "${codeql_state}" ;; @@ -209,6 +214,7 @@ reset_case() { repository_state='{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' workflow_state='{"default_workflow_permissions":"write","can_approve_pull_request_reviews":true}' cp "${source_root}/config/security-merge-protection.json" "${security_merge_config}" + cp "${source_root}/config/codeql-advanced-setup.json" "${codeql_advanced_config}" echo '{"phase":"pending","canary":null}' >"${actions_sha_pinning_config}" actions_state='{"enabled_repositories":"selected","allowed_actions":"all","sha_pinning_required":false}' selected_state=$(cat "${root}/config/actions-selected.json") @@ -255,6 +261,35 @@ if converge_codeql_default_setup >"${temporary}/languages-malformed.txt" 2>&1; t fi assert_no_writes +# Read failures are fatal even without a language minimum, in both setup modes. +for setup_kind in default advanced; do + for http_status in 403 404 500; do + reset_case + jq '.codeql_required_languages = {}' "${security_merge_config}" >"${temporary}/no-minima.json" + cp "${temporary}/no-minima.json" "${security_merge_config}" + if [[ ${setup_kind} == advanced ]]; then + echo '{"repositories":["content-toolkit"]}' >"${codeql_advanced_config}" + fi + codeql_state=http-${http_status} + if converge_codeql_default_setup >"${temporary}/languages-${setup_kind}-${http_status}.txt" 2>&1; then + echo 'expected unreadable default setup to fail publication without a language minimum' >&2 + exit 1 + fi + grep -q 'cannot read existing CodeQL languages' "${temporary}/languages-${setup_kind}-${http_status}.txt" + assert_no_writes + done +done +reset_case +jq '.codeql_required_languages = {}' "${security_merge_config}" >"${temporary}/no-minima.json" +cp "${temporary}/no-minima.json" "${security_merge_config}" +codeql_state='{"state":"configured","query_suite":"default","languages":["python"]}' +converge_codeql_default_setup >"${temporary}/languages-readable-no-minimum.txt" +assert_no_writes +echo '{"repositories":["content-toolkit"]}' >"${codeql_advanced_config}" +codeql_state='{"state":"not-configured","languages":null}' +converge_codeql_default_setup >"${temporary}/languages-readable-advanced-null.txt" +assert_no_writes + # Use the actual dispatch: security producer readiness follows base convergence # for both scopes, and these Team-compatible gates always target repositories. publish_organization_rulesets() { echo organization-rules >>"${temporary}/order"; } @@ -432,6 +467,39 @@ converge_actions_permissions >"${temporary}/sha-keep.txt" assert_no_writes [[ ! -s ${readiness_log} ]] +# Explicit pending canary probes still run when SHA is already enabled. +for producer in ready failed unreadable stale; do + reset_case + actions_state=$(cat "${root}/config/actions-permissions.json") + sha_pinning_canary=atrinik/content-toolkit/99 + export READINESS_STATUS=${producer} + converge_actions_permissions >"${temporary}/sha-enabled-probe-${producer}.txt" + jq -se '. == [{mode:"sha", repository:"atrinik/content-toolkit/99"}]' "${readiness_log}" >/dev/null + jq -se 'all(.[]; .method == "GET" or .method == "PROOF")' "${api_log}" >/dev/null + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == true ]] + if [[ ${producer} == ready ]]; then + grep -q '^READY canary evidence:' "${temporary}/sha-enabled-probe-${producer}.txt" + else + grep -q '^PENDING canary evidence:' "${temporary}/sha-enabled-probe-${producer}.txt" + fi +done +for invalid in foreign/content-toolkit/99 atrinik/unregistered/99 replaced; do + reset_case + actions_state=$(cat "${root}/config/actions-permissions.json") + sha_pinning_canary=${invalid} + if [[ ${invalid} == replaced ]]; then + sha_pinning_canary=atrinik/content-toolkit/99 + repository_state=$(jq '.id = 999' <<<"${repository_state}") + fi + if converge_actions_permissions >"${temporary}/sha-enabled-probe-invalid.txt" 2>&1; then + echo 'expected invalid requested canary identity to fail with SHA already enabled' >&2 + exit 1 + fi + assert_no_writes + [[ ! -s ${readiness_log} ]] + [[ $(jq -r '.sha_pinning_required' <<<"${actions_state}") == true ]] +done + # Simultaneous mode, patterns and SHA drift repairs conservative policy first. for producer in ready failed unreadable; do reset_case From 6115755d456c05903366086750ba211c2c1b7422 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 14:02:16 -0500 Subject: [PATCH 08/22] docs: state the trusted repository writer boundary --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 13a393a..160113e 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,14 @@ It relies on GitHub's dependency graph and submitted snapshots; unsupported or missing manifests cannot provide comprehensive dependency coverage. This is separate from Dependabot's scheduled update and alert capabilities. +Dependency review uses ordinary pull-request CI and trusts repository writers. +The required check context and GitHub Actions app binding do not attest immutable +workflow bytes. A writer can modify the producer within the same PR, including +replacing its review with a successful no-op. The high/critical gate remains in +place for ordinary introduced vulnerabilities in supported dependency data; +it does not defend against malicious repository writers or an owner using the +pull-request bypass. This policy adds no human approval requirement. + GitHub's [Dependabot alert limitations](https://docs.github.com/en/code-security/concepts/supply-chain-security/dependabot-alerts#limitations) exclude advisory alerts for Actions referenced by commit SHA. SHA pinning and a passing dependency review therefore do not prove that an Action commit is From 835cc8f0dcbd2886e8ddce5a31c4774ec44a5d30 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 14:19:41 -0500 Subject: [PATCH 09/22] fix: require complete Classic CodeQL partition coverage --- README.md | 14 ++- bin/validate | 13 ++- config/codeql-advanced-setup.json | 26 +++++- lib/security_readiness.py | 82 ++++++++++++++++-- tests/publish-maintenance-branch.sh | 2 +- tests/security-readiness.py | 127 +++++++++++++++++++++++++--- tests/validate-codeql-advanced.sh | 23 +++++ 7 files changed, 265 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 160113e..4b79122 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,10 @@ Rollout is owned by an organization administrator: revision. Synthetic merge revisions must bind the associated PR's base and head. CodeQL requires current-head coverage for every configured default-setup language plus policy minima. Classic instead requires its successful full - advanced producer and all current source-derived analysis partitions; + advanced producer and all current source-derived analysis partitions. The + advanced inventory independently requires Classic’s client, server, and core + C/C++, Python, and GitHub Actions scope/language pairs, so reducing producer + declarations or executed jobs cannot silently reduce required coverage; changed-path PR subsets cannot establish readiness. The bounded newest-first query stops once current coverage is proven; retired categories and old analysis history do not require manual cleanup. @@ -801,7 +804,14 @@ publisher fails closed if a live repository ID differs from the inventory. Do not add a repository to `config/codeql-advanced-setup.json` merely because default setup needs tuning. The inventory is reserved for a reviewed advanced CodeQL workflow that needs repository-specific build, component, or path -coverage. Both security configurations must remain identical except for their +coverage. Its `required_partitions` records semantic scope/language minima, +while the current producer owns display names and upload categories. Readiness +reads literal partition declarations without executing repository Python and +requires a distinct successful job and matching upload for every declaration, +including additions. Changing action versions needs no coverage-inventory edit. +Remove the matching minima when removing an advanced repository; an empty +inventory with an empty minima object is valid. Both security configurations +must remain identical except for their name, description, and `code_scanning_options.allow_advanced` value. Moving a repository to `archive` is an executable policy change: the next diff --git a/bin/validate b/bin/validate index c4dcca6..8d10ec3 100755 --- a/bin/validate +++ b/bin/validate @@ -272,7 +272,18 @@ jq -e \ jq -e \ --slurpfile repositories "${repositories_config}" ' - (keys == ["repositories"]) and + (keys == ["repositories", "required_partitions"]) and + (.required_partitions | type == "object") and + ((.required_partitions | keys) == (.repositories | sort)) and + all(.required_partitions[]; + type == "array" and length > 0 and + all(.[]; + (keys == ["language", "scope"]) and + (.scope | type == "string" and test("^[a-z][a-z0-9+-]*$")) and + (.language | type == "string" and test("^[a-z][a-z0-9+-]*$")) + ) and + ((map([.scope, .language]) | length) == (map([.scope, .language]) | unique | length)) + ) and (.repositories | type == "array") and ((.repositories | length) == (.repositories | unique | length)) and all( diff --git a/config/codeql-advanced-setup.json b/config/codeql-advanced-setup.json index 114f975..808abec 100644 --- a/config/codeql-advanced-setup.json +++ b/config/codeql-advanced-setup.json @@ -1,5 +1,29 @@ { "repositories": [ "classic" - ] + ], + "required_partitions": { + "classic": [ + { + "scope": "client", + "language": "c-cpp" + }, + { + "scope": "server", + "language": "c-cpp" + }, + { + "scope": "core-cpp", + "language": "c-cpp" + }, + { + "scope": "python", + "language": "python" + }, + { + "scope": "actions", + "language": "actions" + } + ] + } } diff --git a/lib/security_readiness.py b/lib/security_readiness.py index cd83429..3dca77d 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -686,6 +686,67 @@ def classifier_full_events(source): return set(events) & {"push", "schedule", "workflow_dispatch"} +def advanced_partition_minima(policy, repository_name): + """Validate source-owned coverage even when preflight is called directly.""" + inventories = policy.get("required_partitions") + if (not isinstance(inventories, dict) or set(inventories) != set(policy["repositories"]) + or not isinstance(inventories.get(repository_name), list) + or not inventories[repository_name]): + raise EvidenceError("advanced CodeQL required partition inventory malformed/missing") + minima = set() + for item in inventories[repository_name]: + if (not isinstance(item, dict) or set(item) != {"scope", "language"} + or any(not isinstance(item[key], str) + or not re.fullmatch(r"[a-z][a-z0-9+-]*", item[key]) for key in item)): + raise EvidenceError("advanced CodeQL required partition identity malformed") + identity = (item["scope"], canonical_language(item["language"])) + if identity in minima: + raise EvidenceError("advanced CodeQL required partition identity duplicated") + minima.add(identity) + return minima + + +def classifier_partitions(source, minima): + """Inspect literal matrix declarations; never execute repository Python.""" + tree = ast.parse(source) + functions = [node for node in tree.body if isinstance(node, ast.FunctionDef) + and node.name == "write_outputs"] + require(len(functions) == 1, "advanced CodeQL partition declarations missing") + function = functions[0] + assignments = [node for node in ast.walk(function) if isinstance(node, ast.Name) + and node.id == "entries" and isinstance(node.ctx, ast.Store)] + initializers = [node for node in function.body if isinstance(node, ast.Assign) + and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == "entries" and isinstance(node.value, ast.List) + and not node.value.elts] + require(len(assignments) == len(initializers) == 1, + "advanced CodeQL partition declaration shape unsupported") + declarations, identities, categories = {}, set(), set() + for node in ast.walk(function): + if not (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) and node.func.value.id == "entries"): + continue + require(node.func.attr == "append" and len(node.args) == 1 and not node.keywords, + "advanced CodeQL partition declaration shape unsupported") + try: + item = ast.literal_eval(node.args[0]) + except (ValueError, TypeError): + raise Deferred("advanced CodeQL partition declaration is not literal") from None + require(isinstance(item, dict) and set(item) == {"display", "category", "scope", "language", "run"} + and item["run"] is True and all(isinstance(item[key], str) and item[key] + for key in ("display", "category", "scope", "language")), + "advanced CodeQL partition declaration malformed") + identity = (item["scope"], canonical_language(item["language"])) + require(item["display"] not in declarations and item["category"] not in categories + and identity not in identities, "advanced CodeQL partition declarations duplicated") + declarations[item["display"]] = item + categories.add(item["category"]) + identities.add(identity) + require(bool(declarations) and minima <= identities, + "advanced CodeQL producer omits required partition coverage") + return declarations + + def latest_full_codeql_run(coordinate, branch, head, events): value = api(f"repos/{coordinate}/actions/workflows/codeql.yml/runs?head_sha={head}&per_page=100") candidates = [run for run in value["workflow_runs"] if run["event"] in events @@ -697,7 +758,7 @@ def latest_full_codeql_run(coordinate, branch, head, events): return run -def advanced_codeql(coordinate, branch, head, minima): +def advanced_codeql(coordinate, branch, head, minima, partition_minima): sources = {} for path in (".github/workflows/codeql.yml", "tools/ci/classify_changes.py", "tools/ci/require_checks.py"): sources[path] = contents(coordinate, path, branch) @@ -708,6 +769,7 @@ def advanced_codeql(coordinate, branch, head, minima): "python3 tools/ci/require_checks.py optional", "${{ needs.classify.outputs.run }}", "${{ needs.analyze.result }}"): require(marker in workflow, "advanced CodeQL workflow producer contract unsupported") + declarations = classifier_partitions(sources["tools/ci/classify_changes.py"], partition_minima) events = classifier_full_events(sources["tools/ci/classify_changes.py"]) run = latest_full_codeql_run(coordinate, branch, head, events) jobs_endpoint = f"repos/{coordinate}/actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs" @@ -736,7 +798,8 @@ def advanced_codeql(coordinate, branch, head, minima): if match: require(match[1] not in selected, "advanced CodeQL analysis jobs duplicated") selected[match[1]] = job - require(bool(selected), "advanced CodeQL full run selected no analysis jobs") + require(set(selected) == set(declarations), + "advanced CodeQL full run does not cover every declared partition") found, categories, languages = set(), set(), set() for entries in newest_analysis_pages(coordinate, branch): for analysis in entries: @@ -752,12 +815,16 @@ def advanced_codeql(coordinate, branch, head, minima): if display not in selected: continue job = selected[display] + declaration = declarations[display] if analysis["created_at"] < job["started_at"]: continue require(display not in found, "advanced CodeQL selected display has ambiguous upload categories") successful_analysis(analysis, head) category_language = re.search(r"(?:^|/)language:([a-z0-9+-]+)$", category) - require(environment.get("category") == category and category_language is not None + require(environment.get("category") == category == declaration["category"] + and environment.get("scope") == declaration["scope"] + and language == canonical_language(declaration["language"]) + and category_language is not None and canonical_language(category_language[1]) == language and job["started_at"] <= analysis["created_at"] <= job["completed_at"], "advanced CodeQL analysis upload does not belong to selected full-run job") @@ -777,7 +844,7 @@ def advanced_codeql(coordinate, branch, head, minima): for path, source in sources.items(): require(contents(coordinate, path, branch) == source, "advanced CodeQL producer source changed during preflight") - return "current full-event CodeQL producer and every selected partition upload successful" + return "current full-event CodeQL producer covers policy minima and every declared partition upload" raise Deferred("required advanced CodeQL partition upload missing within bounded evidence") @@ -789,9 +856,10 @@ def codeql(coordinate): require(bool(SHA.fullmatch(head)), "default-branch head missing") required = config("security-merge-protection.json")["codeql_required_languages"].get( coordinate.split('/')[1], []) - advanced = config("codeql-advanced-setup.json")["repositories"] - if coordinate.split('/')[1] in advanced: - reason = advanced_codeql(coordinate, branch, head, required) + advanced = config("codeql-advanced-setup.json") + if coordinate.split('/')[1] in advanced["repositories"]: + partition_minima = advanced_partition_minima(advanced, coordinate.split('/')[1]) + reason = advanced_codeql(coordinate, branch, head, required, partition_minima) else: reason = default_codeql(coordinate, branch, head, [canonical_language(value) for value in required]) require(api(branch_endpoint)["commit"]["sha"] == head, "default branch changed during CodeQL preflight") diff --git a/tests/publish-maintenance-branch.sh b/tests/publish-maintenance-branch.sh index aa9468b..c84ea5f 100755 --- a/tests/publish-maintenance-branch.sh +++ b/tests/publish-maintenance-branch.sh @@ -1256,7 +1256,7 @@ cp -R "${root}/config" "${rollback_root}/config" cp -R "${root}/community-health" "${rollback_root}/community-health" mkdir -p "${rollback_root}/.github" cp -R "${root}/.github/workflows" "${rollback_root}/.github/workflows" -jq '.repositories = []' \ +jq '.repositories = [] | .required_partitions = {}' \ "${root}/config/codeql-advanced-setup.json" \ >"${rollback_root}/config/codeql-advanced-setup.json" rollback_log=${temporary}/empty-inventory.jsonl diff --git a/tests/security-readiness.py b/tests/security-readiness.py index 63f76e3..d239c46 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -75,12 +75,36 @@ name: CodeQL validation run: python3 tools/ci/require_checks.py optional '${{ needs.classify.outputs.run }}' '${{ needs.analyze.result }}' """ -CLASSIFIER = b"""FULL_EVENTS = {'push', 'schedule', 'workflow_dispatch', 'merge_group'} +PARTITIONS = [ + {"display": "C/C++ client", "scope": "client", "language": "c-cpp", + "category": "/partition:client/language:c-cpp", "run": True}, + {"display": "C/C++ server", "scope": "server", "language": "c-cpp", + "category": "/partition:server/language:c-cpp", "run": True}, + {"display": "C/C++ core", "scope": "core-cpp", "language": "c-cpp", + "category": "/partition:core/language:c-cpp", "run": True}, + {"display": "Python", "scope": "python", "language": "python", + "category": "/partition:core/language:python", "run": True}, + {"display": "GitHub Actions", "scope": "actions", "language": "actions", + "category": "/partition:core/language:actions", "run": True}, +] + + +def classifier_source(partitions): + source = """FULL_EVENTS = {'push', 'schedule', 'workflow_dispatch', 'merge_group'} def main(): if arguments.event in FULL_EVENTS: full = True paths = [] +def write_outputs(path, result, config): + entries = [] """ + for item in partitions: + source += f" if result[{item['scope']!r}]:\n entries.append({item!r})\n" + source += " matrix = {'include': entries or [{'run': False, 'scope': 'union'}]}\n" + return source.encode() + + +CLASSIFIER = classifier_source(PARTITIONS) def content(data): @@ -230,7 +254,7 @@ def setUp(self): "required_ci": {"example": ["Required checks", "Conventional PR title"]}})) (self.root / "config/security-merge-protection.json").write_text(json.dumps({ "codeql_required_languages": {"example": ["rust"]}, "dependency_review": []})) - (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": []}') + (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": [], "required_partitions": {}}') self.fixture = Fixture() self.root_patch = patch.object(readiness, "ROOT", self.root) self.api_patch = patch.object(readiness, "api", self.fixture) @@ -1099,16 +1123,19 @@ def test_codeql_javascript_typescript_alias_normalized(self): self.fixture.analyses.append(second) self.assertEqual(self.invoke("codeql")[0], 0) - def enable_advanced(self): - (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": ["example"]}') + def enable_advanced(self, partitions=None): + if partitions is None: + partitions = PARTITIONS + # The policy minimum remains independent of the producer declarations. + (self.root / "config/codeql-advanced-setup.json").write_text(json.dumps({ + "repositories": ["example"], "required_partitions": {"example": [ + {"scope": item["scope"], "language": item["language"]} for item in PARTITIONS]}})) + self.set_classifier_source(classifier_source(partitions)) (self.root / "config/security-merge-protection.json").write_text('{"codeql_required_languages": {}}') - displays = [("C/C++ client", "client", "c-cpp"), ("C/C++ server", "server", "c-cpp"), - ("C/C++ core", "core", "c-cpp"), ("Python", "core", "python"), - ("GitHub Actions", "core", "actions")] self.fixture.analyses = [] self.fixture.codeql_jobs = [] - for index, (display, partition, language) in enumerate(displays): - category = f"/partition:{partition}/language:{language}" + for index, item in enumerate(partitions): + display, category, language = item["display"], item["category"], item["language"] job = {"id": index + 1, "name": f"CodeQL analysis ({display})", "head_sha": HEAD, "status": "completed", "conclusion": "success", "started_at": "2026-10-11T11:59:00Z", "completed_at": "2026-10-11T12:01:00Z"} @@ -1116,7 +1143,7 @@ def enable_advanced(self): self.fixture.analyses.append({"id": index + 1, "tool": {"name": "CodeQL"}, "analysis_key": ".github/workflows/codeql.yml:analyze", "category": category, "environment": json.dumps({"display": display, "category": category, "language": language, - "scope": partition, "run": ""}), + "scope": item["scope"], "run": ""}), "commit_sha": HEAD, "ref": "refs/heads/main", "error": "", "rules_count": 58, "created_at": "2026-10-11T12:00:00Z"}) for name in ("CodeQL classification", "CodeQL validation"): @@ -1124,6 +1151,86 @@ def enable_advanced(self): job.update(id=999, name=name, check_run_url="https://api.github.com/repos/atrinik/example/check-runs/999") self.fixture.codeql_jobs.append(job) + def set_classifier_source(self, source): + self.fixture.codeql_sources["tools/ci/classify_changes.py"] = source + self.fixture.codeql_head_sources["tools/ci/classify_changes.py"] = source + + def test_codeql_advanced_reduced_source_and_jobs_below_policy_deferred(self): + self.enable_advanced(PARTITIONS[:1]) + self.assert_deferred("codeql", "omits required partition coverage") + + def test_codeql_advanced_reduced_jobs_with_full_source_deferred(self): + self.enable_advanced() + self.fixture.codeql_jobs = self.fixture.codeql_jobs[:1] + self.fixture.codeql_jobs[-2:] + self.fixture.analyses = self.fixture.analyses[:1] + self.assert_deferred("codeql", "every declared partition") + + def test_codeql_advanced_additional_declared_partition_ready(self): + extra = {"display": "Extra C/C++", "scope": "extra", "language": "c-cpp", + "category": "/partition:extra/language:c-cpp", "run": True} + self.enable_advanced(PARTITIONS + [extra]) + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_advanced_additional_declaration_requires_job(self): + extra = {"display": "Extra C/C++", "scope": "extra", "language": "c-cpp", + "category": "/partition:extra/language:c-cpp", "run": True} + self.enable_advanced(PARTITIONS + [extra]) + del self.fixture.codeql_jobs[5] + self.assert_deferred("codeql", "every declared partition") + + def test_codeql_advanced_category_rename_preserves_semantic_minimum(self): + partitions = copy.deepcopy(PARTITIONS) + partitions[0]["category"] = "/partition:client-renamed/language:c-cpp" + self.enable_advanced(partitions) + self.assertEqual(self.invoke("codeql")[0], 0) + + def test_codeql_advanced_upload_matches_declared_scope_category_and_language(self): + for field, value in (("scope", "server"), ("category", "/other/language:c-cpp"), + ("language", "python")): + with self.subTest(field=field): + self.enable_advanced() + analysis = self.fixture.analyses[0] + environment = json.loads(analysis["environment"]) + environment[field] = value + analysis["environment"] = json.dumps(environment) + if field == "category": + analysis["category"] = value + self.assert_deferred("codeql", "selected full-run job") + + def test_codeql_advanced_duplicate_declarations_rejected(self): + for field in ("display", "category", "scope"): + with self.subTest(field=field): + partitions = copy.deepcopy(PARTITIONS) + partitions[1][field] = partitions[0][field] + self.enable_advanced(partitions) + self.assert_deferred("codeql", "declarations duplicated") + + def test_codeql_advanced_nonliteral_declaration_deferred(self): + self.enable_advanced() + self.set_classifier_source(CLASSIFIER.replace(b"entries.append({", b"entries.append(dict(", 1)) + self.assertEqual(self.invoke("codeql")[0], 2) # Invalid Python is unreadable evidence. + self.set_classifier_source(CLASSIFIER + b"\n") + source = CLASSIFIER.replace(b" matrix =", b" entries.append(dynamic_partition)\n matrix =") + self.set_classifier_source(source) + self.assert_deferred("codeql", "not literal") + + def test_codeql_advanced_unsupported_matrix_mutation_deferred(self): + self.enable_advanced() + source = CLASSIFIER.replace(b" matrix =", b" entries.extend(extra)\n matrix =") + self.set_classifier_source(source) + self.assert_deferred("codeql", "shape unsupported") + + def test_codeql_advanced_missing_or_malformed_minima_unreadable(self): + for minima in (None, [], {}, [{"scope": "client"}], + [{"scope": "client", "language": "c-cpp"}] * 2): + with self.subTest(minima=minima): + self.enable_advanced() + policy = {"repositories": ["example"], "required_partitions": {"example": minima}} + (self.root / "config/codeql-advanced-setup.json").write_text(json.dumps(policy)) + self.assertEqual(self.invoke("codeql")[0], 2) + (self.root / "config/codeql-advanced-setup.json").write_text('{"repositories": ["example"]}') + self.assertEqual(self.invoke("codeql")[0], 2) + def test_codeql_advanced_current_full_producer_all_partitions_ready(self): self.enable_advanced() self.assertEqual(self.invoke("codeql")[0], 0) diff --git a/tests/validate-codeql-advanced.sh b/tests/validate-codeql-advanced.sh index a6e92b1..3fd7ddb 100755 --- a/tests/validate-codeql-advanced.sh +++ b/tests/validate-codeql-advanced.sh @@ -10,6 +10,8 @@ mkdir -p "${temporary}/bin" cp "${root}/bin/validate" "${temporary}/bin/validate" cp -R "${root}/config" "${temporary}/config" cp -R "${root}/community-health" "${temporary}/community-health" +cp -R "${root}/.github" "${temporary}/.github" +"${temporary}/bin/validate" >/dev/null assert_invalid() { local description=$1 @@ -42,6 +44,27 @@ assert_invalid 'an archived advanced CodeQL inventory repository' cp "${root}/config/codeql-advanced-setup.json" \ "${temporary}/config/codeql-advanced-setup.json" +for filter in \ + 'del(.required_partitions)' \ + '.required_partitions = {}' \ + '.required_partitions.classic = []' \ + '.required_partitions.classic = {}' \ + '.required_partitions.classic[0].scope = null' \ + 'del(.required_partitions.classic[0].language)' \ + '.required_partitions.classic += [.required_partitions.classic[0]]' \ + '.required_partitions.other = .required_partitions.classic'; do + rewrite_json "${filter}" "${temporary}/config/codeql-advanced-setup.json" + assert_invalid "malformed or mismatched advanced partition policy: ${filter}" + cp "${root}/config/codeql-advanced-setup.json" \ + "${temporary}/config/codeql-advanced-setup.json" +done + +rewrite_json '.repositories = [] | .required_partitions = {}' \ + "${temporary}/config/codeql-advanced-setup.json" +"${temporary}/bin/validate" >/dev/null +cp "${root}/config/codeql-advanced-setup.json" \ + "${temporary}/config/codeql-advanced-setup.json" + rewrite_json '.secret_scanning = "disabled"' \ "${temporary}/config/code-security-advanced.json" assert_invalid 'advanced security configuration drift from the baseline' From af1bda03c1a82dba1df6ec76d33a4181c220c4db Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 14:45:22 -0500 Subject: [PATCH 10/22] fix: recheck CodeQL analyses before accepting readiness --- lib/security_readiness.py | 144 ++++++++++++++++++++---------------- tests/security-readiness.py | 87 +++++++++++++++++++++- 2 files changed, 164 insertions(+), 67 deletions(-) diff --git a/lib/security_readiness.py b/lib/security_readiness.py index 3dca77d..e61d364 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -634,14 +634,9 @@ def configured_codeql_languages(setup): return {canonical_language(value) for value in values} -def default_codeql(coordinate, branch, head, minima): - endpoint = f"repos/{coordinate}/code-scanning/default-setup" - setup = api(endpoint) - require(setup["state"] == "configured", "CodeQL default setup not configured") - languages = configured_codeql_languages(setup) - require(bool(languages), "CodeQL default setup has no configured languages") - require(set(minima) <= languages, "required CodeQL languages not configured") - found = set() +def default_codeql_coverage(coordinate, branch, head, languages): + """Select only the newest canonical analyses within the bounded history.""" + found = {} for entries in newest_analysis_pages(coordinate, branch): for analysis in entries: if analysis["tool"]["name"] != "CodeQL" or analysis["ref"] != f"refs/heads/{branch}": @@ -649,22 +644,35 @@ def default_codeql(coordinate, branch, head, minima): language, _ = analysis_language(analysis) # Default setup owns one canonical category per configured language. # Retired advanced partitions, even in that language, are historical. - category_language = analysis["category"].removeprefix("/language:") + category_language = canonical_language(analysis["category"].removeprefix("/language:")) if (not analysis["category"].startswith("/language:") - or canonical_language(category_language) != language - or language not in languages or language in found): + or category_language not in languages or category_language in found): continue successful_analysis(analysis, head) - found.add(language) - if found == languages: - fresh = api(endpoint) - require(fresh["state"] == setup["state"] - and configured_codeql_languages(fresh) == languages, - "CodeQL default setup changed during preflight") - return "latest canonical default-setup analyses cover every configured language on current head" + require(language == category_language, "default CodeQL analysis language does not match its category") + found[language] = analysis + if set(found) == languages: + return found raise Deferred("required configured CodeQL language analysis missing within bounded evidence") +def default_codeql(coordinate, branch, head, minima): + endpoint = f"repos/{coordinate}/code-scanning/default-setup" + setup = api(endpoint) + require(setup["state"] == "configured", "CodeQL default setup not configured") + languages = configured_codeql_languages(setup) + require(bool(languages), "CodeQL default setup has no configured languages") + require(set(minima) <= languages, "required CodeQL languages not configured") + coverage = default_codeql_coverage(coordinate, branch, head, languages) + fresh = api(endpoint) + require(fresh["state"] == setup["state"] + and configured_codeql_languages(fresh) == languages, + "CodeQL default setup changed during preflight") + require(default_codeql_coverage(coordinate, branch, head, languages) == coverage, + "CodeQL default-setup analysis coverage changed during preflight") + return "latest canonical default-setup analyses cover every configured language on current head" + + def classifier_full_events(source): """Read the owner's full-event contract without executing fetched Python.""" tree = ast.parse(source) @@ -758,6 +766,47 @@ def latest_full_codeql_run(coordinate, branch, head, events): return run +def advanced_codeql_coverage(coordinate, branch, head, minima, selected, declarations): + """Select current producer uploads, ignoring retired categories and history.""" + found, categories, languages = {}, set(), set() + for entries in newest_analysis_pages(coordinate, branch): + for analysis in entries: + if (analysis["tool"]["name"] != "CodeQL" or analysis["ref"] != f"refs/heads/{branch}" + or analysis.get("analysis_key") != ".github/workflows/codeql.yml:analyze"): + continue + category = analysis["category"] + if category in categories: + continue + categories.add(category) + language, environment = analysis_language(analysis) + display = environment.get("display") + if display not in selected: + continue + job = selected[display] + declaration = declarations[display] + if analysis["created_at"] < job["started_at"]: + continue + require(display not in found, "advanced CodeQL selected display has ambiguous upload categories") + successful_analysis(analysis, head) + category_language = re.search(r"(?:^|/)language:([a-z0-9+-]+)$", category) + require(environment.get("category") == category == declaration["category"] + and environment.get("scope") == declaration["scope"] + and language == canonical_language(declaration["language"]) + and category_language is not None + and canonical_language(category_language[1]) == language + and job["started_at"] <= analysis["created_at"] <= job["completed_at"], + "advanced CodeQL analysis upload does not belong to selected full-run job") + found[display] = analysis + languages.add(language) + if set(found) == set(selected): + break + if set(found) == set(selected): + require({canonical_language(value) for value in minima} <= languages, + "required CodeQL language analysis missing") + return found + raise Deferred("required advanced CodeQL partition upload missing within bounded evidence") + + def advanced_codeql(coordinate, branch, head, minima, partition_minima): sources = {} for path in (".github/workflows/codeql.yml", "tools/ci/classify_changes.py", "tools/ci/require_checks.py"): @@ -800,52 +849,19 @@ def advanced_codeql(coordinate, branch, head, minima, partition_minima): selected[match[1]] = job require(set(selected) == set(declarations), "advanced CodeQL full run does not cover every declared partition") - found, categories, languages = set(), set(), set() - for entries in newest_analysis_pages(coordinate, branch): - for analysis in entries: - if (analysis["tool"]["name"] != "CodeQL" or analysis["ref"] != f"refs/heads/{branch}" - or analysis.get("analysis_key") != ".github/workflows/codeql.yml:analyze"): - continue - category = analysis["category"] - if category in categories: - continue - categories.add(category) - language, environment = analysis_language(analysis) - display = environment.get("display") - if display not in selected: - continue - job = selected[display] - declaration = declarations[display] - if analysis["created_at"] < job["started_at"]: - continue - require(display not in found, "advanced CodeQL selected display has ambiguous upload categories") - successful_analysis(analysis, head) - category_language = re.search(r"(?:^|/)language:([a-z0-9+-]+)$", category) - require(environment.get("category") == category == declaration["category"] - and environment.get("scope") == declaration["scope"] - and language == canonical_language(declaration["language"]) - and category_language is not None - and canonical_language(category_language[1]) == language - and job["started_at"] <= analysis["created_at"] <= job["completed_at"], - "advanced CodeQL analysis upload does not belong to selected full-run job") - found.add(display) - languages.add(language) - if found == set(selected): - break - if found == set(selected): - require({canonical_language(value) for value in minima} <= languages, - "required CodeQL language analysis missing") - fresh = latest_full_codeql_run(coordinate, branch, head, events) - require(all(fresh[key] == run[key] for key in - ("id", "run_attempt", "head_sha", "head_branch", "event", "path", "check_suite_id")), - "advanced CodeQL full run changed during preflight") - require(api(check_endpoint) == check, "advanced CodeQL validation check changed during preflight") - require(pages(jobs_endpoint, "jobs") == jobs, "advanced CodeQL full-run jobs changed during preflight") - for path, source in sources.items(): - require(contents(coordinate, path, branch) == source, - "advanced CodeQL producer source changed during preflight") - return "current full-event CodeQL producer covers policy minima and every declared partition upload" - raise Deferred("required advanced CodeQL partition upload missing within bounded evidence") + coverage = advanced_codeql_coverage(coordinate, branch, head, minima, selected, declarations) + fresh = latest_full_codeql_run(coordinate, branch, head, events) + require(all(fresh[key] == run[key] for key in + ("id", "run_attempt", "head_sha", "head_branch", "event", "path", "check_suite_id")), + "advanced CodeQL full run changed during preflight") + require(api(check_endpoint) == check, "advanced CodeQL validation check changed during preflight") + require(pages(jobs_endpoint, "jobs") == jobs, "advanced CodeQL full-run jobs changed during preflight") + for path, source in sources.items(): + require(contents(coordinate, path, branch) == source, + "advanced CodeQL producer source changed during preflight") + require(advanced_codeql_coverage(coordinate, branch, head, minima, selected, declarations) == coverage, + "advanced CodeQL analysis coverage changed during preflight") + return "current full-event CodeQL producer covers policy minima and every declared partition upload" def codeql(coordinate): diff --git a/tests/security-readiness.py b/tests/security-readiness.py index d239c46..9ec9a9a 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -162,6 +162,8 @@ def __init__(self): self.codeql_check = check("CodeQL validation", identifier=999) self.codeql_check["check_suite"]["id"] = 777 self.codeql_jobs = [] + self.analysis_reads = 0 + self.fresh_analyses = None self.analyses = [{"id": 1, "created_at": "2026-10-11T12:00:00Z", "tool": {"name": "CodeQL"}, "ref": "refs/heads/main", "commit_sha": HEAD, "category": "/language:rust", "environment": '{"language":"rust"}', "error": "", "rules_count": 42}] @@ -235,7 +237,11 @@ def __call__(self, endpoint): return {"commit": {"sha": self.branch_head}} if "/code-scanning/analyses?" in endpoint: page = int(parse_qs(urlparse(endpoint).query)["page"][0]) - return copy.deepcopy(self.analyses[(page - 1) * 100:page * 100]) + if page == 1: + self.analysis_reads += 1 + analyses = (self.fresh_analyses if self.analysis_reads > 1 and self.fresh_analyses is not None + else self.analyses) + return copy.deepcopy(analyses[(page - 1) * 100:page * 100]) if endpoint.endswith("/code-scanning/default-setup"): return copy.deepcopy(self.setup) raise AssertionError("Unexpected read-only API endpoint: " + endpoint) @@ -1104,7 +1110,7 @@ def test_codeql_large_history_stops_after_current_coverage(self): old.update(commit_sha=OLD, category="/partition:retired/language:rust") self.fixture.analyses.extend([old] * 1200) self.assertEqual(self.invoke("codeql")[0], 0) - self.assertEqual(len([call for call in self.fixture.calls if "/analyses?" in call]), 1) + self.assertEqual(len([call for call in self.fixture.calls if "/analyses?" in call]), 2) def test_codeql_retired_category_failure_does_not_gate(self): old = copy.deepcopy(self.fixture.analyses[0]) @@ -1242,7 +1248,7 @@ def test_codeql_advanced_large_retired_history_ready(self): self.fixture.analyses.insert(0, retired) self.fixture.analyses.extend([retired] * 1200) self.assertEqual(self.invoke("codeql")[0], 0) - self.assertEqual(len([call for call in self.fixture.calls if "/analyses?" in call]), 1) + self.assertEqual(len([call for call in self.fixture.calls if "/analyses?" in call]), 2) def test_codeql_advanced_partial_uploads_deferred(self): self.enable_advanced() @@ -1280,6 +1286,81 @@ def test_codeql_advanced_upload_outside_job_window_deferred(self): self.fixture.analyses[0]["created_at"] = "2026-10-11T13:00:00Z" self.assert_deferred("codeql", "selected full-run job") + def prepare_analysis_recheck(self, advanced): + if advanced: + self.enable_advanced() + self.fixture.analysis_reads = 0 + self.fixture.fresh_analyses = copy.deepcopy(self.fixture.analyses) + + def test_codeql_newer_failure_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + failed = copy.deepcopy(self.fixture.analyses[0]) + failed.update(id=1000, error="new analysis failed") + self.fixture.fresh_analyses.insert(0, failed) + self.assert_deferred("codeql", "failed") + self.assertEqual(self.fixture.analysis_reads, 2) + + def test_codeql_newer_zero_rules_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + empty = copy.deepcopy(self.fixture.analyses[0]) + empty.update(id=1000, rules_count=0) + self.fixture.fresh_analyses.insert(0, empty) + self.assert_deferred("codeql", "no executed rules") + + def test_codeql_deleted_selected_analysis_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + del self.fixture.fresh_analyses[0] + self.assert_deferred("codeql", "missing within bounded evidence") + + def test_codeql_replaced_successful_coverage_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + replacement = copy.deepcopy(self.fixture.analyses[0]) + replacement["id"] = 1000 + self.fixture.fresh_analyses.insert(0, replacement) + self.assert_deferred("codeql", "analysis coverage changed during preflight") + + def test_codeql_selected_analysis_mutation_during_preflight_deferred(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + self.fixture.fresh_analyses[0]["rules_count"] += 1 + self.assert_deferred("codeql", "analysis coverage changed during preflight") + + def test_codeql_new_retired_analysis_during_preflight_preserves_readiness(self): + for advanced in (False, True): + with self.subTest(advanced=advanced): + self.prepare_analysis_recheck(advanced) + retired = copy.deepcopy(self.fixture.analyses[0]) + retired.update(id=1000, commit_sha=OLD, category="/retired/language:c-cpp", + environment='{"display":"Retired"}') + self.fixture.fresh_analyses.insert(0, retired) + self.assertEqual(self.invoke("codeql")[0], 0) + self.assertEqual(self.fixture.analysis_reads, 2) + + def test_codeql_default_new_failure_with_changed_language_does_not_fallback(self): + self.prepare_analysis_recheck(False) + failed = copy.deepcopy(self.fixture.analyses[0]) + failed.update(id=1000, error="new analysis failed", environment='{"language":"python"}') + self.fixture.fresh_analyses.insert(0, failed) + self.assert_deferred("codeql", "failed") + + def test_codeql_advanced_new_failure_without_display_does_not_fallback(self): + for environment in ('{}', '{"display":"Renamed"}'): + with self.subTest(environment=environment): + self.prepare_analysis_recheck(True) + failed = copy.deepcopy(self.fixture.analyses[0]) + failed.update(id=1000, error="new analysis failed", environment=environment) + self.fixture.fresh_analyses.insert(0, failed) + self.assert_deferred("codeql", "partition upload missing") + def test_codeql_latest_error_masks_older_success(self): failed = copy.deepcopy(self.fixture.analyses[0]) failed.update(id=2, error="analysis failed") From 9955156fe17c9e707858bba2f70e84f518bae3af Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 14:53:28 -0500 Subject: [PATCH 11/22] fix: align merged governance fixtures and approval guidance --- README.md | 18 ++++++++++-------- tests/publish-maintenance-branch.sh | 6 ++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 853865f..e0a860e 100644 --- a/README.md +++ b/README.md @@ -414,9 +414,9 @@ to update the automation branch, also authorizes Git-reference and release API operations. GitHub likewise has no separate create-or-update-PR permission that excludes review APIs: `pull requests: write` covers both. The App credential therefore cannot by itself prove that tag, release, or review calls are -technically impossible. Because the App owns its generated pull request, it -cannot provide the distinct human approval required by the ordinary gate, and -it has no bypass. The consumer must never review, approve, merge, tag, publish, +technically impossible. Ordinary pull requests require zero human approvals; +required checks and the pull-request process still apply, and the App has no +bypass. The consumer must never review, approve, merge, tag, publish, dispatch a release, write the default branch, or change repository settings. Its reviewed code, exact branch and changed-path checks, ordinary pull-request gate, protected release-tag rules, workflow contract tests, and audit trail @@ -439,10 +439,11 @@ Provision or rotate the key as an Atrinik organization owner: `update-content.yml` and require one App-authored disposable pull request to receive ordinary `Classic validation`, `CodeQL validation`, and `Conventional PR title` checks. -4. Confirm the App-authored pull request cannot satisfy its own human approval - or merge gate, close the disposable pull request, remove its branch, and - only then delete the previous private key. A later scheduled run must also - succeed before rotation is considered complete. +4. Confirm the App-authored pull request receives the required checks, the App + has no bypass, and the consumer does not approve or merge the pull request. + Close the disposable pull request, remove its branch, and only then delete + the previous private key. A later scheduled run must also succeed before + rotation is considered complete. For suspected disclosure or misuse, disable the Classic updater workflow, suspend installation `153045686`, delete the repository secret, and revoke the @@ -1534,7 +1535,8 @@ credential-free preparation verifies exact released versions and registry checksums, resolves dependencies, updates Cargo locks, and runs aggregate validation before requesting an App token in the isolated PR-writing job. They maintain only their owned `automation/released-crate-dependencies` branch and -pull request; ordinary required checks and human review still govern merging. +pull request; ordinary required checks and the pull-request process still govern +merging, with zero required human approvals. Missing registry readiness produces no migration PR. They never upload crates, approve a pull request, merge it, or loosen branch protection. Prove initial App permissions with a disposable/owned PR and current ordinary checks, without diff --git a/tests/publish-maintenance-branch.sh b/tests/publish-maintenance-branch.sh index caede34..fef6b8d 100755 --- a/tests/publish-maintenance-branch.sh +++ b/tests/publish-maintenance-branch.sh @@ -14,7 +14,9 @@ source_root=${root} # Retain the original Classic rollout/recovery scenarios as a bounded fixture. root=${temporary}/classic-policy mkdir -p "${root}/bin" -cp "${source_root}/bin/publish" "${source_root}/bin/validate" "${root}/bin/" +cp "${source_root}/bin/publish" "${source_root}/bin/validate" \ + "${source_root}/bin/security-readiness" "${root}/bin/" +cp -R "${source_root}/lib" "${root}/lib" cp -R "${source_root}/config" "${source_root}/community-health" "${source_root}/.github" "${root}/" jq '.repositories |= map(select(.name == "classic"))' \ "${source_root}/config/immutable-releases.json" >"${root}/config/immutable-releases.json" @@ -370,7 +372,7 @@ repos/atrinik/*/actions/permissions/workflow\|*) printf '{"status":"enforced","configuration":{"id":265376}}\n' ;; "repos/atrinik/protocol/code-scanning/default-setup|"* | "repos/atrinik/renderer/code-scanning/default-setup|"* | "repos/atrinik/content-toolkit/code-scanning/default-setup|"*) - printf '{"state":"configured","query_suite":"default"}\n' + printf '{"state":"configured","query_suite":"default","languages":["rust"]}\n' ;; "repos/atrinik/classic/immutable-releases|"*) if [[ ${GH_IMMUTABLE_VERIFY_FAILURE:-false} == true ]]; then From 5ec53a623e496d40b0dd5ea01293e4e3a730a475 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 14:55:29 -0500 Subject: [PATCH 12/22] fix: complete integrated publication test fixtures --- tests/publish-maintenance-branch.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/publish-maintenance-branch.sh b/tests/publish-maintenance-branch.sh index fef6b8d..28c24dd 100755 --- a/tests/publish-maintenance-branch.sh +++ b/tests/publish-maintenance-branch.sh @@ -1702,8 +1702,9 @@ crate_setup_root() { mkdir -p "${destination}/bin" "${destination}/lib" "${destination}/tests" cp "${source_root}/bin/publish" "${source_root}/bin/validate" \ - "${source_root}/bin/verify-manual-settings" "${destination}/bin/" - cp "${source_root}/lib/verify_crate_workflow.py" "${destination}/lib/" + "${source_root}/bin/verify-manual-settings" "${source_root}/bin/security-readiness" "${destination}/bin/" + cp "${source_root}/lib/verify_crate_workflow.py" "${source_root}/lib/security_readiness.py" \ + "${destination}/lib/" cp -R "${source_root}/config" "${source_root}/community-health" \ "${source_root}/.github" "${destination}/" cp -R "${source_root}/tests/fixtures" "${destination}/tests/" @@ -1799,7 +1800,8 @@ for mode in selected all; do GH_CRATE_IMMUTABLE_INVENTORY=true GH_CRATE_FIXTURE_ROOT=${source_root} \ PATH="${temporary}/bin:${PATH}" ATRINIK_POLICY_SCOPE=organization \ "${pending_setup_root}/bin/publish" --apply >"${temporary}/crate-${mode}-already-selected.out" - jq -s -e 'all(.[]; (.endpoint | contains("/contents/.github/workflows/") | not) and + jq -s -e 'all(.[]; + (.endpoint | test("/contents/\\.github/workflows/(publish-crate|release)\\.yml") | not) and (.endpoint | contains("/environments/crates-io-release") | not))' "${mode_log}" >/dev/null done From 0160d34c416bf1e20ec759fc57b7f376162ce5fe Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 15:33:09 -0500 Subject: [PATCH 13/22] fix(audit): normalize implicit GitHub ruleset defaults --- bin/audit | 29 +++++++++++++++++++++++------ tests/audit-fixtures.py | 32 ++++++++++++++++++++++++++++++++ tests/audit.sh | 9 ++++++++- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/bin/audit b/bin/audit index 1f92f58..c294e99 100755 --- a/bin/audit +++ b/bin/audit @@ -227,6 +227,25 @@ organization_ruleset_scope() { else error("unknown managed ruleset scope") end' } +ruleset_normalization=' +# Omitted request fields are expanded by GitHub to these exact defaults. +# Preserve nondefault, malformed and unknown fields as observable drift. +def normalize_rules: + map(if .type == "pull_request" then + .parameters |= ( + if .required_reviewers == [] then del(.required_reviewers) else . end | + if .require_extra_approval_for_unattributed_changes == true then + del(.require_extra_approval_for_unattributed_changes) else . end | + if (.allowed_merge_methods | type) == "array" and + (.allowed_merge_methods | sort) == ["merge", "rebase", "squash"] then + del(.allowed_merge_methods) else . end) + elif .type == "required_status_checks" then + .parameters |= ( + if .do_not_enforce_on_create == false then del(.do_not_enforce_on_create) else . end | + .required_status_checks |= sort_by(.context, .integration_id)) + else . end) | sort_by(.type); +' + check_ruleset() { local repository=$1 name=$2 template=$3 list=$4 local entry id source endpoint detail wanted scope @@ -275,7 +294,7 @@ check_ruleset() { scope=$(organization_ruleset_scope "${name}" "${repository}") wanted=$(jq --argjson scope "${scope}" '.conditions.repository_name = $scope' <<<"${wanted}") fi - if ! jq -e --argjson wanted "${wanted}" ' + if ! jq -e --argjson wanted "${wanted}" "${ruleset_normalization}"' # GitHub returns null for OrganizationAdmin, whose desired actor ID is 1. def normalize_bypass: map(if .actor_type == "OrganizationAdmin" and .actor_id == null then @@ -286,10 +305,7 @@ check_ruleset() { .enforcement == $wanted.enforcement and .target == $wanted.target and (.bypass_actors | normalize_bypass) == ($wanted.bypass_actors | normalize_bypass) and (.conditions | normalize_conditions) == ($wanted.conditions | normalize_conditions) and - (.rules | map(if .type == "required_status_checks" then - .parameters.required_status_checks |= sort_by(.context, .integration_id) else . end) | sort_by(.type)) == - ($wanted.rules | map(if .type == "required_status_checks" then - .parameters.required_status_checks |= sort_by(.context, .integration_id) else . end) | sort_by(.type)) + (.rules | normalize_rules) == ($wanted.rules | normalize_rules) ' <<<"${detail}" >/dev/null; then record_finding "ruleset scope or policy drift '${name}': ${organization}/${repository}" fi @@ -457,7 +473,8 @@ for repository in "${!expected_active[@]}"; do "${advisory_windows}" >/dev/null; then template="${root}/config/rulesets/pull-request-gate-classic.json"; fi check_ruleset "${repository}" "$(jq -r .name "${template}")" "${template}" "${rulesets}" "${metadata_file}" if ! jq -e --slurpfile desired "${template}" \ - '[.[] | select(.type == "pull_request") | .parameters] == [$desired[0].rules[0].parameters]' \ + "${ruleset_normalization}"'([.[] | select(.type == "pull_request") | {type, parameters}] | + normalize_rules) == ($desired[0].rules | normalize_rules)' \ <<<"${effective}" >/dev/null; then record_finding "effective pull-request policy drift: ${organization}/${repository}" fi diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index c8e2613..5ea2566 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -76,6 +76,38 @@ def rulesets(repository): item['rules'][0]['parameters']['required_status_checks'] = [ {'context': context, 'integration_id': 15368} for context in repositories['required_ci'][repository]] + # Actual ruleset GET responses expand omitted request parameters. + for rule in item['rules']: + if rule['type'] == 'pull_request': + rule['parameters'].update(required_reviewers=[], + require_extra_approval_for_unattributed_changes=True, + allowed_merge_methods=['merge', 'squash', 'rebase']) + deviations = { + 'api-required-reviewers': ('required_reviewers', [{'reviewer': {'id': 42, 'type': 'Team'}, 'minimum_approvals': 1}]), + 'api-extra-approval': ('require_extra_approval_for_unattributed_changes', False), + 'api-restricted-merges': ('allowed_merge_methods', ['squash']), + 'api-duplicate-merges': ('allowed_merge_methods', ['merge', 'squash', 'rebase', 'merge']), + 'api-malformed-reviewers': ('required_reviewers', None), + 'api-malformed-approval': ('require_extra_approval_for_unattributed_changes', 'true'), + 'api-malformed-merges': ('allowed_merge_methods', 'merge,squash,rebase'), + 'api-unknown-pr-field': ('unknown_future_control', False), + } + if scenario in deviations: + key, value = deviations[scenario] + rule['parameters'][key] = value + if scenario == 'api-default-order': + rule['parameters']['allowed_merge_methods'].reverse() + if rule['type'] == 'required_status_checks': + rule['parameters']['do_not_enforce_on_create'] = False + if scenario == 'api-create-exemption': + rule['parameters']['do_not_enforce_on_create'] = True + if scenario == 'api-malformed-create-exemption': + rule['parameters']['do_not_enforce_on_create'] = None + if scenario == 'api-defaults-omitted': + for rule in item['rules']: + for key in ('required_reviewers', 'require_extra_approval_for_unattributed_changes', + 'allowed_merge_methods', 'do_not_enforce_on_create'): + rule.get('parameters', {}).pop(key, None) item.update(id=active.index(repository) * 10 + index, source_type='Repository' if scenario == 'repository-rulesets' or template in ('code-scanning', 'dependency-review') else 'Organization') diff --git a/tests/audit.sh b/tests/audit.sh index 36d5b9d..742c358 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -78,7 +78,7 @@ output=$(run_audit legacy-security) grep -Fq 'Governance audit passed:' <<<"${output}" output=$(run_audit repository-rulesets) grep -Fq 'Governance audit passed:' <<<"${output}" -for scenario in scope-order inherited-tag-unaffected foreign-tag sha-enforced-on languages-readable languages-empty advanced-null-languages; do +for scenario in api-defaults-omitted api-default-order scope-order inherited-tag-unaffected foreign-tag sha-enforced-on languages-readable languages-empty advanced-null-languages; do output=$(run_audit "${scenario}") grep -Fq 'Governance audit passed:' <<<"${output}" done @@ -116,6 +116,13 @@ for scenario in tag-all-scope tag-extra-repo tag-extra-exclude tag-all-nonreleas assert_drift "${scenario}" "ruleset scope or policy drift '04 - Immutable release tags'" done assert_drift tag-repository-nonrelease 'unexpected managed release-tag protection: atrinik/.github' +for scenario in api-required-reviewers api-extra-approval api-restricted-merges api-duplicate-merges \ + api-malformed-reviewers api-malformed-approval api-malformed-merges api-unknown-pr-field; do + assert_drift "${scenario}" "ruleset scope or policy drift '02 - Changes through pull requests'" +done +for scenario in api-create-exemption api-malformed-create-exemption; do + assert_drift "${scenario}" "ruleset scope or policy drift '03 - Required CI - observatory'" +done assert_drift pr-all-scope "ruleset scope or policy drift '02 - Changes through pull requests'" assert_drift default-extra-exclude "ruleset scope or policy drift '01 - Default branch integrity'" assert_drift repository-scope-extra "ruleset scope or policy drift '07 - Dependency review'" From 7486867d18693069db9946197f83c76074d387c4 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 15:58:47 -0500 Subject: [PATCH 14/22] fix(security): persist required gate enforcement intent --- AGENTS.md | 20 ++- CONTRIBUTING.md | 9 +- README.md | 35 +++-- bin/audit | 14 +- bin/publish | 53 +++++++- bin/validate | 7 +- config/security-merge-protection.json | 48 +++++++ tests/audit-fixtures.py | 4 +- tests/audit.sh | 43 +++++- tests/publish-security-merge.sh | 183 ++++++++++++++++++++++---- tests/validate-security-merge.sh | 17 ++- 11 files changed, 373 insertions(+), 60 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1f59a5c..225a6d4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -222,13 +222,17 @@ ownership, policy scope, publisher behavior, required checks, or validation. - `config/security-merge-protection.json` owns the dependency-review and CodeQL - target sets and default-setup required language minima. Keep both sets equal - to the required-CI inventory. Union required languages with live default-setup + target sets, per-gate `pending`/`enforced` activation intent, and default-setup + required language minima. Keep both sets and activation keys equal to the + required-CI inventory. Union required languages with live default-setup languages, fail on unreadable language state, and patch only real drift. - Keep `06 - CodeQL security` and `07 - Dependency review` repository-owned in both policy scopes; Team has no organization code-scanning rule support. - New gates require read-only `bin/security-readiness` evidence for landed and - successful producers; existing active gates survive later PR failures. Keep + Pending gates only probe readiness. Promote intent through source review after + read-only `bin/security-readiness` proves landed successful producers, before + the first authorized live installation. Enforced gates remain required when + deleted or producers fail; audit fails on absence and publication restores + protection without fresh initial qualification. Keep aggregate checks, zero approvals, and owner pull-request-only bypass intact. - `config/actions-sha-pinning.json` persists SHA rollout as `pending` with no canary or `enforced` with an accepted PR coordinate. Promote it through normal @@ -238,8 +242,10 @@ if proof cannot be revalidated; audit reports disabled enforced policy as drift. Preserve existing true settings. Allow action names with `@*`; keep full SHAs in producer workflows instead of duplicate pin catalogs. -- Existing security gates may waive initial producer readiness only when the - effective default-branch API proves the same ruleset and desired tool/context. - Retired targets lose only exact repository-owned managed security rules; +- Adoption of existing security gates into enforced source intent requires the + effective default-branch API to prove the repository-owned ruleset and desired + tool/context. Never infer activation from target membership or silently change + source intent. Retire target and activation entries together. Retired targets + lose only exact repository-owned managed security rules; preserve archived, foreign, and modified same-name rules. Keep repository workflow overrides read-only and unable to approve PRs, with delta-aware writes. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f1a5ca..1aac060 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -134,8 +134,13 @@ Keep `config/security-merge-protection.json` aligned with `required_ci` and preserve ordinary aggregates. New dependency gates require the reviewed workflow on the default branch and a successful corresponding PR run; CodeQL gates require successful current-head analyses, including required languages. -Keep existing gates active on later failures. Tests must cover missing/failed -producers, unreadable evidence, archived targets, language unions, both scopes, +Record successful initial qualification in the policy PR and promote the gate's +`activation` entry from `pending` to `enforced` through source review before its +first live installation. Pending publication only probes readiness. Enforced +intent survives missing rules and later producer failures, allowing restoration +without fresh initial proof; it does not claim live installation success. Remove +activation entries with retired targets, preserving exact owned-rule rollback. +Tests must cover missing/failed producers, unreadable evidence, archived targets, language unions, both scopes, and zero security mutations on a converged rerun. SHA enforcement must remain pending until an ordinary action-update canary proves successful PR-bound current-head CI without an allowlist edit and the persistent rollout source is diff --git a/README.md b/README.md index e0a860e..5dc1b6c 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,10 @@ available. `config/security-merge-protection.json` targets the 21 repositories in `required_ci`, including `service-updater`; `.github` and `web-platform` are -excluded. Existing aggregate checks and zero required approvals remain intact. +excluded. Its `activation` map records each target gate's reviewed `pending` or +`enforced` intent. `enforced` means protection is required, including when the +live rule is missing; it is not evidence of a successful installation. Existing +aggregate checks and zero required approvals remain intact. The existing publisher restores `service-updater`'s already inventoried PR, `Updater validation`/title CI, and release-tag protections. @@ -212,8 +215,9 @@ Rollout is owned by an organization administrator: `Dependency review` jobs. Merge this policy and review `bin/publish`'s plan. 2. Apply only with separate live-settings authorization. Existing governance, the wildcard action allowance, and the Rust language correction can converge - while security gates remain `PENDING`. The publisher creates each gate only - after `bin/security-readiness` verifies its producer. Dependency readiness + while security gates remain `PENDING`. Pending gates only probe + `bin/security-readiness`; even a successful probe cannot activate protection + or silently promote source intent. Dependency readiness requires the reviewed workflow contract on the default branch, a successful latest PR run/job, and the same workflow at the check suite's executed revision. Synthetic merge revisions must bind the associated PR's base and @@ -226,16 +230,25 @@ Rollout is owned by an organization administrator: changed-path PR subsets cannot establish readiness. The bounded newest-first query stops once current coverage is proven; retired categories and old analysis history do not require manual cleanup. -3. Rerun the plan after the pending analyses complete. Existing active gates - remain enforced even when a later PR fails a security check. Scope migrations +3. After successful initial producer qualification, promote that gate's + `activation` entry to `enforced` through source review and merge, retaining + the qualification evidence in the policy PR. Adoption of existing protection + instead requires read-only evidence of an active, repository-owned gate with + the intended tool/context applying to the live default branch. Desired target + membership alone never proves activation. Review the next plan and apply only + with live-settings authorization. Persisted intent precedes the first write, + so interrupted installation or later deletion cannot revert to pending. + The publisher creates or restores enforced protection without requiring a + currently healthy producer; audit reports missing or modified protection as + drift even when producer evidence fails or cannot be read. Scope migrations preserve these repository-owned gates. A converged security gate or language configuration causes no mutation or security-configuration reattachment. - Moving a rule from another branch onto the default branch requires fresh - producer proof; only an already effective gate with the same tool or - dependency context can retain protection after a later producer failure. - Retiring a target removes its required-CI and security inventory entries - together. The publisher removes only matching repository-owned security - rules from active retired targets, preserving foreign or modified same-name + Existing-rule repair accepts only recognized enforcement and ref-scope drift + on the exact repository-owned rule. Changed bypass, tool, context, severity, + or unknown policy fields are retained and reported for explicit review. + Retiring a target removes its required-CI, security target, and activation + entries together through source review. The publisher removes only matching + repository-owned security rules from active retired targets, preserving foreign or modified same-name rules for explicit review and leaving archived repositories untouched. 4. Desired organization `sha_pinning_required` is true. Its durable rollout contract is `config/actions-sha-pinning.json`: the initial `pending` phase diff --git a/bin/audit b/bin/audit index c294e99..7cbcff7 100755 --- a/bin/audit +++ b/bin/audit @@ -424,14 +424,17 @@ for repository in "${!expected_active[@]}"; do dependency_required=false if jq -e --arg repository "${repository}" '.dependency_review | index($repository) != null' \ "${root}/config/security-merge-protection.json" >/dev/null; then - if jq -e 'any(.[]; .type == "required_status_checks" and + if jq -e --arg repository "${repository}" \ + '.activation.dependency_review[$repository] == "enforced"' \ + "${root}/config/security-merge-protection.json" >/dev/null || + jq -e 'any(.[]; .type == "required_status_checks" and any(.parameters.required_status_checks[]; .context == "Dependency review"))' \ <<<"${effective}" >/dev/null || jq -e 'any(.[]; .name == "07 - Dependency review")' <<<"${rulesets}" >/dev/null; then dependency_required=true elif "${root}/bin/security-readiness" dependency "${organization}/${repository}" \ >"${temporary}/${repository}.readiness" 2>&1; then - dependency_required=true + pending+=("Dependency review merge protection awaits reviewed activation intent: ${organization}/${repository}") else readiness_status=$? if ((readiness_status == 1)); then @@ -496,12 +499,15 @@ for repository in "${!expected_active[@]}"; do if jq -e --arg repository "${repository}" '.code_scanning | index($repository) != null' \ "${root}/config/security-merge-protection.json" >/dev/null; then codeql_required=false - if jq -e 'any(.[]; .type == "code_scanning")' <<<"${effective}" >/dev/null || + if jq -e --arg repository "${repository}" \ + '.activation.code_scanning[$repository] == "enforced"' \ + "${root}/config/security-merge-protection.json" >/dev/null || + jq -e 'any(.[]; .type == "code_scanning")' <<<"${effective}" >/dev/null || jq -e 'any(.[]; .name == "06 - CodeQL security")' <<<"${rulesets}" >/dev/null; then codeql_required=true elif "${root}/bin/security-readiness" codeql "${organization}/${repository}" \ >"${temporary}/${repository}.codeql-readiness" 2>&1; then - codeql_required=true + pending+=("CodeQL merge protection awaits reviewed activation intent: ${organization}/${repository}") else readiness_status=$? if ((readiness_status == 1)); then diff --git a/bin/publish b/bin/publish index ce16639..62daf64 100755 --- a/bin/publish +++ b/bin/publish @@ -1077,7 +1077,13 @@ security_ruleset_matches() { .bypass_actors |= (map( if .actor_type == "OrganizationAdmin" and .actor_id == null then .actor_id = 1 - else . end) | sort_by(.actor_type, .actor_id, .bypass_mode)); + else . end) | sort_by(.actor_type, .actor_id, .bypass_mode)) | + # GitHub expands this omitted request parameter to false. Keep true, + # null and unknown parameters visible so modified gates are not retired. + .rules |= map(if .type == "required_status_checks" and + .parameters.do_not_enforce_on_create == false then + .parameters |= del(.do_not_enforce_on_create) + else . end); normalize as $current | all($desired[0] | normalize | to_entries[]; $current[.key] == .value) ' <<<"${current}" >/dev/null @@ -1148,7 +1154,7 @@ delete_removed_security_merge_rulesets() { # These repository rules remain repository-owned even in organization scope: # GitHub Team does not support organization-level code-scanning rules. publish_security_merge_rulesets() { - local repository kind template name existing id current reason status readiness_mode + local repository kind template name existing id current reason status readiness_mode phase metadata delete_removed_security_merge_rulesets for kind in code_scanning dependency_review; do if [[ ${kind} == code_scanning ]]; then @@ -1164,6 +1170,12 @@ publish_security_merge_rulesets() { echo "SKIP ${organization}/${repository} is not active; ${name}" continue fi + phase=$(jq -er --arg kind "${kind}" --arg repository "${repository}" \ + '.activation[$kind][$repository] | select(. == "pending" or . == "enforced")' \ + "${security_merge_config}") || { + echo "error: invalid security gate activation intent: ${repository} ${kind}" >&2 + return 1 + } existing=$(github_api \ "repos/${organization}/${repository}/rulesets?includes_parents=false") if ! jq -e --arg name "${name}" \ @@ -1176,20 +1188,49 @@ publish_security_merge_rulesets() { current='{}' if [[ -n ${id} ]]; then current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") + if ! jq -e --argjson id "${id}" --arg name "${name}" \ + --arg source "${organization}/${repository}" ' + .id == $id and .name == $name and + .source_type == "Repository" and .source == $source and + (.enforcement | IN("active", "disabled", "evaluate")) + ' <<<"${current}" >/dev/null || + ! security_ruleset_matches "${template}" "$(jq --slurpfile desired "${template}" ' + .enforcement = $desired[0].enforcement | + .conditions.ref_name.include = $desired[0].conditions.ref_name.include | + .conditions.ref_name.exclude = $desired[0].conditions.ref_name.exclude + ' <<<"${current}")"; then + echo "error: security gate ownership or managed rule identity changed; retained: ${repository} ${name}" >&2 + return 1 + fi if security_ruleset_matches "${template}" "${current}"; then echo "KEEP /repos/${organization}/${repository}/rulesets/${id} ${name}" continue fi fi - # Preserve an already active gate when a later pull request fails it. - if [[ -z ${id} ]] || ! security_gate_applies_to_default_branch "${repository}" "${kind}" "${id}"; then + # Pending qualification cannot create protection before reviewed intent is + # persisted. Existing effective gates still survive producer failures. + if [[ ${phase} == pending ]] && { [[ -z ${id} ]] || + ! security_gate_applies_to_default_branch "${repository}" "${kind}" "${id}"; }; then status=0 reason=$("${root}/bin/security-readiness" "${readiness_mode}" \ "${organization}/${repository}") || status=$? - if ((status != 0)); then + if ((status == 0)); then + echo "READY ${organization}/${repository} ${name}: promote reviewed activation intent before installation" + else echo "PENDING ${organization}/${repository} ${name}: ${reason}" - continue fi + continue + fi + # Enforced source is durable intent, not a claim about current producer + # health. Restore it only on the same active repository and owned gate. + metadata=$(github_api "repos/${organization}/${repository}") || return 1 + if ! jq -e --argjson id "${active_repository_ids["${repository}"]}" \ + --arg name "${organization}/${repository}" ' + .id == $id and .full_name == $name and .archived == false and + (.default_branch | type == "string" and length > 0) + ' <<<"${metadata}" >/dev/null; then + echo "error: security gate repository identity or archive state changed: ${repository}" >&2 + return 1 fi if [[ -n ${id} ]]; then run_api PUT "repos/${organization}/${repository}/rulesets/${id}" "${template}" diff --git a/bin/validate b/bin/validate index 86b338d..3eaed82 100755 --- a/bin/validate +++ b/bin/validate @@ -53,7 +53,12 @@ jq -e ' jq -e --slurpfile repositories "${repositories_config}" \ --slurpfile advanced "${advanced_codeql_config}" ' - (keys == ["code_scanning", "codeql_required_languages", "dependency_review"]) and + . as $policy | + (keys == ["activation", "code_scanning", "codeql_required_languages", "dependency_review"]) and + (.activation | type == "object" and keys == ["code_scanning", "dependency_review"]) and + all(["code_scanning", "dependency_review"][]; . as $kind | + ($policy.activation[$kind] | type == "object" and keys == $policy[$kind]) and + all($policy.activation[$kind][]; IN("pending", "enforced"))) and .dependency_review == ($repositories[0].required_ci | keys) and .code_scanning == .dependency_review and (.codeql_required_languages | type == "object") and diff --git a/config/security-merge-protection.json b/config/security-merge-protection.json index 56ea14b..72a10c3 100644 --- a/config/security-merge-protection.json +++ b/config/security-merge-protection.json @@ -49,5 +49,53 @@ "content-toolkit": [ "rust" ] + }, + "activation": { + "code_scanning": { + "agent-integrations": "pending", + "atrinik": "pending", + "classic": "pending", + "client": "pending", + "content": "pending", + "content-toolkit": "pending", + "deploy-control": "pending", + "devcontainer": "pending", + "editor": "pending", + "github-settings": "pending", + "metaserver-worker": "pending", + "observatory": "pending", + "playtester": "pending", + "protocol": "pending", + "renderer": "pending", + "resources": "pending", + "server": "pending", + "service-updater": "pending", + "sound": "pending", + "tools": "pending", + "website": "pending" + }, + "dependency_review": { + "agent-integrations": "pending", + "atrinik": "pending", + "classic": "pending", + "client": "pending", + "content": "pending", + "content-toolkit": "pending", + "deploy-control": "pending", + "devcontainer": "pending", + "editor": "pending", + "github-settings": "pending", + "metaserver-worker": "pending", + "observatory": "pending", + "playtester": "pending", + "protocol": "pending", + "renderer": "pending", + "resources": "pending", + "server": "pending", + "service-updater": "pending", + "sound": "pending", + "tools": "pending", + "website": "pending" + } } } diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index 5ea2566..6c1a186 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -61,9 +61,9 @@ def rulesets(repository): if repository in repositories['release_tags'] or scenario in ( 'inherited-tag-unaffected', 'tag-extra-nonrelease', 'tag-all-nonrelease', 'tag-repository-nonrelease'): specs.append(('immutable-release-tags', None)) - if repository in policy['code_scanning'] and scenario not in ('pending-codeql', 'ready-missing-codeql', 'codeql-readiness-unreadable'): + if repository in policy['code_scanning'] and scenario not in ('pending-codeql', 'ready-missing-codeql', 'codeql-readiness-unreadable', 'enforced-missing-codeql', 'enforced-unreadable-codeql'): specs.append(('code-scanning', None)) - if repository in policy['dependency_review'] and scenario not in ('pending', 'ready-missing', 'dependency-readiness-unreadable'): + if repository in policy['dependency_review'] and scenario not in ('pending', 'ready-missing', 'dependency-readiness-unreadable', 'enforced-missing-dependency', 'enforced-unreadable-dependency'): specs.append(('dependency-review', None)) if scenario == 'foreign-tag': specs.append(('immutable-release-tags', 'Foreign tag retention')) diff --git a/tests/audit.sh b/tests/audit.sh index 742c358..da707e2 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -12,9 +12,10 @@ cp "${root}/bin/audit" "${temporary}/full/bin/audit" printf '#!/usr/bin/env bash\nexit 0\n' >"${temporary}/full/bin/validate" cat >"${temporary}/full/bin/security-readiness" <<'EOF' #!/usr/bin/env bash +printf '%s\n' "$1" >>"${AUDIT_READINESS_LOG}" case ${GH_TEST_SCENARIO:-pass} in - pending|pending-codeql|pending-wrong-dependency-scope) exit 1 ;; - readiness-unreadable|dependency-readiness-unreadable|codeql-readiness-unreadable) exit 2 ;; + pending|pending-codeql|pending-wrong-dependency-scope|enforced-missing-dependency|enforced-missing-codeql) exit 1 ;; + readiness-unreadable|dependency-readiness-unreadable|codeql-readiness-unreadable|enforced-unreadable-dependency|enforced-unreadable-codeql) exit 2 ;; *) exit 0 ;; esac EOF @@ -63,7 +64,22 @@ run_audit() { >"${temporary}/${fixture}/config/actions-sha-pinning.json" ;; esac + cp "${root}/config/security-merge-protection.json" "${temporary}/${fixture}/config/security-merge-protection.json" + case ${scenario} in + enforced-missing-dependency|enforced-unreadable-dependency) + jq '.activation.dependency_review.observatory = "enforced"' \ + "${temporary}/${fixture}/config/security-merge-protection.json" >"${temporary}/activation.json" + mv "${temporary}/activation.json" "${temporary}/${fixture}/config/security-merge-protection.json" + ;; + enforced-missing-codeql|enforced-unreadable-codeql) + jq '.activation.code_scanning.observatory = "enforced"' \ + "${temporary}/${fixture}/config/security-merge-protection.json" >"${temporary}/activation.json" + mv "${temporary}/activation.json" "${temporary}/${fixture}/config/security-merge-protection.json" + ;; + esac + : >"${temporary}/${scenario}.readiness-log" GH_TEST_SCENARIO=${scenario} \ + AUDIT_READINESS_LOG="${temporary}/${scenario}.readiness-log" \ PATH="${temporary}/mock:${PATH}" \ AUDIT_CONFIG_ROOT="${temporary}/${fixture}" \ AUDIT_API_LOG="${temporary}/api.log" \ @@ -174,8 +190,27 @@ assert_drift private-reporting 'private vulnerability reporting disabled or unre assert_drift secret-baseline 'exposed security baseline drift' assert_drift effective-unreadable 'effective default-branch rules could not be read' assert_drift ruleset-unreadable 'ruleset contents could not be read' -assert_drift ready-missing 'effective required CI contexts or app drift' -assert_drift ready-missing-codeql 'effective CodeQL merge protection drift' +for scenario in enforced-missing-dependency enforced-unreadable-dependency; do + assert_drift "${scenario}" 'effective required CI contexts or app drift' + [[ ! -s ${temporary}/${scenario}.readiness-log ]] + if grep -Fq 'PENDING Dependency review' "${temporary}/${scenario}.out"; then + echo 'enforced dependency protection must never fall back to pending' >&2 + exit 1 + fi +done +for scenario in enforced-missing-codeql enforced-unreadable-codeql; do + assert_drift "${scenario}" 'effective CodeQL merge protection drift' + [[ ! -s ${temporary}/${scenario}.readiness-log ]] + if grep -Fq 'PENDING CodeQL' "${temporary}/${scenario}.out"; then + echo 'enforced CodeQL protection must never fall back to pending' >&2 + exit 1 + fi +done +for scenario in ready-missing ready-missing-codeql; do + output=$(run_audit "${scenario}") + grep -Fq 'merge protection awaits reviewed activation intent' <<<"${output}" + grep -Fq 'Governance audit baseline passed with' <<<"${output}" +done assert_drift dependency-readiness-unreadable 'Dependency review readiness could not be read' assert_drift codeql-readiness-unreadable 'CodeQL readiness could not be read' output=$(run_audit pending-codeql) diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index 185ffbe..56c0057 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -118,7 +118,20 @@ github_api() { repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) local template=code-scanning [[ ${endpoint##*/} != 7 ]] || template=dependency-review - if [[ ${ruleset_state} == exact || ${ruleset_state} == removal-owned ]]; then + if [[ ${ruleset_state} == api-default-* ]]; then + jq --argjson id "${endpoint##*/}" --arg state "${ruleset_state}" ' + . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + .bypass_actors[0].actor_id = null | + if $id == 7 then + .rules[0].parameters.do_not_enforce_on_create = + (if $state == "api-default-false" then false + elif $state == "api-default-true" then true else null end) | + if $state == "api-default-unknown" then + .rules[0].parameters.do_not_enforce_on_create = false | + .rules[0].parameters.unknown_future_control = false + else . end + else . end' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == exact || ${ruleset_state} == removal-owned ]]; then jq --argjson id "${endpoint##*/}" \ '. + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"}' \ "${root}/config/rulesets/${template}.json" @@ -140,11 +153,32 @@ github_api() { "${root}/config/rulesets/${template}.json" elif [[ ${ruleset_state} == owner-null ]]; then jq '.bypass_actors[0].actor_id = null' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == foreign-repository-source ]]; then + jq '.source = "foreign/content-toolkit"' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == extra-rule ]]; then + jq '.rules += [{type: "deletion"}]' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == unknown-rule-parameter ]]; then + jq '.rules[0].parameters.unknown_future_control = true' \ + "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == unknown-condition ]]; then + jq '.conditions.unknown_future_scope = {}' \ + "${root}/config/rulesets/${template}.json" elif [[ ${ruleset_state} == active-wrong-ref ]]; then jq '.conditions.ref_name.include = ["refs/heads/feature"]' \ "${root}/config/rulesets/${template}.json" elif [[ ${ruleset_state} == active-drift ]]; then + jq '.conditions.ref_name.include = ["refs/heads/main"]' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == active-bypass-drift ]]; then jq '.bypass_actors = []' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == unknown-ref-condition ]]; then + jq '.conditions.ref_name.unknown_future_scope = true' "${root}/config/rulesets/${template}.json" + elif [[ ${ruleset_state} == changed-severity ]]; then + if [[ ${template} == code-scanning ]]; then + jq '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' \ + "${root}/config/rulesets/${template}.json" + else + cat "${root}/config/rulesets/${template}.json" + fi elif [[ ${ruleset_state} == active-unrelated-gate ]]; then if [[ ${template} == code-scanning ]]; then jq '.rules[0].parameters.code_scanning_tools[0].tool = "Another scanner"' \ @@ -157,7 +191,8 @@ github_api() { jq '.rules = []' "${root}/config/rulesets/${template}.json" else jq '.enforcement = "disabled"' "${root}/config/rulesets/${template}.json" - fi + fi | jq --argjson id "${endpoint##*/}" \ + '{id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} + .' ;; orgs/atrinik/actions/permissions) printf '%s\n' "${actions_state}" ;; orgs/atrinik/actions/permissions/selected-actions) @@ -304,15 +339,9 @@ for policy_scope in organization repository; do # shellcheck source=/dev/null source "${temporary}/dispatch.sh" >"${temporary}/${policy_scope}-${producer}.txt" [[ $(cat "${temporary}/order") == "${policy_scope}-rules"$'\n'"${policy_scope}-security" ]] + assert_no_writes if [[ ${producer} == ready ]]; then - jq -se --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ - --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' - [.[] | select(.method != "GET")] == [ - {method: "POST", endpoint: "repos/atrinik/content-toolkit/rulesets", payload: $codeql[0]}, - {method: "POST", endpoint: "repos/atrinik/content-toolkit/rulesets", payload: $dependency[0]} - ]' "${api_log}" >/dev/null - else - assert_no_writes + [[ $(grep -c '^READY ' "${temporary}/${policy_scope}-${producer}.txt") == 2 ]] fi jq -se '[.[].mode] == ["codeql", "dependency"] and all(.[]; .repository == "atrinik/content-toolkit")' "${readiness_log}" >/dev/null @@ -321,6 +350,82 @@ for policy_scope in organization repository; do done done +# Reviewed enforced intent precedes initial installation and survives deletion, +# disablement, and ref drift independently of current producer health. +for policy_scope in organization repository; do + for producer in failed unreadable; do + for state in missing disabled active-wrong-ref; do + reset_case + ruleset_state=${state} + effective_state=absent + export READINESS_STATUS=${producer} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/enforced-${policy_scope}-${producer}-${state}.txt" + method=PUT + [[ ${state} != missing ]] || method=POST + jq -se --arg method "${method}" \ + --slurpfile codeql "${root}/config/rulesets/code-scanning.json" \ + --slurpfile dependency "${root}/config/rulesets/dependency-review.json" ' + [.[] | select(.method != "GET")] | + [.[].payload] == [$codeql[0], $dependency[0]] and + all(.[]; .method == $method)' "${api_log}" >/dev/null + [[ ! -s ${readiness_log} ]] + done + done +done + +# The two gate phases are independent; qualification does not mutate source. +reset_case +export READINESS_STATUS=failed +jq '.activation.code_scanning["content-toolkit"] = "enforced"' \ + "${security_merge_config}" >"${temporary}/mixed.json" +cp "${temporary}/mixed.json" "${security_merge_config}" +publish_security_merge_rulesets >"${temporary}/mixed.txt" +jq -se '[.[] | select(.method != "GET")] | length == 1 and + .[0].payload.name == "06 - CodeQL security"' "${api_log}" >/dev/null +jq -se '[.[].mode] == ["dependency"]' "${readiness_log}" >/dev/null +cmp "${temporary}/mixed.json" "${security_merge_config}" + +# Enforced restoration preserves foreign/ambiguous same-name rules and rejects +# archived or replaced repositories at the point of mutation. +for state in removal-foreign removal-id-drift removal-name-collision active-unrelated-gate active-without-gate unknown-rule-parameter unknown-condition unknown-ref-condition active-bypass-drift changed-severity foreign-repository-source extra-rule; do + reset_case + ruleset_state=${state} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + if publish_security_merge_rulesets >"${temporary}/enforced-collision-${state}.txt" 2>&1; then + echo 'expected enforced gate ownership collision to fail closed' >&2 + exit 1 + fi + assert_no_writes + [[ ! -s ${readiness_log} ]] +done +for kind in code_scanning dependency_review; do + for metadata in \ + '{"id":123,"full_name":"atrinik/content-toolkit","archived":true,"default_branch":"main"}' \ + '{"id":999,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' \ + '{"id":123,"full_name":"atrinik/renamed-toolkit","archived":false,"default_branch":"main"}' \ + '{"id":123,"full_name":"foreign/content-toolkit","archived":false,"default_branch":"main"}'; do + reset_case + export READINESS_STATUS=failed + repository_state=${metadata} + jq --arg kind "${kind}" '.activation[$kind]["content-toolkit"] = "enforced"' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + if publish_security_merge_rulesets >"${temporary}/enforced-metadata-${kind}.txt" 2>&1; then + echo 'expected enforced repository identity fence to fail closed' >&2 + exit 1 + fi + assert_no_writes + jq -se --arg mode "$([[ ${kind} == code_scanning ]] && echo codeql || echo dependency)" \ + 'all(.[]; .mode != $mode)' "${readiness_log}" >/dev/null + done +done + reset_case ruleset_state=exact export READINESS_STATUS=failed @@ -341,15 +446,14 @@ jq -se '[.[] | select(.method != "GET")] | length == 2 and all(.[]; .method == "PUT" and .payload.enforcement == "active")' \ "${api_log}" >/dev/null [[ ! -s ${readiness_log} ]] -for ruleset_state in disabled active-without-gate active-unrelated-gate; do - effective_state=absent - : >"${api_log}" - : >"${readiness_log}" - export READINESS_STATUS=failed - publish_security_merge_rulesets >"${temporary}/${ruleset_state}.txt" - assert_no_writes - jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null -done +ruleset_state=disabled +effective_state=absent +: >"${api_log}" +: >"${readiness_log}" +export READINESS_STATUS=failed +publish_security_merge_rulesets >"${temporary}/${ruleset_state}.txt" +assert_no_writes +jq -se '[.[].mode] == ["codeql", "dependency"]' "${readiness_log}" >/dev/null # An active rule on another ref does not grandfather a new default-branch gate. for effective_state in absent different-id; do @@ -363,7 +467,7 @@ for effective_state in absent different-id; do done reset_case ruleset_state=active-drift -repository_state='{"id":123,"archived":false,"default_branch":"release/current"}' +repository_state='{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"release/current"}' export READINESS_STATUS=failed publish_security_merge_rulesets >"${temporary}/encoded-ref.txt" jq -se 'any(.[]; .endpoint == "repos/atrinik/content-toolkit/rules/branches/release%2Fcurrent") and @@ -386,7 +490,7 @@ done for removal in removal-owned removal-owner-null removal-foreign removal-id-drift removal-name-collision; do reset_case ruleset_state=${removal} - jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"]' \ + jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"] | del(.activation.code_scanning["content-toolkit"], .activation.dependency_review["content-toolkit"])' \ "${security_merge_config}" >"${temporary}/removed.json" cp "${temporary}/removed.json" "${security_merge_config}" publish_security_merge_rulesets >"${temporary}/${removal}.txt" @@ -402,12 +506,47 @@ for removal in removal-owned removal-owner-null removal-foreign removal-id-drift [[ ! -s ${readiness_log} ]] done +# GitHub expands an omitted create-exemption flag to false. Both publisher +# scopes must keep exact active API payloads and retire only exact owned gates. +for policy_scope in organization repository; do + reset_case + ruleset_state=api-default-false + export READINESS_STATUS=failed + : >"${temporary}/order" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/${policy_scope}-api-default-keep.txt" + assert_no_writes + [[ ! -s ${readiness_log} ]] + grep -Fq 'KEEP /repos/atrinik/content-toolkit/rulesets/7 07 - Dependency review' \ + "${temporary}/${policy_scope}-api-default-keep.txt" + for default_state in false true null unknown; do + reset_case + ruleset_state=api-default-${default_state} + jq '.dependency_review -= ["content-toolkit"] | del(.activation.dependency_review["content-toolkit"])' \ + "${security_merge_config}" >"${temporary}/removed.json" + cp "${temporary}/removed.json" "${security_merge_config}" + : >"${temporary}/order" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/${policy_scope}-api-default-${default_state}.txt" + if [[ ${default_state} == false ]]; then + jq -se '[.[] | select(.method != "GET")] == [ + {method: "DELETE", endpoint: "repos/atrinik/content-toolkit/rulesets/7"} + ]' "${api_log}" >/dev/null + else + assert_no_writes + grep -Fq '07 - Dependency review: unrecognized ownership or payload; retained' \ + "${temporary}/${policy_scope}-api-default-${default_state}.txt" + fi + [[ ! -s ${readiness_log} ]] + done +done + # Retirements also fence a repository archived or replaced after enumeration. for retired_state in '{"id":123,"archived":true}' '{"id":999,"archived":false}'; do reset_case repository_state=${retired_state} ruleset_state="removal-owned" - jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"]' \ + jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"] | del(.activation.code_scanning["content-toolkit"], .activation.dependency_review["content-toolkit"])' \ "${security_merge_config}" >"${temporary}/removed.json" cp "${temporary}/removed.json" "${security_merge_config}" publish_security_merge_rulesets >"${temporary}/removed-state-changed.txt" diff --git a/tests/validate-security-merge.sh b/tests/validate-security-merge.sh index 561f815..5efadea 100755 --- a/tests/validate-security-merge.sh +++ b/tests/validate-security-merge.sh @@ -25,6 +25,15 @@ reject security-merge-protection.json '.dependency_review += ["web-platform"]' reject security-merge-protection.json '.codeql_required_languages["content-toolkit"] = []' reject security-merge-protection.json '.codeql_required_languages.classic = ["rust"]' reject security-merge-protection.json '.codeql_required_languages.server = ["invented"]' +for filter in 'del(.activation)' '.activation = null' '.activation = []' \ + '.activation.unexpected = {}' 'del(.activation.code_scanning)' \ + '.activation.code_scanning = []' '.activation.code_scanning = null' \ + 'del(.activation.code_scanning.classic)' '.activation.code_scanning.classic = null' \ + '.activation.code_scanning.classic = "unknown"' '.activation.code_scanning.classic = true' \ + '.activation.dependency_review.classic = {}' '.activation.dependency_review.classic = "ready"' \ + '.activation.dependency_review.nawerhals = "enforced"'; do + reject security-merge-protection.json "${filter}" +done reject rulesets/code-scanning.json '.bypass_actors[0].bypass_mode = "always"' reject rulesets/code-scanning.json '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' reject rulesets/code-scanning.json '.rules[0].parameters.code_scanning_tools[0].alerts_threshold = "errors"' @@ -45,10 +54,16 @@ jq '.phase = "enforced" | .canary = "atrinik/github-settings/99"' \ "${temporary}/bin/validate" >/dev/null cp "${root}/config/actions-sha-pinning.json" "${temporary}/config/actions-sha-pinning.json" +# Source review can promote gates independently without a version catalog. +jq '.activation.code_scanning.classic = "enforced" | .activation.dependency_review.atrinik = "enforced"' \ + "${root}/config/security-merge-protection.json" >"${temporary}/config/security-merge-protection.json" +"${temporary}/bin/validate" >/dev/null +cp "${root}/config/security-merge-protection.json" "${temporary}/config/security-merge-protection.json" + # A coordinated target retirement is valid; unrelated governed policies stay. jq 'del(.required_ci["service-updater"])' "${root}/config/repositories.json" \ >"${temporary}/config/repositories.json" -jq '.code_scanning -= ["service-updater"] | .dependency_review -= ["service-updater"]' \ +jq '.code_scanning -= ["service-updater"] | .dependency_review -= ["service-updater"] | del(.activation.code_scanning["service-updater"], .activation.dependency_review["service-updater"])' \ "${root}/config/security-merge-protection.json" >"${temporary}/config/security-merge-protection.json" "${temporary}/bin/validate" >/dev/null From e794287a749f34ab72e4efb985582cddea6399df Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 16:31:55 -0500 Subject: [PATCH 15/22] fix: qualify security producers and retire exact CI rules --- .github/workflows/audit.yml | 3 + .github/workflows/publish-planning.yml | 3 + AGENTS.md | 11 + CONTRIBUTING.md | 14 + README.md | 31 ++- bin/publish | 67 ++++- bin/validate | 20 ++ config/retired-required-ci.json | 3 + lib/security_readiness.py | 216 ++++++++++++++- requirements-validation.txt | 2 +- tests/publish-retired-required-ci.sh | 145 ++++++++++ tests/security-readiness.py | 352 ++++++++++++++++++++++++- tests/validate-retired-required-ci.sh | 28 ++ 13 files changed, 878 insertions(+), 17 deletions(-) create mode 100644 config/retired-required-ci.json create mode 100755 tests/publish-retired-required-ci.sh create mode 100755 tests/validate-retired-required-ci.sh diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 5cf295a..e3230c7 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -23,6 +23,9 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install workflow verification dependencies + run: python3 -m pip install --requirement requirements-validation.txt + - name: Compare desired state with GitHub env: GH_TOKEN: ${{ secrets.ATRINIK_SETTINGS_TOKEN }} diff --git a/.github/workflows/publish-planning.yml b/.github/workflows/publish-planning.yml index 32ae3d6..914f3ee 100644 --- a/.github/workflows/publish-planning.yml +++ b/.github/workflows/publish-planning.yml @@ -18,6 +18,9 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install workflow verification dependencies + run: python3 -m pip install --requirement requirements-validation.txt + - name: Verify settings credential inventory env: GH_TOKEN: ${{ secrets.ATRINIK_SETTINGS_TOKEN }} diff --git a/AGENTS.md b/AGENTS.md index 225a6d4..2c6da3a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -234,6 +234,12 @@ deleted or producers fail; audit fails on absence and publication restores protection without fresh initial qualification. Keep aggregate checks, zero approvals, and owner pull-request-only bypass intact. +- Advanced CodeQL readiness parses executable YAML with the declared validation + dependency: bind classification outputs, matrix, scoped config, official + full-SHA init/analyze actions, and the always-running aggregate. Reject + duplicate keys, disconnected or disabled steps, and failure suppression. + Producer helpers remain trusted; preserve owner-controlled action updates, + category names and paths without a duplicate version/path catalog. - `config/actions-sha-pinning.json` persists SHA rollout as `pending` with no canary or `enforced` with an accepted PR coordinate. Promote it through normal source review only after ordinary PR-bound action-update evidence passes. @@ -249,3 +255,8 @@ lose only exact repository-owned managed security rules; preserve archived, foreign, and modified same-name rules. Keep repository workflow overrides read-only and unable to approve PRs, with delta-aware writes. +- Repository-scope `03 - Required CI` retirement requires reviewed exact prior + repository/ruleset IDs and contexts in `config/retired-required-ci.json`. + Delete only the unchanged Actions-bound template after fresh identity/archive + and payload checks. Preserve modified/foreign rules and missing-provenance + rules with actionable `SKIP`; retain records for idempotent reruns. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1aac060..88e6eb2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -140,6 +140,10 @@ first live installation. Pending publication only probes readiness. Enforced intent survives missing rules and later producer failures, allowing restoration without fresh initial proof; it does not claim live installation success. Remove activation entries with retired targets, preserving exact owned-rule rollback. +For an existing repository-owned `03 - Required CI` rule, also record exact repository +and ruleset IDs and its prior contexts in `config/retired-required-ci.json`. +Validate retained/modified/foreign rules and fresh identity checks; keep records +for idempotent retirement. Missing provenance must retain the rule. Tests must cover missing/failed producers, unreadable evidence, archived targets, language unions, both scopes, and zero security mutations on a converged rerun. SHA enforcement must remain pending until an ordinary action-update canary proves successful PR-bound @@ -150,6 +154,16 @@ restoring it. Test rule retirement, effective default-branch applicability, and repository workflow permission overrides. Action allowances use owner/action wildcards; full commit pins belong in workflows, not redundant version assertions. +Advanced CodeQL readiness must validate executable YAML wiring from classifier +invocation and outputs through the dynamic analysis matrix, scoped generated +configuration, official full-SHA CodeQL actions, and the unconditional aggregate. +Use the existing validation dependency with duplicate-key rejection. Test +comments-only markers, disconnected inputs, disabled or failure-suppressed steps, +checkout revision drift, and ambiguous YAML; preserve full-SHA updates, formatting +changes, additional partitions, and category renames. Classification path +semantics remain in trusted producer helpers; do not add a central source-path or +action-version catalog or claim resistance to malicious repository writers. + Crates.io publication contracts bind exact repository/workflow/environment and official auth-action identities with full immutable workflow SHA references; preserve the reviewed `rust-lang/crates-io-auth-action@*` allowance for normal diff --git a/README.md b/README.md index 5dc1b6c..adfd283 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,17 @@ for those Dependabot PRs; it does not inherit those CodeQL exceptions. CodeQL me on the alert's affected lines appearing in the PR diff. Ordinary quality alerts remain excluded by `alerts_threshold: none`. +Classic readiness parses the executable workflow with the declared PyYAML +validation dependency. It binds classification outputs to the analysis matrix, +regenerates each selected scope's configuration, and follows that output into +the official CodeQL initialization and analysis steps and the always-running +validation aggregate. Disabled steps, failure suppression, disconnected inputs, +and ambiguous YAML fail closed. The current producer uses `build-mode: none`; +full-SHA action updates remain producer-owned, and classifier source owns path +selection and category names. This structural check trusts repository writers +and their Python helpers; it does not attest that arbitrary writer-controlled +classifier or aggregate code behaves honestly. It adds no approval requirement. + Rollout is owned by an organization administrator: 1. Merge the dependency workflow PRs and let ordinary pull requests prove their @@ -254,6 +265,10 @@ Rollout is owned by an organization administrator: contract is `config/actions-sha-pinning.json`: the initial `pending` phase has no accepted canary and never enables the field. Verify an ordinary action-pin update PR with `bin/security-readiness sha OWNER/REPOSITORY/PR`. + Qualifying changes update action pins only. Reusable workflow references are + [excluded from GitHub's action SHA enforcement](https://docs.github.com/en/organizations/managing-organization-settings/disabling-or-limiting-github-actions-for-your-organization), so changing one cannot qualify + a canary, even alongside an action update. Unchanged reusable references and + real action subpaths remain supported. The optional publisher `--sha-pinning-canary` argument only reports readiness while pending. After successful PR evidence, promote the source to `{"phase":"enforced","canary":"OWNER/REPOSITORY/PR"}` through the normal @@ -274,6 +289,16 @@ Rollout is owned by an organization administrator: as drift, never as initial rollout pending. A live true setting is always preserved. Full SHA pinning does not prove an allowed action is trustworthy. +Repository-scope retirement of an existing `03 - Required CI` rule also needs a +reviewed `config/retired-required-ci.json` record with its exact repository and +ruleset IDs and previous required contexts. Remove the repository from +`required_ci` and both security target/activation inventories in the same source +change. Publication deletes only the unchanged GitHub Actions-bound template +after fresh repository identity, archive-state and rule-payload checks. Missing +provenance or modified/foreign rules are retained with an actionable `SKIP`; +keep retirement records for idempotent reruns. This does not delete branches or +ordinary branch protections. + Read-only preflight failures and unfinished producers are explicit pending controls, never successful rollout evidence. The audit reports pending controls separately and rejects unreadable evidence. No paid subscription is introduced. @@ -283,8 +308,10 @@ merges. Do not remove a gate merely because a vulnerable PR fails it. ## Usage -Requirements: `git`, `gh`, `jq`, `python3`, and an authenticated organization-owner -account. +Requirements: `git`, `gh`, `jq`, `python3`, the packages in +`requirements-validation.txt`, and an authenticated organization-owner account. +Install the declared validation dependencies before local publisher, audit, or +advanced CodeQL readiness checks; the owning workflows install them too. ```sh gh auth refresh -h github.com -s admin:org,project,repo diff --git a/bin/publish b/bin/publish index 62daf64..4883b07 100755 --- a/bin/publish +++ b/bin/publish @@ -581,6 +581,65 @@ delete_managed_organization_rulesets() { done < <(github_api "orgs/${organization}/rulesets" | jq -c '.[]') } +retired_required_ci_matches() { + local repository=$1 id=$2 template=$3 current=$4 + jq -e --argjson id "${id}" --arg source "${organization}/${repository}" ' + .id == $id and .source_type == "Repository" and .source == $source + ' <<<"${current}" >/dev/null && security_ruleset_matches "${template}" "${current}" +} + +delete_retired_repository_required_ci() { + local repository listed record id template current metadata fresh + for repository in "${active_repositories[@]}"; do + [[ -v active_repository_ids["${repository}"] ]] || continue + if jq -e --arg repository "${repository}" '.required_ci | has($repository)' \ + "${repositories_config}" >/dev/null; then + continue + fi + listed=$(github_api "repos/${organization}/${repository}/rulesets?includes_parents=false") + listed=$(jq -c '[.[] | select(.name == "03 - Required CI")]' <<<"${listed}") + [[ $(jq length <<<"${listed}") != 0 ]] || continue + record=$(jq -c --arg repository "${repository}" \ + '[.retirements[] | select(.repository == $repository)]' "${root}/config/retired-required-ci.json") + if [[ $(jq length <<<"${listed}") != 1 || $(jq length <<<"${record}") != 1 ]]; then + echo "SKIP ${organization}/${repository} 03 - Required CI: retained; requires one reviewed config/retired-required-ci.json record and unique live rule" + continue + fi + record=$(jq -c '.[0]' <<<"${record}") + id=$(jq -r '.ruleset_id' <<<"${record}") + if ! jq -e --argjson id "${id}" '.[0].id == $id' <<<"${listed}" >/dev/null; then + echo "SKIP ${organization}/${repository} 03 - Required CI: recorded ruleset identity differs; retained" + continue + fi + template=$(mktemp) + temporary_files+=("${template}") + jq --argjson record "${record}" ' + .rules[0].parameters.required_status_checks = + ($record.required_ci | map({context: ., integration_id: 15368})) + ' "${root}/config/rulesets/required-ci.json" >"${template}" + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") + if ! retired_required_ci_matches "${repository}" "${id}" "${template}" "${current}"; then + echo "SKIP ${organization}/${repository} 03 - Required CI: modified or foreign rule; retained" + continue + fi + metadata=$(github_api "repos/${organization}/${repository}") + if ! jq -e --argjson record "${record}" --argjson enumerated "${active_repository_ids["${repository}"]}" \ + --arg full_name "${organization}/${repository}" ' + .id == $record.repository_id and .id == $enumerated and + .full_name == $full_name and .archived == false + ' <<<"${metadata}" >/dev/null; then + echo "SKIP ${organization}/${repository} 03 - Required CI: repository identity or archive state changed; retained" + continue + fi + fresh=$(github_api "repos/${organization}/${repository}/rulesets/${id}") + if ! retired_required_ci_matches "${repository}" "${id}" "${template}" "${fresh}"; then + echo "SKIP ${organization}/${repository} 03 - Required CI: rule changed during retirement preflight; retained" + continue + fi + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" + done +} + publish_repository_rulesets() { local repository ruleset_config ci_payload maintenance branch required_ci local input @@ -649,6 +708,7 @@ publish_repository_rulesets() { upsert_repository_ruleset "${repository}" "${input}" done < <(jq -c '.maintenance_branches[]' "${repositories_config}") + delete_retired_repository_required_ci delete_stale_repository_maintenance_rulesets delete_stale_repository_advisory_rulesets @@ -1080,9 +1140,10 @@ security_ruleset_matches() { else . end) | sort_by(.actor_type, .actor_id, .bypass_mode)) | # GitHub expands this omitted request parameter to false. Keep true, # null and unknown parameters visible so modified gates are not retired. - .rules |= map(if .type == "required_status_checks" and - .parameters.do_not_enforce_on_create == false then - .parameters |= del(.do_not_enforce_on_create) + .rules |= map(if .type == "required_status_checks" then + .parameters |= ( + if .do_not_enforce_on_create == false then del(.do_not_enforce_on_create) else . end | + .required_status_checks |= sort_by(.context, .integration_id)) else . end); normalize as $current | all($desired[0] | normalize | to_entries[]; $current[.key] == .value) diff --git a/bin/validate b/bin/validate index 3eaed82..73bf412 100755 --- a/bin/validate +++ b/bin/validate @@ -216,6 +216,26 @@ jq -e ' ($branches | length) == ($branches | unique | length)) ' "${repositories_config}" >/dev/null +jq -e --slurpfile repositories "${repositories_config}" ' + (keys == ["retirements"]) and (.retirements | type == "array") and + all(.retirements[]; + (keys == ["repository", "repository_id", "required_ci", "ruleset_id"]) and + (.repository | type == "string" and length > 0) and + (.repository_id | type == "number" and floor == . and . > 0) and + (.ruleset_id | type == "number" and floor == . and . > 0) and + (.required_ci | type == "array" and length > 0) and + all(.required_ci[]; type == "string" and test("[^[:space:]]")) and + ((.required_ci | length) == (.required_ci | unique | length)) and + (.repository as $repository | + ($repositories[0].required_ci | has($repository) | not) and + ($repositories[0].pull_request_gate | index($repository)) != null and + ($repositories[0].archive | index($repository)) == null) + ) and + ((.retirements | map(.repository) | length) == (.retirements | map(.repository) | unique | length)) and + ((.retirements | map(.repository_id) | length) == (.retirements | map(.repository_id) | unique | length)) and + ((.retirements | map(.ruleset_id) | length) == (.retirements | map(.ruleset_id) | unique | length)) +' "${root}/config/retired-required-ci.json" >/dev/null + jq -e \ --slurpfile repositories "${repositories_config}" ' (keys == ["retirements"]) and diff --git a/config/retired-required-ci.json b/config/retired-required-ci.json new file mode 100644 index 0000000..80b8b75 --- /dev/null +++ b/config/retired-required-ci.json @@ -0,0 +1,3 @@ +{ + "retirements": [] +} diff --git a/lib/security_readiness.py b/lib/security_readiness.py index e61d364..0981148 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -10,6 +10,7 @@ import json from pathlib import Path import re +import shlex import subprocess import sys from urllib.parse import quote, urlencode @@ -347,6 +348,11 @@ def pin_changes(before, after, selected): require(left_match[key] == right_match[key], "canary changes action structure") require(left_match["sha"] != right_match["sha"], "canary contains unrelated comment changes") action = right_match["action"] + # GitHub's action SHA requirement excludes reusable workflow refs. + # The canary admits only action-pin changes, including in mixed PRs. + require(re.fullmatch(r"[^/]+/[^/]+/\.github/workflows/[^/]+\.ya?ml", action, + flags=re.IGNORECASE) is None, + "canary changes a reusable workflow reference, not only action pins") identity = f"{action}@{right_match['sha']}" require((selected["github_owned_allowed"] and action.split("/")[0].lower() in {"actions", "github"}) @@ -807,17 +813,215 @@ def advanced_codeql_coverage(coordinate, branch, head, minima, selected, declara raise Deferred("required advanced CodeQL partition upload missing within bounded evidence") +def advanced_workflow_wiring(source): + """Check executable dataflow, not the honesty of repository-owned Python. + + The supported ordinary-job shape is deliberately bounded. Producer-owned + action SHAs and classifier path/category declarations are not central pins. + """ + try: + import yaml + except ImportError as exc: + raise EvidenceError("install requirements-validation.txt for CodeQL workflow verification") from exc + + class UniqueLoader(yaml.SafeLoader): + # Actions uses `on` as a mapping key. Preserve scalar booleans as text, + # as in the existing publication verifier, and validate them explicitly. + yaml_implicit_resolvers = { + key: [(tag, regex) for tag, regex in values if tag != "tag:yaml.org,2002:bool"] + for key, values in yaml.SafeLoader.yaml_implicit_resolvers.items() + } + + def unique_mapping(loader, node, deep=False): + result = {} + for key_node, value_node in node.value: + key = loader.construct_object(key_node, deep=deep) + if key in result: + raise ValueError(f"duplicate YAML key: {key}") + result[key] = loader.construct_object(value_node, deep=deep) + return result + + UniqueLoader.add_constructor("tag:yaml.org,2002:map", unique_mapping) + try: + workflow = yaml.load(source, Loader=UniqueLoader) + except (yaml.YAMLError, ValueError, TypeError) as exc: + raise EvidenceError("advanced CodeQL workflow YAML malformed or ambiguous") from exc + + def check(condition, detail): + require(condition, "advanced CodeQL executable workflow contract: " + detail) + + def expression(value, expected, wrapped=True): + if not isinstance(value, str): + return False + value = value.strip() + if value.startswith("${{") and value.endswith("}}"): + value = value[3:-2].strip() + elif wrapped: + return False + # Ignore token spacing, preserving string contents (including spaces). + def tokens(text): + return re.findall(r"'(?:[^']|'')*'|[A-Za-z_][A-Za-z_0-9]*|==|\S", text) + return tokens(value) == tokens(expected) + + def execution_settings(value, context): + check(isinstance(value, dict), context + " must be a mapping") + check(not value.get("env") and not value.get("defaults"), context + " execution overrides") + check(value.get("continue-on-error", "false") == "false", context + " suppresses failure") + check(not any(key in value for key in ("shell", "working-directory", "container", "services")), + context + " changes execution context") + + def step(value, context, condition=None): + execution_settings(value, context) + check(set(value) <= {"name", "id", "uses", "with", "run", "env", "if", "continue-on-error"}, + context + " unsupported step execution") + check(("if" not in value) if condition is None else expression(value.get("if"), condition, False), + context + " condition disconnected") + + def action(value, name): + check(isinstance(value.get("uses"), str) + and re.fullmatch(re.escape(name) + r"@[0-9a-fA-F]{40}", value["uses"]) is not None + and "run" not in value, "expected official full-SHA " + name) + + def checkout(value): + step(value, "checkout") + action(value, "actions/checkout") + inputs = value.get("with", {}) + check(isinstance(inputs, dict) + and set(inputs) <= {"fetch-depth", "ref", "repository", "persist-credentials"}, + "checkout must use the current complete workspace") + check("ref" not in inputs or expression(inputs["ref"], "github.sha"), + "checkout revision differs from run") + check("repository" not in inputs or expression(inputs["repository"], "github.repository"), + "checkout repository differs from run") + + def command(value, prefix): + check(isinstance(value.get("run"), str) and "uses" not in value and "with" not in value, + "helper must execute a run step") + try: + # Retain shell quoting as well as argument values: single-quoted or + # escaped $EVENT_NAME is a literal, not the bound event environment. + source = value["run"].replace("\\\r\n", "").replace("\\\n", "") + raw = shlex.split(source, comments=True, posix=False) + decoded = [shlex.split(token) for token in raw] + check(all(len(item) == 1 for item in decoded), "helper shell arguments unsupported") + tokens = [item[0] for item in decoded] + except ValueError: + raw, tokens = [], [] + check(tokens[:len(prefix)] == prefix, "helper invocation disconnected") + arguments, raw_arguments = tokens[len(prefix):], raw[len(prefix):] + check(len(arguments) % 2 == 0, "helper arguments malformed") + options, quoted_options = {}, {} + for index in range(0, len(arguments), 2): + key, argument = arguments[index:index + 2] + check(key.startswith("--") and key not in options, "helper arguments ambiguous") + options[key] = argument + quoted_options[key] = raw_arguments[index + 1] + return options, quoted_options + + def classifier(value, scoped=False): + expected_env = {"BASE_SHA": "github.event.pull_request.base.sha", + "EVENT_NAME": "github.event_name", "HEAD_SHA": "github.event.pull_request.head.sha"} + environment = value.get("env") + check(isinstance(environment, dict) and set(environment) == set(expected_env) + and all(expression(environment[key], expected) for key, expected in expected_env.items()), + "classifier event/revision environment disconnected") + # Environment is validated above; every other execution override is rejected. + step({key: item for key, item in value.items() if key != "env"}, "classifier") + check(value.get("id") == "classify", "classifier step identity disconnected") + options, quoted = command(value, ["python3", "tools/ci/classify_changes.py"]) + expected = {"--event": "EVENT_NAME", "--base": "BASE_SHA", "--head": "HEAD_SHA", + "--github-output": "GITHUB_OUTPUT"} + check(set(options) == set(expected) | {"--codeql-config"} | ({"--codeql-scope"} if scoped else set()), + "classifier arguments disconnected") + check(all(quoted[key] in ("$" + variable, "${" + variable + "}", + '"$' + variable + '"', '"${' + variable + '}"') + for key, variable in expected.items()), "classifier event/output arguments disconnected") + path = options["--codeql-config"] + check(re.fullmatch(r"(?:[A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\.ya?ml", path) is not None + and ".." not in path.split("/"), "classifier config destination unsupported") + if scoped: + check(expression(options["--codeql-scope"], "matrix.scope"), "classifier scope disconnected") + return path + + execution_settings(workflow, "workflow") + jobs = workflow.get("jobs") + check(isinstance(jobs, dict) and set(jobs) == {"classify", "analyze", "codeql-validation"}, + "requires ordinary classification, analysis and validation jobs") + for name, job in jobs.items(): + execution_settings(job, name) + check(set(job) <= {"name", "runs-on", "timeout-minutes", "outputs", "steps", "if", "needs", + "strategy", "continue-on-error"}, name + " unsupported job execution") + runner = job.get("runs-on") + check((isinstance(runner, str) and bool(runner)) or + (isinstance(runner, list) and bool(runner) and + all(isinstance(item, str) and item for item in runner)), name + " runner missing") + steps = job.get("steps") + check(isinstance(steps, list) and all(isinstance(item, dict) for item in steps), + name + " executable steps missing") + identifiers = [item["id"] for item in steps if "id" in item] + check(all(isinstance(item, str) for item in identifiers) + and len(identifiers) == len(set(identifiers)), name + " duplicate step identities") + classify, analyze, validation = (jobs[name] for name in ("classify", "analyze", "codeql-validation")) + check(classify.get("name") == "CodeQL classification" + and not any(key in classify for key in ("if", "needs", "strategy")) + and len(classify["steps"]) == 2, "classification job execution disconnected") + outputs = classify.get("outputs", {}) + check(isinstance(outputs, dict) and set(outputs) == {"matrix", "run"} + and expression(outputs["matrix"], "steps.classify.outputs.codeql_matrix") + and expression(outputs["run"], "steps.classify.outputs.codeql_run"), "classifier outputs disconnected") + checkout(classify["steps"][0]) + config_path = classifier(classify["steps"][1]) + + check(analyze.get("needs") in ("classify", ["classify"]) + and expression(analyze.get("if"), "needs.classify.outputs.run == 'true'", False), + "analysis dependency/condition disconnected") + name = analyze.get("name", "") + match = re.fullmatch(r"CodeQL analysis \((\$\{\{.*\}\})\)", name) if isinstance(name, str) else None + check(match is not None and expression(match[1], "matrix.display"), "analysis display disconnected") + strategy = analyze.get("strategy", {}) + check(isinstance(strategy, dict) and set(strategy) == {"fail-fast", "matrix"} + and strategy["fail-fast"] == "false" + and expression(strategy["matrix"], "fromJSON(needs.classify.outputs.matrix)"), + "analysis matrix disconnected") + check("outputs" not in analyze and len(analyze["steps"]) == 4, "analysis step order unsupported") + checkout(analyze["steps"][0]) + check(classifier(analyze["steps"][1], scoped=True) == config_path, "analysis config destination disconnected") + initialize, analyze_step = analyze["steps"][2:] + step(initialize, "CodeQL init", "matrix.run == true") + action(initialize, "github/codeql-action/init") + inputs = initialize.get("with", {}) + check(isinstance(inputs, dict) and set(inputs) == {"build-mode", "config-file", "languages"} + and inputs["build-mode"] == "none" + and expression(inputs["config-file"], "steps.classify.outputs.codeql_config") + and expression(inputs["languages"], "matrix.language"), "CodeQL init/config/language disconnected") + step(analyze_step, "CodeQL analysis", "matrix.run == true") + action(analyze_step, "github/codeql-action/analyze") + inputs = analyze_step.get("with", {}) + check(isinstance(inputs, dict) and set(inputs) == {"category"} + and expression(inputs["category"], "matrix.category"), "CodeQL analysis category disconnected") + + check(validation.get("name") == "CodeQL validation" + and expression(validation.get("if"), "always()", False) + and isinstance(validation.get("needs"), list) and sorted(validation["needs"]) == ["analyze", "classify"] + and not any(key in validation for key in ("strategy", "outputs")) + and len(validation["steps"]) == 2, "validation dependency/condition disconnected") + checkout(validation["steps"][0]) + aggregate = validation["steps"][1] + step(aggregate, "validation aggregate") + options, _ = command(aggregate, ["python3", "tools/ci/require_checks.py", "optional"]) + check(set(options) == {"--label", "--classifier-result", "--required", "--result"} + and options["--label"] == "CodeQL" + and expression(options["--classifier-result"], "needs.classify.result") + and expression(options["--required"], "needs.classify.outputs.run") + and expression(options["--result"], "needs.analyze.result"), "validation result arguments disconnected") + + def advanced_codeql(coordinate, branch, head, minima, partition_minima): sources = {} for path in (".github/workflows/codeql.yml", "tools/ci/classify_changes.py", "tools/ci/require_checks.py"): sources[path] = contents(coordinate, path, branch) require(contents(coordinate, path, head) == sources[path], "advanced CodeQL producer source is stale") - workflow = sources[".github/workflows/codeql.yml"].decode() - for marker in ("name: CodeQL analysis (${{ matrix.display }})", "languages: ${{ matrix.language }}", - "category: ${{ matrix.category }}", "python3 tools/ci/classify_changes.py", - "python3 tools/ci/require_checks.py optional", "${{ needs.classify.outputs.run }}", - "${{ needs.analyze.result }}"): - require(marker in workflow, "advanced CodeQL workflow producer contract unsupported") + advanced_workflow_wiring(sources[".github/workflows/codeql.yml"]) declarations = classifier_partitions(sources["tools/ci/classify_changes.py"], partition_minima) events = classifier_full_events(sources["tools/ci/classify_changes.py"]) run = latest_full_codeql_run(coordinate, branch, head, events) diff --git a/requirements-validation.txt b/requirements-validation.txt index 39990fd..ad78371 100644 --- a/requirements-validation.txt +++ b/requirements-validation.txt @@ -1,2 +1,2 @@ -# YAML parsing for exact-main executable publication workflow verification. +# YAML parsing for executable publication and CodeQL workflow verification. PyYAML==6.0.3 diff --git a/tests/publish-retired-required-ci.sh b/tests/publish-retired-required-ci.sh new file mode 100755 index 0000000..0680fa8 --- /dev/null +++ b/tests/publish-retired-required-ci.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash + +# Production functions consume these fixture globals. +# shellcheck disable=SC2034 +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +source_root=${ATRINIK_PUBLISH_TEST_ROOT:-${root}} +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +sed -n '/^converge_organization_settings() {/,/^organization_metadata=/p' \ + "${source_root}/bin/publish" | sed '$d' >"${temporary}/functions.sh" +# shellcheck source=/dev/null +source "${temporary}/functions.sh" +root=${temporary}/fixture +mkdir -p "${root}/config" +cp -R "${source_root}/config/rulesets" "${root}/config/rulesets" +organization=atrinik +organization_plan=free +repositories_config=${root}/config/repositories.json +advisory_merge_windows_config=${root}/config/advisory-merge-windows.json +printf '{"repositories":[]}\n' >"${advisory_merge_windows_config}" +active_repositories=(web-platform) +declare -A active_repository_ids=([web-platform]=123) +temporary_files=() +log=${temporary}/api.jsonl +repository_has_advisory_merge_window() { return 1; } +upsert_repository_ruleset() { :; } +delete_stale_repository_maintenance_rulesets() { :; } +delete_stale_repository_advisory_rulesets() { :; } +github_api() { + jq -cn --arg endpoint "$1" '{method:"GET",endpoint:$endpoint}' >>"${log}" + case $1 in + 'repos/atrinik/web-platform/rulesets?includes_parents=false') + if [[ -e ${temporary}/deleted ]]; then echo '[]'; else cat "${temporary}/list.json"; fi ;; + repos/atrinik/web-platform) + cat "${temporary}/metadata.json" ;; + repos/atrinik/web-platform/rulesets/3) + if [[ -e ${temporary}/read-once && -e ${temporary}/changed.json ]]; then + cat "${temporary}/changed.json" + else + cat "${temporary}/rule.json" + fi + touch "${temporary}/read-once" ;; + *) echo "unexpected endpoint: $1" >&2; return 99 ;; + esac +} +run_api() { + [[ $1 == DELETE && $2 == repos/atrinik/web-platform/rulesets/3 ]] + if ${apply}; then + jq -cn --arg endpoint "$2" '{method:"DELETE",endpoint:$endpoint}' >>"${log}" + touch "${temporary}/deleted" + else + jq -cn --arg endpoint "$2" '{method:"PLAN",endpoint:$endpoint}' >>"${log}" + fi +} +reset_case() { + : >"${log}" + rm -f "${temporary}/deleted" "${temporary}/read-once" "${temporary}/changed.json" + apply=true + active_repository_ids=([web-platform]=123) + printf '{"pull_request_gate":["web-platform"],"required_ci":{},"release_tags":[],"maintenance_branches":[]}\n' >"${repositories_config}" + printf '{"retirements":[{"repository":"web-platform","repository_id":123,"ruleset_id":3,"required_ci":["Platform validation","Conventional PR title"]}]}\n' >"${root}/config/retired-required-ci.json" + printf '[{"id":1,"name":"01 - Default branch integrity"},{"id":3,"name":"03 - Required CI"},{"id":5,"name":"Foreign CI"}]\n' >"${temporary}/list.json" + printf '{"id":123,"full_name":"atrinik/web-platform","archived":false}\n' >"${temporary}/metadata.json" + jq '. + {id:3,source_type:"Repository",source:"atrinik/web-platform"} | + .bypass_actors[0].actor_id = null | + .rules[0].parameters += {do_not_enforce_on_create:false, + required_status_checks:[{context:"Conventional PR title",integration_id:15368}, + {context:"Platform validation",integration_id:15368}]}' \ + "${root}/config/rulesets/required-ci.json" >"${temporary}/rule.json" +} +assert_retained() { + publish_repository_rulesets >"${temporary}/output" + jq -se 'all(.[]; .method == "GET")' "${log}" >/dev/null + grep -Fq 'retained' "${temporary}/output" +} +reset_case +publish_repository_rulesets >"${temporary}/output" +jq -se '[.[] | select(.method == "DELETE")] == [{method:"DELETE",endpoint:"repos/atrinik/web-platform/rulesets/3"}]' "${log}" >/dev/null +# Converged retirement makes no second deletion. +: >"${log}" +publish_repository_rulesets >"${temporary}/output" +jq -se 'all(.[]; .method == "GET")' "${log}" >/dev/null +reset_case +jq 'del(.rules[0].parameters.do_not_enforce_on_create)' \ + "${temporary}/rule.json" >"${temporary}/implicit-default.json" +mv "${temporary}/implicit-default.json" "${temporary}/rule.json" +publish_repository_rulesets >"${temporary}/output" +jq -se 'any(.[]; .method == "DELETE")' "${log}" >/dev/null +reset_case +apply=false +publish_repository_rulesets >"${temporary}/output" +jq -se '[.[] | select(.method != "GET")] == [{method:"PLAN",endpoint:"repos/atrinik/web-platform/rulesets/3"}]' "${log}" >/dev/null +[[ ! -e ${temporary}/deleted ]] +for filter in \ + '.id = 999' '.source_type = "Organization"' '.source = "foreign/web-platform"' \ + '.name = "Foreign CI"' '.target = "tag"' '.enforcement = "disabled"' \ + '.bypass_actors[0].bypass_mode = "always"' '.bypass_actors[0].actor_id = 2' \ + '.conditions.ref_name.include = ["refs/heads/other"]' \ + '.rules[0].parameters.required_status_checks[0].integration_id = 9' \ + '.rules[0].parameters.required_status_checks[0].context = "Unknown check"' \ + '.rules[0].parameters.strict_required_status_checks_policy = false' \ + '.rules[0].parameters.do_not_enforce_on_create = true' \ + '.rules[0].parameters.do_not_enforce_on_create = null' \ + '.rules[0].parameters.unknown_option = false' '.rules += [{type:"deletion"}]'; do + reset_case + jq "${filter}" "${temporary}/rule.json" >"${temporary}/changed-rule.json" + mv "${temporary}/changed-rule.json" "${temporary}/rule.json" + assert_retained +done +for filter in '.id = 999' '.full_name = "foreign/web-platform"' '.archived = true'; do + reset_case + jq "${filter}" "${temporary}/metadata.json" >"${temporary}/changed-metadata.json" + mv "${temporary}/changed-metadata.json" "${temporary}/metadata.json" + assert_retained +done +reset_case +jq '.rules[0].parameters.required_status_checks[0].context = "Concurrent change"' \ + "${temporary}/rule.json" >"${temporary}/changed.json" +assert_retained +reset_case +jq '.retirements[0].repository_id = 999' "${root}/config/retired-required-ci.json" >"${temporary}/wrong-record.json" +mv "${temporary}/wrong-record.json" "${root}/config/retired-required-ci.json" +assert_retained +reset_case +printf '{"retirements":[]}\n' >"${root}/config/retired-required-ci.json" +assert_retained +grep -Fq 'config/retired-required-ci.json' "${temporary}/output" +reset_case +printf '[{"id":3,"name":"03 - Required CI"},{"id":4,"name":"03 - Required CI"}]\n' >"${temporary}/list.json" +assert_retained +reset_case +printf '[{"id":4,"name":"03 - Required CI"}]\n' >"${temporary}/list.json" +assert_retained +# Still-active CI and repositories excluded from the active identity map are untouched. +reset_case +jq '.required_ci["web-platform"] = ["Platform validation"]' "${repositories_config}" >"${temporary}/active.json" +mv "${temporary}/active.json" "${repositories_config}" +publish_repository_rulesets >"${temporary}/output" +[[ ! -s ${log} ]] +reset_case +active_repository_ids=() +publish_repository_rulesets >"${temporary}/output" +[[ ! -s ${log} ]] +echo 'Required-CI retirement publisher tests passed.' diff --git a/tests/security-readiness.py b/tests/security-readiness.py index 9ec9a9a..fcaa5a1 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -64,16 +64,113 @@ """ SELECTED = {"github_owned_allowed": True, "verified_allowed": False, "patterns_allowed": ["rust-lang/crates-io-auth-action@*", "docker/setup-buildx-action@*"]} +# Executable Classic workflow snapshot; action pins remain producer-owned. CODEQL_WORKFLOW = b"""name: CodeQL + +on: + push: + branches: + - main + pull_request: + branches: + - main + merge_group: + workflow_dispatch: + schedule: + - cron: "17 4 * * 2" + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: classic-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: + classify: + name: CodeQL classification + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: ${{ steps.classify.outputs.codeql_matrix }} + run: ${{ steps.classify.outputs.codeql_run }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - id: classify + name: Select analysis languages and paths + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + python3 tools/ci/classify_changes.py \\ + --event "${EVENT_NAME}" \\ + --base "${BASE_SHA}" \\ + --head "${HEAD_SHA}" \\ + --github-output "${GITHUB_OUTPUT}" \\ + --codeql-config build/codeql-config.yml + analyze: name: CodeQL analysis (${{ matrix.display }}) - run: python3 tools/ci/classify_changes.py - languages: ${{ matrix.language }} - category: ${{ matrix.category }} - validation: + needs: classify + if: needs.classify.outputs.run == 'true' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.classify.outputs.matrix) }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - id: classify + name: Recreate the trusted path configuration + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + EVENT_NAME: ${{ github.event_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + python3 tools/ci/classify_changes.py \\ + --event "${EVENT_NAME}" \\ + --base "${BASE_SHA}" \\ + --head "${HEAD_SHA}" \\ + --github-output "${GITHUB_OUTPUT}" \\ + --codeql-config build/codeql-config.yml \\ + --codeql-scope "${{ matrix.scope }}" + - name: Initialize CodeQL + if: matrix.run == true + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + build-mode: none + config-file: ${{ steps.classify.outputs.codeql_config }} + languages: ${{ matrix.language }} + - name: Analyze + if: matrix.run == true + uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + category: ${{ matrix.category }} + + codeql-validation: name: CodeQL validation - run: python3 tools/ci/require_checks.py optional '${{ needs.classify.outputs.run }}' '${{ needs.analyze.result }}' + if: always() + needs: + - classify + - analyze + runs-on: ubuntu-24.04 + timeout-minutes: 2 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Require every selected analysis + run: | + python3 tools/ci/require_checks.py optional \\ + --label CodeQL \\ + --classifier-result '${{ needs.classify.result }}' \\ + --required '${{ needs.classify.outputs.run }}' \\ + --result '${{ needs.analyze.result }}' """ PARTITIONS = [ {"display": "C/C++ client", "scope": "client", "language": "c-cpp", @@ -841,6 +938,51 @@ def test_sha_general_github_owned_pin_update(self): self.fixture.after = f"steps:\n - uses: actions/checkout@{NEW} # v6\n".encode() self.assertEqual(self.invoke("sha")[0], 0) + def allow_canary_action_family(self, pattern): + self.fixture.live["patterns_allowed"].append(pattern) + (self.root / "config/actions-selected.json").write_text(json.dumps(self.fixture.live)) + + def test_sha_reusable_workflow_update_is_not_action_enforcement_evidence(self): + self.allow_canary_action_family("atrinik/shared/*") + for suffix in ("yml", "yaml"): + with self.subTest(suffix=suffix): + workflow = ("name: Check\non: pull_request\njobs:\n validation:\n" + f" uses: atrinik/shared/.github/workflows/check.{suffix}@{{pin}}\n") + self.fixture.before = workflow.format(pin=OLD).encode() + self.fixture.after = workflow.format(pin=NEW).encode() + self.assert_deferred("sha", "reusable workflow reference") + # GitHub-owned allowances do not turn a reusable workflow into an action. + self.fixture.before = self.fixture.before.replace(b"atrinik/shared", b"github/shared") + self.fixture.after = self.fixture.after.replace(b"atrinik/shared", b"github/shared") + self.assert_deferred("sha", "reusable workflow reference") + + def test_sha_mixed_reusable_and_action_updates_rejected(self): + self.allow_canary_action_family("atrinik/shared/*") + workflow = ("name: Check\non: pull_request\njobs:\n validation:\n" + " uses: atrinik/shared/.github/workflows/check.yml@{workflow_pin}\n" + " local:\n runs-on: ubuntu-24.04\n steps:\n" + " - uses: actions/checkout@{action_pin}\n") + self.fixture.before = workflow.format(workflow_pin=OLD, action_pin=OLD).encode() + self.fixture.after = workflow.format(workflow_pin=NEW, action_pin=NEW).encode() + self.assert_deferred("sha", "reusable workflow reference") + # An unchanged reusable reference is ordinary context, not changed proof. + self.fixture.after = workflow.format(workflow_pin=OLD, action_pin=NEW).encode() + result, message = self.invoke("sha") + self.assertEqual(result, 0, message) + + def test_sha_real_action_subpath_update_remains_eligible(self): + self.allow_canary_action_family("atrinik/shared/*") + for action in ("atrinik/shared/actions/check", "atrinik/shared/.github/actions/check", + "github/codeql-action/init"): + with self.subTest(action=action): + workflow = ("name: Check\non: pull_request\njobs:\n validation:\n" + " runs-on: ubuntu-24.04\n steps:\n" + f" - uses: {action}@{{pin}}\n") + self.fixture.before = workflow.format(pin=OLD).encode() + self.fixture.after = workflow.format(pin=NEW).encode() + result, message = self.invoke("sha") + self.assertEqual(result, 0, message) + def test_sha_live_old_exact_pin_deferred(self): self.fixture.live["patterns_allowed"][-1] = "rust-lang/crates-io-auth-action@" + OLD self.assert_deferred("sha", "live selected-actions") @@ -1161,6 +1303,206 @@ def set_classifier_source(self, source): self.fixture.codeql_sources["tools/ci/classify_changes.py"] = source self.fixture.codeql_head_sources["tools/ci/classify_changes.py"] = source + def set_codeql_workflow(self, source): + self.fixture.codeql_sources[".github/workflows/codeql.yml"] = source + self.fixture.codeql_head_sources[".github/workflows/codeql.yml"] = source + + def codeql_document(self): + import yaml + # Preserve Actions' on key when constructing modified fixtures. + return yaml.safe_load(CODEQL_WORKFLOW.replace(b"\non:", b"\n'on':")) + + def set_codeql_document(self, document): + import yaml + self.set_codeql_workflow(yaml.safe_dump(document, sort_keys=False).encode()) + + def test_codeql_yaml_dependency_missing_fails_closed_without_affecting_other_modes(self): + self.enable_advanced() + with patch.dict(sys.modules, {"yaml": None}): + result, message = self.invoke("codeql") + self.assertEqual(result, 2, message) + self.assertIn("install requirements-validation.txt", message) + self.assertEqual(self.invoke("dependency")[0], 0) + + def test_codeql_advanced_comments_cannot_prove_executable_wiring(self): + self.enable_advanced() + source = b"name: CodeQL\njobs:\n fake:\n runs-on: ubuntu-24.04\n steps:\n - run: true\n" + source += b"\n".join(b"# " + line for line in CODEQL_WORKFLOW.splitlines()) + self.set_codeql_workflow(source) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_disconnected_wiring_rejected_despite_green_uploads(self): + changes = [ + (("classify", "outputs", "matrix"), '${{ steps.other.outputs.codeql_matrix }}'), + (("classify", "outputs", "run"), 'true'), + (("classify", "steps", 1, "id"), 'other'), + (("classify", "steps", 1, "env", "EVENT_NAME"), 'push'), + (("classify", "steps", 1, "env", "HEAD_SHA"), '${{ github.event.pull_request.base.sha }}'), + (("analyze", "needs"), 'other'), + (("analyze", "if"), 'false'), + (("analyze", "name"), 'CodeQL analysis (C/C++ client)'), + (("analyze", "strategy", "matrix"), {"include": PARTITIONS}), + (("analyze", "strategy", "matrix"), '${{ fromJSON(needs.other.outputs.matrix) }}'), + (("analyze", "steps", 2, "with", "config-file"), 'unrelated-codeql.yml'), + (("analyze", "steps", 2, "with", "config-file"), '${{ steps.other.outputs.codeql_config }}'), + (("analyze", "steps", 2, "with", "languages"), 'c-cpp'), + (("analyze", "steps", 2, "with", "build-mode"), 'autobuild'), + (("analyze", "steps", 3, "with", "category"), '/partition:client/language:c-cpp'), + (("codeql-validation", "if"), 'success()'), + (("codeql-validation", "needs"), ['classify']), + (("codeql-validation", "strategy"), {"matrix": {"fake": [1]}}), + ] + for path, value in changes: + with self.subTest(path=path, value=value): + self.enable_advanced() + document = self.codeql_document() + target = document["jobs"] + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_classifier_and_aggregate_arguments_are_executable(self): + mutations = [ + ('classify', '--event "${EVENT_NAME}"', '--event push'), + ('classify', '--event "${EVENT_NAME}"', "--event '${EVENT_NAME}'"), + ('classify', '--event "${EVENT_NAME}"', '--event "\\${EVENT_NAME}"'), + ('classify', '--github-output "${GITHUB_OUTPUT}"', '--github-output /dev/null'), + ('analyze', '--codeql-scope "${{ matrix.scope }}"', '--codeql-scope client'), + ('analyze', 'build/codeql-config.yml', 'build/disconnected-config.yml'), + ('codeql-validation', "'${{ needs.classify.result }}'", 'success'), + ('codeql-validation', "'${{ needs.classify.outputs.run }}'", 'true'), + ('codeql-validation', "'${{ needs.analyze.result }}'", 'success'), + ] + for name, before, after in mutations: + with self.subTest(name=name, before=before): + self.enable_advanced() + document = self.codeql_document() + step = document['jobs'][name]['steps'][1] + self.assertIn(before, step['run']) + step['run'] = step['run'].replace(before, after) + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + for name in ('classify', 'analyze', 'codeql-validation'): + for mutation in ('comment-only', 'trailing-command', 'leading-command', 'substitution', 'duplicate-option'): + with self.subTest(name=name, mutation=mutation): + self.enable_advanced() + document = self.codeql_document() + step = document['jobs'][name]['steps'][1] + run = step['run'] + if mutation == 'comment-only': + step['run'] = '\n'.join('# ' + line for line in run.splitlines()) + '\ntrue\n' + elif mutation == 'trailing-command': + step['run'] = run.rstrip() + '; true\n' + elif mutation == 'leading-command': + step['run'] = 'true\n' + run + elif mutation == 'substitution': + step['run'] = run.replace('python3', '$(echo python3)', 1) + else: + step['run'] = run.rstrip() + ' --event push\n' + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_disabled_or_overridden_execution_rejected(self): + targets = [('classify', None), ('analyze', None), ('codeql-validation', None), + ('classify', 1), ('analyze', 1), ('analyze', 2), ('analyze', 3), + ('codeql-validation', 1)] + for name, index in targets: + for key, value in [('continue-on-error', True), ('if', 'false'), + ('env', {'PYTHONPATH': 'untrusted'}), + ('defaults', {'run': {'working-directory': 'elsewhere'}}), + ('shell', 'bash {0}'), ('working-directory', 'elsewhere')]: + with self.subTest(name=name, index=index, override=key): + self.enable_advanced() + document = self.codeql_document() + target = document['jobs'][name] + if index is not None: + target = target['steps'][index] + target[key] = value + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + for key, value in [('env', {'GITHUB_OUTPUT': '/dev/null'}), + ('defaults', {'run': {'shell': 'true {0}'}})]: + with self.subTest(workflow_override=key): + self.enable_advanced() + document = self.codeql_document() + document[key] = value + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + self.enable_advanced() + document = self.codeql_document() + document['jobs']['analyze'] = {'uses': 'foreign/repo/.github/workflows/codeql.yml@' + NEW} + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_current_complete_checkout_required(self): + for name in ('classify', 'analyze', 'codeql-validation'): + for key, value in [('ref', 'main'), ('ref', OLD), ('repository', 'foreign/repo'), + ('path', 'elsewhere'), ('sparse-checkout', 'only-one-file')]: + with self.subTest(name=name, key=key, value=value): + self.enable_advanced() + document = self.codeql_document() + document['jobs'][name]['steps'][0].setdefault('with', {})[key] = value + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_step_order_actions_and_duplicate_ids_rejected(self): + for mutation in ('order', 'extra', 'duplicate-id', 'mutable-init', 'foreign-analyze'): + with self.subTest(mutation=mutation): + self.enable_advanced() + document = self.codeql_document() + steps = document['jobs']['analyze']['steps'] + if mutation == 'order': + steps[2], steps[3] = steps[3], steps[2] + elif mutation == 'extra': + steps.insert(2, {'run': 'echo overwrite generated config'}) + elif mutation == 'duplicate-id': + steps[0]['id'] = 'classify' + elif mutation == 'mutable-init': + steps[2]['uses'] = 'github/codeql-action/init@v4' + else: + steps[3]['uses'] = 'foreign/codeql-action/analyze@' + NEW + self.set_codeql_document(document) + self.assert_deferred("codeql", "executable workflow contract") + + def test_codeql_advanced_duplicate_yaml_is_unreadable(self): + for source in (CODEQL_WORKFLOW + b'\njobs: {}\n', + CODEQL_WORKFLOW.replace(b' - id: classify', b' - id: classify\n id: other', 1), + CODEQL_WORKFLOW.replace(b' if: always()', b' if: false\n if: always()'), + b'jobs: [unclosed'): + with self.subTest(source=source[-50:]): + self.enable_advanced() + self.set_codeql_workflow(source) + result, message = self.invoke('codeql') + self.assertEqual(result, 2, message) + self.assertIn('YAML malformed or ambiguous', message) + + def test_codeql_advanced_formatting_and_owner_action_updates_ready(self): + self.enable_advanced() + document = self.codeql_document() + for job in document['jobs'].values(): + for step in job['steps']: + if 'uses' in step: + step['uses'] = step['uses'].split('@')[0] + '@' + NEW + job['steps'][0].setdefault('with', {}).update( + ref='${{github.sha}}', repository='${{ github.repository }}') + for name in ('classify', 'analyze'): + step = document['jobs'][name]['steps'][1] + step['run'] = ('# executable invocation follows\npython3 tools/ci/classify_changes.py ' + '--codeql-config build/new-config.yml --github-output "$GITHUB_OUTPUT" ' + '--head "$HEAD_SHA" --base "$BASE_SHA" --event "$EVENT_NAME"') + if name == 'analyze': + step['run'] += ' --codeql-scope "${{matrix.scope}}"' + document['jobs']['analyze']['if'] = "${{needs.classify.outputs.run=='true'}}" + document['jobs']['analyze']['strategy']['matrix'] = '${{ fromJSON ( needs.classify.outputs.matrix ) }}' + document['jobs']['analyze']['steps'][2]['with']['languages'] = '${{matrix.language}}' + document['jobs']['analyze']['steps'][3]['with']['category'] = '${{ matrix.category}}' + document['jobs']['codeql-validation']['if'] = '${{ always ( ) }}' + self.set_codeql_document(document) + result, message = self.invoke('codeql') + self.assertEqual(result, 0, message) + def test_codeql_advanced_reduced_source_and_jobs_below_policy_deferred(self): self.enable_advanced(PARTITIONS[:1]) self.assert_deferred("codeql", "omits required partition coverage") diff --git a/tests/validate-retired-required-ci.sh b/tests/validate-retired-required-ci.sh new file mode 100755 index 0000000..8bfcc2e --- /dev/null +++ b/tests/validate-retired-required-ci.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +mkdir -p "${temporary}/bin" +cp "${root}/bin/validate" "${temporary}/bin/" +cp -R "${root}/config" "${root}/community-health" "${root}/.github" "${temporary}/" +"${temporary}/bin/validate" >/dev/null +printf '{"retirements":[{"repository":"web-platform","repository_id":123,"ruleset_id":3,"required_ci":["Platform validation","Conventional PR title"]}]}\n' >"${temporary}/valid.json" +cp "${temporary}/valid.json" "${temporary}/config/retired-required-ci.json" +"${temporary}/bin/validate" >/dev/null +for filter in \ + '.retirements[0].repository = "client"' '.retirements[0].repository = "legacy-client"' \ + '.retirements[0].repository = "unknown"' '.retirements[0].required_ci = []' \ + '.retirements[0].required_ci = [""]' '.retirements[0].required_ci = [" "]' \ + '.retirements[0].required_ci = ["Duplicate", "Duplicate"]' \ + '.retirements[0].required_ci = [null]' '.retirements[0].repository_id = 0' \ + '.retirements[0].ruleset_id = -1' '.retirements[0].ruleset_id = "3"' \ + '.retirements[0].repository_id = 1.5' '.retirements += [.retirements[0]]' \ + '.retirements[0].unknown_field = true' 'del(.retirements[0].ruleset_id)'; do + jq "${filter}" "${temporary}/valid.json" >"${temporary}/config/retired-required-ci.json" + if "${temporary}/bin/validate" >/dev/null 2>&1; then + echo "error: accepted invalid required-CI retirement: ${filter}" >&2 + exit 1 + fi +done +echo 'Required-CI retirement schema tests passed.' From 9f21a63a4aa13de2c9879b941b4d62edff441c01 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 16:53:45 -0500 Subject: [PATCH 16/22] fix: verify PR triggers and layered branch protections --- AGENTS.md | 9 +++ CONTRIBUTING.md | 10 ++- README.md | 13 ++++ bin/audit | 32 ++++++++++ bin/publish | 41 ++++++++---- lib/security_readiness.py | 40 +++++++++++- tests/audit-fixtures.py | 54 ++++++++++++++++ tests/audit.sh | 32 ++++++++++ tests/publish-security-merge.sh | 107 +++++++++++++++++++++++++++++++- tests/security-readiness.py | 92 ++++++++++++++++++++++++++- 10 files changed, 411 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2c6da3a..18f4b02 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -238,6 +238,8 @@ dependency: bind classification outputs, matrix, scoped config, official full-SHA init/analyze actions, and the always-running aggregate. Reject duplicate keys, disconnected or disabled steps, and failure suppression. + Require ordinary pull-request triggers covering the live default branch and + opened/synchronize/reopened events without path filters. Producer helpers remain trusted; preserve owner-controlled action updates, category names and paths without a duplicate version/path catalog. - `config/actions-sha-pinning.json` persists SHA rollout as `pending` with no @@ -260,3 +262,10 @@ Delete only the unchanged Actions-bound template after fresh identity/archive and payload checks. Preserve modified/foreign rules and missing-provenance rules with actionable `SKIP`; retain records for idempotent reruns. + +- Before updating an existing security gate, re-read and revalidate its identity + and recognized payload, retaining concurrent modifications. Audit full legacy + protection on active default and declared maintenance branches; any presence + is drift under the canonical ruleset contract. Only verified unprotected + responses count as absent; unreadable evidence fails closed and audit never + removes protections. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 88e6eb2..5c0d9aa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -157,13 +157,21 @@ full commit pins belong in workflows, not redundant version assertions. Advanced CodeQL readiness must validate executable YAML wiring from classifier invocation and outputs through the dynamic analysis matrix, scoped generated configuration, official full-SHA CodeQL actions, and the unconditional aggregate. -Use the existing validation dependency with duplicate-key rejection. Test +Require the live default branch's ordinary pull-request trigger without path +filters, including opened, synchronize and reopened events. Use the existing +validation dependency with duplicate-key rejection. Test comments-only markers, disconnected inputs, disabled or failure-suppressed steps, checkout revision drift, and ambiguous YAML; preserve full-SHA updates, formatting changes, additional partitions, and category renames. Classification path semantics remain in trusted producer helpers; do not add a central source-path or action-version catalog or claim resistance to malicious repository writers. +Security gate updates must re-read and validate ownership and the recognized +payload immediately before PUT, preserving concurrent modifications. Audit +legacy protection on active default and maintenance branches as drift under the +canonical ruleset contract; distinguish verified unprotected responses from +unreadable evidence, and never remove protection during audit. + Crates.io publication contracts bind exact repository/workflow/environment and official auth-action identities with full immutable workflow SHA references; preserve the reviewed `rust-lang/crates-io-auth-action@*` allowance for normal diff --git a/README.md b/README.md index adfd283..0cfcaf2 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,19 @@ Rollout is owned by an organization administrator: as drift, never as initial rollout pending. A live true setting is always preserved. Full SHA pinning does not prove an allowed action is trustworthy. +Advanced CodeQL qualification also requires ordinary pull-request triggers +covering the live default branch and opened, synchronize and reopened events, +without workflow path filters. Successful manual or scheduled analysis cannot +qualify a missing or retargeted pull-request producer. Before updating an +existing security gate, publication rechecks its ownership and recognized +payload immediately before the write, retaining concurrent policy changes. + +Rulesets are the canonical branch protection contract. Audit also reads legacy +protection on every active default branch and declared maintenance branch. Any +legacy protection is drift requiring separate reviewed manual remediation; +unreadable responses fail closed. Only a verified unprotected-branch response +counts as absent. The audit never removes branch protections. + Repository-scope retirement of an existing `03 - Required CI` rule also needs a reviewed `config/retired-required-ci.json` record with its exact repository and ruleset IDs and previous required contexts. Remove the repository from diff --git a/bin/audit b/bin/audit index 7cbcff7..d4273b3 100755 --- a/bin/audit +++ b/bin/audit @@ -311,6 +311,32 @@ check_ruleset() { fi } +# Rulesets are the declared protection contract in both publisher scopes. +# Legacy protection layers additional requirements outside the rules endpoint; +# retain it for manual review rather than silently treating rulesets as complete. +audit_legacy_protection() { + local repository=$1 branch=$2 encoded response status=0 body + encoded=$(jq -rn --arg branch "${branch}" '$branch | @uri') + response=$(github_api --include \ + "repos/${organization}/${repository}/branches/${encoded}/protection" \ + 2>"${temporary}/legacy-protection.err") || status=$? + response=${response//$'\r'/} + body=${response#*$'\n\n'} + if [[ ${status} == 1 && ${response%%$'\n'*} =~ ^HTTP/[0-9.]+\ 404(\ |$) ]] && + jq -se 'length == 1 and (.[0] | type == "object" and + .message == "Branch not protected" and .status == "404")' \ + <<<"${body}" >/dev/null 2>&1; then + return + fi + if [[ ${status} == 0 && ${response%%$'\n'*} =~ ^HTTP/[0-9.]+\ 200(\ |$) ]] && + jq -se 'length == 1 and (.[0] | type == "object")' \ + <<<"${body}" >/dev/null 2>&1; then + record_finding "unexpected legacy branch protection requires manual review: ${organization}/${repository}/${branch}" + else + record_finding "legacy branch protection could not be read: ${organization}/${repository}/${branch}" + fi +} + audit_security() { local repository=$1 desired="${root}/config/code-security.json" setup="${root}/config/codeql-default-setup.json" local current languages @@ -413,6 +439,12 @@ for repository in "${!expected_active[@]}"; do rulesets='[]' fi branch=$(jq -r '.default_branch' "${repository_defaults}") + audit_legacy_protection "${repository}" "${branch}" + while IFS= read -r maintenance_branch; do + audit_legacy_protection "${repository}" "${maintenance_branch}" + done < <(jq -r --arg repository "${repository}" ' + .maintenance_branches[] | select(.repository == $repository) | .branch + ' "${repositories_config}") effective='[]' if ! effective=$(github_api --paginate --slurp \ "repos/${organization}/${repository}/rules/branches/${branch}?per_page=100" | jq -e 'add'); then diff --git a/bin/publish b/bin/publish index 4883b07..6c030a4 100755 --- a/bin/publish +++ b/bin/publish @@ -1150,6 +1150,21 @@ security_ruleset_matches() { ' <<<"${current}" >/dev/null } +security_ruleset_restoration_allowed() { + local repository=$1 id=$2 template=$3 current=$4 + jq -e --argjson id "${id}" --slurpfile desired "${template}" \ + --arg source "${organization}/${repository}" ' + .id == $id and .name == $desired[0].name and + .source_type == "Repository" and .source == $source and + (.enforcement | IN("active", "disabled", "evaluate")) + ' <<<"${current}" >/dev/null && + security_ruleset_matches "${template}" "$(jq --slurpfile desired "${template}" ' + .enforcement = $desired[0].enforcement | + .conditions.ref_name.include = $desired[0].conditions.ref_name.include | + .conditions.ref_name.exclude = $desired[0].conditions.ref_name.exclude + ' <<<"${current}")" +} + security_gate_applies_to_default_branch() { local repository=$1 kind=$2 id=$3 metadata branch effective metadata=$(github_api "repos/${organization}/${repository}") || return 1 @@ -1249,17 +1264,7 @@ publish_security_merge_rulesets() { current='{}' if [[ -n ${id} ]]; then current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") - if ! jq -e --argjson id "${id}" --arg name "${name}" \ - --arg source "${organization}/${repository}" ' - .id == $id and .name == $name and - .source_type == "Repository" and .source == $source and - (.enforcement | IN("active", "disabled", "evaluate")) - ' <<<"${current}" >/dev/null || - ! security_ruleset_matches "${template}" "$(jq --slurpfile desired "${template}" ' - .enforcement = $desired[0].enforcement | - .conditions.ref_name.include = $desired[0].conditions.ref_name.include | - .conditions.ref_name.exclude = $desired[0].conditions.ref_name.exclude - ' <<<"${current}")"; then + if ! security_ruleset_restoration_allowed "${repository}" "${id}" "${template}" "${current}"; then echo "error: security gate ownership or managed rule identity changed; retained: ${repository} ${name}" >&2 return 1 fi @@ -1294,6 +1299,20 @@ publish_security_merge_rulesets() { return 1 fi if [[ -n ${id} ]]; then + # Re-read after producer and repository checks: a concurrent policy edit + # must not be overwritten with the earlier restoration decision. + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || { + echo "error: security gate final read failed; retained: ${repository} ${name}" >&2 + return 1 + } + if ! security_ruleset_restoration_allowed "${repository}" "${id}" "${template}" "${current}"; then + echo "error: security gate changed during restoration preflight; retained: ${repository} ${name}" >&2 + return 1 + fi + if security_ruleset_matches "${template}" "${current}"; then + echo "KEEP /repos/${organization}/${repository}/rulesets/${id} ${name}" + continue + fi run_api PUT "repos/${organization}/${repository}/rulesets/${id}" "${template}" else run_api POST "repos/${organization}/${repository}/rulesets" "${template}" diff --git a/lib/security_readiness.py b/lib/security_readiness.py index 0981148..40a451e 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -813,7 +813,7 @@ def advanced_codeql_coverage(coordinate, branch, head, minima, selected, declara raise Deferred("required advanced CodeQL partition upload missing within bounded evidence") -def advanced_workflow_wiring(source): +def advanced_workflow_wiring(source, default_branch): """Check executable dataflow, not the honesty of repository-owned Python. The supported ordinary-job shape is deliberately bounded. Producer-owned @@ -944,6 +944,42 @@ def classifier(value, scoped=False): return path execution_settings(workflow, "workflow") + check(isinstance(default_branch, str) and bool(default_branch), "default branch missing") + events = workflow.get("on") + if isinstance(events, str): + events = {events: None} + elif isinstance(events, list): + check(all(isinstance(event, str) for event in events) + and len(events) == len(set(events)), "workflow events malformed") + events = dict.fromkeys(events) + check(isinstance(events, dict) and "pull_request" in events, + "pull_request trigger missing") + pull_request = events["pull_request"] + if pull_request is None: + pull_request = {} + # Do not partially interpret GitHub glob/exclusion or workflow path filters: + # they can leave a required producer pending. Path selection belongs in the + # trusted classifier after the workflow has started for the pull request. + check(isinstance(pull_request, dict) and set(pull_request) <= {"branches", "types"}, + "pull_request filters unsupported") + if "branches" in pull_request: + branches = pull_request["branches"] + check(isinstance(branches, list) and bool(branches) + and all(isinstance(branch, str) and branch + and not any(char in branch for char in "*?[]!+\\") for branch in branches) + and default_branch in branches, "pull_request does not cover the default branch") + activities = {"opened", "synchronize", "reopened"} + if "types" in pull_request: + # Values need an explicit enum: the Actions loader preserves YAML + # booleans as text, so an invalid `false` must not become an event name. + supported = activities | {"assigned", "unassigned", "labeled", "unlabeled", "edited", "closed", + "converted_to_draft", "locked", "unlocked", "enqueued", "dequeued", + "milestoned", "demilestoned", "ready_for_review", "review_requested", + "review_request_removed", "auto_merge_enabled", "auto_merge_disabled"} + types = pull_request["types"] + check(isinstance(types, list) and all(isinstance(value, str) and value in supported for value in types) + and len(types) == len(set(types)) and activities <= set(types), + "pull_request activity coverage incomplete") jobs = workflow.get("jobs") check(isinstance(jobs, dict) and set(jobs) == {"classify", "analyze", "codeql-validation"}, "requires ordinary classification, analysis and validation jobs") @@ -1021,7 +1057,7 @@ def advanced_codeql(coordinate, branch, head, minima, partition_minima): for path in (".github/workflows/codeql.yml", "tools/ci/classify_changes.py", "tools/ci/require_checks.py"): sources[path] = contents(coordinate, path, branch) require(contents(coordinate, path, head) == sources[path], "advanced CodeQL producer source is stale") - advanced_workflow_wiring(sources[".github/workflows/codeql.yml"]) + advanced_workflow_wiring(sources[".github/workflows/codeql.yml"], branch) declarations = classifier_partitions(sources["tools/ci/classify_changes.py"], partition_minima) events = classifier_full_events(sources["tools/ci/classify_changes.py"]) run = latest_full_codeql_run(coordinate, branch, head, events) diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index 6c1a186..411117a 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -24,6 +24,7 @@ def reject(message): if not args or args.pop(0) != 'api': reject('API only') slurp = False +include = False endpoint = None while args: arg = args.pop(0) @@ -36,6 +37,8 @@ def reject(message): pass elif arg == '--slurp': slurp = True + elif arg == '--include': + include = True elif arg.startswith('-'): reject(f'unexpected or mutating API flag: {arg}') else: @@ -236,6 +239,57 @@ def rulesets(repository): payload['default_workflow_permissions'] = 'write' if scenario == 'repository-approval' and target: payload['can_approve_pull_request_reviews'] = True + elif path.startswith('branches/') and path.endswith('/protection'): + if not include: + reject('legacy protection requires an observable HTTP status') + branch = path.removeprefix('branches/').removesuffix('/protection') + if branch not in ('main', 'release%2F2.x', 'release%2Fnext%23candidate'): + reject('unexpected or unencoded branch: ' + branch) + code, payload, result = 404, {'message': 'Branch not protected', 'status': '404'}, 1 + selected = target and branch == 'main' + if scenario.startswith('legacy-maintenance-'): + selected = target and branch == 'release%2Fnext%23candidate' + if selected: + variant = scenario.removeprefix('legacy-maintenance-').removeprefix('legacy-') + policies = { + 'extra-context': {'required_status_checks': {'checks': [ + {'context': 'Unconfigured check', 'app_id': 15368}]}}, + 'wrong-app': {'required_status_checks': {'checks': [ + {'context': 'Conventional PR title', 'app_id': 99}]}}, + 'approval': {'required_pull_request_reviews': {'required_approving_review_count': 1}}, + 'force-push': {'allow_force_pushes': {'enabled': True}}, + 'matching': {'required_status_checks': {'checks': [ + {'context': context, 'app_id': 15368} + for context in repositories['required_ci'][repository]]}}, + 'empty': {}, + } + if variant in policies: + code, payload, result = 200, policies[variant], 0 + elif variant == 'forbidden': + code, payload = 403, {'message': 'Resource not accessible by integration', 'status': '403'} + elif variant == 'not-found': + payload['message'] = 'Not Found' + elif variant == 'wrong-status': + payload['status'] = '403' + elif variant == 'mismatched-http': + code = 403 + elif variant == 'transport-error': + reject('connection failed') + elif variant == 'wrong-exit': + result = 2 + elif variant == 'success-error': + code, result = 200, 0 + elif variant == 'non-json': + payload = 'service unavailable' + elif variant == 'success-non-json': + code, payload, result = 200, 'service unavailable', 0 + elif variant == 'null': + payload = None + elif variant == 'array': + payload = [payload] + print(f'HTTP/2.0 {code} Fixture\r\nContent-Type: application/json\r\n\r') + print(payload if isinstance(payload, str) else json.dumps(payload)) + sys.exit(result) elif path == 'properties/values': payload = [{'property_name': key, 'value': value} for key, value in properties['repositories'][repository].items()] diff --git a/tests/audit.sh b/tests/audit.sh index da707e2..0b788f4 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -39,10 +39,19 @@ for repository in .github classic observatory content-toolkit service-updater we "${temporary}/full/config/repository-properties.json" >"${fixture}/config/repository-properties.json" done +cp -R "${temporary}/fixture-observatory" "${temporary}/fixture-legacy-branches" +jq '.maintenance_branches = [ + {repository: "observatory", branch: "release/2.x", required_ci: []}, + {repository: "observatory", branch: "release/next#candidate", required_ci: []}, + {repository: "classic", branch: "foreign/maintenance", required_ci: []} +]' "${temporary}/full/config/repositories.json" \ + >"${temporary}/fixture-legacy-branches/config/repositories.json" + run_audit() { local scenario=$1 fixture=${2:-focused} repository=observatory if [[ ${fixture} == focused ]]; then case ${scenario} in + legacy-maintenance-*|legacy-multiple) repository=legacy-branches ;; wrong-pr-scope|unexpected-codeql|tag-extra-nonrelease|tag-all-nonrelease|tag-repository-nonrelease|inherited-tag-unaffected|foreign-tag) repository=.github ;; advanced-config|advanced-setup|advanced-null-languages) repository=classic ;; missing-rust) repository=content-toolkit ;; @@ -120,6 +129,29 @@ assert_drift() { fi } +# Only the precise GitHub absence response is accepted; legacy policies are +# manual drift even when their contexts match the declared ruleset contract. +for scenario in extra-context wrong-app approval force-push matching empty success-error; do + assert_drift "legacy-${scenario}" 'unexpected legacy branch protection requires manual review: atrinik/observatory/main' +done +for scenario in forbidden not-found wrong-status mismatched-http transport-error wrong-exit non-json success-non-json null array; do + assert_drift "legacy-${scenario}" 'legacy branch protection could not be read: atrinik/observatory/main' +done +for scenario in extra-context approval; do + assert_drift "legacy-maintenance-${scenario}" 'unexpected legacy branch protection requires manual review: atrinik/observatory/release/next#candidate' +done +assert_drift legacy-maintenance-forbidden 'legacy branch protection could not be read: atrinik/observatory/release/next#candidate' +: >"${temporary}/api.log" +output=$(run_audit legacy-multiple) +grep -Fq 'Governance audit passed:' <<<"${output}" +for branch in main release%2F2.x release%2Fnext%23candidate; do + [[ $(grep -Fxc "repos/atrinik/observatory/branches/${branch}/protection" "${temporary}/api.log") == 1 ]] +done +if grep -Fq 'foreign' "${temporary}/api.log"; then + echo 'maintenance protection must remain scoped to its repository' >&2 + exit 1 +fi + assert_drift unmanaged 'active repository is not in the desired-state inventory' assert_drift merge-drift 'repository merge policy drift' assert_drift properties-drift 'repository custom-property drift' diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index 56c0057..49604c7 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -75,7 +75,12 @@ github_api() { '{method: "GET", endpoint: $endpoint}' >>"${api_log}" case ${endpoint} in repos/atrinik/content-toolkit) - printf '%s\n' "${repository_state}" + if [[ -n ${gate_metadata_filter} ]]; then + [[ ${gate_metadata_filter} != unreadable ]] || return 1 + jq "${gate_metadata_filter}" <<<"${repository_state}" + else + printf '%s\n' "${repository_state}" + fi ;; repos/atrinik/content-toolkit/rules/branches/main | repos/atrinik/content-toolkit/rules/branches/release%2Fcurrent) if [[ ${effective_state} == covering ]]; then @@ -118,7 +123,19 @@ github_api() { repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) local template=code-scanning [[ ${endpoint##*/} != 7 ]] || template=dependency-review - if [[ ${ruleset_state} == api-default-* ]]; then + if [[ ${ruleset_state} == restoration-race ]]; then + local filter=. + if [[ ${endpoint##*/} == "${gate_update_id}" ]] && + (( $(jq -s --arg endpoint "${endpoint}" '[.[] | select(.endpoint == $endpoint)] | length' "${api_log}") >= 2 )); then + [[ ${gate_update_filter} != unreadable ]] || return 1 + filter=${gate_update_filter} + fi + jq --argjson id "${endpoint##*/}" --argjson target "${gate_update_id}" ' + . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + if $id == $target then .enforcement = "disabled" else . end' \ + "${root}/config/rulesets/${template}.json" | jq "${filter}" + return + elif [[ ${ruleset_state} == api-default-* ]]; then jq --argjson id "${endpoint##*/}" --arg state "${ruleset_state}" ' . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | .bypass_actors[0].actor_id = null | @@ -245,6 +262,9 @@ reset_case() { : >"${readiness_log}" codeql_state='{"state":"configured","query_suite":"default","languages":["python","javascript-typescript"]}' ruleset_state=missing + gate_update_id=6 + gate_update_filter=. + gate_metadata_filter= effective_state=covering repository_state='{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' workflow_state='{"default_workflow_permissions":"write","can_approve_pull_request_reviews":true}' @@ -377,6 +397,89 @@ for policy_scope in organization repository; do done done +# Changes made after the initial GET cannot be erased by restoration PUT. +for policy_scope in organization repository; do + for gate_update_target in 6 7; do + common_filters=( + '.id = 999' '.source_type = "Organization"' '.source = "foreign/content-toolkit"' + '.name = "Renamed gate"' '.target = "tag"' '.enforcement = "unknown"' + '.bypass_actors[0].bypass_mode = "always"' '.bypass_actors = []' + '.rules[0].parameters.unknown_future_control = true' + '.conditions.ref_name.unknown_future_scope = true' '.rules += [{type:"deletion"}]' + 'unreadable' + ) + if [[ ${gate_update_target} == 6 ]]; then + common_filters+=( + '.rules[0].parameters.code_scanning_tools[0].tool = "Other tool"' + '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' + ) + else + common_filters+=( + '.rules[0].parameters.required_status_checks[0].context = "Unrelated context"' + '.rules[0].parameters.required_status_checks[0].integration_id = 9' + '.rules[0].parameters.do_not_enforce_on_create = true' + '.rules[0].parameters.do_not_enforce_on_create = null' + ) + fi + for filter in "${common_filters[@]}"; do + reset_case + ruleset_state=restoration-race + gate_update_id=${gate_update_target} + gate_update_filter=${filter} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + if source "${temporary}/dispatch.sh" >"${temporary}/concurrent-gate.txt" 2>&1; then + echo "expected concurrent gate change to be retained: ${filter}" >&2 + exit 1 + fi + assert_no_writes + grep -Fq 'retained:' "${temporary}/concurrent-gate.txt" + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${gate_update_id}" \ + '[.[] | select(.endpoint == $endpoint)] | length == 2' "${api_log}" >/dev/null + done + # Unchanged ownership, explicit API defaults, and allowed enforcement/ref + # restoration still converge; a concurrent completed repair causes KEEP. + for filter in '.' \ + '.bypass_actors[0].actor_id = null | if .rules[0].type == "required_status_checks" then .rules[0].parameters.do_not_enforce_on_create = false else . end' \ + '.enforcement = "evaluate" | .conditions.ref_name.include = ["refs/heads/other"]' \ + '.enforcement = "active"'; do + reset_case + ruleset_state=restoration-race + gate_update_id=${gate_update_target} + gate_update_filter=${filter} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/safe-concurrent-gate.txt" + if [[ ${filter} == '.enforcement = "active"' ]]; then + assert_no_writes + else + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${gate_update_id}" ' + [.[] | select(.method != "GET")] | length == 1 and + .[0].method == "PUT" and .[0].endpoint == $endpoint' "${api_log}" >/dev/null + fi + done + for filter in '.archived = true' '.id = 999' '.full_name = "foreign/content-toolkit"' unreadable; do + reset_case + ruleset_state=restoration-race + gate_update_id=${gate_update_target} + gate_metadata_filter=${filter} + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + if source "${temporary}/dispatch.sh" >"${temporary}/fresh-repository.txt" 2>&1; then + echo 'expected fresh repository identity/archive/read failure to block PUT' >&2 + exit 1 + fi + assert_no_writes + done + done +done + # The two gate phases are independent; qualification does not mutate source. reset_case export READINESS_STATUS=failed diff --git a/tests/security-readiness.py b/tests/security-readiness.py index fcaa5a1..9eaa924 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -12,7 +12,7 @@ import tempfile import unittest from unittest.mock import patch -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, quote, urlparse ROOT = Path(__file__).resolve().parent.parent @@ -278,7 +278,8 @@ def __call__(self, endpoint): return content(self.before if "ref=" + self.pr["base"]["sha"] in endpoint else self.after) if "/contents/" in endpoint: path = endpoint.split("/contents/")[1].split("?")[0] - sources = self.codeql_sources if "ref=main" in endpoint else self.codeql_head_sources + ref = parse_qs(urlparse(endpoint).query).get("ref", [None])[0] + sources = self.codeql_sources if ref == self.metadata["default_branch"] else self.codeql_head_sources return content(sources[path]) if "/actions/workflows/codeql.yml/runs" in endpoint: return {"workflow_runs": [copy.deepcopy(self.codeql_run)]} @@ -330,7 +331,7 @@ def __call__(self, endpoint): value = copy.deepcopy(self.suite) value["id"] = int(endpoint.rsplit("/", 1)[1]) return value - if "/branches/main" in endpoint: + if endpoint == "repos/atrinik/example/branches/" + quote(self.metadata["default_branch"], safe=""): return {"commit": {"sha": self.branch_head}} if "/code-scanning/analyses?" in endpoint: page = int(parse_qs(urlparse(endpoint).query)["page"][0]) @@ -1324,6 +1325,91 @@ def test_codeql_yaml_dependency_missing_fails_closed_without_affecting_other_mod self.assertIn("install requirements-validation.txt", message) self.assertEqual(self.invoke("dependency")[0], 0) + def test_codeql_advanced_full_run_cannot_replace_pull_request_trigger(self): + for event in ('push', 'schedule', 'workflow_dispatch'): + for triggers in (None, 'push', ['push', 'workflow_dispatch'], + {'push': {'branches': ['main']}, 'schedule': [{'cron': '0 0 * * *'}], + 'workflow_dispatch': None}, {'pull_request_target': None}): + with self.subTest(event=event, triggers=triggers): + self.enable_advanced() + self.fixture.codeql_run['event'] = event + document = self.codeql_document() + if triggers is None: + del document['on'] + else: + document['on'] = triggers + self.set_codeql_document(document) + self.assert_deferred('codeql', 'pull_request trigger missing') + + def test_codeql_advanced_pull_request_filters_cannot_skip_required_producer(self): + rejected = [False, [], 'main', {'branches': ['develop']}, + {'branches': []}, {'branches': 'main'}, {'branches': None}, + {'branches': ['main', '!main']}, {'branches': ['**']}, + {'branches': ['main', 3]}, {'branches-ignore': ['main']}, + {'branches-ignore': ['unrelated']}, {'paths': ['client/**']}, + {'paths': ['**']}, {'paths-ignore': ['docs/**']}, + {'branches': ['main'], 'paths-ignore': []}, + {'tags': ['v*']}, {'types': None}, {'types': 'opened'}, + {'types': []}, {'types': ['opened', 'reopened']}, + {'types': ['synchronize', 'reopened']}, {'types': ['opened', 'synchronize']}, + {'types': ['opened', 'synchronize', 'reopened', False]}, + {'types': ['opened', 'synchronize', 'reopened', 'invented']}, + {'types': ['opened', 'synchronize', 'reopened', 'opened']}] + for trigger in rejected: + with self.subTest(trigger=trigger): + self.enable_advanced() + document = self.codeql_document() + document['on']['pull_request'] = trigger + self.set_codeql_document(document) + self.assert_deferred('codeql', 'pull_request') + + def test_codeql_advanced_pull_request_event_forms_and_additional_types_ready(self): + for triggers in ('pull_request', ['push', 'pull_request', 'workflow_dispatch'], + {'pull_request': None}, {'pull_request': {}}, + {'pull_request': {'branches': ['main', 'maintenance']}}, + {'pull_request': {'types': ['opened', 'synchronize', 'reopened']}}, + {'pull_request': {'branches': ['main'], 'types': [ + 'ready_for_review', 'edited', 'reopened', 'opened', 'synchronize']}}): + with self.subTest(triggers=triggers): + self.enable_advanced() + document = self.codeql_document() + document['on'] = triggers + self.set_codeql_document(document) + result, message = self.invoke('codeql') + self.assertEqual(result, 0, message) + + def test_codeql_advanced_pull_request_uses_fresh_default_branch(self): + self.enable_advanced() + branch = 'release/current' + self.fixture.metadata['default_branch'] = branch + self.fixture.codeql_run['head_branch'] = branch + for analysis in self.fixture.analyses: + analysis['ref'] = 'refs/heads/' + branch + self.assert_deferred('codeql', 'pull_request does not cover the default branch') + document = self.codeql_document() + document['on']['pull_request']['branches'] = [branch] + document['on']['push']['branches'] = [branch] + self.set_codeql_document(document) + result, message = self.invoke('codeql') + self.assertEqual(result, 0, message) + self.assertIn('repos/atrinik/example/branches/release%2Fcurrent', self.fixture.calls) + + def test_codeql_advanced_default_branch_change_during_preflight_deferred(self): + self.enable_advanced() + reads = 0 + + def drift(endpoint): + nonlocal reads + result = self.fixture(endpoint) + if endpoint == 'repos/atrinik/example': + reads += 1 + if reads > 1: + result = dict(result, default_branch='release/current') + return result + + with patch.object(readiness, 'api', drift): + self.assert_deferred('codeql', 'repository default branch changed') + def test_codeql_advanced_comments_cannot_prove_executable_wiring(self): self.enable_advanced() source = b"name: CodeQL\njobs:\n fake:\n runs-on: ubuntu-24.04\n steps:\n - run: true\n" From 0c17b40a56389af8e80124ed827c1f27c32208d9 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 17:06:24 -0500 Subject: [PATCH 17/22] fix: verify legacy absence when REST protection is disabled --- AGENTS.md | 5 ++- CONTRIBUTING.md | 6 +++- README.md | 6 +++- bin/audit | 56 +++++++++++++++++++++++++++++++-- tests/audit-fixtures.py | 68 +++++++++++++++++++++++++++++++++++++---- tests/audit.sh | 13 ++++++++ 6 files changed, 142 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 18f4b02..f9f0998 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -268,4 +268,7 @@ protection on active default and declared maintenance branches; any presence is drift under the canonical ruleset contract. Only verified unprotected responses count as absent; unreadable evidence fails closed and audit never - removes protections. + removes protections. Only the exact repository-disabled REST response may + fall back to a fixed read-only GraphQL query with matching repository/branch + identity, active state and explicit null legacy-rule association. Missing or + conflicting evidence fails closed; preserve historical rules on other refs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c0d9aa..d4038e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -170,7 +170,11 @@ Security gate updates must re-read and validate ownership and the recognized payload immediately before PUT, preserving concurrent modifications. Audit legacy protection on active default and maintenance branches as drift under the canonical ruleset contract; distinguish verified unprotected responses from -unreadable evidence, and never remove protection during audit. +unreadable evidence, and never remove protection during audit. The exact +repository-disabled REST response may use the fixed read-only GraphQL fallback; +require matching repository identity and branch, active state and an explicit +null legacy-rule association. Errors and missing or conflicting fields fail +closed; preserve unrelated historical rules. Crates.io publication contracts bind exact repository/workflow/environment and official auth-action identities with full immutable workflow SHA references; diff --git a/README.md b/README.md index 0cfcaf2..cd5fab4 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,11 @@ Rulesets are the canonical branch protection contract. Audit also reads legacy protection on every active default branch and declared maintenance branch. Any legacy protection is drift requiring separate reviewed manual remediation; unreadable responses fail closed. Only a verified unprotected-branch response -counts as absent. The audit never removes branch protections. +counts as absent. For the exact repository-disabled REST response, a fixed +read-only GraphQL query may instead prove that the identified active repository +and exact existing branch have an explicitly null legacy-rule association. +Missing, conflicting or unreadable evidence remains drift. Historical rules on +other branches remain untouched. The audit never removes branch protections. Repository-scope retirement of an existing `03 - Required CI` rule also needs a reviewed `config/retired-required-ci.json` record with its exact repository and diff --git a/bin/audit b/bin/audit index d4273b3..a2a292c 100755 --- a/bin/audit +++ b/bin/audit @@ -315,7 +315,7 @@ check_ruleset() { # Legacy protection layers additional requirements outside the rules endpoint; # retain it for manual review rather than silently treating rulesets as complete. audit_legacy_protection() { - local repository=$1 branch=$2 encoded response status=0 body + local repository=$1 branch=$2 metadata=$3 encoded response status=0 body encoded=$(jq -rn --arg branch "${branch}" '$branch | @uri') response=$(github_api --include \ "repos/${organization}/${repository}/branches/${encoded}/protection" \ @@ -328,6 +328,15 @@ audit_legacy_protection() { <<<"${body}" >/dev/null 2>&1; then return fi + # Some repositories report disabled REST protection despite retaining legacy + # rules for other branches. Only an exact-ref GraphQL read can prove absence. + if [[ ${status} == 1 && ${response%%$'\n'*} =~ ^HTTP/[0-9.]+\ 404(\ |$) ]] && + jq -se 'length == 1 and (.[0] | type == "object" and + .message == "Branch protection has been disabled on this repository." and + .status == "404")' <<<"${body}" >/dev/null 2>&1; then + audit_disabled_legacy_protection "${repository}" "${branch}" "${metadata}" + return + fi if [[ ${status} == 0 && ${response%%$'\n'*} =~ ^HTTP/[0-9.]+\ 200(\ |$) ]] && jq -se 'length == 1 and (.[0] | type == "object")' \ <<<"${body}" >/dev/null 2>&1; then @@ -337,6 +346,47 @@ audit_legacy_protection() { fi } +audit_disabled_legacy_protection() { + local repository=$1 branch=$2 metadata=$3 repository_id response + # GraphQL variables are literal; gh supplies their values as separate fields. + # shellcheck disable=SC2016 + local query='query LegacyBranchProtection($owner: String!, $name: String!, $qualifiedName: String!) { repository(owner: $owner, name: $name) { databaseId nameWithOwner isArchived ref(qualifiedName: $qualifiedName) { name branchProtectionRule { id } } } }' + # Fence rename/transfer/recreation against both the fresh REST metadata and + # the enumeration used to select this active repository. + if ! repository_id=$(jq -er --arg repository "${repository}" \ + --arg coordinate "${organization}/${repository}" \ + --slurpfile inventory "${temporary}/inventory.json" ' + . as $metadata | + [$inventory[0][] | select(.name == $repository)] as $entries | + if ($entries | length) == 1 and $entries[0].id == .id and + $entries[0].archived == false and .archived == false and + .name == $repository and .full_name == $coordinate and + (.id | type == "number" and . > 0 and floor == .) + then .id else error("repository identity drift") end + ' "${metadata}" 2>/dev/null) || + ! response=$(gh api --method POST -H "X-GitHub-Api-Version: ${api_version}" \ + graphql -f "query=${query}" -f "owner=${organization}" \ + -f "name=${repository}" -f "qualifiedName=refs/heads/${branch}" \ + 2>"${temporary}/legacy-protection.err") || + ! jq -se --argjson repository_id "${repository_id}" \ + --arg coordinate "${organization}/${repository}" --arg branch "${branch}" ' + length == 1 and (.[0] | type == "object" and (has("errors") | not) and + (.data.repository | type == "object" and + .databaseId == $repository_id and .nameWithOwner == $coordinate and + .isArchived == false and + (.ref | type == "object" and .name == $branch and + has("branchProtectionRule") and + (.branchProtectionRule == null or + (.branchProtectionRule | type == "object" and + (.id | type == "string" and length > 0)))))) + ' <<<"${response}" >/dev/null 2>&1; then + record_finding "legacy branch protection could not be read: ${organization}/${repository}/${branch}" + elif ! jq -e '.data.repository.ref.branchProtectionRule == null' \ + <<<"${response}" >/dev/null; then + record_finding "unexpected legacy branch protection requires manual review: ${organization}/${repository}/${branch}" + fi +} + audit_security() { local repository=$1 desired="${root}/config/code-security.json" setup="${root}/config/codeql-default-setup.json" local current languages @@ -439,9 +489,9 @@ for repository in "${!expected_active[@]}"; do rulesets='[]' fi branch=$(jq -r '.default_branch' "${repository_defaults}") - audit_legacy_protection "${repository}" "${branch}" + audit_legacy_protection "${repository}" "${branch}" "${metadata_file}" while IFS= read -r maintenance_branch; do - audit_legacy_protection "${repository}" "${maintenance_branch}" + audit_legacy_protection "${repository}" "${maintenance_branch}" "${metadata_file}" done < <(jq -r --arg repository "${repository}" ' .maintenance_branches[] | select(.repository == $repository) | .branch ' "${repositories_config}") diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index 411117a..0c4dfc8 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -26,11 +26,12 @@ def reject(message): slurp = False include = False endpoint = None +method = None +fields = {} while args: arg = args.pop(0) if arg == '--method': - if args.pop(0) != 'GET': - reject('audit attempted API mutation') + method = args.pop(0) elif arg == '-H': args.pop(0) elif arg == '--paginate': @@ -39,12 +40,27 @@ def reject(message): slurp = True elif arg == '--include': include = True + elif arg == '-f': + key, separator, value = args.pop(0).partition('=') + if not separator or key in fields: + reject('invalid or duplicated GraphQL variable') + fields[key] = value elif arg.startswith('-'): reject(f'unexpected or mutating API flag: {arg}') else: endpoint = arg if endpoint is None: reject('endpoint missing') +query = 'query LegacyBranchProtection($owner: String!, $name: String!, $qualifiedName: String!) { repository(owner: $owner, name: $name) { databaseId nameWithOwner isArchived ref(qualifiedName: $qualifiedName) { name branchProtectionRule { id } } } }' +if endpoint == 'graphql': + if (method != 'POST' or set(fields) != {'query', 'owner', 'name', 'qualifiedName'} + or fields['query'] != query or fields['owner'] != 'atrinik' + or fields['name'] != 'observatory' + or fields['qualifiedName'] not in ('refs/heads/main', 'refs/heads/release/next#candidate') + or not scenario.startswith(('legacy-disabled-', 'legacy-maintenance-disabled-'))): + reject('unexpected or mutating GraphQL request') +elif method != 'GET' or fields: + reject('audit attempted API mutation') with open(os.environ['AUDIT_API_LOG'], 'a', encoding='utf-8') as log: log.write(endpoint + '\n') @@ -182,13 +198,49 @@ def rulesets(repository): payload = None -if endpoint.startswith('orgs/atrinik/repos?'): +if endpoint == 'graphql': + variant = scenario.removeprefix('legacy-maintenance-').removeprefix('legacy-').removeprefix('disabled-') + repository = {'databaseId': active.index('observatory') + 1, 'nameWithOwner': 'atrinik/observatory', + 'isArchived': False, 'ref': {'name': fields['qualifiedName'].removeprefix('refs/heads/'), + 'branchProtectionRule': None}} + payload = {'data': {'repository': repository}} + if variant == 'present': + repository['ref']['branchProtectionRule'] = {'id': 'BPR_fixture'} + elif variant == 'wrong-id': + repository['databaseId'] += 100 + elif variant == 'wrong-owner': + repository['nameWithOwner'] = 'foreign/observatory' + elif variant == 'wrong-name': + repository['ref']['name'] = 'master' + elif variant == 'archived': + repository['isArchived'] = True + elif variant == 'missing-field': + del repository['ref']['branchProtectionRule'] + elif variant == 'null-ref': + repository['ref'] = None + elif variant == 'null-repository': + payload['data']['repository'] = None + elif variant == 'missing-data': + del payload['data'] + elif variant == 'malformed-rule': + repository['ref']['branchProtectionRule'] = False + elif variant == 'errors': + payload['errors'] = [{'message': 'read denied'}] + elif variant == 'unreadable': + reject('GraphQL request denied') + elif variant == 'non-json': + print('unavailable') + sys.exit(0) +elif endpoint.startswith('orgs/atrinik/repos?'): if scenario == 'inventory-unreadable': reject('inventory read denied') - payload = [dict(name=name, archived=value['lifecycle'] == 'archived', private=False, + payload = [dict(name=name, id=active.index(name) + 1 if name in active else 1000, + archived=value['lifecycle'] == 'archived', private=False, visibility='public', disabled=False, fork=False, has_issues=True, has_projects=True, has_wiki=True) for name, value in properties['repositories'].items()] + if scenario == 'legacy-disabled-inventory-id': + next(item for item in payload if item['name'] == 'observatory')['id'] += 100 if scenario == 'unmanaged': payload.append(dict(name='unmanaged', archived=False, private=False, visibility='public', disabled=False, fork=False, has_issues=True, has_projects=True, has_wiki=True)) @@ -263,7 +315,9 @@ def rulesets(repository): for context in repositories['required_ci'][repository]]}}, 'empty': {}, } - if variant in policies: + if variant.startswith('disabled-'): + payload['message'] = 'Branch protection has been disabled on this repository.' + elif variant in policies: code, payload, result = 200, policies[variant], 0 elif variant == 'forbidden': code, payload = 403, {'message': 'Resource not accessible by integration', 'status': '403'} @@ -386,10 +440,12 @@ def rulesets(repository): payload = {'enabled': not (scenario == 'private-reporting' and target)} elif path == '': payload = dict(config('repository-defaults.json'), id=active.index(repository) + 1, - name=repository, archived=False, security_and_analysis={ + name=repository, full_name='atrinik/' + repository, archived=False, security_and_analysis={ name: {'status': 'enabled'} for name in ( 'secret_scanning', 'secret_scanning_push_protection', 'secret_scanning_validity_checks', 'dependabot_security_updates')}) + if scenario == 'legacy-disabled-metadata-name' and target: + payload['full_name'] = 'foreign/observatory' if scenario == 'merge-drift' and target: payload['allow_merge_commit'] = True if scenario == 'secret-baseline' and target: diff --git a/tests/audit.sh b/tests/audit.sh index 0b788f4..49c380d 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -152,6 +152,19 @@ if grep -Fq 'foreign' "${temporary}/api.log"; then exit 1 fi +# GitHub's disabled-protection response requires independent exact-ref proof; +# historical rules on another branch must neither create drift nor prove absence. +for scenario in legacy-disabled-absent legacy-maintenance-disabled-absent; do + output=$(run_audit "${scenario}") + grep -Fq 'Governance audit passed:' <<<"${output}" +done +assert_drift legacy-disabled-present 'unexpected legacy branch protection requires manual review: atrinik/observatory/main' +assert_drift legacy-maintenance-disabled-present 'unexpected legacy branch protection requires manual review: atrinik/observatory/release/next#candidate' +for scenario in wrong-id wrong-owner wrong-name archived missing-field null-ref null-repository missing-data malformed-rule errors unreadable non-json inventory-id metadata-name; do + assert_drift "legacy-disabled-${scenario}" 'legacy branch protection could not be read: atrinik/observatory/main' +done +assert_drift legacy-maintenance-disabled-wrong-name 'legacy branch protection could not be read: atrinik/observatory/release/next#candidate' + assert_drift unmanaged 'active repository is not in the desired-state inventory' assert_drift merge-drift 'repository merge policy drift' assert_drift properties-drift 'repository custom-property drift' From 8870bef13c635753797731d1a67182214a8cc944 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 17:32:19 -0500 Subject: [PATCH 18/22] fix(security): fence retirement and audit retained gates --- AGENTS.md | 5 +- CONTRIBUTING.md | 4 ++ README.md | 6 ++- bin/audit | 23 ++++++++- bin/publish | 34 +++++++++---- tests/audit-fixtures.py | 45 ++++++++++++++++- tests/audit.sh | 33 +++++++++++++ tests/publish-security-merge.sh | 86 +++++++++++++++++++++++++++++++-- 8 files changed, 217 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f9f0998..3c4e039 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -255,7 +255,10 @@ tool/context. Never infer activation from target membership or silently change source intent. Retire target and activation entries together. Retired targets lose only exact repository-owned managed security rules; - preserve archived, foreign, and modified same-name rules. Keep repository + preserve archived, foreign, and modified same-name rules. Re-read and validate + the exact rule payload after the repository metadata fence immediately before + deletion. Audit retained named security rules on non-target repositories as + drift even when disabled, modified or scoped to another ref. Keep repository workflow overrides read-only and unable to approve PRs, with delta-aware writes. - Repository-scope `03 - Required CI` retirement requires reviewed exact prior repository/ruleset IDs and contexts in `config/retired-required-ci.json`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4038e9..877e3d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -140,6 +140,10 @@ first live installation. Pending publication only probes readiness. Enforced intent survives missing rules and later producer failures, allowing restoration without fresh initial proof; it does not claim live installation success. Remove activation entries with retired targets, preserving exact owned-rule rollback. +Re-read the exact security-rule payload after the repository metadata fence +immediately before deletion, retaining concurrent modifications. Audit retained +named security rules on non-target repositories as drift regardless of their +enforcement or ref applicability. For an existing repository-owned `03 - Required CI` rule, also record exact repository and ruleset IDs and its prior contexts in `config/retired-required-ci.json`. Validate retained/modified/foreign rules and fresh identity checks; keep records diff --git a/README.md b/README.md index cd5fab4..bfccc75 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,11 @@ Rollout is owned by an organization administrator: Retiring a target removes its required-CI, security target, and activation entries together through source review. The publisher removes only matching repository-owned security rules from active retired targets, preserving foreign or modified same-name - rules for explicit review and leaving archived repositories untouched. + rules for explicit review and leaving archived repositories untouched. It re-reads + the exact rule payload after the repository metadata fence immediately before + deletion to retain concurrent modifications. Audit reports retained named + security rules on non-target repositories as drift even when disabled, + modified or scoped to another ref. 4. Desired organization `sha_pinning_required` is true. Its durable rollout contract is `config/actions-sha-pinning.json`: the initial `pending` phase has no accepted canary and never enables the field. Verify an ordinary diff --git a/bin/audit b/bin/audit index a2a292c..ce35b7c 100755 --- a/bin/audit +++ b/bin/audit @@ -484,10 +484,31 @@ for repository in "${!expected_active[@]}"; do rulesets='[]' if ! rulesets=$(github_api --paginate --slurp \ - "repos/${organization}/${repository}/rulesets?includes_parents=true&per_page=100" | jq -e 'add'); then + "repos/${organization}/${repository}/rulesets?includes_parents=true&per_page=100" | jq -e ' + if type == "array" and length > 0 and all(.[]; type == "array") then add + else error("invalid ruleset pages") end | + if type == "array" and all(.[]; + type == "object" and (.name | type == "string" and length > 0)) then . + else error("invalid ruleset inventory") end'); then record_finding "inherited rulesets could not be read: ${organization}/${repository}" rulesets='[]' fi + # Retired/non-target security gates must be absent from the listing, even + # when disabled or scoped away from the default branch. Presence is manual + # drift regardless of ownership or payload; this audit never removes a rule. + for security_kind in dependency_review code_scanning; do + case ${security_kind} in + dependency_review) security_name='07 - Dependency review' ;; + code_scanning) security_name='06 - CodeQL security' ;; + esac + if ! jq -e --arg repository "${repository}" --arg kind "${security_kind}" \ + '.[$kind] | index($repository) != null' \ + "${root}/config/security-merge-protection.json" >/dev/null && + jq -e --arg name "${security_name}" 'any(.[]; .name == $name)' \ + <<<"${rulesets}" >/dev/null; then + record_finding "unexpected retained security ruleset '${security_name}' requires manual review: ${organization}/${repository}" + fi + done branch=$(jq -r '.default_branch' "${repository_defaults}") audit_legacy_protection "${repository}" "${branch}" "${metadata_file}" while IFS= read -r maintenance_branch; do diff --git a/bin/publish b/bin/publish index 6c030a4..d8ade75 100755 --- a/bin/publish +++ b/bin/publish @@ -1187,6 +1187,13 @@ security_gate_applies_to_default_branch() { ' <<<"${effective}" >/dev/null } +owned_security_ruleset_matches() { + local repository=$1 id=$2 template=$3 current=$4 + jq -e --argjson id "${id}" --arg source "${organization}/${repository}" ' + .id == $id and .source_type == "Repository" and .source == $source + ' <<<"${current}" >/dev/null && security_ruleset_matches "${template}" "${current}" +} + delete_removed_security_merge_rulesets() { local repository kind template name existing rule id current metadata for repository in "${active_repositories[@]}"; do @@ -1204,18 +1211,25 @@ delete_removed_security_merge_rulesets() { template=${root}/config/rulesets/dependency-review.json fi name=$(jq -r '.name' "${template}") - existing=$(github_api "repos/${organization}/${repository}/rulesets?includes_parents=false") + existing=$(github_api "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 while IFS= read -r rule; do id=$(jq -r '.id' <<<"${rule}") - current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") - if jq -e --argjson id "${id}" --arg source "${organization}/${repository}" ' - .id == $id and .source_type == "Repository" and .source == $source - ' <<<"${current}" >/dev/null && security_ruleset_matches "${template}" "${current}"; then - metadata=$(github_api "repos/${organization}/${repository}") - if jq -e --argjson id "${active_repository_ids["${repository}"]}" ' - .id == $id and .archived == false + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || return 1 + if owned_security_ruleset_matches "${repository}" "${id}" "${template}" "${current}"; then + metadata=$(github_api "repos/${organization}/${repository}") || return 1 + if jq -e --argjson id "${active_repository_ids["${repository}"]}" \ + --arg full_name "${organization}/${repository}" ' + .id == $id and .full_name == $full_name and .archived == false ' <<<"${metadata}" >/dev/null; then - run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || { + echo "error: security gate final retirement read failed; retained: ${repository} ${name}" >&2 + return 1 + } + if ! owned_security_ruleset_matches "${repository}" "${id}" "${template}" "${current}"; then + echo "SKIP ${organization}/${repository} ${name}: rule changed during retirement preflight; retained" + continue + fi + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" || return 1 else echo "SKIP ${organization}/${repository} ${name}: repository identity or archive state changed; retained" fi @@ -1231,7 +1245,7 @@ delete_removed_security_merge_rulesets() { # GitHub Team does not support organization-level code-scanning rules. publish_security_merge_rulesets() { local repository kind template name existing id current reason status readiness_mode phase metadata - delete_removed_security_merge_rulesets + delete_removed_security_merge_rulesets || return 1 for kind in code_scanning dependency_review; do if [[ ${kind} == code_scanning ]]; then template=${root}/config/rulesets/code-scanning.json diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index 0c4dfc8..df66e19 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -86,6 +86,14 @@ def rulesets(repository): specs.append(('dependency-review', None)) if scenario == 'foreign-tag': specs.append(('immutable-release-tags', 'Foreign tag retention')) + retained_template = None + if scenario.startswith(('retained-', 'retired-')): + kind = scenario.split('-')[1] + retained_template = {'codeql': 'code-scanning', 'dependency': 'dependency-review'}.get(kind) + if retained_template: + specs.append((retained_template, None)) + if scenario.endswith('-duplicate'): + specs.append((retained_template, None)) result = [] for index, (template, name) in enumerate(specs, 1): item = config('rulesets/' + template + '.json') @@ -193,6 +201,19 @@ def rulesets(repository): item['rules'][0]['parameters']['code_scanning_tools'][0]['tool'] = 'other-scanner' if scenario == 'codeql-quality' and repository == 'observatory' and template == 'code-scanning': item['rules'][0]['parameters']['code_scanning_tools'][0]['alerts_threshold'] = 'errors' + if template == retained_template: + if scenario.endswith(('-disabled', '-foreign-name', '-duplicate')): + item['enforcement'] = 'disabled' + if scenario.endswith('-moved'): + item['conditions']['ref_name']['include'] = ['refs/heads/other'] + if scenario.endswith('-modified'): + item['rules'] = [{'type': 'creation'}] + item['enforcement'] = 'disabled' + if scenario.endswith('-foreign-owner'): + item['source_type'] = 'Organization' + item['enforcement'] = 'disabled' + if scenario.endswith('-foreign-name'): + item['name'] = 'Foreign security policy' result.append(item) return result @@ -312,7 +333,7 @@ def rulesets(repository): 'force-push': {'allow_force_pushes': {'enabled': True}}, 'matching': {'required_status_checks': {'checks': [ {'context': context, 'app_id': 15368} - for context in repositories['required_ci'][repository]]}}, + for context in repositories['required_ci'].get(repository, [])]}}, 'empty': {}, } if variant.startswith('disabled-'): @@ -350,8 +371,22 @@ def rulesets(repository): if scenario == 'properties-drift' and target: payload[0]['value'] = 'drift' elif path.startswith('rulesets?'): + if scenario == 'retained-list-unreadable': + reject('ruleset list read denied') + if scenario == 'retained-list-non-json': + print('unavailable') + sys.exit(0) payload = [{'name': item['name'], 'id': item['id'], 'source_type': item['source_type']} for item in rulesets(repository)] + if scenario == 'retained-list-object': + payload = {'message': 'unreadable'} + elif scenario == 'retained-list-missing-name': + payload.append({'id': 999, 'source_type': 'Repository'}) + elif scenario == 'retained-list-null-item': + payload.append(None) + elif scenario == 'retained-list-partial-page': + print(json.dumps([payload, None])) + sys.exit(0) elif path.startswith('rulesets/'): identifier = int(path.rsplit('/', 1)[1]) payload = next((item for item in rulesets(repository) if item['id'] == identifier), None) @@ -365,7 +400,13 @@ def rulesets(repository): scope = item['conditions'].get('repository_name', {'include': [repository], 'exclude': []}) included = any(pattern == '~ALL' or fnmatch.fnmatchcase(repository, pattern) for pattern in scope['include']) excluded = any(pattern == '~ALL' or fnmatch.fnmatchcase(repository, pattern) for pattern in scope['exclude']) - if item['target'] == 'branch' and included and not excluded: + refs = item['conditions']['ref_name'] + branch_included = any(pattern == '~DEFAULT_BRANCH' or + fnmatch.fnmatchcase('refs/heads/main', pattern) for pattern in refs['include']) + branch_excluded = any(pattern == '~DEFAULT_BRANCH' or + fnmatch.fnmatchcase('refs/heads/main', pattern) for pattern in refs['exclude']) + if (item['target'] == 'branch' and item['enforcement'] == 'active' + and included and not excluded and branch_included and not branch_excluded): payload.extend(copy.deepcopy(item['rules'])) for rule in payload: if target and rule['type'] == 'required_status_checks': diff --git a/tests/audit.sh b/tests/audit.sh index 49c380d..a86917c 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -47,10 +47,16 @@ jq '.maintenance_branches = [ ]' "${temporary}/full/config/repositories.json" \ >"${temporary}/fixture-legacy-branches/config/repositories.json" +cp -R "${temporary}/fixture-observatory" "${temporary}/fixture-retired-observatory" +jq 'del(.required_ci.observatory)' "${temporary}/full/config/repositories.json" \ + >"${temporary}/fixture-retired-observatory/config/repositories.json" + run_audit() { local scenario=$1 fixture=${2:-focused} repository=observatory if [[ ${fixture} == focused ]]; then case ${scenario} in + retained-*) repository=.github ;; + retired-*) repository=retired-observatory ;; legacy-maintenance-*|legacy-multiple) repository=legacy-branches ;; wrong-pr-scope|unexpected-codeql|tag-extra-nonrelease|tag-all-nonrelease|tag-repository-nonrelease|inherited-tag-unaffected|foreign-tag) repository=.github ;; advanced-config|advanced-setup|advanced-null-languages) repository=classic ;; @@ -75,6 +81,12 @@ run_audit() { esac cp "${root}/config/security-merge-protection.json" "${temporary}/${fixture}/config/security-merge-protection.json" case ${scenario} in + retired-*) + jq '.dependency_review -= ["observatory"] | .code_scanning -= ["observatory"] | + del(.activation.dependency_review.observatory, .activation.code_scanning.observatory)' \ + "${temporary}/${fixture}/config/security-merge-protection.json" >"${temporary}/activation.json" + mv "${temporary}/activation.json" "${temporary}/${fixture}/config/security-merge-protection.json" + ;; enforced-missing-dependency|enforced-unreadable-dependency) jq '.activation.dependency_review.observatory = "enforced"' \ "${temporary}/${fixture}/config/security-merge-protection.json" >"${temporary}/activation.json" @@ -165,6 +177,27 @@ for scenario in wrong-id wrong-owner wrong-name archived missing-field null-ref done assert_drift legacy-maintenance-disabled-wrong-name 'legacy branch protection could not be read: atrinik/observatory/release/next#candidate' +# Retained named rules remain drift after target removal, regardless of whether +# their current rules apply to main. Unrelated names remain outside ownership. +for scope in retained retired; do + for kind in codeql dependency; do + case ${kind} in + codeql) name='06 - CodeQL security' ;; + dependency) name='07 - Dependency review' ;; + esac + for variant in disabled moved modified duplicate foreign-owner; do + assert_drift "${scope}-${kind}-${variant}" "unexpected retained security ruleset '${name}' requires manual review" + done + output=$(run_audit "${scope}-${kind}-foreign-name") + grep -Fq 'Governance audit passed:' <<<"${output}" + done + output=$(run_audit "${scope}-absent") + grep -Fq 'Governance audit passed:' <<<"${output}" +done +for variant in unreadable non-json object missing-name null-item partial-page; do + assert_drift "retained-list-${variant}" 'inherited rulesets could not be read' +done + assert_drift unmanaged 'active repository is not in the desired-state inventory' assert_drift merge-drift 'repository merge policy drift' assert_drift properties-drift 'repository custom-property drift' diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index 49604c7..db3422b 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -123,16 +123,16 @@ github_api() { repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) local template=code-scanning [[ ${endpoint##*/} != 7 ]] || template=dependency-review - if [[ ${ruleset_state} == restoration-race ]]; then + if [[ ${ruleset_state} == restoration-race || ${ruleset_state} == retirement-race ]]; then local filter=. if [[ ${endpoint##*/} == "${gate_update_id}" ]] && (( $(jq -s --arg endpoint "${endpoint}" '[.[] | select(.endpoint == $endpoint)] | length' "${api_log}") >= 2 )); then [[ ${gate_update_filter} != unreadable ]] || return 1 filter=${gate_update_filter} fi - jq --argjson id "${endpoint##*/}" --argjson target "${gate_update_id}" ' + jq --argjson id "${endpoint##*/}" --argjson target "${gate_update_id}" --arg state "${ruleset_state}" ' . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | - if $id == $target then .enforcement = "disabled" else . end' \ + if $id == $target and $state == "restoration-race" then .enforcement = "disabled" else . end' \ "${root}/config/rulesets/${template}.json" | jq "${filter}" return elif [[ ${ruleset_state} == api-default-* ]]; then @@ -644,8 +644,72 @@ for policy_scope in organization repository; do done done +# The final retirement GET must still prove exact managed ownership/payload +# after the repository fence, including for changes that PUT may restore. +for policy_scope in organization repository; do + for retired_gate_id in 6 7; do + retired_kind=code_scanning + [[ ${retired_gate_id} != 7 ]] || retired_kind=dependency_review + retirement_filters=( + '.id = 999' '.source_type = "Organization"' '.source = "foreign/content-toolkit"' + '.name = "Renamed gate"' '.enforcement = "disabled"' '.enforcement = "evaluate"' + '.conditions.ref_name.include = ["refs/heads/other"]' + '.conditions.ref_name.exclude = ["~DEFAULT_BRANCH"]' + '.bypass_actors[0].bypass_mode = "always"' '.bypass_actors = []' + '.rules[0].parameters.unknown_future_control = true' '.rules += [{type:"deletion"}]' + 'unreadable' + ) + if [[ ${retired_gate_id} == 6 ]]; then + retirement_filters+=( + '.rules[0].parameters.code_scanning_tools[0].tool = "Other tool"' + '.rules[0].parameters.code_scanning_tools[0].security_alerts_threshold = "critical"' + ) + else + retirement_filters+=( + '.rules[0].parameters.required_status_checks[0].context = "Changed context"' + '.rules[0].parameters.required_status_checks[0].integration_id = 9' + '.rules[0].parameters.do_not_enforce_on_create = true' + '.rules[0].parameters.do_not_enforce_on_create = null' + ) + fi + for filter in "${retirement_filters[@]}" '.' \ + '.bypass_actors[0].actor_id = null | if .rules[0].type == "required_status_checks" then .rules[0].parameters.do_not_enforce_on_create = false else . end'; do + reset_case + ruleset_state=retirement-race + gate_update_id=${retired_gate_id} + gate_update_filter=${filter} + jq --arg kind "${retired_kind}" '.[$kind] -= ["content-toolkit"] | + del(.activation[$kind]["content-toolkit"])' "${security_merge_config}" >"${temporary}/removed.json" + cp "${temporary}/removed.json" "${security_merge_config}" + status=0 + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/retirement-race.txt" 2>&1 || status=$? + if [[ ${filter} == unreadable ]]; then + [[ ${status} != 0 ]] + assert_no_writes + grep -Fq 'final retirement read failed; retained' "${temporary}/retirement-race.txt" + elif [[ ${filter} == '.' || ${filter} == '.bypass_actors[0].actor_id = null |'* ]]; then + [[ ${status} == 0 ]] + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${retired_gate_id}" ' + [.[] | select(.method != "GET")] == [{method:"DELETE",endpoint:$endpoint}]' "${api_log}" >/dev/null + else + [[ ${status} == 0 ]] + assert_no_writes + grep -Fq 'rule changed during retirement preflight; retained' "${temporary}/retirement-race.txt" + fi + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${retired_gate_id}" ' + [to_entries[] | select(.value.endpoint == $endpoint and .value.method == "GET") | .key] as $reads | + ($reads | length) == 2 and + any(to_entries[]; .key > $reads[0] and .key < $reads[1] and + .value.endpoint == "repos/atrinik/content-toolkit")' "${api_log}" >/dev/null + [[ ! -s ${readiness_log} ]] + done + done +done + # Retirements also fence a repository archived or replaced after enumeration. -for retired_state in '{"id":123,"archived":true}' '{"id":999,"archived":false}'; do +for retired_state in '{"id":123,"archived":true}' '{"id":999,"archived":false}' \ + '{"id":123,"full_name":"foreign/content-toolkit","archived":false}'; do reset_case repository_state=${retired_state} ruleset_state="removal-owned" @@ -657,6 +721,20 @@ for retired_state in '{"id":123,"archived":true}' '{"id":999,"archived":false}'; [[ ! -s ${readiness_log} ]] done +# Read errors must propagate even when publication is called conditionally. +reset_case +ruleset_state="removal-owned" +gate_metadata_filter=unreadable +jq '.code_scanning -= ["content-toolkit"] | .dependency_review -= ["content-toolkit"] | + del(.activation.code_scanning["content-toolkit"], .activation.dependency_review["content-toolkit"])' \ + "${security_merge_config}" >"${temporary}/removed.json" +cp "${temporary}/removed.json" "${security_merge_config}" +if publish_security_merge_rulesets >"${temporary}/retirement-metadata-unreadable.txt" 2>&1; then + echo 'expected unreadable retirement repository metadata to fail closed' >&2 + exit 1 +fi +assert_no_writes + # An archived or missing repository is absent from the active identity map. reset_case active_repository_ids=() From feb5fe64734c8f1ed3d908e0a975e406af49466d Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 17:57:49 -0500 Subject: [PATCH 19/22] fix(security): bind pending gates to the proven default branch --- AGENTS.md | 4 +- CONTRIBUTING.md | 4 +- README.md | 3 ++ bin/publish | 13 +++++-- tests/publish-security-merge.sh | 67 ++++++++++++++++++++++++++++++++- 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b5502ee..d17876e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -272,7 +272,9 @@ rules with actionable `SKIP`; retain records for idempotent reruns. - Before updating an existing security gate, re-read and revalidate its identity - and recognized payload, retaining concurrent modifications. Audit full legacy + and recognized payload, retaining concurrent modifications. Bind the final + metadata default branch to the branch used for pending-gate qualification + or existing effective-gate proof; changed branches must not activate a gate. Audit full legacy protection on active default and declared maintenance branches; any presence is drift under the canonical ruleset contract. Only verified unprotected responses count as absent; unreadable evidence fails closed and audit never diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 877e3d8..6e0feac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -171,7 +171,9 @@ semantics remain in trusted producer helpers; do not add a central source-path o action-version catalog or claim resistance to malicious repository writers. Security gate updates must re-read and validate ownership and the recognized -payload immediately before PUT, preserving concurrent modifications. Audit +payload immediately before PUT, preserving concurrent modifications. Bind the +final metadata default branch to the branch used for pending-gate qualification +or existing effective-gate proof; a changed branch must not activate a gate. Audit legacy protection on active default and maintenance branches as drift under the canonical ruleset contract; distinguish verified unprotected responses from unreadable evidence, and never remove protection during audit. The exact diff --git a/README.md b/README.md index 28e4b18..1a726ba 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,9 @@ without workflow path filters. Successful manual or scheduled analysis cannot qualify a missing or retargeted pull-request producer. Before updating an existing security gate, publication rechecks its ownership and recognized payload immediately before the write, retaining concurrent policy changes. +Pending-gate qualification and existing effective-gate proof bind the same +default branch as the final metadata fence; a changed branch cannot activate +a gate using proof from the earlier branch. Rulesets are the canonical branch protection contract. Audit also reads legacy protection on every active default branch and declared maintenance branch. Any diff --git a/bin/publish b/bin/publish index d8ade75..54bff52 100755 --- a/bin/publish +++ b/bin/publish @@ -1184,7 +1184,8 @@ security_gate_applies_to_default_branch() { any(.parameters.required_status_checks[]?; .context == "Dependency review" and .integration_id == 15368) end) - ' <<<"${effective}" >/dev/null + ' <<<"${effective}" >/dev/null || return 1 + printf '%s\n' "${branch}" } owned_security_ruleset_matches() { @@ -1244,7 +1245,7 @@ delete_removed_security_merge_rulesets() { # These repository rules remain repository-owned even in organization scope: # GitHub Team does not support organization-level code-scanning rules. publish_security_merge_rulesets() { - local repository kind template name existing id current reason status readiness_mode phase metadata + local repository kind template name existing id current reason status readiness_mode phase metadata effective_branch delete_removed_security_merge_rulesets || return 1 for kind in code_scanning dependency_review; do if [[ ${kind} == code_scanning ]]; then @@ -1289,8 +1290,9 @@ publish_security_merge_rulesets() { fi # Pending qualification cannot create protection before reviewed intent is # persisted. Existing effective gates still survive producer failures. + effective_branch= if [[ ${phase} == pending ]] && { [[ -z ${id} ]] || - ! security_gate_applies_to_default_branch "${repository}" "${kind}" "${id}"; }; then + ! effective_branch=$(security_gate_applies_to_default_branch "${repository}" "${kind}" "${id}"); }; then status=0 reason=$("${root}/bin/security-readiness" "${readiness_mode}" \ "${organization}/${repository}") || status=$? @@ -1312,6 +1314,11 @@ publish_security_merge_rulesets() { echo "error: security gate repository identity or archive state changed: ${repository}" >&2 return 1 fi + if [[ ${phase} == pending ]] && + [[ $(jq -r '.default_branch | @uri' <<<"${metadata}") != "${effective_branch}" ]]; then + echo "error: default branch changed after existing security gate proof; retained: ${repository} ${name}" >&2 + return 1 + fi if [[ -n ${id} ]]; then # Re-read after producer and repository checks: a concurrent policy edit # must not be overwritten with the earlier restoration decision. diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index db3422b..3211b0e 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -75,7 +75,11 @@ github_api() { '{method: "GET", endpoint: $endpoint}' >>"${api_log}" case ${endpoint} in repos/atrinik/content-toolkit) - if [[ -n ${gate_metadata_filter} ]]; then + if [[ -n ${gate_metadata_after_proof_filter} ]] && + jq -se 'any(.[]; .endpoint | contains("/rules/branches/"))' "${api_log}" >/dev/null; then + [[ ${gate_metadata_after_proof_filter} != unreadable ]] || return 1 + jq "${gate_metadata_after_proof_filter}" <<<"${repository_state}" + elif [[ -n ${gate_metadata_filter} ]]; then [[ ${gate_metadata_filter} != unreadable ]] || return 1 jq "${gate_metadata_filter}" <<<"${repository_state}" else @@ -123,7 +127,14 @@ github_api() { repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) local template=code-scanning [[ ${endpoint##*/} != 7 ]] || template=dependency-review - if [[ ${ruleset_state} == restoration-race || ${ruleset_state} == retirement-race ]]; then + if [[ ${ruleset_state} == pending-branch-race ]]; then + jq --argjson id "${endpoint##*/}" --argjson target "${gate_update_id}" \ + --arg branch "$(jq -r '.default_branch' <<<"${repository_state}")" ' + . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | + if $id == $target then .conditions.ref_name.include = [("refs/heads/" + $branch)] else . end' \ + "${root}/config/rulesets/${template}.json" + return + elif [[ ${ruleset_state} == restoration-race || ${ruleset_state} == retirement-race ]]; then local filter=. if [[ ${endpoint##*/} == "${gate_update_id}" ]] && (( $(jq -s --arg endpoint "${endpoint}" '[.[] | select(.endpoint == $endpoint)] | length' "${api_log}") >= 2 )); then @@ -265,6 +276,7 @@ reset_case() { gate_update_id=6 gate_update_filter=. gate_metadata_filter= + gate_metadata_after_proof_filter= effective_state=covering repository_state='{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' workflow_state='{"default_workflow_permissions":"write","can_approve_pull_request_reviews":true}' @@ -480,6 +492,57 @@ for policy_scope in organization repository; do done done +# Pending existing-gate restoration is authorized only for the exact default +# branch whose effective protection was checked, even if another branch is valid. +for policy_scope in organization repository; do + for pending_gate_id in 6 7; do + for filter in '.default_branch = "new-main"' '.default_branch = "release/next"' \ + '.default_branch = null' 'del(.default_branch)' unreadable '.'; do + reset_case + ruleset_state=pending-branch-race + gate_update_id=${pending_gate_id} + gate_metadata_after_proof_filter=${filter} + export READINESS_STATUS=failed + jq '.activation |= with_entries(.value |= with_entries(.value = "pending"))' \ + "${security_merge_config}" >"${temporary}/pending.json" + cp "${temporary}/pending.json" "${security_merge_config}" + status=0 + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/pending-branch-fence.txt" 2>&1 || status=$? + if [[ ${filter} == '.' ]]; then + [[ ${status} == 0 ]] + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${pending_gate_id}" ' + [.[] | select(.method != "GET")] | length == 1 and + .[0].method == "PUT" and .[0].endpoint == $endpoint' "${api_log}" >/dev/null + else + [[ ${status} != 0 ]] + assert_no_writes + if [[ ${filter} == '.default_branch = "'* ]]; then + grep -Fq 'default branch changed after existing security gate proof; retained' \ + "${temporary}/pending-branch-fence.txt" + fi + fi + jq -se 'any(.[]; .endpoint == "repos/atrinik/content-toolkit/rules/branches/main")' \ + "${api_log}" >/dev/null + [[ ! -s ${readiness_log} ]] + cmp "${temporary}/pending.json" "${security_merge_config}" + done + done + # Durable enforced intent still restores missing gates on the current default. + reset_case + ruleset_state=missing + gate_metadata_filter='.default_branch = "new-main"' + export READINESS_STATUS=failed + jq '.activation |= with_entries(.value |= with_entries(.value = "enforced"))' \ + "${security_merge_config}" >"${temporary}/enforced.json" + cp "${temporary}/enforced.json" "${security_merge_config}" + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/enforced-current-default.txt" + jq -se '[.[] | select(.method != "GET")] | length == 2 and all(.[]; .method == "POST")' \ + "${api_log}" >/dev/null + [[ ! -s ${readiness_log} ]] +done + # The two gate phases are independent; qualification does not mutate source. reset_case export READINESS_STATUS=failed From 5bf0ce9221113d5a2abb5716ba14eda917852f94 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 18:11:37 -0500 Subject: [PATCH 20/22] fix(security): reject revoked pending gate proof --- AGENTS.md | 5 ++- CONTRIBUTING.md | 4 ++- README.md | 5 ++- bin/publish | 15 +++++++- tests/publish-security-merge.sh | 63 ++++++++++++++++++++++++++++++++- 5 files changed, 87 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d17876e..82ab52e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -274,7 +274,10 @@ - Before updating an existing security gate, re-read and revalidate its identity and recognized payload, retaining concurrent modifications. Bind the final metadata default branch to the branch used for pending-gate qualification - or existing effective-gate proof; changed branches must not activate a gate. Audit full legacy + or existing effective-gate proof; changed branches must not activate a gate. + Pending existing-gate proof also requires unchanged enforcement and ref + applicability in the final rule read. Enforced intent retains recognized + restoration. Audit full legacy protection on active default and declared maintenance branches; any presence is drift under the canonical ruleset contract. Only verified unprotected responses count as absent; unreadable evidence fails closed and audit never diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e0feac..0018a68 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -173,7 +173,9 @@ action-version catalog or claim resistance to malicious repository writers. Security gate updates must re-read and validate ownership and the recognized payload immediately before PUT, preserving concurrent modifications. Bind the final metadata default branch to the branch used for pending-gate qualification -or existing effective-gate proof; a changed branch must not activate a gate. Audit +or existing effective-gate proof; a changed branch must not activate a gate. +Pending existing-gate proof requires unchanged enforcement and ref applicability +in the final rule read; enforced intent retains recognized restoration. Audit legacy protection on active default and maintenance branches as drift under the canonical ruleset contract; distinguish verified unprotected responses from unreadable evidence, and never remove protection during audit. The exact diff --git a/README.md b/README.md index 1a726ba..c93df51 100644 --- a/README.md +++ b/README.md @@ -301,7 +301,10 @@ existing security gate, publication rechecks its ownership and recognized payload immediately before the write, retaining concurrent policy changes. Pending-gate qualification and existing effective-gate proof bind the same default branch as the final metadata fence; a changed branch cannot activate -a gate using proof from the earlier branch. +a gate using proof from the earlier branch. Pending existing-gate proof also +requires unchanged enforcement and ref applicability in the final rule read; +concurrent disabling or retargeting invalidates that proof. Enforced intent +retains its recognized restoration behavior. Rulesets are the canonical branch protection contract. Audit also reads legacy protection on every active default branch and declared maintenance branch. Any diff --git a/bin/publish b/bin/publish index 54bff52..8060851 100755 --- a/bin/publish +++ b/bin/publish @@ -1245,7 +1245,7 @@ delete_removed_security_merge_rulesets() { # These repository rules remain repository-owned even in organization scope: # GitHub Team does not support organization-level code-scanning rules. publish_security_merge_rulesets() { - local repository kind template name existing id current reason status readiness_mode phase metadata effective_branch + local repository kind template name existing id current reason status readiness_mode phase metadata effective_branch proven_current delete_removed_security_merge_rulesets || return 1 for kind in code_scanning dependency_review; do if [[ ${kind} == code_scanning ]]; then @@ -1291,6 +1291,7 @@ publish_security_merge_rulesets() { # Pending qualification cannot create protection before reviewed intent is # persisted. Existing effective gates still survive producer failures. effective_branch= + proven_current=${current} if [[ ${phase} == pending ]] && { [[ -z ${id} ]] || ! effective_branch=$(security_gate_applies_to_default_branch "${repository}" "${kind}" "${id}"); }; then status=0 @@ -1330,6 +1331,18 @@ publish_security_merge_rulesets() { echo "error: security gate changed during restoration preflight; retained: ${repository} ${name}" >&2 return 1 fi + # Pending intent may preserve a proven effective gate, but cannot + # restore protection revoked after that proof. Only enforced intent + # authorizes enforcement or ref-scope restoration without qualification. + if [[ ${phase} == pending ]] && ! jq -e --argjson proven "${proven_current}" ' + def protection: + {enforcement, ref_name: (.conditions.ref_name | + .include |= sort | .exclude |= sort)}; + protection == ($proven | protection) + ' <<<"${current}" >/dev/null; then + echo "error: existing security gate protection changed after effective proof; retained: ${repository} ${name}" >&2 + return 1 + fi if security_ruleset_matches "${template}" "${current}"; then echo "KEEP /repos/${organization}/${repository}/rulesets/${id} ${name}" continue diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index 3211b0e..81ccf05 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -128,11 +128,18 @@ github_api() { local template=code-scanning [[ ${endpoint##*/} != 7 ]] || template=dependency-review if [[ ${ruleset_state} == pending-branch-race ]]; then + local pending_filter=. pending_initial_filter=. + [[ ${endpoint##*/} != "${gate_update_id}" ]] || pending_initial_filter=${gate_initial_filter} + if [[ ${endpoint##*/} == "${gate_update_id}" ]] && + (( $(jq -s --arg endpoint "${endpoint}" '[.[] | select(.endpoint == $endpoint)] | length' "${api_log}") >= 2 )); then + [[ ${gate_update_filter} != unreadable ]] || return 1 + pending_filter=${gate_update_filter} + fi jq --argjson id "${endpoint##*/}" --argjson target "${gate_update_id}" \ --arg branch "$(jq -r '.default_branch' <<<"${repository_state}")" ' . + {id: $id, source_type: "Repository", source: "atrinik/content-toolkit"} | if $id == $target then .conditions.ref_name.include = [("refs/heads/" + $branch)] else . end' \ - "${root}/config/rulesets/${template}.json" + "${root}/config/rulesets/${template}.json" | jq "${pending_initial_filter}" | jq "${pending_filter}" return elif [[ ${ruleset_state} == restoration-race || ${ruleset_state} == retirement-race ]]; then local filter=. @@ -275,6 +282,7 @@ reset_case() { ruleset_state=missing gate_update_id=6 gate_update_filter=. + gate_initial_filter=. gate_metadata_filter= gate_metadata_after_proof_filter= effective_state=covering @@ -543,6 +551,59 @@ for policy_scope in organization repository; do [[ ! -s ${readiness_log} ]] done +# Pending grandfathering cannot restore a gate revoked after its effective +# proof. Enforced intent deliberately permits the same restoration changes. +for policy_scope in organization repository; do + for pending_gate_id in 6 7; do + for phase in pending enforced; do + for filter in '.enforcement = "disabled"' '.enforcement = "evaluate"' \ + '.conditions.ref_name.include = ["refs/heads/other"]' \ + '.conditions.ref_name.exclude = ["refs/heads/main"]' \ + 'unreadable' '.' \ + '.bypass_actors[0].actor_id = null | if .rules[0].type == "required_status_checks" then .rules[0].parameters.do_not_enforce_on_create = false else . end' \ + '.conditions.ref_name.include |= reverse | .conditions.ref_name.exclude |= reverse'; do + reset_case + ruleset_state=pending-branch-race + gate_update_id=${pending_gate_id} + gate_update_filter=${filter} + if [[ ${filter} == *'|= reverse'* ]]; then + gate_initial_filter='.conditions.ref_name.include += ["refs/heads/release/*"] | .conditions.ref_name.exclude = ["refs/heads/excluded-a", "refs/heads/excluded-b"]' + fi + export READINESS_STATUS=failed + jq --arg phase "${phase}" '.activation |= with_entries(.value |= with_entries(.value = $phase))' \ + "${security_merge_config}" >"${temporary}/proof-phase.json" + cp "${temporary}/proof-phase.json" "${security_merge_config}" + status=0 + # shellcheck source=/dev/null + source "${temporary}/dispatch.sh" >"${temporary}/pending-proof-fence.txt" 2>&1 || status=$? + if [[ ${filter} == unreadable ]] || { [[ ${phase} == pending ]] && + [[ ${filter} == '.enforcement ='* || ${filter} == '.conditions.ref_name.include ='* || + ${filter} == '.conditions.ref_name.exclude ='* ]]; }; then + [[ ${status} != 0 ]] + assert_no_writes + if [[ ${filter} != unreadable ]]; then + grep -Fq 'existing security gate protection changed after effective proof; retained' \ + "${temporary}/pending-proof-fence.txt" + fi + else + [[ ${status} == 0 ]] + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${pending_gate_id}" ' + [.[] | select(.method != "GET")] | length == 1 and + .[0].method == "PUT" and .[0].endpoint == $endpoint' "${api_log}" >/dev/null + fi + jq -se --arg endpoint "repos/atrinik/content-toolkit/rulesets/${pending_gate_id}" ' + [.[] | select(.endpoint == $endpoint and .method == "GET")] | length == 2' "${api_log}" >/dev/null + if [[ ${phase} == pending ]]; then + jq -se 'any(.[]; .endpoint == "repos/atrinik/content-toolkit/rules/branches/main")' \ + "${api_log}" >/dev/null + fi + [[ ! -s ${readiness_log} ]] + cmp "${temporary}/proof-phase.json" "${security_merge_config}" + done + done + done +done + # The two gate phases are independent; qualification does not mutate source. reset_case export READINESS_STATUS=failed From 1e6423c7fc53ab8429f5b7f793905ee306a098eb Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 18:37:25 -0500 Subject: [PATCH 21/22] fix(security): verify legacy absence and paginate ruleset inventories --- AGENTS.md | 5 +- CONTRIBUTING.md | 4 + README.md | 7 +- bin/publish | 149 +++++++------- lib/security_readiness.py | 85 ++++++-- tests/publish-maintenance-retirement.sh | 3 + tests/publish-retired-required-ci.sh | 8 +- tests/publish-ruleset-pagination.sh | 251 ++++++++++++++++++++++++ tests/publish-security-merge.sh | 1 + tests/security-readiness.py | 148 +++++++++++++- 10 files changed, 572 insertions(+), 89 deletions(-) create mode 100755 tests/publish-ruleset-pagination.sh diff --git a/AGENTS.md b/AGENTS.md index 82ab52e..e282c73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -254,7 +254,10 @@ enforcement. Enforced policy uses its stored proof to restore drift and fails if proof cannot be revalidated; audit reports disabled enforced policy as drift. Preserve existing true settings. Allow action names with `@*`; keep - full SHAs in producer workflows instead of duplicate pin catalogs. + full SHAs in producer workflows instead of duplicate pin catalogs. Canary + legacy-check absence requires verified protection evidence; a bare or + ambiguous 404 is unreadable. Enumerate and validate every ruleset page before + publisher identity, absence, update or retirement decisions. - Adoption of existing security gates into enforced source intent requires the effective default-branch API to prove the repository-owned ruleset and desired tool/context. Never infer activation from target membership or silently change diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0018a68..d5d2fb2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -157,6 +157,10 @@ not pending rollout; ordinary publication revalidates the recorded proof before restoring it. Test rule retirement, effective default-branch applicability, and repository workflow permission overrides. Action allowances use owner/action wildcards; full commit pins belong in workflows, not redundant version assertions. +Canary legacy-check absence requires verified protection evidence; bare or +ambiguous 404 responses fail closed. Publisher identity and absence decisions +require a complete validated ruleset inventory across all pages, including +retirement and ordinary repository/organization rules. Advanced CodeQL readiness must validate executable YAML wiring from classifier invocation and outputs through the dynamic analysis matrix, scoped generated diff --git a/README.md b/README.md index c93df51..a0e57fd 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,8 @@ Rollout is owned by an organization administrator: repository enablement, exact PR changes, and successful PR-bound current-head checks. These include configured CI, desired Dependency review for targeted repositories, and all effective ruleset and legacy branch-protection status - contexts, with trusted app identities. This proves current policy and PR-run + contexts, with trusted app identities. Legacy-check absence requires verified + protection evidence; bare or ambiguous 404 responses fail closed. This proves current policy and PR-run evidence; it does not attest historical organization settings. Ordinary subsequent publication uses that stored canary. It first converges selected mode, then allowed-action patterns, then revalidates the canary @@ -316,6 +317,10 @@ and exact existing branch have an explicitly null legacy-rule association. Missing, conflicting or unreadable evidence remains drift. Historical rules on other branches remain untouched. The audit never removes branch protections. +Publisher identity, absence, update and retirement decisions use the complete +validated ruleset inventory across all pages, including ordinary organization +and repository rules. Unreadable pages cannot count as absent rules. + Repository-scope retirement of an existing `03 - Required CI` rule also needs a reviewed `config/retired-required-ci.json` record with its exact repository and ruleset IDs and previous required contexts. Remove the repository from diff --git a/bin/publish b/bin/publish index 8060851..f669bff 100755 --- a/bin/publish +++ b/bin/publish @@ -74,6 +74,31 @@ github_api() { gh api -H "X-GitHub-Api-Version: ${api_version}" "$@" } +# Capture every page before any caller treats a ruleset as present or absent. +# gh 2.46 emits one JSON array per page; --slurp is unavailable on that host. +list_rulesets() { + local endpoint=$1 pages + pages=$(github_api "${endpoint}" --paginate) || { + echo "error: cannot read complete ruleset inventory: ${endpoint}" >&2 + return 1 + } + jq -ecs ' + if length > 0 and all(.[]; + type == "array" and all(.[]; + type == "object" and + (.id | type == "number" and . > 0 and floor == .) and + (.name | type == "string" and length > 0))) then + add | + if (map(.id) | length == (unique | length)) and + (map(.name) | length == (unique | length)) then . + else error("duplicate ruleset identity or name") end + else error("invalid ruleset page") end + ' <<<"${pages}" || { + echo "error: invalid or ambiguous complete ruleset inventory: ${endpoint}" >&2 + return 1 + } +} + run_api() { local method=$1 local endpoint=$2 @@ -177,7 +202,7 @@ retire_maintenance_branch_ruleset() { exit 1 fi - rulesets=$(github_api "orgs/${organization}/rulesets") + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 while IFS=$'\t' read -r preserved_id preserved_name; do if ! jq -e --argjson id "${preserved_id}" --arg name "${preserved_name}" ' any(.[]; .id == $id and .name == $name and .enforcement == "active") @@ -250,7 +275,7 @@ retire_maintenance_branch_ruleset() { run_api DELETE "orgs/${organization}/rulesets/${ruleset_id}" if ${apply}; then - rulesets=$(github_api "orgs/${organization}/rulesets") + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 if jq -e --argjson id "${ruleset_id}" 'any(.[]; .id == $id)' \ <<<"${rulesets}" >/dev/null; then echo "error: retirement ruleset still exists after apply" >&2 @@ -346,13 +371,11 @@ is_temporary_security_advisory_repository() { upsert_repository_ruleset() { local repository=$1 local input=$2 - local name id + local name id rulesets name=$(jq -r '.name' "${input}") - id=$( - github_api "repos/${organization}/${repository}/rulesets?includes_parents=false" | - jq -r --arg name "${name}" 'map(select(.name == $name))[0].id // empty' - ) + rulesets=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 + id=$(jq -r --arg name "${name}" 'map(select(.name == $name))[0].id // empty' <<<"${rulesets}") if [[ -n ${id} ]]; then run_api PUT "repos/${organization}/${repository}/rulesets/${id}" "${input}" @@ -363,13 +386,11 @@ upsert_repository_ruleset() { upsert_organization_ruleset() { local input=$1 - local name id + local name id rulesets name=$(jq -r '.name' "${input}") - id=$( - github_api "orgs/${organization}/rulesets" | - jq -r --arg name "${name}" 'map(select(.name == $name))[0].id // empty' - ) + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 + id=$(jq -r --arg name "${name}" 'map(select(.name == $name))[0].id // empty' <<<"${rulesets}") if [[ -n ${id} ]]; then run_api PUT "orgs/${organization}/rulesets/${id}" "${input}" @@ -475,9 +496,10 @@ maintenance_ruleset_is_desired() { } delete_stale_repository_maintenance_rulesets() { - local repository ruleset name id + local repository ruleset name id rulesets for repository in "${active_repositories[@]}"; do + rulesets=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -485,24 +507,21 @@ delete_stale_repository_maintenance_rulesets() { continue fi if ! maintenance_ruleset_is_desired "${repository}" "${name}"; then - run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" || return 1 fi - done < <( - github_api \ - "repos/${organization}/${repository}/rulesets?includes_parents=false" | - jq -c '.[]' - ) + done < <(jq -c '.[]' <<<"${rulesets}") done } delete_stale_repository_advisory_rulesets() { - local repository ruleset name id window_active + local repository ruleset name id window_active rulesets while IFS= read -r repository; do window_active=false if repository_has_advisory_merge_window "${repository}"; then window_active=true fi + rulesets=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -512,7 +531,7 @@ delete_stale_repository_advisory_rulesets() { "01 - Default branch linear history" | \ "02 - Changes through pull requests") run_api DELETE \ - "repos/${organization}/${repository}/rulesets/${id}" + "repos/${organization}/${repository}/rulesets/${id}" || return 1 ;; esac else @@ -521,22 +540,19 @@ delete_stale_repository_advisory_rulesets() { "01 - Default branch linear history - ${repository}" | \ "02 - Changes through pull requests - ${repository}") run_api DELETE \ - "repos/${organization}/${repository}/rulesets/${id}" + "repos/${organization}/${repository}/rulesets/${id}" || return 1 ;; esac fi - done < <( - github_api \ - "repos/${organization}/${repository}/rulesets?includes_parents=false" | - jq -c '.[]' - ) + done < <(jq -c '.[]' <<<"${rulesets}") done < <(jq -r '.pull_request_gate[]' "${repositories_config}") } delete_managed_repository_rulesets() { local repository=$1 - local ruleset name id + local ruleset name id rulesets + rulesets=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -550,18 +566,16 @@ delete_managed_repository_rulesets() { "03 - Required CI" | \ "04 - Immutable release tags" | \ "05 - Maintenance branch - "*) - run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" || return 1 ;; esac - done < <( - github_api "repos/${organization}/${repository}/rulesets?includes_parents=false" | - jq -c '.[]' - ) + done < <(jq -c '.[]' <<<"${rulesets}") } delete_managed_organization_rulesets() { - local ruleset name id + local ruleset name id rulesets + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -575,10 +589,10 @@ delete_managed_organization_rulesets() { "04 - Immutable release tags" | \ "03 - Required CI - "* | \ "05 - Maintenance branch - "*) - run_api DELETE "orgs/${organization}/rulesets/${id}" + run_api DELETE "orgs/${organization}/rulesets/${id}" || return 1 ;; esac - done < <(github_api "orgs/${organization}/rulesets" | jq -c '.[]') + done < <(jq -c '.[]' <<<"${rulesets}") } retired_required_ci_matches() { @@ -596,7 +610,7 @@ delete_retired_repository_required_ci() { "${repositories_config}" >/dev/null; then continue fi - listed=$(github_api "repos/${organization}/${repository}/rulesets?includes_parents=false") + listed=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 listed=$(jq -c '[.[] | select(.name == "03 - Required CI")]' <<<"${listed}") [[ $(jq length <<<"${listed}") != 0 ]] || continue record=$(jq -c --arg repository "${repository}" \ @@ -617,12 +631,12 @@ delete_retired_repository_required_ci() { .rules[0].parameters.required_status_checks = ($record.required_ci | map({context: ., integration_id: 15368})) ' "${root}/config/rulesets/required-ci.json" >"${template}" - current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") + current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || return 1 if ! retired_required_ci_matches "${repository}" "${id}" "${template}" "${current}"; then echo "SKIP ${organization}/${repository} 03 - Required CI: modified or foreign rule; retained" continue fi - metadata=$(github_api "repos/${organization}/${repository}") + metadata=$(github_api "repos/${organization}/${repository}") || return 1 if ! jq -e --argjson record "${record}" --argjson enumerated "${active_repository_ids["${repository}"]}" \ --arg full_name "${organization}/${repository}" ' .id == $record.repository_id and .id == $enumerated and @@ -631,12 +645,12 @@ delete_retired_repository_required_ci() { echo "SKIP ${organization}/${repository} 03 - Required CI: repository identity or archive state changed; retained" continue fi - fresh=$(github_api "repos/${organization}/${repository}/rulesets/${id}") + fresh=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || return 1 if ! retired_required_ci_matches "${repository}" "${id}" "${template}" "${fresh}"; then echo "SKIP ${organization}/${repository} 03 - Required CI: rule changed during retirement preflight; retained" continue fi - run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" + run_api DELETE "repos/${organization}/${repository}/rulesets/${id}" || return 1 done } @@ -651,7 +665,7 @@ publish_repository_rulesets() { ruleset_config=${root}/config/rulesets/default-branch-linear-history-classic.json fi upsert_repository_ruleset "${repository}" \ - "${ruleset_config}" + "${ruleset_config}" || return 1 done for repository in "${active_repositories[@]}"; do @@ -660,7 +674,7 @@ publish_repository_rulesets() { ruleset_config=${root}/config/rulesets/default-branch-integrity-classic.json fi upsert_repository_ruleset "${repository}" \ - "${ruleset_config}" + "${ruleset_config}" || return 1 done while IFS= read -r repository; do @@ -669,7 +683,7 @@ publish_repository_rulesets() { ruleset_config=${root}/config/rulesets/pull-request-gate-classic.json fi upsert_repository_ruleset "${repository}" \ - "${ruleset_config}" + "${ruleset_config}" || return 1 done < <(jq -r '.pull_request_gate[]' "${repositories_config}") while IFS= read -r repository; do @@ -690,12 +704,12 @@ publish_repository_rulesets() { ($repositories[0].required_ci[$repository] | map({context: ., integration_id: $app_id})) ' "${root}/config/rulesets/required-ci.json" >"${ci_payload}" - upsert_repository_ruleset "${repository}" "${ci_payload}" + upsert_repository_ruleset "${repository}" "${ci_payload}" || return 1 done < <(jq -r '.required_ci | keys[]' "${repositories_config}") while IFS= read -r repository; do upsert_repository_ruleset "${repository}" \ - "${root}/config/rulesets/immutable-release-tags.json" + "${root}/config/rulesets/immutable-release-tags.json" || return 1 done < <(jq -r '.release_tags[]' "${repositories_config}") while IFS= read -r maintenance; do @@ -705,22 +719,22 @@ publish_repository_rulesets() { input=$(make_maintenance_ruleset \ "${repository}" "${branch}" "${required_ci}" repository) temporary_files+=("${input}") - upsert_repository_ruleset "${repository}" "${input}" + upsert_repository_ruleset "${repository}" "${input}" || return 1 done < <(jq -c '.maintenance_branches[]' "${repositories_config}") - delete_retired_repository_required_ci - delete_stale_repository_maintenance_rulesets - delete_stale_repository_advisory_rulesets + delete_retired_repository_required_ci || return 1 + delete_stale_repository_maintenance_rulesets || return 1 + delete_stale_repository_advisory_rulesets || return 1 if [[ ${organization_plan} != free ]]; then - delete_managed_organization_rulesets + delete_managed_organization_rulesets || return 1 fi } publish_organization_rulesets() { local input repository ci_payload maintenance branch required_ci local -a desired_rulesets - local repositories ruleset name id desired + local repositories ruleset name id desired rulesets desired_rulesets=( "01 - Default branch integrity" @@ -741,12 +755,12 @@ publish_organization_rulesets() { "${root}/config/rulesets/default-branch-integrity-classic.json" \ "01 - Default branch integrity - classic" '["classic"]') temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 input=$(make_organization_ruleset \ "${root}/config/rulesets/default-branch-linear-history-classic.json" \ "01 - Default branch linear history - classic" '["classic"]') temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 fi repositories=$(jq -c '.repositories' "${advisory_merge_windows_config}") @@ -755,13 +769,13 @@ publish_organization_rulesets() { "${root}/config/rulesets/default-branch-linear-history.json" \ "01 - Default branch linear history" '["~ALL"]' "${repositories}") temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 input=$(make_organization_ruleset \ "${root}/config/rulesets/default-branch-integrity.json" \ "01 - Default branch integrity" '["~ALL"]' "${repositories}") temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 # Establish the Classic-specific policy before excluding it from the shared # pull-request policy. @@ -770,7 +784,7 @@ publish_organization_rulesets() { "${root}/config/rulesets/pull-request-gate-classic.json" \ "02 - Changes through pull requests - classic" '["classic"]') temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 fi repositories=$(jq -c \ @@ -781,14 +795,14 @@ publish_organization_rulesets() { "${root}/config/rulesets/pull-request-gate.json" \ "02 - Changes through pull requests" "${repositories}") temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 repositories=$(jq -c '.release_tags' "${repositories_config}") input=$(make_organization_ruleset \ "${root}/config/rulesets/immutable-release-tags.json" \ "04 - Immutable release tags" "${repositories}") temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 while IFS= read -r repository; do name="03 - Required CI - ${repository}" @@ -817,7 +831,7 @@ publish_organization_rulesets() { ($repositories[0].required_ci[$repository] | map({context: ., integration_id: $app_id})) ' "${root}/config/rulesets/required-ci.json" >"${ci_payload}" - upsert_organization_ruleset "${ci_payload}" + upsert_organization_ruleset "${ci_payload}" || return 1 done < <(jq -r '.required_ci | keys[]' "${repositories_config}") while IFS= read -r maintenance; do @@ -829,9 +843,10 @@ publish_organization_rulesets() { input=$(make_maintenance_ruleset \ "${repository}" "${branch}" "${required_ci}" organization) temporary_files+=("${input}") - upsert_organization_ruleset "${input}" + upsert_organization_ruleset "${input}" || return 1 done < <(jq -c '.maintenance_branches[]' "${repositories_config}") + rulesets=$(list_rulesets "orgs/${organization}/rulesets") || return 1 while IFS= read -r ruleset; do name=$(jq -r '.name' <<<"${ruleset}") id=$(jq -r '.id' <<<"${ruleset}") @@ -853,12 +868,12 @@ publish_organization_rulesets() { fi done if ! ${desired}; then - run_api DELETE "orgs/${organization}/rulesets/${id}" + run_api DELETE "orgs/${organization}/rulesets/${id}" || return 1 fi - done < <(github_api "orgs/${organization}/rulesets" | jq -c '.[]') + done < <(jq -c '.[]' <<<"${rulesets}") for repository in "${active_repositories[@]}"; do - delete_managed_repository_rulesets "${repository}" + delete_managed_repository_rulesets "${repository}" || return 1 done } @@ -1212,7 +1227,7 @@ delete_removed_security_merge_rulesets() { template=${root}/config/rulesets/dependency-review.json fi name=$(jq -r '.name' "${template}") - existing=$(github_api "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 + existing=$(list_rulesets "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 while IFS= read -r rule; do id=$(jq -r '.id' <<<"${rule}") current=$(github_api "repos/${organization}/${repository}/rulesets/${id}") || return 1 @@ -1267,8 +1282,8 @@ publish_security_merge_rulesets() { echo "error: invalid security gate activation intent: ${repository} ${kind}" >&2 return 1 } - existing=$(github_api \ - "repos/${organization}/${repository}/rulesets?includes_parents=false") + existing=$(list_rulesets \ + "repos/${organization}/${repository}/rulesets?includes_parents=false") || return 1 if ! jq -e --arg name "${name}" \ '[.[] | select(.name == $name)] | length <= 1' <<<"${existing}" >/dev/null; then echo "error: ambiguous managed ruleset ${name} on ${repository}" >&2 diff --git a/lib/security_readiness.py b/lib/security_readiness.py index 40a451e..e34618a 100644 --- a/lib/security_readiness.py +++ b/lib/security_readiness.py @@ -40,8 +40,9 @@ class EvidenceError(Exception): class APIError(EvidenceError): - def __init__(self, endpoint, status=None): + def __init__(self, endpoint, status=None, response=None): self.status = status + self.response = response super().__init__(f"API unreadable: {endpoint.split('?')[0]}" + (f" (HTTP {status})" if status else "")) @@ -58,7 +59,11 @@ def api(endpoint): raise APIError(endpoint) from exc if result.returncode: status = re.search(r"HTTP (\d{3})", result.stderr) - raise APIError(endpoint, int(status[1]) if status else None) + try: + response = json.loads(result.stdout) + except json.JSONDecodeError: + response = None + raise APIError(endpoint, int(status[1]) if status else None, response) try: return json.loads(result.stdout) except json.JSONDecodeError as exc: @@ -447,21 +452,77 @@ def canary_actions_policy(coordinate, metadata): "repository": repository_policy, "selected": selected} -def canary_required_checks(coordinate, branch): +def disabled_legacy_absent(coordinate, metadata): + """Prove absence only for the exact REST repository-disabled response.""" + branch = metadata["default_branch"] + fresh = repository(coordinate) + if (type(metadata["id"]) is not int or metadata["id"] <= 0 + or fresh["id"] != metadata["id"] or type(fresh["id"]) is not int + or fresh["full_name"] != coordinate or metadata["full_name"] != coordinate + or fresh["default_branch"] != branch): + raise EvidenceError("legacy protection repository identity/default branch changed") + # This fixed query is read-only. GraphQL requires POST; variables remain + # separate literal fields and cannot add operations to the query. + query = ("query LegacyBranchProtection($owner: String!, $name: String!, $qualifiedName: String!) " + "{ repository(owner: $owner, name: $name) { databaseId nameWithOwner isArchived " + "ref(qualifiedName: $qualifiedName) { name branchProtectionRule { id } } } }") + owner, name = coordinate.split("/") + try: + result = subprocess.run( + ["gh", "api", "--method", "POST", "-H", f"X-GitHub-Api-Version: {VERSION}", + "graphql", "-f", f"query={query}", "-f", f"owner={owner}", "-f", f"name={name}", + "-f", f"qualifiedName=refs/heads/{branch}"], + capture_output=True, text=True, timeout=30, check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise EvidenceError("legacy protection GraphQL unreadable") from exc + if result.returncode: + raise EvidenceError("legacy protection GraphQL unreadable") + try: + response = json.loads(result.stdout) + repo = response["data"]["repository"] + ref = repo["ref"] + valid = ("errors" not in response and type(repo["databaseId"]) is int + and repo["databaseId"] == metadata["id"] and repo["nameWithOwner"] == coordinate + and repo["isArchived"] is False and ref["name"] == branch + and "branchProtectionRule" in ref and ref["branchProtectionRule"] is None) + except (json.JSONDecodeError, KeyError, TypeError): + valid = False + if not valid: + raise EvidenceError("legacy protection absence not proven for repository/default branch") + + +def canary_legacy_checks(coordinate, metadata): + branch = quote(metadata["default_branch"], safe="") + try: + protection = api(f"repos/{coordinate}/branches/{branch}/protection") + except APIError as exc: + body = exc.response + if exc.status != 404 or not isinstance(body, dict) or body.get("status") != "404": + raise + if body.get("message") == "Branch not protected": + return None + if body.get("message") == "Branch protection has been disabled on this repository.": + disabled_legacy_absent(coordinate, metadata) + return None + raise + if not isinstance(protection, dict) or "required_status_checks" not in protection: + raise EvidenceError("legacy branch protection malformed") + return protection["required_status_checks"] + + +def canary_required_checks(coordinate, metadata): + branch = metadata["default_branch"] rules = pages(f"repos/{coordinate}/rules/branches/{quote(branch, safe='')}") entries = [] for rule in rules: if rule["type"] == "required_status_checks": entries.extend((item["context"], item["integration_id"]) for item in rule["parameters"]["required_status_checks"]) - try: - legacy = api(f"repos/{coordinate}/branches/{quote(branch, safe='')}/protection/required_status_checks") - except APIError as exc: - if exc.status != 404: - raise - legacy = None + legacy = canary_legacy_checks(coordinate, metadata) if legacy is not None: - if (not isinstance(legacy["checks"], list) or not isinstance(legacy["contexts"], list) + if (not isinstance(legacy, dict) or not isinstance(legacy["checks"], list) + or not isinstance(legacy["contexts"], list) or not all(isinstance(value, str) and value for value in legacy["contexts"])): raise EvidenceError("legacy required status checks malformed") entries.extend((item["context"], item["app_id"]) for item in legacy["checks"]) @@ -494,7 +555,7 @@ def sha(coordinate, number): require(bool(SHA.fullmatch(pr["head"]["sha"])), "canary head SHA missing") actions_policy = canary_actions_policy(coordinate, metadata) selected = config("actions-selected.json") - required, required_evidence = canary_required_checks(coordinate, metadata["default_branch"]) + required, required_evidence = canary_required_checks(coordinate, metadata) files = pages(f"repos/{coordinate}/pulls/{number}/files") require(bool(files) and len(files) == pr["changed_files"], "canary file evidence incomplete") updates = 0 @@ -593,7 +654,7 @@ def check_snapshot(values): and (fresh["merged"] or not fresh["draft"]), "canary changed during preflight") require(canary_actions_policy(coordinate, metadata) == actions_policy, "live Actions permissions changed during preflight") - require(canary_required_checks(coordinate, metadata["default_branch"]) + require(canary_required_checks(coordinate, metadata) == (required, required_evidence), "effective required checks changed during preflight") fresh_metadata = repository(coordinate) require(fresh_metadata["default_branch"] == metadata["default_branch"] diff --git a/tests/publish-maintenance-retirement.sh b/tests/publish-maintenance-retirement.sh index 587504c..7b40304 100755 --- a/tests/publish-maintenance-retirement.sh +++ b/tests/publish-maintenance-retirement.sh @@ -21,6 +21,9 @@ while (($#)); do -H | --header) shift 2 ;; + --paginate) + shift + ;; --method) method=$2 shift 2 diff --git a/tests/publish-retired-required-ci.sh b/tests/publish-retired-required-ci.sh index 0680fa8..bc128d4 100755 --- a/tests/publish-retired-required-ci.sh +++ b/tests/publish-retired-required-ci.sh @@ -9,6 +9,7 @@ temporary=$(mktemp -d) trap 'rm -rf "${temporary}"' EXIT sed -n '/^converge_organization_settings() {/,/^organization_metadata=/p' \ "${source_root}/bin/publish" | sed '$d' >"${temporary}/functions.sh" +sed -n '/^list_rulesets() {/,/^}/p' "${source_root}/bin/publish" >>"${temporary}/functions.sh" # shellcheck source=/dev/null source "${temporary}/functions.sh" root=${temporary}/fixture @@ -128,7 +129,12 @@ assert_retained grep -Fq 'config/retired-required-ci.json' "${temporary}/output" reset_case printf '[{"id":3,"name":"03 - Required CI"},{"id":4,"name":"03 - Required CI"}]\n' >"${temporary}/list.json" -assert_retained +if publish_repository_rulesets >"${temporary}/output" 2>&1; then + echo 'expected ambiguous live ruleset inventory to fail' >&2 + exit 1 +fi +jq -se 'all(.[]; .method == "GET")' "${log}" >/dev/null +grep -Fq 'ambiguous complete ruleset inventory' "${temporary}/output" reset_case printf '[{"id":4,"name":"03 - Required CI"}]\n' >"${temporary}/list.json" assert_retained diff --git a/tests/publish-ruleset-pagination.sh b/tests/publish-ruleset-pagination.sh new file mode 100755 index 0000000..610f85b --- /dev/null +++ b/tests/publish-ruleset-pagination.sh @@ -0,0 +1,251 @@ +#!/usr/bin/env bash + +# Sourced production functions consume these fixture globals. +# shellcheck disable=SC2034 +set -euo pipefail +source_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +source_root=${ATRINIK_PUBLISH_TEST_ROOT:-${source_root}} +temporary=$(mktemp -d) +trap 'rm -rf "${temporary}"' EXIT +root=${temporary}/fixture +mkdir -p "${root}/config" +cp -R "${source_root}/config/." "${root}/config/" +sed -n '/^converge_organization_settings() {/,/^organization_metadata=/p' \ + "${source_root}/bin/publish" | sed '$d' >"${temporary}/functions.sh" +sed -n '/^list_rulesets() {/,/^}/p' "${source_root}/bin/publish" >>"${temporary}/functions.sh" +# shellcheck source=/dev/null +source "${temporary}/functions.sh" +organization=atrinik +organization_plan=free +apply=true +temporary_files=() +repositories_config=${root}/config/repositories.json +security_merge_config=${root}/config/security-merge-protection.json +advisory_merge_windows_config=${root}/config/advisory-merge-windows.json +api_log=${temporary}/api.jsonl +active_repositories=(content-toolkit) +declare -A active_repository_ids=([content-toolkit]=123) +# The first 30 entries are foreign policies. Managed entries occur only on the +# next page, matching GitHub's default page size and the reported duplicate bug. +jq -cn '[range(100;130) | {id:.,name:("Foreign policy " + tostring)}]' >"${temporary}/first-page.json" +github_api() { + local endpoint=$1 + shift + jq -cn --arg endpoint "${endpoint}" '{method:"GET",endpoint:$endpoint}' >>"${api_log}" + case ${endpoint} in + orgs/atrinik/rulesets | 'repos/atrinik/content-toolkit/rulesets?includes_parents=false') + [[ $# == 1 && $1 == --paginate ]] || { + echo 'inventory must use supported --paginate without --slurp' >&2 + return 99 + } + [[ ${page_state} != empty ]] || return 0 + cat "${temporary}/first-page.json" + case ${page_state} in + valid) cat "${temporary}/second-page.json" ;; + failed) echo 'gh: later ruleset page failed (HTTP 503)' >&2; return 1 ;; + malformed) echo '{' ;; + object) echo '{"message":"API error"}' ;; + null) echo 'null' ;; + nested) echo '[[{"id":8,"name":"Nested"}]]' ;; + invalid-id) echo '[{"id":null,"name":"Invalid"}]' ;; + fractional-id) echo '[{"id":8.5,"name":"Invalid"}]' ;; + invalid-name) echo '[{"id":8,"name":null}]' ;; + duplicate-id) echo '[{"id":100,"name":"Changed name"}]' ;; + duplicate-name) echo '[{"id":8,"name":"Foreign policy 100"}]' ;; + *) return 99 ;; + esac + ;; + repos/atrinik/content-toolkit) + echo '{"id":123,"full_name":"atrinik/content-toolkit","archived":false,"default_branch":"main"}' ;; + repos/atrinik/content-toolkit/rulesets/3 | \ + repos/atrinik/content-toolkit/rulesets/6 | repos/atrinik/content-toolkit/rulesets/7) + cat "${temporary}/rule-${endpoint##*/}.json" ;; + *) echo "unexpected API endpoint: ${endpoint}" >&2; return 99 ;; + esac +} +run_api() { + local payload=null + [[ -z ${3:-} ]] || payload=$(cat "$3") + jq -cn --arg method "$1" --arg endpoint "$2" --argjson payload "${payload}" \ + '{method:$method,endpoint:$endpoint,payload:$payload}' >>"${api_log}" + [[ ${write_failed} != true ]] +} +reset_case() { + : >"${api_log}" + page_state=valid + write_failed=false + active_repositories=(content-toolkit) + active_repository_ids=([content-toolkit]=123) + echo '[]' >"${temporary}/second-page.json" + echo '{"repositories":[]}' >"${advisory_merge_windows_config}" + echo '{"pull_request_gate":["content-toolkit"],"required_ci":{"content-toolkit":["Toolkit validation"]},"release_tags":[],"maintenance_branches":[]}' >"${repositories_config}" + echo '{"dependency_review":["content-toolkit"],"code_scanning":["content-toolkit"],"activation":{"dependency_review":{"content-toolkit":"enforced"},"code_scanning":{"content-toolkit":"enforced"}}}' >"${security_merge_config}" + echo '{"retirements":[{"repository":"content-toolkit","repository_id":123,"ruleset_id":3,"required_ci":["Toolkit validation"]}]}' >"${root}/config/retired-required-ci.json" + for pair in '3 required-ci' '6 code-scanning' '7 dependency-review'; do + read -r id template <<<"${pair}" + jq --argjson id "${id}" '. + {id:$id,source_type:"Repository",source:"atrinik/content-toolkit"} | + if $id == 3 then .rules[0].parameters.required_status_checks = [{context:"Toolkit validation",integration_id:15368}] else . end' \ + "${root}/config/rulesets/${template}.json" >"${temporary}/rule-${id}.json" + done +} +assert_no_writes() { + jq -se 'all(.[]; .method == "GET")' "${api_log}" >/dev/null +} +# Ordinary repository/org upserts must update the existing off-page identity; +# complete absence alone permits POST. +for scope in repository organization; do + for found in true false; do + reset_case + if ${found}; then + echo '[{"id":31,"name":"01 - Default branch integrity"}]' >"${temporary}/second-page.json" + fi + if [[ ${scope} == repository ]]; then + upsert_repository_ruleset content-toolkit "${root}/config/rulesets/default-branch-integrity.json" + endpoint=repos/atrinik/content-toolkit/rulesets + else + upsert_organization_ruleset "${root}/config/rulesets/default-branch-integrity.json" + endpoint=orgs/atrinik/rulesets + fi + method=POST + if ${found}; then method=PUT; endpoint+=/31; fi + jq -se --arg method "${method}" --arg endpoint "${endpoint}" ' + [.[] | select(.method != "GET")] | length == 1 and .[0].method == $method and .[0].endpoint == $endpoint' "${api_log}" >/dev/null + done +done +# Security gate convergence, restoration, and retirement share full discovery +# in both scopes. Foreign same-name detail remains retained after discovery. +for policy_scope in organization repository; do + for state in converged disabled retired foreign; do + reset_case + echo '[{"id":6,"name":"06 - CodeQL security"},{"id":7,"name":"07 - Dependency review"}]' >"${temporary}/second-page.json" + if [[ ${state} == disabled ]]; then + for id in 6 7; do + jq '.enforcement = "disabled"' "${temporary}/rule-${id}.json" >"${temporary}/changed.json" + mv "${temporary}/changed.json" "${temporary}/rule-${id}.json" + done + elif [[ ${state} == retired || ${state} == foreign ]]; then + echo '{"dependency_review":[],"code_scanning":[],"activation":{"dependency_review":{},"code_scanning":{}}}' >"${security_merge_config}" + if [[ ${state} == foreign ]]; then + for id in 6 7; do + jq '.source = "foreign/content-toolkit"' "${temporary}/rule-${id}.json" >"${temporary}/changed.json" + mv "${temporary}/changed.json" "${temporary}/rule-${id}.json" + done + fi + fi + publish_security_merge_rulesets >"${temporary}/output" + case ${state} in + converged | foreign) assert_no_writes ;; + disabled | retired) + method=PUT + [[ ${state} != retired ]] || method=DELETE + jq -se --arg method "${method}" ' + [.[] | select(.method != "GET")] | length == 2 and + all(.[]; .method == $method) and + ([.[].endpoint] | sort) == ["repos/atrinik/content-toolkit/rulesets/6","repos/atrinik/content-toolkit/rulesets/7"]' "${api_log}" >/dev/null ;; + esac + done +done +# A recorded aggregate on a later page is retired only when its exact payload +# and repository provenance still match, leaving all foreign policies intact. +for state in owned foreign; do + reset_case + echo '[{"id":3,"name":"03 - Required CI"}]' >"${temporary}/second-page.json" + echo '{"pull_request_gate":[],"required_ci":{},"release_tags":[],"maintenance_branches":[]}' >"${repositories_config}" + if [[ ${state} == foreign ]]; then + jq '.source = "foreign/content-toolkit"' "${temporary}/rule-3.json" >"${temporary}/changed.json" + mv "${temporary}/changed.json" "${temporary}/rule-3.json" + fi + delete_retired_repository_required_ci >"${temporary}/output" + if [[ ${state} == foreign ]]; then assert_no_writes; else + jq -se '[.[] | select(.method != "GET")] | length == 1 and .[0].method == "DELETE" and .[0].endpoint == "repos/atrinik/content-toolkit/rulesets/3"' "${api_log}" >/dev/null + fi +done +# Scope migration and stale maintenance/advisory removal also see later pages. +for consumer in delete_managed_repository_rulesets delete_managed_organization_rulesets \ + delete_stale_repository_maintenance_rulesets delete_stale_repository_advisory_rulesets; do + reset_case + case ${consumer} in + delete_stale_repository_maintenance_rulesets) name='05 - Maintenance branch - content-toolkit - old' ;; + delete_stale_repository_advisory_rulesets) name='02 - Changes through pull requests - content-toolkit' ;; + *) name='01 - Default branch integrity' ;; + esac + jq -cn --arg name "${name}" '[{id:31,name:$name}]' >"${temporary}/second-page.json" + "${consumer}" content-toolkit + jq -se '[.[] | select(.method != "GET")] | length == 1 and .[0].method == "DELETE" and (.[0].endpoint | endswith("/31"))' "${api_log}" >/dev/null +done +# Propagating inventory failures through parent functions must also preserve +# API write failures instead of continuing a loop after a failed deletion. +for consumer in delete_managed_repository_rulesets delete_managed_organization_rulesets \ + delete_stale_repository_maintenance_rulesets delete_stale_repository_advisory_rulesets \ + delete_retired_repository_required_ci; do + reset_case + write_failed=true + case ${consumer} in + delete_stale_repository_maintenance_rulesets) name='05 - Maintenance branch - content-toolkit - old' ;; + delete_stale_repository_advisory_rulesets) name='02 - Changes through pull requests - content-toolkit' ;; + delete_retired_repository_required_ci) + name='03 - Required CI' + echo '{"required_ci":{}}' >"${repositories_config}" ;; + *) name='01 - Default branch integrity' ;; + esac + jq -cn --arg name "${name}" '[{id:3,name:$name},{id:32,name:"05 - Maintenance branch - content-toolkit - older"}]' >"${temporary}/second-page.json" + if "${consumer}" content-toolkit >"${temporary}/failure" 2>&1; then + echo "expected ${consumer} to propagate the failed DELETE" >&2 + exit 1 + fi + jq -se '[.[] | select(.method != "GET")] | length == 1 and .[0].method == "DELETE"' "${api_log}" >/dev/null +done +# The organization publisher's final stale-policy scan must not miss an +# off-page aggregate, nor mistake off-page desired rules for missing rules. +reset_case +active_repositories=() +echo '{"pull_request_gate":[],"required_ci":{},"release_tags":[],"maintenance_branches":[]}' >"${repositories_config}" +echo '[{"id":1,"name":"01 - Default branch integrity"},{"id":2,"name":"01 - Default branch linear history"},{"id":4,"name":"02 - Changes through pull requests"},{"id":5,"name":"04 - Immutable release tags"},{"id":31,"name":"03 - Required CI - retired"}]' >"${temporary}/second-page.json" +publish_organization_rulesets +jq -se 'all(.[]; .method != "POST") and + ([.[] | select(.method == "DELETE") | .endpoint] == ["orgs/atrinik/rulesets/31"])' "${api_log}" >/dev/null +# No inventory consumer may use an incomplete, malformed or ambiguous page +# stream as proof of absence, even when called from a shell conditional. +for invalid in failed malformed object null nested invalid-id fractional-id invalid-name duplicate-id duplicate-name empty; do + for consumer in upsert_repository_ruleset upsert_organization_ruleset \ + publish_repository_rulesets publish_organization_rulesets publish_security_merge_rulesets \ + delete_retired_repository_required_ci delete_removed_security_merge_rulesets \ + delete_managed_repository_rulesets delete_managed_organization_rulesets \ + delete_stale_repository_maintenance_rulesets delete_stale_repository_advisory_rulesets; do + reset_case + page_state=${invalid} + case ${consumer} in + upsert_repository_ruleset) arguments=(content-toolkit "${root}/config/rulesets/default-branch-integrity.json") ;; + upsert_organization_ruleset) arguments=("${root}/config/rulesets/default-branch-integrity.json") ;; + delete_retired_repository_required_ci) + arguments=() + echo '{"required_ci":{}}' >"${repositories_config}" ;; + delete_removed_security_merge_rulesets) + arguments=() + echo '{"code_scanning":[],"dependency_review":[]}' >"${security_merge_config}" ;; + *) arguments=(content-toolkit) ;; + esac + if "${consumer}" "${arguments[@]}" >"${temporary}/failure" 2>&1; then + echo "expected ${consumer} to reject ${invalid} later-page evidence" >&2 + exit 1 + fi + assert_no_writes + grep -Eq 'cannot read complete ruleset inventory|invalid or ambiguous complete ruleset inventory' "${temporary}/failure" + done +done +# Even an actionable first-page gate cannot be deleted while a later page is +# unreadable: pagination is a complete preflight, not a streaming mutation loop. +cp "${temporary}/first-page.json" "${temporary}/foreign-page.json" +for invalid in failed malformed; do + reset_case + page_state=${invalid} + jq '. + [{id:6,name:"06 - CodeQL security"},{id:7,name:"07 - Dependency review"}]' \ + "${temporary}/foreign-page.json" >"${temporary}/first-page.json" + echo '{"code_scanning":[],"dependency_review":[]}' >"${security_merge_config}" + if delete_removed_security_merge_rulesets >"${temporary}/failure" 2>&1; then + echo 'expected failed full enumeration to prevent first-page retirement' >&2 + exit 1 + fi + assert_no_writes +done +printf 'Ruleset pagination publisher tests passed.\n' diff --git a/tests/publish-security-merge.sh b/tests/publish-security-merge.sh index 81ccf05..badb91b 100755 --- a/tests/publish-security-merge.sh +++ b/tests/publish-security-merge.sh @@ -18,6 +18,7 @@ cp "${source_root}/config/rulesets/"*.json "${root}/config/rulesets/" # live publication. Every API call is replaced below by a fail-closed mock. sed -n '/^converge_organization_settings() {/,/^organization_metadata=/p' \ "${source_root}/bin/publish" | sed '$d' >"${temporary}/functions.sh" +sed -n '/^list_rulesets() {/,/^}/p' "${source_root}/bin/publish" >>"${temporary}/functions.sh" # shellcheck source=/dev/null source "${temporary}/functions.sh" # Exercise the production scope dispatch, including its following gate call. diff --git a/tests/security-readiness.py b/tests/security-readiness.py index 9eaa924..81787a3 100644 --- a/tests/security-readiness.py +++ b/tests/security-readiness.py @@ -307,10 +307,8 @@ def __call__(self, endpoint): "repositories": copy.deepcopy(self.enabled_repositories)} if "/rules/branches/" in endpoint: return copy.deepcopy(self.effective_rules) - if endpoint.endswith("/protection/required_status_checks"): - if self.legacy_checks is None: - raise readiness.APIError(endpoint, 404) - return copy.deepcopy(self.legacy_checks) + if endpoint.endswith("/protection"): + return {"required_status_checks": copy.deepcopy(self.legacy_checks)} if "/actions/permissions/selected-actions" in endpoint: return copy.deepcopy(self.live) if "/pulls/12/files" in endpoint: @@ -793,6 +791,131 @@ def test_sha_legacy_protection_context_required(self): self.fixture.checks[HEAD].append(check("Legacy check", identifier=30)) self.assertEqual(self.invoke("sha")[0], 0) + def legacy_error(self, message, status="404"): + def response(endpoint): + if endpoint.endswith("/protection"): + raise readiness.APIError(endpoint, 404, {"message": message, "status": status}) + return self.fixture(endpoint) + return patch.object(readiness, "api", response) + + def disabled_legacy_response(self): + return {"data": {"repository": {"databaseId": 42, "nameWithOwner": "atrinik/example", + "isArchived": False, "ref": {"name": "main", "branchProtectionRule": None}}}} + + def test_sha_only_exact_unprotected_error_proves_absence(self): + for message, status, expected in (("Branch not protected", "404", 0), + ("Resource not found", "404", 2), ("Not Found", "404", 2), + ("Resource not accessible by integration", "404", 2), + ("Branch not protected", None, 2), ("Branch not protected", 404, 2)): + with self.subTest(message=message, status=status), self.legacy_error(message, status): + self.assertEqual(self.invoke("sha")[0], expected) + + def test_sha_missing_or_malformed_full_protection_not_absence(self): + for value in ({}, [], None, {"required_status_checks": False}, + {"required_status_checks": {}}, + {"required_status_checks": {"checks": None, "contexts": []}}, + {"required_status_checks": {"checks": [], "contexts": None}}): + def response(endpoint): + return value if endpoint.endswith("/protection") else self.fixture(endpoint) + with self.subTest(value=value), patch.object(readiness, "api", response): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_disabled_legacy_requires_exact_readonly_identity_proof(self): + result = subprocess.CompletedProcess([], 0, stdout=json.dumps(self.disabled_legacy_response()), stderr="") + with self.legacy_error("Branch protection has been disabled on this repository."), \ + patch.object(readiness.subprocess, "run", return_value=result) as mocked: + self.assertEqual(self.invoke("sha")[0], 0) + self.assertEqual(mocked.call_count, 2) + args = mocked.call_args.args[0] + self.assertEqual(args[args.index("--method") + 1], "POST") + self.assertIn("graphql", args) + self.assertIn("owner=atrinik", args) + self.assertIn("name=example", args) + self.assertIn("qualifiedName=refs/heads/main", args) + query = next(arg for arg in args if arg.startswith("query=")) + self.assertTrue(query.startswith("query=query LegacyBranchProtection(")) + self.assertNotIn("mutation", query) + + def test_sha_disabled_legacy_ambiguous_identity_or_presence_rejected(self): + mutations = [lambda r: r.update(errors=[]), + lambda r: r["data"].update(repository=None), + lambda r: r["data"]["repository"].update(databaseId=43), + lambda r: r["data"]["repository"].update(databaseId=True), + lambda r: r["data"]["repository"].update(nameWithOwner="foreign/example"), + lambda r: r["data"]["repository"].update(isArchived=True), + lambda r: r["data"]["repository"].update(ref=None), + lambda r: r["data"]["repository"]["ref"].update(name="other"), + lambda r: r["data"]["repository"]["ref"].pop("branchProtectionRule"), + lambda r: r["data"]["repository"]["ref"].update(branchProtectionRule={"id": "rule"})] + for index, mutate in enumerate(mutations): + response = self.disabled_legacy_response() + mutate(response) + result = subprocess.CompletedProcess([], 0, stdout=json.dumps(response), stderr="") + with self.subTest(index=index), self.legacy_error("Branch protection has been disabled on this repository."), \ + patch.object(readiness.subprocess, "run", return_value=result): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_disabled_legacy_unreadable_or_invalid_graphql_rejected(self): + for result in (subprocess.CompletedProcess([], 1, stdout="", stderr="denied"), + subprocess.CompletedProcess([], 0, stdout="not json", stderr="")): + with self.subTest(result=result), self.legacy_error("Branch protection has been disabled on this repository."), \ + patch.object(readiness.subprocess, "run", return_value=result): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_disabled_legacy_fresh_rest_identity_required(self): + for field, value in (("id", 43), ("full_name", "foreign/example"), + ("default_branch", "other"), ("archived", True)): + count = 0 + def response(endpoint): + nonlocal count + if endpoint.endswith("/protection"): + raise readiness.APIError(endpoint, 404, { + "message": "Branch protection has been disabled on this repository.", "status": "404"}) + current = self.fixture(endpoint) + if endpoint == "repos/atrinik/example": + count += 1 + if count > 1: + current[field] = value + return current + with self.subTest(field=field), patch.object(readiness, "api", response), \ + patch.object(readiness.subprocess, "run") as mocked: + self.assertNotEqual(self.invoke("sha")[0], 0) + mocked.assert_not_called() + + def test_sha_disabled_legacy_proof_rechecked_before_acceptance(self): + initial = self.disabled_legacy_response() + changed = copy.deepcopy(initial) + changed["data"]["repository"]["ref"]["branchProtectionRule"] = {"id": "new-rule"} + results = [subprocess.CompletedProcess([], 0, stdout=json.dumps(value), stderr="") + for value in (initial, changed)] + with self.legacy_error("Branch protection has been disabled on this repository."), \ + patch.object(readiness.subprocess, "run", side_effect=results): + self.assertEqual(self.invoke("sha")[0], 2) + + def test_sha_full_legacy_checks_array_requires_all_bound_contexts(self): + self.fixture.legacy_checks = {"contexts": [], "checks": [ + {"context": "Legacy check", "app_id": 15368}, + {"context": "Second legacy check", "app_id": None}]} + self.assert_deferred("sha", "Legacy check") + self.fixture.checks[HEAD].append(check("Legacy check", identifier=30)) + self.assert_deferred("sha", "Second legacy check") + self.fixture.checks[HEAD].append(check("Second legacy check", identifier=31)) + self.assertEqual(self.invoke("sha")[0], 0) + self.fixture.checks[HEAD][-1]["conclusion"] = "failure" + self.assert_deferred("sha", "Second legacy check") + + def test_sha_legacy_absence_must_remain_readable_at_final_fence(self): + count = 0 + def response(endpoint): + nonlocal count + if endpoint.endswith("/protection"): + count += 1 + if count > 1: + raise readiness.APIError(endpoint, 404, {"message": "Not Found", "status": "404"}) + return self.fixture(endpoint) + with patch.object(readiness, "api", response): + self.assertEqual(self.invoke("sha")[0], 2) + def test_sha_non_actions_app_binding_never_satisfied_by_same_named_actions_check(self): self.fixture.effective_rules = [self.effective_check("Required checks", 999)] self.assert_deferred("sha", "non-Actions app") @@ -808,7 +931,7 @@ def test_sha_unbound_effective_context_still_requires_trusted_actions(self): self.assert_deferred("sha", "Required checks") def test_sha_effective_rule_or_legacy_changes_during_preflight_rejected(self): - for endpoint_part in ("/rules/branches/", "/protection/required_status_checks"): + for endpoint_part in ("/rules/branches/", "/protection"): with self.subTest(endpoint=endpoint_part): count = 0 def drift(endpoint): @@ -818,13 +941,13 @@ def drift(endpoint): if count > 1: if "rules/branches" in endpoint: return [self.effective_check("New gate")] - return {"contexts": ["New gate"], "checks": []} + return {"required_status_checks": {"contexts": ["New gate"], "checks": []}} return self.fixture(endpoint) with patch.object(readiness, "api", drift): self.assert_deferred("sha", "effective required checks changed") def test_sha_effective_rules_and_legacy_unreadable_fail_closed(self): - for endpoint_part in ("/rules/branches/", "/protection/required_status_checks"): + for endpoint_part in ("/rules/branches/", "/protection"): with self.subTest(endpoint=endpoint_part): def unreadable(endpoint): if endpoint_part in endpoint: @@ -1825,6 +1948,17 @@ def test_api_subprocess_uses_only_get_and_version(self): self.assertEqual(args[args.index("--method") + 1], "GET") self.assertIn("X-GitHub-Api-Version: 2026-03-10", args) + def test_api_error_retains_structured_response_without_printing_body(self): + self.api_patch.stop() + for body in ('{"message":"Branch not protected","status":"404"}', "not json", "null"): + result = subprocess.CompletedProcess([], 1, stdout=body, stderr="gh: error (HTTP 404)") + with self.subTest(body=body), patch.object(readiness.subprocess, "run", return_value=result): + with self.assertRaises(readiness.APIError) as raised: + readiness.api("repos/atrinik/example/branches/main/protection") + self.assertEqual(raised.exception.status, 404) + self.assertEqual(raised.exception.response, json.loads(body) if body != "not json" else None) + self.assertNotIn(body, str(raised.exception)) + def test_executable_wrapper_is_bash_and_delegates_python(self): wrapper = ROOT / "bin/security-readiness" self.assertEqual(wrapper.read_text().splitlines()[0], "#!/usr/bin/env bash") From bd9ae640a3528817d02e26301e19ae3722f388b8 Mon Sep 17 00:00:00 2001 From: Zoey Rose Date: Sun, 11 Oct 2026 18:59:31 -0500 Subject: [PATCH 22/22] fix(security): audit the complete effective branch policy --- AGENTS.md | 4 ++- CONTRIBUTING.md | 5 ++- README.md | 4 ++- bin/audit | 67 ++++++++++++++++++++++++++++++++++++++--- tests/audit-fixtures.py | 61 ++++++++++++++++++++++++++++++++++--- tests/audit.sh | 21 +++++++++++++ 6 files changed, 150 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e282c73..a3f436c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -284,7 +284,9 @@ protection on active default and declared maintenance branches; any presence is drift under the canonical ruleset contract. Only verified unprotected responses count as absent; unreadable evidence fails closed and audit never - removes protections. Only the exact repository-disabled REST response may + removes protections. Compare the complete effective default and maintenance branch policy with + declared allowed rule types and parameters; undeclared integrity or restriction types are drift, + even when every desired rule is present. Only the exact repository-disabled REST response may fall back to a fixed read-only GraphQL query with matching repository/branch identity, active state and explicit null legacy-rule association. Missing or conflicting evidence fails closed; preserve historical rules on other refs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5d2fb2..5145d20 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -182,7 +182,10 @@ Pending existing-gate proof requires unchanged enforcement and ref applicability in the final rule read; enforced intent retains recognized restoration. Audit legacy protection on active default and maintenance branches as drift under the canonical ruleset contract; distinguish verified unprotected responses from -unreadable evidence, and never remove protection during audit. The exact +unreadable evidence, and never remove protection during audit. Compare the +complete effective default and maintenance branch policy with declared allowed +rule types and parameters; additional +undeclared integrity or restriction types are drift. The exact repository-disabled REST response may use the fixed read-only GraphQL fallback; require matching repository identity and branch, active state and an explicit null legacy-rule association. Errors and missing or conflicting fields fail diff --git a/README.md b/README.md index a0e57fd..8399d2c 100644 --- a/README.md +++ b/README.md @@ -315,7 +315,9 @@ counts as absent. For the exact repository-disabled REST response, a fixed read-only GraphQL query may instead prove that the identified active repository and exact existing branch have an explicitly null legacy-rule association. Missing, conflicting or unreadable evidence remains drift. Historical rules on -other branches remain untouched. The audit never removes branch protections. +other branches remain untouched. Audit compares the complete effective default and maintenance branch +policy with declared allowed rule types and parameters; extra integrity or restriction types remain +drift even when every desired rule is present. The audit never removes branch protections. Publisher identity, absence, update and retirement decisions use the complete validated ruleset inventory across all pages, including ordinary organization diff --git a/bin/audit b/bin/audit index ce35b7c..657a0fd 100755 --- a/bin/audit +++ b/bin/audit @@ -246,6 +246,42 @@ def normalize_rules: else . end) | sort_by(.type); ' +read_effective_rules() { + local repository=$1 branch=$2 encoded + encoded=$(jq -rn --arg branch "${branch}" '$branch | @uri') + github_api --paginate --slurp \ + "repos/${organization}/${repository}/rules/branches/${encoded}?per_page=100" | jq -e ' + if type == "array" and length > 0 and all(.[]; type == "array") then add + else error("invalid effective-rule pages") end | + if type == "array" and all(.[]; type == "object" and + (.type | type == "string" and length > 0)) then . + else error("invalid effective-rule inventory") end' +} + +audit_effective_policy() { + local repository=$1 branch=$2 effective=$3 desired=$4 + # API provenance is diagnostic metadata. Every policy field, including + # unknown future parameters, must match the declared contract. CI and + # Dependency review may contribute separate rules with the same parameters. + if ! jq -e --argjson desired "${desired}" "${ruleset_normalization}"' + def policy_fields: map(del(.ruleset_id, .ruleset_source, .ruleset_source_type)); + def without_checks: + if .type == "required_status_checks" then .parameters.required_status_checks = [] + else . end; + def checks: + [.[] | select(.type == "required_status_checks") | .parameters.required_status_checks[]] | + unique_by(.context, .integration_id) | sort_by(.context, .integration_id); + ($desired | normalize_rules) as $wanted | + (policy_fields | normalize_rules) as $current | + ([$current[].type] | unique) == ([$wanted[].type] | unique) and + all($current[]; . as $rule | + any($wanted[]; without_checks == ($rule | without_checks))) and + ($current | checks) == ($wanted | checks) + ' <<<"${effective}" >/dev/null 2>&1; then + record_finding "effective branch policy drift requires manual review: ${organization}/${repository}/${branch}" + fi +} + check_ruleset() { local repository=$1 name=$2 template=$3 list=$4 local entry id source endpoint detail wanted scope @@ -511,14 +547,26 @@ for repository in "${!expected_active[@]}"; do done branch=$(jq -r '.default_branch' "${repository_defaults}") audit_legacy_protection "${repository}" "${branch}" "${metadata_file}" - while IFS= read -r maintenance_branch; do + while IFS= read -r maintenance; do + maintenance_branch=$(jq -r .branch <<<"${maintenance}") audit_legacy_protection "${repository}" "${maintenance_branch}" "${metadata_file}" - done < <(jq -r --arg repository "${repository}" ' - .maintenance_branches[] | select(.repository == $repository) | .branch + if ! maintenance_effective=$(read_effective_rules "${repository}" "${maintenance_branch}"); then + record_finding "effective maintenance-branch rules could not be read: ${organization}/${repository}/${maintenance_branch}" + maintenance_effective='[]' + fi + maintenance_desired=$(jq --argjson checks "$(jq -c .required_ci <<<"${maintenance}")" ' + .rules | map(if .type == "required_status_checks" then + if ($checks | length) == 0 then empty + else .parameters.required_status_checks = + ($checks | map({context: ., integration_id: 15368})) end + else . end) + ' "${root}/config/rulesets/maintenance-branch.json") + audit_effective_policy "${repository}" "${maintenance_branch}" "${maintenance_effective}" "${maintenance_desired}" + done < <(jq -c --arg repository "${repository}" ' + .maintenance_branches[] | select(.repository == $repository) ' "${repositories_config}") effective='[]' - if ! effective=$(github_api --paginate --slurp \ - "repos/${organization}/${repository}/rules/branches/${branch}?per_page=100" | jq -e 'add'); then + if ! effective=$(read_effective_rules "${repository}" "${branch}"); then record_finding "effective default-branch rules could not be read: ${organization}/${repository}" effective='[]' fi @@ -560,6 +608,10 @@ for repository in "${!expected_active[@]}"; do ' <<<"${effective}" >/dev/null; then record_finding "effective required CI contexts or app drift: ${organization}/${repository}" fi + desired_effective=$(jq --argjson checks "${expected_checks}" ' + if ($checks | length) == 0 then [] + else .rules | map(.parameters.required_status_checks = $checks) end + ' "${root}/config/rulesets/required-ci.json") for policy in default-branch-integrity default-branch-linear-history; do template="${root}/config/rulesets/${policy}.json" if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ @@ -567,6 +619,7 @@ for repository in "${!expected_active[@]}"; do template="${root}/config/rulesets/${policy}-classic.json" fi check_ruleset "${repository}" "$(jq -r .name "${template}")" "${template}" "${rulesets}" "${metadata_file}" + desired_effective=$(jq --argjson prior "${desired_effective}" '$prior + .rules' "${template}") if ! jq -e --slurpfile desired "${template}" \ '. as $effective | all($desired[0].rules[]; . as $rule | any($effective[]; .type == $rule.type))' \ <<<"${effective}" >/dev/null; then @@ -578,6 +631,7 @@ for repository in "${!expected_active[@]}"; do if jq -e --arg repository "${repository}" '.repositories | index($repository) != null' \ "${advisory_windows}" >/dev/null; then template="${root}/config/rulesets/pull-request-gate-classic.json"; fi check_ruleset "${repository}" "$(jq -r .name "${template}")" "${template}" "${rulesets}" "${metadata_file}" + desired_effective=$(jq --argjson prior "${desired_effective}" '$prior + .rules' "${template}") if ! jq -e --slurpfile desired "${template}" \ "${ruleset_normalization}"'([.[] | select(.type == "pull_request") | {type, parameters}] | normalize_rules) == ($desired[0].rules | normalize_rules)' \ @@ -622,6 +676,8 @@ for repository in "${!expected_active[@]}"; do if ${codeql_required}; then check_ruleset "${repository}" '06 - CodeQL security' \ "${root}/config/rulesets/code-scanning.json" "${rulesets}" "${metadata_file}" + desired_effective=$(jq --argjson prior "${desired_effective}" '$prior + .rules' \ + "${root}/config/rulesets/code-scanning.json") if ! jq -e --slurpfile desired "${root}/config/rulesets/code-scanning.json" \ '[.[] | select(.type == "code_scanning") | {type, parameters}] == $desired[0].rules' \ <<<"${effective}" >/dev/null; then @@ -631,6 +687,7 @@ for repository in "${!expected_active[@]}"; do elif jq -e 'any(.[]; .type == "code_scanning")' <<<"${effective}" >/dev/null; then record_finding "unexpected effective CodeQL merge protection: ${organization}/${repository}" fi + audit_effective_policy "${repository}" "${branch}" "${effective}" "${desired_effective}" audit_security "${repository}" if ! jq -e ' diff --git a/tests/audit-fixtures.py b/tests/audit-fixtures.py index df66e19..6b8e82b 100755 --- a/tests/audit-fixtures.py +++ b/tests/audit-fixtures.py @@ -6,6 +6,7 @@ import os import sys from pathlib import Path +from urllib.parse import unquote root = Path(os.environ['AUDIT_CONFIG_ROOT']) scenario = os.environ.get('GH_TEST_SCENARIO', 'pass') @@ -393,6 +394,7 @@ def rulesets(repository): if payload is None: reject('repository ruleset missing') elif path.startswith('rules/branches/'): + branch = unquote(path.removeprefix('rules/branches/').partition('?')[0]) if scenario == 'effective-unreadable' and target: reject('effective branch read denied') payload = [] @@ -401,13 +403,24 @@ def rulesets(repository): included = any(pattern == '~ALL' or fnmatch.fnmatchcase(repository, pattern) for pattern in scope['include']) excluded = any(pattern == '~ALL' or fnmatch.fnmatchcase(repository, pattern) for pattern in scope['exclude']) refs = item['conditions']['ref_name'] - branch_included = any(pattern == '~DEFAULT_BRANCH' or - fnmatch.fnmatchcase('refs/heads/main', pattern) for pattern in refs['include']) - branch_excluded = any(pattern == '~DEFAULT_BRANCH' or - fnmatch.fnmatchcase('refs/heads/main', pattern) for pattern in refs['exclude']) + branch_included = any((pattern == '~DEFAULT_BRANCH' and branch == 'main') or + fnmatch.fnmatchcase('refs/heads/' + branch, pattern) for pattern in refs['include']) + branch_excluded = any((pattern == '~DEFAULT_BRANCH' and branch == 'main') or + fnmatch.fnmatchcase('refs/heads/' + branch, pattern) for pattern in refs['exclude']) if (item['target'] == 'branch' and item['enforcement'] == 'active' and included and not excluded and branch_included and not branch_excluded): payload.extend(copy.deepcopy(item['rules'])) + for maintenance in repositories['maintenance_branches']: + if maintenance['repository'] == repository and maintenance['branch'] == branch: + rules = config('rulesets/maintenance-branch.json')['rules'] + for rule in rules: + if rule['type'] == 'required_status_checks': + if not maintenance['required_ci']: + continue + rule['parameters']['required_status_checks'] = [ + {'context': context, 'integration_id': 15368} + for context in maintenance['required_ci']] + payload.append(rule) for rule in payload: if target and rule['type'] == 'required_status_checks': if scenario == 'wrong-app': @@ -427,6 +440,46 @@ def rulesets(repository): payload.extend(config('rulesets/code-scanning.json')['rules']) if scenario == 'unexpected-pr' and repository == 'web-platform': payload.append(config('rulesets/pull-request-gate.json')['rules'][0]) + if scenario.startswith('closure-'): + _, selected_branch, variant = scenario.split('-', 2) + selected = (selected_branch == 'default' and branch == 'main') or ( + selected_branch == 'maintenance' and branch == 'release/next#candidate') + if selected: + extra = { + 'signatures': {'type': 'required_signatures'}, + 'deployments': {'type': 'required_deployments', 'parameters': {'required_deployment_environments': ['production']}}, + 'update': {'type': 'update', 'parameters': {'update_allows_fetch_and_merge': False}}, + 'unknown': {'type': 'future_unconfigured_rule', 'parameters': {'enabled': True}}, + } + if variant in extra: + payload.append(extra[variant]) + elif variant == 'integrity-parameter': + next(rule for rule in payload if rule['type'] == 'deletion')['parameters'] = {'unexpected': True} + elif variant == 'status-parameter': + next(rule for rule in payload if rule['type'] == 'required_status_checks')['parameters']['unexpected'] = True + elif variant == 'top-field': + payload[0]['unknown_policy_field'] = True + elif variant == 'missing-type': + payload.append({'parameters': {}}) + elif variant == 'null-item': + payload.append(None) + elif variant == 'malformed-parameters': + next(rule for rule in payload if rule['type'] == 'deletion')['parameters'] = [] + elif variant == 'null-page': + print(json.dumps([payload, None])) + sys.exit(0) + elif variant == 'unreadable': + reject('effective branch read denied') + elif variant == 'non-json': + print('unavailable') + sys.exit(0) + elif variant in ('metadata-organization', 'metadata-repository'): + for rule in payload: + rule.update(ruleset_id=99, ruleset_source='atrinik' if variant.endswith('organization') else 'atrinik/observatory', + ruleset_source_type='Organization' if variant.endswith('organization') else 'Repository') + elif variant == 'signatures-inherited': + payload.append({'type': 'required_signatures', 'ruleset_id': 99, + 'ruleset_source': 'atrinik', 'ruleset_source_type': 'Organization'}) elif path == 'code-security-configuration': advanced = repository in config('codeql-advanced-setup.json')['repositories'] if scenario == 'configuration-unreadable' and target: diff --git a/tests/audit.sh b/tests/audit.sh index a86917c..f8c3223 100755 --- a/tests/audit.sh +++ b/tests/audit.sh @@ -51,10 +51,16 @@ cp -R "${temporary}/fixture-observatory" "${temporary}/fixture-retired-observato jq 'del(.required_ci.observatory)' "${temporary}/full/config/repositories.json" \ >"${temporary}/fixture-retired-observatory/config/repositories.json" +cp -R "${temporary}/fixture-legacy-branches" "${temporary}/fixture-effective-branches" +jq '(.maintenance_branches[] | select(.branch == "release/next#candidate") | .required_ci) = ["Conventional PR title"]' \ + "${temporary}/fixture-legacy-branches/config/repositories.json" \ + >"${temporary}/fixture-effective-branches/config/repositories.json" + run_audit() { local scenario=$1 fixture=${2:-focused} repository=observatory if [[ ${fixture} == focused ]]; then case ${scenario} in + closure-*) repository=effective-branches ;; retained-*) repository=.github ;; retired-*) repository=retired-observatory ;; legacy-maintenance-*|legacy-multiple) repository=legacy-branches ;; @@ -198,6 +204,21 @@ for variant in unreadable non-json object missing-name null-item partial-page; d assert_drift "retained-list-${variant}" 'inherited rulesets could not be read' done +# Check the entire effective policy on main and exact maintenance refs. Desired +# CI, Dependency review and CodeQL coexist; API provenance is metadata only. +for branch in default maintenance; do + for variant in baseline metadata-organization metadata-repository; do + output=$(run_audit "closure-${branch}-${variant}") + grep -Fq 'Governance audit passed:' <<<"${output}" + done + for variant in signatures deployments update unknown integrity-parameter status-parameter top-field malformed-parameters signatures-inherited; do + assert_drift "closure-${branch}-${variant}" 'effective branch policy drift requires manual review' + done + for variant in missing-type null-item null-page unreadable non-json; do + assert_drift "closure-${branch}-${variant}" "effective ${branch}-branch rules could not be read" + done +done + assert_drift unmanaged 'active repository is not in the desired-state inventory' assert_drift merge-drift 'repository merge policy drift' assert_drift properties-drift 'repository custom-property drift'