diff --git a/bin/check-project-health b/bin/check-project-health index 5744bd3..d939e3d 100755 --- a/bin/check-project-health +++ b/bin/check-project-health @@ -57,60 +57,84 @@ if [[ -n ${GITHUB_REPOSITORY:-} && ${GITHUB_REPOSITORY} != "${repository}" ]]; t exit 1 fi -github_api() { +github_api_to_file() { local operation=$1 - local response + local response_file=$2 local status - shift - if response=$(gh api -H "X-GitHub-Api-Version: ${api_version}" "$@"); then + shift 2 + if gh api -H "X-GitHub-Api-Version: ${api_version}" "$@" \ + >"${response_file}"; then : else status=$? echo "error: GitHub API operation failed: ${operation}" >&2 return "${status}" fi - if ! jq -e '.' <<<"${response}" >/dev/null; then + if ! jq -e '.' "${response_file}" >/dev/null; then echo "error: GitHub API operation returned invalid JSON: ${operation}" >&2 return 1 fi - printf '%s\n' "${response}" } -if runs=$(github_api "read ${sync_workflow} workflow runs" \ - "repos/${repository}/actions/workflows/${sync_workflow}/runs?per_page=100"); then +github_api() { + local operation=$1 + local response_file=${temporary}/github-api-response + + shift + github_api_to_file "${operation}" "${response_file}" "$@" || return + cat "${response_file}" +} + +workflow_runs_file=${temporary}/workflow-runs.json +successful_runs_file=${temporary}/successful-workflow-runs.json +if github_api_to_file "read ${sync_workflow} workflow runs" \ + "${workflow_runs_file}" \ + "repos/${repository}/actions/workflows/${sync_workflow}/runs?per_page=100"; then : else status=$? exit "${status}" fi -if successful_runs=$(github_api "read latest successful ${sync_workflow} run" \ - "repos/${repository}/actions/workflows/${sync_workflow}/runs?status=success&per_page=1"); then +if github_api_to_file "read latest successful ${sync_workflow} run" \ + "${successful_runs_file}" \ + "repos/${repository}/actions/workflows/${sync_workflow}/runs?status=success&per_page=1"; then : else status=$? exit "${status}" fi -if ! jq -e '.workflow_runs | type == "array"' <<<"${runs}" >/dev/null; then +if ! jq -se ' + length == 1 and (.[0].workflow_runs | type == "array") +' "${workflow_runs_file}" >/dev/null; then echo "error: workflow-runs response omitted workflow_runs" >&2 exit 1 fi -if ! jq -e '.workflow_runs | type == "array"' \ - <<<"${successful_runs}" >/dev/null; then +if ! jq -se ' + length == 1 and (.[0].workflow_runs | type == "array") +' "${successful_runs_file}" >/dev/null; then echo "error: successful workflow-runs response omitted workflow_runs" >&2 exit 1 fi -latest_run=$(jq -c '.workflow_runs[0] // null' <<<"${runs}") +latest_run=$(jq -c ' + .workflow_runs[0] // null | + if . == null then null + else {html_url, conclusion, status, head_sha} + end +' "${workflow_runs_file}") # The filtered endpoint can lag the unfiltered history. Compare completion # timestamps so an older filtered result cannot hide a recent successful sync; # retain it as a fallback when success is outside the recent 100-run window. -last_success=$(jq -cn --argjson recent "${runs}" \ - --argjson successful "${successful_runs}" ' +last_success=$(jq -cs ' + .[0] as $recent | .[1] as $successful | [$recent.workflow_runs[], $successful.workflow_runs[] | select(.status == "completed" and .conclusion == "success")] | max_by(.updated_at | fromdateiso8601) // null -') + | if . == null then null + else {html_url, updated_at, head_sha} + end +' "${workflow_runs_file}" "${successful_runs_file}") consecutive_failures=$(jq -r ' reduce .workflow_runs[] as $run ( {count: 0, stopped: false}; @@ -121,7 +145,7 @@ consecutive_failures=$(jq -r ' .count += 1 else . end ) | .count -' <<<"${runs}") +' "${workflow_runs_file}") latest_url=$(jq -r '.html_url // "none"' <<<"${latest_run}") latest_conclusion=$(jq -r '.conclusion // .status // "none"' <<<"${latest_run}") diff --git a/tests/check-project-health.sh b/tests/check-project-health.sh index 09a8f41..b9c7709 100755 --- a/tests/check-project-health.sh +++ b/tests/check-project-health.sh @@ -59,9 +59,74 @@ fi case ${endpoint} in "repos/atrinik/github-settings/actions/workflows/sync-project.yml/runs?per_page=100") case ${FAKE_GH_SCENARIO} in + large-healthy) + jq -n --argjson size "${FAKE_LARGE_RESPONSE_BYTES}" '{workflow_runs: [{ + status: "completed", conclusion: "success", + created_at: "2026-08-10T04:49:00Z", + updated_at: "2026-08-10T04:50:00Z", + html_url: "https://github.com/atrinik/github-settings/actions/runs/100", + head_sha: "SUCCESS", + log: ("x" * $size) + }]}' + ;; + large-repeated-failure) + jq -n --argjson size "${FAKE_LARGE_RESPONSE_BYTES}" '{workflow_runs: [ + { + status: "completed", conclusion: "failure", + updated_at: "2026-08-10T04:59:00Z", + html_url: "https://github.com/atrinik/github-settings/actions/runs/102", + head_sha: "FAIL2", + log: ("x" * $size) + }, + { + status: "completed", conclusion: "failure", + updated_at: "2026-08-10T04:49:00Z", + html_url: "https://github.com/atrinik/github-settings/actions/runs/101", + head_sha: "FAIL1" + }, + { + status: "completed", conclusion: "success", + updated_at: "2026-08-10T04:40:00Z", + html_url: "https://github.com/atrinik/github-settings/actions/runs/100", + head_sha: "SUCCESS" + } + ]}' + ;; + large-invalid-response) + jq -nr --argjson size "${FAKE_LARGE_RESPONSE_BYTES}" \ + '"{\"workflow_runs\":[" + ("x" * $size)' + ;; + runs-malformed) + printf '{' + ;; + runs-empty) + : + ;; + runs-whitespace) + printf ' \n\t' + ;; + runs-null) + jq -n 'null' + ;; + runs-false) + jq -n 'false' + ;; + runs-object) + jq -n '{}' + ;; + runs-wrong-type) + jq -n '{workflow_runs: {}}' + ;; + multi-document-runs) + jq -n '{workflow_runs: []}' + jq -n '{workflow_runs: []}' + ;; healthy | missing-settings | recovery | recurrence-before | reopen | \ incident-api-failure | filtered-stale | filtered-empty | filtered-newer | \ - filtered-recovery | filtered-before-episode | filtered-pending | future-success) + filtered-recovery | filtered-before-episode | filtered-pending | future-success | \ + multi-document-success | success-malformed | success-empty | \ + success-whitespace | success-null | success-false | success-object | \ + success-wrong-type) jq -n '{workflow_runs: [{ status: "completed", conclusion: "success", created_at: "2026-08-10T04:49:00Z", @@ -121,7 +186,9 @@ case ${endpoint} in } ]}' ;; - stale | sync-failure | foreign | multiple-managed | alert-api-failure) + stale | sync-failure | foreign | multiple-managed | alert-api-failure | \ + alert-malformed-response | alert-empty-response | alert-null-response | \ + alert-false-response) jq -n '{workflow_runs: [{ status: "completed", conclusion: "success", created_at: "2026-08-10T01:59:00Z", @@ -165,8 +232,43 @@ case ${endpoint} in ;; "repos/atrinik/github-settings/actions/workflows/sync-project.yml/runs?status=success&per_page=1") case ${FAKE_GH_SCENARIO} in + large-healthy | large-repeated-failure) + jq -n --argjson size "${FAKE_LARGE_RESPONSE_BYTES}" '{workflow_runs: [{ + status: "completed", conclusion: "success", + created_at: "2026-08-10T04:39:00Z", + updated_at: "2026-08-10T04:40:00Z", + html_url: "https://github.com/atrinik/github-settings/actions/runs/100", + head_sha: "SUCCESS", + log: ("x" * $size) + }]}' + ;; + multi-document-success) + jq -n '{workflow_runs: []}' + jq -n '{workflow_runs: []}' + ;; + success-malformed) + printf '{' + ;; + success-empty) + : + ;; + success-whitespace) + printf ' \n\t' + ;; + success-null) + jq -n 'null' + ;; + success-false) + jq -n 'false' + ;; + success-object) + jq -n '{}' + ;; + success-wrong-type) + jq -n '{workflow_runs: {}}' + ;; healthy | missing-settings | recovery | recurrence-before | reopen | \ - incident-api-failure) + incident-api-failure | multi-document-runs | runs-object | runs-wrong-type) jq -n '{workflow_runs: [{ status: "completed", conclusion: "success", created_at: "2026-08-10T04:49:00Z", @@ -185,6 +287,8 @@ case ${endpoint} in }]}' ;; stale | sync-failure | foreign | multiple-managed | alert-api-failure | \ + alert-malformed-response | alert-empty-response | alert-null-response | \ + alert-false-response | \ filtered-stale | filtered-recovery | filtered-before-episode | filtered-pending) jq -n '{workflow_runs: [{ status: "completed", conclusion: "success", @@ -291,7 +395,13 @@ case ${endpoint} in ;; repos/atrinik/github-settings/issues) [[ ${method} == POST ]] - jq -n '{number: 70, state: "open"}' + case ${FAKE_GH_SCENARIO} in + alert-malformed-response) printf '{' ;; + alert-empty-response) : ;; + alert-null-response) jq -n 'null' ;; + alert-false-response) jq -n 'false' ;; + *) jq -n '{number: 70, state: "open"}' ;; + esac ;; repos/atrinik/github-settings/issues/70) [[ ${method} == PATCH ]] @@ -336,6 +446,8 @@ PLAN EOF chmod +x "${temporary}/bin/sync-plan" +large_response_bytes=$(($(getconf ARG_MAX) + 65536)) + run_health() { local gh_scenario=$1 local sync_scenario=$2 @@ -345,6 +457,7 @@ run_health() { FAKE_GH_LOG="${temporary}/gh.log" \ FAKE_GH_BODY="${temporary}/gh-body" \ FAKE_GH_SCENARIO="${gh_scenario}" \ + FAKE_LARGE_RESPONSE_BYTES="${large_response_bytes}" \ FAKE_SYNC_LOG="${temporary}/sync.log" \ FAKE_SYNC_SCENARIO="${sync_scenario}" \ GITHUB_ACTIONS=true \ @@ -367,6 +480,67 @@ grep -Fq 'Convergence: converged; mutations 0' <<<"${output}" grep -Fq 'State: **healthy**' "${temporary}/step-summary" [[ $(wc -l <"${temporary}/gh.log") == 2 ]] +# Both workflow-run endpoints can exceed the host's entire command argument +# limit when run metadata contains large logs. The monitor must parse those +# responses without placing either JSON document on a child-process argv. +: >"${temporary}/gh.log" +output=$(run_health large-healthy zero) +grep -Fq 'State: **healthy**' <<<"${output}" +grep -Fq 'Last successful run: [2026-08-10T04:50:00Z]' <<<"${output}" + +: >"${temporary}/gh.log" +if run_health large-repeated-failure zero --apply \ + >"${temporary}/large-failure.out" 2>"${temporary}/large-failure.err"; then + echo "error: health check accepted large repeated synchronization failures" >&2 + exit 1 +fi +grep -Fq '2 consecutive synchronization runs failed' \ + "${temporary}/large-failure.out" +grep -Fq 'ALERT created managed Project health incident' \ + "${temporary}/large-failure.out" + +: >"${temporary}/gh.log" +: >"${temporary}/sync.log" +if run_health large-invalid-response zero \ + >"${temporary}/large-invalid.out" 2>"${temporary}/large-invalid.err"; then + echo "error: health check accepted a large invalid API response" >&2 + exit 1 +fi +grep -Fq 'GitHub API operation returned invalid JSON' \ + "${temporary}/large-invalid.err" +[[ ! -s ${temporary}/sync.log ]] + +for scenario in runs-malformed runs-empty runs-whitespace runs-null runs-false \ + success-malformed success-empty success-whitespace success-null success-false; do + : >"${temporary}/gh.log" + : >"${temporary}/sync.log" + if run_health "${scenario}" zero \ + >"${temporary}/${scenario}.out" 2>"${temporary}/${scenario}.err"; then + echo "error: health check accepted ${scenario} API response" >&2 + exit 1 + fi + grep -Fq 'GitHub API operation returned invalid JSON' \ + "${temporary}/${scenario}.err" + [[ ! -s ${temporary}/sync.log ]] +done + +for scenario in multi-document-runs runs-object runs-wrong-type \ + multi-document-success success-object success-wrong-type; do + : >"${temporary}/gh.log" + : >"${temporary}/sync.log" + if run_health "${scenario}" zero \ + >"${temporary}/${scenario}.out" 2>"${temporary}/${scenario}.err"; then + echo "error: health check accepted ${scenario} workflow response" >&2 + exit 1 + fi + expected_error='workflow-runs response omitted workflow_runs' + if [[ ${scenario} == *success* ]]; then + expected_error='successful workflow-runs response omitted workflow_runs' + fi + grep -Fq "${expected_error}" "${temporary}/${scenario}.err" + [[ ! -s ${temporary}/sync.log ]] +done + # Recent successful runs must win over stale or empty filtered history, while # a newer filtered response must still win if the run completed between reads. for scenario in filtered-stale filtered-empty filtered-newer unordered-successes; do @@ -446,6 +620,18 @@ grep -Fq 'list Project health incidents (page 1)' \ grep -Fq 'create Project health incident' \ "${temporary}/alert-api-failure.err" +for scenario in alert-malformed-response alert-empty-response \ + alert-null-response alert-false-response; do + : >"${temporary}/gh.log" + if run_health "${scenario}" zero --apply \ + >"${temporary}/${scenario}.out" 2>"${temporary}/${scenario}.err"; then + echo "error: health check accepted ${scenario} from incident creation" >&2 + exit 1 + fi + grep -Fq 'GitHub API operation returned invalid JSON: create Project health incident' \ + "${temporary}/${scenario}.err" +done + : >"${temporary}/gh.log" : >"${temporary}/sync.log" if run_health repeated-failure zero --apply \