Skip to content

fix(projects): handle large synchronization histories #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zoeyrose merged 1 commit into from
Oct 4, 2026
Merged

fix(projects): handle large synchronization histories #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 42 additions & 18 deletions bin/check-project-health
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,60 +57,84 @@ if [[ -n ${GITHUB_REPOSITORY:-} && ${GITHUB_REPOSITORY} != "${repository}" ]]; t
exit 1
fi

github_api() {
github_api_to_file() {
local operation=$1
local response
local response_file=$2
local status

shift
if response=$(gh api -H "X-GitHub-Api-Version: ${api_version}" "$@"); then
shift 2
if gh api -H "X-GitHub-Api-Version: ${api_version}" "$@" \
>"${response_file}"; then
:
else
status=$?
echo "error: GitHub API operation failed: ${operation}" >&2
return "${status}"
fi
if ! jq -e '.' <<<"${response}" >/dev/null; then
if ! jq -e '.' "${response_file}" >/dev/null; then
echo "error: GitHub API operation returned invalid JSON: ${operation}" >&2
return 1
fi
printf '%s\n' "${response}"
}

if runs=$(github_api "read ${sync_workflow} workflow runs" \
"repos/${repository}/actions/workflows/${sync_workflow}/runs?per_page=100"); then
github_api() {
local operation=$1
local response_file=${temporary}/github-api-response

shift
github_api_to_file "${operation}" "${response_file}" "$@" || return
cat "${response_file}"
}

workflow_runs_file=${temporary}/workflow-runs.json
successful_runs_file=${temporary}/successful-workflow-runs.json
if github_api_to_file "read ${sync_workflow} workflow runs" \
"${workflow_runs_file}" \
"repos/${repository}/actions/workflows/${sync_workflow}/runs?per_page=100"; then
:
else
status=$?
exit "${status}"
fi
if successful_runs=$(github_api "read latest successful ${sync_workflow} run" \
"repos/${repository}/actions/workflows/${sync_workflow}/runs?status=success&per_page=1"); then
if github_api_to_file "read latest successful ${sync_workflow} run" \
"${successful_runs_file}" \
"repos/${repository}/actions/workflows/${sync_workflow}/runs?status=success&per_page=1"; then
:
else
status=$?
exit "${status}"
fi
if ! jq -e '.workflow_runs | type == "array"' <<<"${runs}" >/dev/null; then
if ! jq -se '
length == 1 and (.[0].workflow_runs | type == "array")
' "${workflow_runs_file}" >/dev/null; then
echo "error: workflow-runs response omitted workflow_runs" >&2
exit 1
fi
if ! jq -e '.workflow_runs | type == "array"' \
<<<"${successful_runs}" >/dev/null; then
if ! jq -se '
length == 1 and (.[0].workflow_runs | type == "array")
' "${successful_runs_file}" >/dev/null; then
echo "error: successful workflow-runs response omitted workflow_runs" >&2
exit 1
fi

latest_run=$(jq -c '.workflow_runs[0] // null' <<<"${runs}")
latest_run=$(jq -c '
.workflow_runs[0] // null |
if . == null then null
else {html_url, conclusion, status, head_sha}
end
' "${workflow_runs_file}")
# The filtered endpoint can lag the unfiltered history. Compare completion
# timestamps so an older filtered result cannot hide a recent successful sync;
# retain it as a fallback when success is outside the recent 100-run window.
last_success=$(jq -cn --argjson recent "${runs}" \
--argjson successful "${successful_runs}" '
last_success=$(jq -cs '
.[0] as $recent | .[1] as $successful |
[$recent.workflow_runs[], $successful.workflow_runs[] |
select(.status == "completed" and .conclusion == "success")]
| max_by(.updated_at | fromdateiso8601) // null
')
| if . == null then null
else {html_url, updated_at, head_sha}
end
' "${workflow_runs_file}" "${successful_runs_file}")
consecutive_failures=$(jq -r '
reduce .workflow_runs[] as $run (
{count: 0, stopped: false};
Expand All @@ -121,7 +145,7 @@ consecutive_failures=$(jq -r '
.count += 1
else . end
) | .count
' <<<"${runs}")
' "${workflow_runs_file}")

latest_url=$(jq -r '.html_url // "none"' <<<"${latest_run}")
latest_conclusion=$(jq -r '.conclusion // .status // "none"' <<<"${latest_run}")
Expand Down
194 changes: 190 additions & 4 deletions tests/check-project-health.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,74 @@ fi
case ${endpoint} in
"repos/atrinik/github-settings/actions/workflows/sync-project.yml/runs?per_page=100")
case ${FAKE_GH_SCENARIO} in
large-healthy)
jq -n --argjson size "${FAKE_LARGE_RESPONSE_BYTES}" '{workflow_runs: [{
status: "completed", conclusion: "success",
created_at: "2026-08-10T04:49:00Z",
updated_at: "2026-08-10T04:50:00Z",
html_url: "https://github.com/atrinik/github-settings/actions/runs/100",
head_sha: "SUCCESS",
log: ("x" * $size)
}]}'
;;
large-repeated-failure)
jq -n --argjson size "${FAKE_LARGE_RESPONSE_BYTES}" '{workflow_runs: [
{
status: "completed", conclusion: "failure",
updated_at: "2026-08-10T04:59:00Z",
html_url: "https://github.com/atrinik/github-settings/actions/runs/102",
head_sha: "FAIL2",
log: ("x" * $size)
},
{
status: "completed", conclusion: "failure",
updated_at: "2026-08-10T04:49:00Z",
html_url: "https://github.com/atrinik/github-settings/actions/runs/101",
head_sha: "FAIL1"
},
{
status: "completed", conclusion: "success",
updated_at: "2026-08-10T04:40:00Z",
html_url: "https://github.com/atrinik/github-settings/actions/runs/100",
head_sha: "SUCCESS"
}
]}'
;;
large-invalid-response)
jq -nr --argjson size "${FAKE_LARGE_RESPONSE_BYTES}" \
'"{\"workflow_runs\":[" + ("x" * $size)'
;;
runs-malformed)
printf '{'
;;
runs-empty)
:
;;
runs-whitespace)
printf ' \n\t'
;;
runs-null)
jq -n 'null'
;;
runs-false)
jq -n 'false'
;;
runs-object)
jq -n '{}'
;;
runs-wrong-type)
jq -n '{workflow_runs: {}}'
;;
multi-document-runs)
jq -n '{workflow_runs: []}'
jq -n '{workflow_runs: []}'
;;
healthy | missing-settings | recovery | recurrence-before | reopen | \
incident-api-failure | filtered-stale | filtered-empty | filtered-newer | \
filtered-recovery | filtered-before-episode | filtered-pending | future-success)
filtered-recovery | filtered-before-episode | filtered-pending | future-success | \
multi-document-success | success-malformed | success-empty | \
success-whitespace | success-null | success-false | success-object | \
success-wrong-type)
jq -n '{workflow_runs: [{
status: "completed", conclusion: "success",
created_at: "2026-08-10T04:49:00Z",
Expand Down Expand Up @@ -121,7 +186,9 @@ case ${endpoint} in
}
]}'
;;
stale | sync-failure | foreign | multiple-managed | alert-api-failure)
stale | sync-failure | foreign | multiple-managed | alert-api-failure | \
alert-malformed-response | alert-empty-response | alert-null-response | \
alert-false-response)
jq -n '{workflow_runs: [{
status: "completed", conclusion: "success",
created_at: "2026-08-10T01:59:00Z",
Expand Down Expand Up @@ -165,8 +232,43 @@ case ${endpoint} in
;;
"repos/atrinik/github-settings/actions/workflows/sync-project.yml/runs?status=success&per_page=1")
case ${FAKE_GH_SCENARIO} in
large-healthy | large-repeated-failure)
jq -n --argjson size "${FAKE_LARGE_RESPONSE_BYTES}" '{workflow_runs: [{
status: "completed", conclusion: "success",
created_at: "2026-08-10T04:39:00Z",
updated_at: "2026-08-10T04:40:00Z",
html_url: "https://github.com/atrinik/github-settings/actions/runs/100",
head_sha: "SUCCESS",
log: ("x" * $size)
}]}'
;;
multi-document-success)
jq -n '{workflow_runs: []}'
jq -n '{workflow_runs: []}'
;;
success-malformed)
printf '{'
;;
success-empty)
:
;;
success-whitespace)
printf ' \n\t'
;;
success-null)
jq -n 'null'
;;
success-false)
jq -n 'false'
;;
success-object)
jq -n '{}'
;;
success-wrong-type)
jq -n '{workflow_runs: {}}'
;;
healthy | missing-settings | recovery | recurrence-before | reopen | \
incident-api-failure)
incident-api-failure | multi-document-runs | runs-object | runs-wrong-type)
jq -n '{workflow_runs: [{
status: "completed", conclusion: "success",
created_at: "2026-08-10T04:49:00Z",
Expand All @@ -185,6 +287,8 @@ case ${endpoint} in
}]}'
;;
stale | sync-failure | foreign | multiple-managed | alert-api-failure | \
alert-malformed-response | alert-empty-response | alert-null-response | \
alert-false-response | \
filtered-stale | filtered-recovery | filtered-before-episode | filtered-pending)
jq -n '{workflow_runs: [{
status: "completed", conclusion: "success",
Expand Down Expand Up @@ -291,7 +395,13 @@ case ${endpoint} in
;;
repos/atrinik/github-settings/issues)
[[ ${method} == POST ]]
jq -n '{number: 70, state: "open"}'
case ${FAKE_GH_SCENARIO} in
alert-malformed-response) printf '{' ;;
alert-empty-response) : ;;
alert-null-response) jq -n 'null' ;;
alert-false-response) jq -n 'false' ;;
*) jq -n '{number: 70, state: "open"}' ;;
esac
;;
repos/atrinik/github-settings/issues/70)
[[ ${method} == PATCH ]]
Expand Down Expand Up @@ -336,6 +446,8 @@ PLAN
EOF
chmod +x "${temporary}/bin/sync-plan"

large_response_bytes=$(($(getconf ARG_MAX) + 65536))

run_health() {
local gh_scenario=$1
local sync_scenario=$2
Expand All @@ -345,6 +457,7 @@ run_health() {
FAKE_GH_LOG="${temporary}/gh.log" \
FAKE_GH_BODY="${temporary}/gh-body" \
FAKE_GH_SCENARIO="${gh_scenario}" \
FAKE_LARGE_RESPONSE_BYTES="${large_response_bytes}" \
FAKE_SYNC_LOG="${temporary}/sync.log" \
FAKE_SYNC_SCENARIO="${sync_scenario}" \
GITHUB_ACTIONS=true \
Expand All @@ -367,6 +480,67 @@ grep -Fq 'Convergence: converged; mutations 0' <<<"${output}"
grep -Fq 'State: **healthy**' "${temporary}/step-summary"
[[ $(wc -l <"${temporary}/gh.log") == 2 ]]

# Both workflow-run endpoints can exceed the host's entire command argument
# limit when run metadata contains large logs. The monitor must parse those
# responses without placing either JSON document on a child-process argv.
: >"${temporary}/gh.log"
output=$(run_health large-healthy zero)
grep -Fq 'State: **healthy**' <<<"${output}"
grep -Fq 'Last successful run: [2026-08-10T04:50:00Z]' <<<"${output}"

: >"${temporary}/gh.log"
if run_health large-repeated-failure zero --apply \
>"${temporary}/large-failure.out" 2>"${temporary}/large-failure.err"; then
echo "error: health check accepted large repeated synchronization failures" >&2
exit 1
fi
grep -Fq '2 consecutive synchronization runs failed' \
"${temporary}/large-failure.out"
grep -Fq 'ALERT created managed Project health incident' \
"${temporary}/large-failure.out"

: >"${temporary}/gh.log"
: >"${temporary}/sync.log"
if run_health large-invalid-response zero \
>"${temporary}/large-invalid.out" 2>"${temporary}/large-invalid.err"; then
echo "error: health check accepted a large invalid API response" >&2
exit 1
fi
grep -Fq 'GitHub API operation returned invalid JSON' \
"${temporary}/large-invalid.err"
[[ ! -s ${temporary}/sync.log ]]

for scenario in runs-malformed runs-empty runs-whitespace runs-null runs-false \
success-malformed success-empty success-whitespace success-null success-false; do
: >"${temporary}/gh.log"
: >"${temporary}/sync.log"
if run_health "${scenario}" zero \
>"${temporary}/${scenario}.out" 2>"${temporary}/${scenario}.err"; then
echo "error: health check accepted ${scenario} API response" >&2
exit 1
fi
grep -Fq 'GitHub API operation returned invalid JSON' \
"${temporary}/${scenario}.err"
[[ ! -s ${temporary}/sync.log ]]
done

for scenario in multi-document-runs runs-object runs-wrong-type \
multi-document-success success-object success-wrong-type; do
: >"${temporary}/gh.log"
: >"${temporary}/sync.log"
if run_health "${scenario}" zero \
>"${temporary}/${scenario}.out" 2>"${temporary}/${scenario}.err"; then
echo "error: health check accepted ${scenario} workflow response" >&2
exit 1
fi
expected_error='workflow-runs response omitted workflow_runs'
if [[ ${scenario} == *success* ]]; then
expected_error='successful workflow-runs response omitted workflow_runs'
fi
grep -Fq "${expected_error}" "${temporary}/${scenario}.err"
[[ ! -s ${temporary}/sync.log ]]
done

# Recent successful runs must win over stale or empty filtered history, while
# a newer filtered response must still win if the run completed between reads.
for scenario in filtered-stale filtered-empty filtered-newer unordered-successes; do
Expand Down Expand Up @@ -446,6 +620,18 @@ grep -Fq 'list Project health incidents (page 1)' \
grep -Fq 'create Project health incident' \
"${temporary}/alert-api-failure.err"

for scenario in alert-malformed-response alert-empty-response \
alert-null-response alert-false-response; do
: >"${temporary}/gh.log"
if run_health "${scenario}" zero --apply \
>"${temporary}/${scenario}.out" 2>"${temporary}/${scenario}.err"; then
echo "error: health check accepted ${scenario} from incident creation" >&2
exit 1
fi
grep -Fq 'GitHub API operation returned invalid JSON: create Project health incident' \
"${temporary}/${scenario}.err"
done

: >"${temporary}/gh.log"
: >"${temporary}/sync.log"
if run_health repeated-failure zero --apply \
Expand Down
Loading