From 849cc0e88d33fa95719a75057edc2b10010934a0 Mon Sep 17 00:00:00 2001 From: Ege Demir Date: Tue, 13 Oct 2026 13:30:37 +0000 Subject: [PATCH] Read the whole journal when waiting for startup grep -q stops at the first match, journalctl then gets SIGPIPE, and pipefail turned a found startup line into a failure. Production's first automatic deploy started in a minute and was still reported as not starting. Co-Authored-By: Claude Opus 5.5 --- docs/staging.md | 5 +++++ infra/activate_release.sh | 11 +++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/staging.md b/docs/staging.md index 4f180d89..2c5553ca 100644 --- a/docs/staging.md +++ b/docs/staging.md @@ -15,6 +15,11 @@ Merging to `main` deploys to production, so staging is where a change is tested 2. Use the changed behavior on `app.staging.tin.computer` with a staging test project. 3. Write what you did and saw under **Tested on staging** in the pull request. +A change to the deploy scripts themselves (`infra/`) cannot be tried this way, because +deploy-staging always runs `main`'s scripts. Run the branch's `infra/deploy.sh` with +`TIN_LITE_DEPLOY_TARGET=staging` from a workstation, and a maintainer sets the `staging` +status by hand. + Deploying marks that exact commit with the `staging` status that `main` requires. A new push resets it. Changes that cannot reach the running service (docs, tests, evals, Markdown) pass without a deploy. Staging has one slot: whoever deployed last owns it until diff --git a/infra/activate_release.sh b/infra/activate_release.sh index e95d189f..fa1c4a62 100755 --- a/infra/activate_release.sh +++ b/infra/activate_release.sh @@ -100,8 +100,15 @@ rm -f "${upload_dir}/tin-lite-release.tar.gz" "${upload_dir}/tin-lite.env" # startup line and the restart counter. A failure stops here and is reported; nothing is # rolled back automatically, because a release may have changed the catalog for good. deadline=$((SECONDS + ${TIN_LITE_STARTUP_TIMEOUT:-420})) -until journalctl -u tin-lite-switchboard --since "${restarted_at}" --no-pager -q | - grep -q "Application startup complete"; do +# grep -c reads the whole journal: grep -q would stop early, journalctl would get SIGPIPE, +# and pipefail would turn a found line into a failure. +started() { + local count + count="$(journalctl -u tin-lite-switchboard --since "${restarted_at}" --no-pager -q | + grep -c "Application startup complete" || true)" + (( count > 0 )) +} +until started; do if [[ "$(systemctl show -p NRestarts --value tin-lite-switchboard)" != "${restarts_before}" ]]; then echo "the new release is restarting; it did not start cleanly (previous: ${previous})" >&2 journalctl -u tin-lite-switchboard --since "${restarted_at}" --no-pager -q | tail -n 40 >&2