diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c17d85..578d567 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -384,9 +384,13 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.ios-plan.outputs.matrix) }} runs-on: macos-26 - timeout-minutes: 45 + # Two attempts: the first, and one retry after host trouble (docs/releasing.md). + timeout-minutes: 70 env: DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer + # The simulator log of one attempt: the app, the touch path and WebDriverAgent. + IOS_LOG_PREDICATE: >- + subsystem == "com.swipeabledemo.app" OR process == "SwipeableDemo" OR process == "backboardd" OR process == "SpringBoard" OR process == "WebDriverAgentRunner-Runner" OR process == "testmanagerd" OR eventMessage CONTAINS[c] "WebDriverAgent" steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -477,10 +481,11 @@ jobs: CODE_SIGNING_ALLOWED=NO \ -quiet + # A shard gets at most one retry, in this step, and only after host trouble in attempt 1 + # (docs/releasing.md). Each attempt keeps its own evidence in e2e/logs/attempt-N. - name: Run E2E tests id: wdio env: - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e/logs/ios-evidence IOS_DEVICE_NAME: ${{ steps.sim.outputs.device_name }} IOS_UDID: ${{ steps.sim.outputs.udid }} IOS_APP_PATH: ${{ github.workspace }}/example/ios/build/Build/Products/Release-iphonesimulator/SwipeableDemo.app @@ -488,29 +493,79 @@ jobs: IOS_WDA_DERIVED_DATA_PATH: ${{ github.workspace }}/wda-build IOS_GREP: ${{ inputs.ios-grep }} IOS_VIDEO: ${{ inputs.ios-video }} + IOS_SHARD: ${{ matrix.id }} + IOS_SPEC: ${{ matrix.spec }} run: | - # The simulator can stall early in this step. Keep its logs from this point on, - # and sample host load for the first two minutes, to explain a lost touch. - echo "IOS_TEST_START=$(date '+%Y-%m-%d %H:%M:%S')" >> "$GITHUB_ENV" - mkdir -p "$E2E_ARTIFACT_DIR" - top -l 24 -s 5 -o cpu -n 15 > "$E2E_ARTIFACT_DIR/host-load.txt" 2>&1 & - TOP_PID=$! + TOP_PID= VIDEO_PID= - if [ "$IOS_VIDEO" = true ]; then - mkdir -p "$RUNNER_TEMP/ios-video" - xcrun simctl io "$IOS_UDID" recordVideo --codec h264 --force "$RUNNER_TEMP/ios-video/simulator.mp4" & - VIDEO_PID=$! - fi - # SIGINT lets simctl finish the movie file. - trap 'kill "$TOP_PID" 2>/dev/null || true; [ -n "$VIDEO_PID" ] && kill -INT "$VIDEO_PID" 2>/dev/null && wait "$VIDEO_PID" || true' EXIT - cd e2e + stop_samplers() { + # Wait for top to exit, so that host-load.txt is complete before a reader opens it. + if [ -n "$TOP_PID" ]; then kill "$TOP_PID" 2>/dev/null && wait "$TOP_PID" || true; fi + # SIGINT lets simctl finish the movie file. + if [ -n "$VIDEO_PID" ]; then kill -INT "$VIDEO_PID" 2>/dev/null && wait "$VIDEO_PID" || true; fi + TOP_PID= + VIDEO_PID= + } + trap stop_samplers EXIT + GREP_ARGS=() if [ -n "$IOS_GREP" ]; then # Mocha reads grep as a regular expression. Match the text of the input only. - GREP_PATTERN=$(node helpers/test-filter.mjs "$IOS_GREP") - npm run test:ios -- --spec "${{ matrix.spec }}" --maxInstances 1 --mochaOpts.grep "$GREP_PATTERN" - else - npm run test:ios -- --spec "${{ matrix.spec }}" --maxInstances 1 + GREP_PATTERN=$(node e2e/helpers/test-filter.mjs "$IOS_GREP") + GREP_ARGS=(--mochaOpts.grep "$GREP_PATTERN") + fi + # The simulator can stall in an attempt, and a relaunch can time out under load. + # Keep the simulator log from the attempt start on, and sample host load for the + # whole attempt, to explain a lost touch or a slow relaunch. Each evidence time is + # UTC, because the gate classifies the evidence again on another host. The host + # class reads only the load and CPU lines of top, so top prints no process list. + run_attempt() { + export IOS_ATTEMPT_DIR="$GITHUB_WORKSPACE/e2e/logs/attempt-$1" + export E2E_ARTIFACT_DIR="$IOS_ATTEMPT_DIR/ios-evidence" + IOS_TEST_START=$(TZ=UTC date '+%Y-%m-%d %H:%M:%S+0000') + printf 'IOS_ATTEMPT=%s\nIOS_TEST_START=%s\n' "$1" "$IOS_TEST_START" >> "$GITHUB_ENV" + mkdir -p "$E2E_ARTIFACT_DIR" + TZ=UTC top -l 0 -s 5 -n 0 > "$E2E_ARTIFACT_DIR/host-load.txt" 2>&1 & + TOP_PID=$! + if [ "$IOS_VIDEO" = true ]; then + mkdir -p "$RUNNER_TEMP/ios-video" + xcrun simctl io "$IOS_UDID" recordVideo --codec h264 --force "$RUNNER_TEMP/ios-video/attempt-$1.mp4" & + VIDEO_PID=$! + fi + local status=0 + (cd e2e && npm run test:ios -- --spec "$IOS_SPEC" --maxInstances 1 "${GREP_ARGS[@]}") || status=$? + stop_samplers + return "$status" + } + if run_attempt 1; then exit 0; fi + # A filtered run is diagnosis only, and the gate rejects it. It never gets a retry. + if [ -n "$IOS_GREP" ]; then exit 1; fi + # The retry check needs the receipt and the simulator log of attempt 1. + # Maestro runs after this step, so it did not run for attempt 1. + ATTEMPT_1="$GITHUB_WORKSPACE/e2e/logs/attempt-1" + IOS_WDIO_OUTCOME=failure IOS_MAESTRO_OUTCOME=skipped \ + node e2e/helpers/ios-shards.mjs receipt "$IOS_SHARD" ios-plan/manifest.json "$ATTEMPT_1" + xcrun simctl spawn "$IOS_UDID" log show --timezone UTC --start "$IOS_TEST_START" --style compact \ + --predicate "$IOS_LOG_PREDICATE" > "$ATTEMPT_1/ios-evidence/simulator-log.txt" 2>&1 || true + if ! node e2e/helpers/host-health.mjs retry ios-plan/manifest.json "$IOS_SHARD" "$ATTEMPT_1"; then + exit 1 fi + # A new boot ends the app, WebDriverAgent and the stalled SpringBoard of attempt 1. + # The new WDIO run then starts a new Appium server and session. + xcrun simctl shutdown "$IOS_UDID" || true + xcrun simctl bootstatus "$IOS_UDID" -b + run_attempt 2 + + # The WDIO step stops its samplers. If the runner ends that step without its EXIT trap, + # for example on a cancel, this step stops them before a later step reads the evidence. + - name: Stop host samplers + if: always() + run: | + pkill -f '^top -l 0 -s 5 -n 0$' || true + pkill -INT -f 'simctl io .* recordVideo' || true + for _ in $(seq 30); do + pgrep -f '^top -l 0 -s 5 -n 0$|simctl io .* recordVideo' > /dev/null || break + sleep 1 + done - name: Upload simulator video if: always() && inputs.ios-video @@ -544,29 +599,36 @@ jobs: export PATH="$PATH:$HOME/.maestro/bin" maestro test e2e/maestro/seekbar-regression.yaml + # These steps record the last attempt. The WDIO step records an attempt before its retry. - name: Record shard step outcomes if: always() env: IOS_WDIO_OUTCOME: ${{ steps.wdio.outcome }} IOS_MAESTRO_OUTCOME: ${{ steps.maestro.outcome }} - run: >- - node e2e/helpers/ios-shards.mjs receipt "${{ matrix.id }}" - ios-plan/manifest.json e2e/logs/ios-shard-receipt.json + run: | + ATTEMPT_DIR="e2e/logs/attempt-${IOS_ATTEMPT:-1}" + if [ ! -e "$ATTEMPT_DIR/ios-shard-receipt.json" ]; then + node e2e/helpers/ios-shards.mjs receipt "${{ matrix.id }}" ios-plan/manifest.json "$ATTEMPT_DIR" + fi - name: Capture simulator logs if: always() run: | - if [ -n "${IOS_TEST_START:-}" ]; then WINDOW=(--start "$IOS_TEST_START"); else WINDOW=(--last 10m); fi - mkdir -p e2e/logs/ios-evidence - xcrun simctl spawn "${{ steps.sim.outputs.udid }}" log show "${WINDOW[@]}" --style compact \ - --predicate 'subsystem == "com.swipeabledemo.app" OR process == "SwipeableDemo" OR process == "backboardd" OR process == "SpringBoard" OR process == "WebDriverAgentRunner-Runner" OR process == "testmanagerd" OR eventMessage CONTAINS[c] "WebDriverAgent"' \ - > e2e/logs/ios-evidence/simulator-log.txt 2>&1 || true + EVIDENCE="e2e/logs/attempt-${IOS_ATTEMPT:-1}/ios-evidence" + mkdir -p "$EVIDENCE" + if [ ! -e "$EVIDENCE/simulator-log.txt" ]; then + if [ -n "${IOS_TEST_START:-}" ]; then WINDOW=(--start "$IOS_TEST_START"); else WINDOW=(--last 10m); fi + xcrun simctl spawn "${{ steps.sim.outputs.udid }}" log show --timezone UTC "${WINDOW[@]}" \ + --style compact --predicate "$IOS_LOG_PREDICATE" > "$EVIDENCE/simulator-log.txt" 2>&1 || true + fi - # Classifies the host for triage. It never changes the shard or gate result. + # Classifies the host for triage. Only the WDIO step uses a host class: for its one retry. - name: Classify host health if: always() continue-on-error: true - run: node e2e/helpers/host-health.mjs report ios-plan/manifest.json "${{ matrix.id }}" e2e/logs + run: >- + node e2e/helpers/host-health.mjs report ios-plan/manifest.json "${{ matrix.id }}" + "e2e/logs/attempt-${IOS_ATTEMPT:-1}" - name: Upload Appium logs and required shard evidence if: always() diff --git a/docs/releasing.md b/docs/releasing.md index bdd3019..7063499 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -22,8 +22,11 @@ checks cannot replace a successful run of the prepared source. publishing workflow never bumps versions or commits source changes. 4. Run full CI once as a fresh `workflow_dispatch` run on this exact `main` SHA with both mobile suites enabled. The run must completely succeed on attempt 1. - A failed-jobs rerun is not release evidence. Do not change assertions, retries, - the 120-second case deadlines, or the 45-minute mobile job deadlines. + A failed-jobs rerun is not release evidence. Do not change assertions, Mocha + retries, the 120-second case deadlines, or the mobile job deadlines (45 minutes, + and 70 minutes for an iOS shard, which can run two attempts). + The only retry is the one [shard retry after host trouble](#inconclusive-host-runs) + inside an iOS shard job. If the [physical iPhone check](#physical-iphone-check) applies, do it on the same SHA. @@ -68,42 +71,87 @@ The Prepare step 4 CI run is the only check that the tools still work with an ov ## Inconclusive host runs -Each iOS shard writes `ios-evidence/host-health.json`. It compares the simulator log -with the NDJSON results in `ios-results/`. Each failure has a window: a failed test -from its start to its result, a failed hook, or a failed warm-up. A test without a -result fails, and its window ends at the next record. A gesture counts in the window -that contains its start. The `e2e-ios` job summary -shows the gate checks and one host class for each shard. The gate checks column -uses the same checks as the gate for that shard: the step outcomes, the warm-up -and the full case inventory. It lists each condition that fails. - -- `host trouble`: the gate rejects the shard only for failed cases, and the window - of each failed case contains at least one gesture that went to latent system - gesture destinations of the app scene. The simulator sent these gestures to - SpringBoard first, and the app did not get them. A Maestro step that a failed WDIO - step skipped is part of the failed cases. The worker result is part of them only when - it is one Mocha failure for each failed case, or when the worker ends directly after a - failed last case and its worker-end record shows a quarantine that started in that - case. A failed Maestro step or warm-up, a skipped, pending or unfinished case, a - failed hook, a failed worker or a missing case is a real failure, and the shard is - never `host trouble`. -- `host mixed`: some failure windows contain latent-routed gestures, and other - failure windows do not, or the gate also rejects the shard for another reason. - The failures without them, and the other rejections, are real failures. -- `host healthy`: no failure window contains a latent-routed gesture, or the shard - has no failure. A latent-routed gesture during a passed warm-up or test does not count. -- `host unknown`: the shard has no usable simulator log, or it has latent-routed - gestures and no test results. - -The summary shows the latent-routed gestures in failure windows and in total. -The host class never changes the `e2e-ios` result. A run with a failed case fails. - -1. If each failed shard of a failed run shows `host trouble`, the run is - "inconclusive (host)". It is not release evidence. One failure without a - latent-routed gesture in its window makes the run a real failure. -2. Never retry individual tests or failed jobs. +Each iOS shard job runs its spec in an attempt directory: `attempt-1/`, and +`attempt-2/` after a retry. Each attempt has its own receipt, NDJSON results in +`ios-results/`, Appium log, simulator log, host load and `ios-evidence/host-health.json`. +The host load has a `top` sample every 5 seconds for the whole attempt. The simulator +log and the host load give their times in UTC, so the gate reads them the same way on +any host. + +`host-health.json` compares the simulator log and the host load with the NDJSON +results. Each failure has a window: a failed test from its start to its result, a +failed hook, or a failed warm-up. A test without a result fails, and its window ends +at the next record. A gesture counts in the window that contains its start. A load +sample counts in the window that contains its time. The `e2e-ios` job summary shows +the gate checks and one host class for each attempt. The gate checks column uses the +same checks as the gate for that attempt: the step outcomes, the warm-up and the full +case inventory. It lists each condition that fails. + +- `host trouble`: the gate rejects the attempt only for failed cases and touch + readiness timeouts, and host trouble explains each of them. + - A failed case needs at least one gesture in its window that went to latent + system gesture destinations of the app scene. The simulator sent these + gestures to SpringBoard first, and the app did not get them. + - A touch readiness timeout is a hook that fails in the touch readiness gate + because the list screen seekbar did not appear, or because the seekbar thumb + never moved. An example is the relaunch in `afterEach`. If the app left the list + screen, or the thumb did not return to rest, the hook failure is a real failure. + The window of a timeout needs at least one load sample, and each sample in it + needs a 1-minute load average of 100 or more. It needs no simulator log, and a + latent-routed gesture in its window does not count. + The runner has 3 vCPUs. At load 100, a relaunch that needs 1 CPU second on an + idle host needs about 33 seconds. That is more than the 30-second phase of the + readiness gate. A host that keeps up has a load near 3. In the stored shard + evidence of three runs, the overload never fell below load 136. + + These effects of the failures are part of them: + - A Maestro step that a failed WDIO step skipped. + - A worker result of one Mocha failure for each failure. + - A larger worker result, when the worker ends directly after the last failure + and its worker-end record shows a quarantine that started in that failure. + - The cases that never started, when a touch readiness timeout ends the worker + and the cases that started are the first cases of the plan. Mocha starts no + later case of the suite after a failed hook. + + A failed Maestro step or warm-up, a skipped, pending or unfinished case, another + failed hook, a failed worker or any other missing case is a real failure. Then + the attempt is never `host trouble`. +- `host mixed`: host trouble explains some failure windows and not others, or the + gate also rejects the attempt for another reason. The other failures and + rejections are real failures. +- `host healthy`: host trouble explains no failure window, or the attempt has no + failure. A latent-routed gesture during a passed warm-up or test does not count. +- `host unknown`: the attempt has no usable simulator log and a failure other than + a touch readiness timeout, or it has latent-routed gestures and no test results. + +The summary shows the latent-routed gestures in the windows of failed cases and in +total. Host trouble never makes a case pass. It only permits one retry of a shard: + +1. If attempt 1 fails, the `Run E2E tests` step classifies it. The step retries the + shard once, in the same job, only when all of these are true: + - The run has no `ios-grep` filter. A filtered run never gets a retry. + - The WDIO run of attempt 1 failed, and Maestro did not run. + - The classifier finds `host trouble` in the raw evidence of attempt 1. + - The `host-health.json` of attempt 1 says `host trouble`. +2. Before attempt 2, the step boots the simulator again. Attempt 2 starts a new + Appium server, WebDriverAgent session and app. +3. Attempt 2 must pass every case of the shard, with no Mocha retry. Maestro runs + once, after the WDIO run that passed. A Maestro failure never gets a retry. +4. The receipt of attempt 2 records `attempt: 2` and its `retryReason`: the host + class of attempt 1. The shard artifact keeps both attempts. +5. A shard never gets a second retry. If attempt 2 fails, the shard fails. + +The iOS shard job deadline is 70 minutes, so that two attempts fit. A job that +reaches its deadline fails. + +1. If the last attempt of each failed shard shows `host trouble`, the run is + "inconclusive (host)". It is not release evidence. One failure that host trouble + does not explain makes the run a real failure. +2. Never rerun failed jobs, and never retry individual tests. The shard retry above + is the only retry. 3. Dispatch a fresh full run only after an inconclusive (host) run. -4. Release evidence is a fully green full run. +4. Release evidence is a fully green full run. A shard that passes in its retry is + green. ## Evidence interface @@ -130,10 +178,18 @@ this artifact in their clean consumer builds; publishing uses that same tarball, not a new local build or pack. iOS uses the existing `ios-plan---1` and -`ios-shard---1-` artifacts. The gate rediscovers the maintained +`ios-shard---1-` artifacts. Each shard artifact has `attempt-1/`, +and `attempt-2/` after a retry. The gate rediscovers the maintained inventory from the prepared checkout, compares source hashes and case identities, -then validates raw NDJSON, worker completion, receipts, zero retries/skips, -all 62 cases and Maestro. Records must belong to the run's time interval. +then validates raw NDJSON, worker completion, receipts, zero Mocha retries and +skips, all 62 cases and Maestro. A shard passes with one green attempt, or with a +green attempt 2 after a [retry](#inconclusive-host-runs). For a retry, the gate +classifies attempt 1 again from its raw NDJSON, host load and simulator log. Both +that verdict and the stored `host-health.json` must say `host trouble`, and the +receipt of attempt 2 must give the same retry reason. A third attempt, a retry +after an attempt 1 that host trouble does not explain, and an incomplete or retried +attempt 2 fail closed. The report lists each retried shard and its reason in +`iosRetries`. Records of every attempt must belong to the run's time interval. Android uses the actual `e2e-android` job log: five completed spec workers, the complete WDIO summary and every passing case title (including duplicate counts) are required. A success badge, hand-entered count or summary without case output cannot pass. diff --git a/e2e/README.md b/e2e/README.md index 2d99927..2f7bc02 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -67,7 +67,8 @@ These helpers change only E2E observation overhead, not library behavior or API. - Sample an element's rectangle once when both its location and size are needed. Keep the sample local to that observation; read it again after UI changes or on the next poll. Do not reduce assertions or cache geometry across mutations. -- Keep the existing 120-second test and 45-minute job budgets. A partial run is +- Keep the existing 120-second test budget, the 45-minute Android job budget and + the 70-minute iOS shard budget (two attempts). A partial run is not acceptance, and increasing deadlines is not a substitute for diagnosis. - A failed or expired attempt loses permission to send commands. Evidence callbacks also have bounded, revocable ownership; remote evidence is not @@ -94,9 +95,11 @@ does not authorize connecting to or changing any device. ## Isolated iOS CI coverage -The opt-in iOS E2E workflow runs performance, RTL, swipe-to-go-back, single-side swipeable regression, and two-sided specs in separate jobs with matrix fail-fast disabled. Each job retains the 45-minute limit; test assertions, retries, and per-test deadlines are unchanged. The swipeable shard also runs the required Maestro seekbar regression. +The opt-in iOS E2E workflow runs performance, RTL, swipe-to-go-back, single-side swipeable regression, and two-sided specs in separate jobs with matrix fail-fast disabled. Each job has a 70-minute limit, so that a retry after host trouble fits. Test assertions, Mocha retries, and per-test deadlines are unchanged. The swipeable shard also runs the required Maestro seekbar regression. -The final `e2e-ios` gate requires all five jobs to succeed, all 62 registered cases to pass exactly once, completed workers, and successful Maestro coverage. Missing, cancelled, skipped, duplicate, failed, or unfinished evidence fails the gate. Artifacts are separated by source SHA, workflow run, attempt, and shard. Registration discovery rejects inventory changes until the reviewed expected spec counts are updated. +A shard gets at most one retry, inside its job, and only when host trouble explains every failure of attempt 1. The rule is in [Inconclusive host runs](../docs/releasing.md#inconclusive-host-runs). Each attempt writes its results, Appium log, simulator log, host load and host class to `e2e/logs/attempt-N/`. `IOS_ATTEMPT_DIR` sets that directory. Without it, a local run writes to `e2e/logs/`. + +The final `e2e-ios` gate requires all five jobs to succeed, all 62 registered cases to pass exactly once in the last attempt of their shard, completed workers, and successful Maestro coverage. A retry passes only after the gate classifies attempt 1 again as host trouble. Missing, cancelled, skipped, duplicate, failed, or unfinished evidence fails the gate. Artifacts are separated by source SHA, workflow run, workflow run attempt, and shard. Registration discovery rejects inventory changes until the reviewed expected spec counts are updated. Run `cd e2e && npm run test:harness` for the offline shard-evidence regressions alongside the existing harness tests. Offline success does not establish simulator acceptance. Job isolation prevents one serial run from leaving later suites unstarted; it does not change or excuse individual test timeouts. diff --git a/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-host-load.txt b/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-host-load.txt new file mode 100644 index 0000000..9caa928 --- /dev/null +++ b/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-host-load.txt @@ -0,0 +1,96 @@ +Processes: 767 total, 145 running, 29 stuck, 593 sleeping, 2984 threads +2026/10/19 15:16:55 +Load Avg: 385.07, 289.20, 152.36 +CPU usage: 39.88% user, 60.11% sys, 0.0% idle +Processes: 770 total, 6 running, 151 stuck, 613 sleeping, 2872 threads +2026/10/19 15:17:00 +Load Avg: 383.46, 290.46, 153.61 +CPU usage: 49.26% user, 44.76% sys, 5.97% idle +Processes: 773 total, 4 running, 157 stuck, 612 sleeping, 2820 threads +2026/10/19 15:17:05 +Load Avg: 352.99, 285.68, 152.72 +CPU usage: 32.29% user, 27.19% sys, 40.50% idle +Processes: 787 total, 4 running, 163 stuck, 620 sleeping, 2765 threads +2026/10/19 15:17:10 +Load Avg: 299.18, 276.37, 150.97 +CPU usage: 30.60% user, 44.42% sys, 24.96% idle +Processes: 791 total, 4 running, 162 stuck, 625 sleeping, 2834 threads +2026/10/19 15:17:16 +Load Avg: 275.30, 271.80, 150.09 +CPU usage: 18.35% user, 32.31% sys, 49.33% idle +Processes: 790 total, 102 running, 60 stuck, 628 sleeping, 2866 threads +2026/10/19 15:17:21 +Load Avg: 253.34, 267.31, 149.22 +CPU usage: 30.34% user, 34.32% sys, 35.33% idle +Processes: 769 total, 192 running, 13 stuck, 564 sleeping, 2982 threads +2026/10/19 15:17:53 +Load Avg: 330.75, 284.43, 159.50 +CPU usage: 60.48% user, 39.7% sys, 0.43% idle +Processes: 773 total, 208 running, 10 stuck, 555 sleeping, 3013 threads +2026/10/19 15:18:01 +Load Avg: 353.03, 290.68, 163.18 +CPU usage: 55.22% user, 44.77% sys, 0.0% idle +Processes: 766 total, 122 running, 80 stuck, 564 sleeping, 2975 threads +2026/10/19 15:18:07 +Load Avg: 345.66, 290.19, 163.75 +CPU usage: 45.42% user, 54.57% sys, 0.0% idle +Processes: 760 total, 13 running, 120 stuck, 627 sleeping, 2810 threads +2026/10/19 15:18:19 +Load Avg: 296.51, 281.52, 162.15 +CPU usage: 33.75% user, 45.92% sys, 20.32% idle +Processes: 769 total, 2 running, 134 stuck, 633 sleeping, 2854 threads +2026/10/19 15:18:24 +Load Avg: 278.85, 278.11, 161.64 +CPU usage: 29.34% user, 51.91% sys, 18.73% idle +Processes: 772 total, 4 running, 134 stuck, 634 sleeping, 2783 threads +2026/10/19 15:18:29 +Load Avg: 256.60, 273.51, 160.70 +CPU usage: 23.20% user, 33.61% sys, 43.18% idle +Processes: 763 total, 2 running, 138 stuck, 623 sleeping, 2759 threads +2026/10/19 15:18:36 +Load Avg: 218.66, 264.81, 158.94 +CPU usage: 28.59% user, 34.51% sys, 36.88% idle +Processes: 749 total, 141 running, 12 stuck, 596 sleeping, 2783 threads +2026/10/19 15:18:47 +Load Avg: 229.91, 265.60, 160.45 +CPU usage: 45.95% user, 52.42% sys, 1.62% idle +Processes: 739 total, 226 running, 2 stuck, 511 sleeping, 2791 threads +2026/10/19 15:19:02 +Load Avg: 282.96, 275.53, 165.82 +CPU usage: 59.56% user, 40.43% sys, 0.0% idle +Processes: 730 total, 219 running, 2 stuck, 509 sleeping, 2788 threads +2026/10/19 15:19:08 +Load Avg: 298.18, 278.81, 167.62 +CPU usage: 55.34% user, 44.65% sys, 0.0% idle +Processes: 728 total, 118 running, 72 stuck, 538 sleeping, 2802 threads +2026/10/19 15:19:13 +Load Avg: 309.69, 281.52, 169.23 +CPU usage: 58.61% user, 41.38% sys, 0.0% idle +Processes: 726 total, 72 running, 72 stuck, 582 sleeping, 2758 threads +2026/10/19 15:19:19 +Load Avg: 297.79, 279.52, 169.18 +CPU usage: 56.64% user, 43.35% sys, 0.0% idle +Processes: 726 total, 156 running, 3 stuck, 567 sleeping, 2697 threads +2026/10/19 15:19:31 +Load Avg: 289.25, 278.19, 169.99 +CPU usage: 56.19% user, 43.71% sys, 0.8% idle +Processes: 734 total, 25 running, 134 stuck, 575 sleeping, 2823 threads +2026/10/19 15:19:36 +Load Avg: 276.67, 276.08, 170.52 +CPU usage: 46.44% user, 53.55% sys, 0.0% idle +Processes: 735 total, 2 running, 152 stuck, 581 sleeping, 2693 threads +2026/10/19 15:19:41 +Load Avg: 264.84, 273.64, 170.28 +CPU usage: 31.30% user, 38.14% sys, 30.54% idle +Processes: 734 total, 3 running, 142 stuck, 589 sleeping, 2677 threads +2026/10/19 15:19:46 +Load Avg: 243.80, 269.13, 169.29 +CPU usage: 39.45% user, 36.40% sys, 24.14% idle +Processes: 730 total, 5 running, 130 stuck, 595 sleeping, 2664 threads +2026/10/19 15:19:56 +Load Avg: 224.43, 264.70, 168.31 +CPU usage: 36.86% user, 42.66% sys, 20.47% idle +Processes: 734 total, 153 running, 4 stuck, 577 sleeping, 2731 threads +2026/10/19 15:20:01 +Load Avg: 226.94, 263.86, 169.14 +CPU usage: 52.12% user, 46.72% sys, 1.15% idle diff --git a/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-receipt.json b/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-receipt.json new file mode 100644 index 0000000..f9efacf --- /dev/null +++ b/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-receipt.json @@ -0,0 +1,11 @@ +{ + "version": 1, + "sha": "d06730521c230f685bc09a9980ca92d957e47546", + "runId": "39273407706", + "runAttempt": "1", + "id": "swipeable", + "spec": "specs/swipeable.spec.ts", + "sourceHash": "caab14e149b2d150d01259d86038ca90a1302ff469f821494827566abf747d8c", + "wdioOutcome": "failure", + "maestroOutcome": "skipped" +} diff --git a/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-results.ndjson b/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-results.ndjson new file mode 100644 index 0000000..5a573b2 --- /dev/null +++ b/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-results.ndjson @@ -0,0 +1,20 @@ +{"timestamp":"2026-10-19T15:21:21.801Z","pid":11891,"worker":"0-0","event":"worker-start","specs":["file:///Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts"]} +{"timestamp":"2026-10-19T15:22:19.082Z","pid":11891,"worker":"0-0","event":"warm-up","ok":true,"drags":2,"duration":57274} +{"timestamp":"2026-10-19T15:22:19.104Z","pid":11891,"worker":"0-0","event":"test-start","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should open and close swipeable row","parent":"Test 0: Basic Swipe Open/Close","attempt":0} +{"timestamp":"2026-10-19T15:22:46.428Z","pid":11891,"worker":"0-0","event":"test-result","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should open and close swipeable row","parent":"Test 0: Basic Swipe Open/Close","attempt":0,"status":"passed","duration":27306,"retries":{"attempts":0,"limit":0}} +{"timestamp":"2026-10-19T15:23:05.854Z","pid":11891,"worker":"0-0","event":"test-start","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should swipe from left when leading mode enabled","parent":"Test 1: Swipe from left direction","attempt":0} +{"timestamp":"2026-10-19T15:23:44.432Z","pid":11891,"worker":"0-0","event":"test-result","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should swipe from left when leading mode enabled","parent":"Test 1: Swipe from left direction","attempt":0,"status":"passed","duration":38576,"retries":{"attempts":0,"limit":0}} +{"timestamp":"2026-10-19T15:23:57.915Z","pid":11891,"worker":"0-0","event":"test-start","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should trigger action when tapping revealed button","parent":"Test 2: Swipe Action Trigger","attempt":0} +{"timestamp":"2026-10-19T15:24:31.147Z","pid":11891,"worker":"0-0","event":"test-result","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should trigger action when tapping revealed button","parent":"Test 2: Swipe Action Trigger","attempt":0,"status":"passed","duration":33217,"retries":{"attempts":0,"limit":0}} +{"timestamp":"2026-10-19T15:24:44.407Z","pid":11891,"worker":"0-0","event":"test-start","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should close all rows via Close All button","parent":"Test 3: Ref Close API","attempt":0} +{"timestamp":"2026-10-19T15:25:09.232Z","pid":11891,"worker":"0-0","event":"test-result","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should close all rows via Close All button","parent":"Test 3: Ref Close API","attempt":0,"status":"passed","duration":24822,"retries":{"attempts":0,"limit":0}} +{"timestamp":"2026-10-19T15:25:18.947Z","pid":11891,"worker":"0-0","event":"test-start","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should persist swipeable state through list recycling","parent":"Test 4: Recycling State Persistence","attempt":0} +{"timestamp":"2026-10-19T15:25:39.876Z","pid":11891,"worker":"0-0","event":"test-result","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should persist swipeable state through list recycling","parent":"Test 4: Recycling State Persistence","attempt":0,"status":"passed","duration":20919,"retries":{"attempts":0,"limit":0}} +{"timestamp":"2026-10-19T15:25:51.986Z","pid":11891,"worker":"0-0","event":"test-start","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should auto-close and show correct reply preview on swipe","parent":"Test 5: AutoClose (Chat Demo)","attempt":0} +{"timestamp":"2026-10-19T15:26:17.464Z","pid":11891,"worker":"0-0","event":"test-result","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should auto-close and show correct reply preview on swipe","parent":"Test 5: AutoClose (Chat Demo)","attempt":0,"status":"passed","duration":25460,"retries":{"attempts":0,"limit":0}} +{"timestamp":"2026-10-19T15:26:29.462Z","pid":11891,"worker":"0-0","event":"test-start","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should keep keyboard open during autoclose swipe without flicker","parent":"Test 6: Keyboard Persistence","attempt":0} +{"timestamp":"2026-10-19T15:26:48.178Z","pid":11891,"worker":"0-0","event":"test-result","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should keep keyboard open during autoclose swipe without flicker","parent":"Test 6: Keyboard Persistence","attempt":0,"status":"passed","duration":18701,"retries":{"attempts":0,"limit":0}} +{"timestamp":"2026-10-19T15:26:58.160Z","pid":11891,"worker":"0-0","event":"test-start","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should close all rows via Close All after navigating away and back","parent":"Test 7: CloseAll after navigation round-trip","attempt":0} +{"timestamp":"2026-10-19T15:27:35.902Z","pid":11891,"worker":"0-0","event":"test-result","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"should close all rows via Close All after navigating away and back","parent":"Test 7: CloseAll after navigation round-trip","attempt":0,"status":"passed","duration":37727,"retries":{"attempts":0,"limit":0}} +{"timestamp":"2026-10-19T15:28:12.857Z","pid":11891,"worker":"0-0","event":"hook-failure","hook":"afterEach","file":"/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts","title":"\"after each\" hook: wrappedHook for \"should close all rows via Close All after navigating away and back\"","parent":"Swipeable E2E Tests","duration":36899,"error":{"name":"Error","message":"Touch readiness gate failed after 0 drag(s) in 30128 ms: the list screen seekbar did not appear","stack":"Error: Touch readiness gate failed after 0 drag(s) in 30128 ms: the list screen seekbar did not appear\n at failure (/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/helpers/touch-readiness.ts:96:5)\n at waitForTouchReadiness (/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/helpers/touch-readiness.ts:102:11)\n at async relaunchApp (/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/helpers/touch-readiness.ts:143:33)\n at async Context. (/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts:23:5)\n at async (/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/helpers/command-scope.ts:307:18)\n at async (/Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/helpers/command-scope.ts:307:18)"}} +{"timestamp":"2026-10-19T15:28:12.939Z","pid":11891,"worker":"0-0","event":"worker-end","result":2,"specs":["file:///Users/runner/work/react-native-swipeable-actions/react-native-swipeable-actions/e2e/specs/swipeable.spec.ts"],"quarantine":{"reason":"unresolved request when \"Swipeable E2E Tests \"after each\" hook: wrappedHook for \"should close all rows via Close All after navigating away and back\"\" ended","runnable":"Swipeable E2E Tests \"after each\" hook: wrappedHook for \"should close all rows via Close All after navigating away and back\""}} diff --git a/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-simulator-log.txt b/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-simulator-log.txt new file mode 100644 index 0000000..597756a --- /dev/null +++ b/e2e/helpers/fixtures/host-health/run-39273407706/swipeable-simulator-log.txt @@ -0,0 +1,7 @@ +Timestamp Ty Process[PID:TID] +2026-10-19 15:24:01.034 Df backboardd[2348:11839] [com.apple.BackBoard:TouchEvents] init: +2026-10-19 15:24:01.210 Df backboardd[2348:11839] [com.apple.BackBoard:TouchEvents] cancel -- digitizer did disappear:>)> +2026-10-19 15:24:01.210 Df backboardd[2348:11839] [com.apple.BackBoard:TouchEvents] invalidate: >)> +2026-10-19 15:27:30.397 Df backboardd[2348:1686c] [com.apple.BackBoard:TouchEvents] init: +2026-10-19 15:27:30.575 Df backboardd[2348:1686c] [com.apple.BackBoard:TouchEvents] cancel -- digitizer did disappear: +2026-10-19 15:27:30.575 Df backboardd[2348:1686c] [com.apple.BackBoard:TouchEvents] invalidate: diff --git a/e2e/helpers/fixtures/host-trouble.mjs b/e2e/helpers/fixtures/host-trouble.mjs new file mode 100644 index 0000000..e96b527 --- /dev/null +++ b/e2e/helpers/fixtures/host-trouble.mjs @@ -0,0 +1,110 @@ +// Synthetic attempt evidence for the gate and classifier tests. It is never release evidence. +import { classifyAttempt, READINESS_FAILURE } from '../host-health.mjs' + +const pad = (value, width = 2) => String(value).padStart(width, '0') +// CI prints each evidence time in UTC: `TZ=UTC top` and `log show --timezone UTC`. +const utcDate = (date) => + `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}` +const utcTime = (date) => + `${pad(date.getUTCHours())}:${pad(date.getUTCMinutes())}:${pad(date.getUTCSeconds())}` + +/** `top -l -n 0` output: one sample for each [time in ms, 1-minute load average], in UTC. */ +export function topSamples(samples) { + return samples + .map(([at, load]) => { + const date = new Date(at) + return [ + 'Processes: 800 total, 80 running, 120 stuck, 600 sleeping, 3000 threads ', + `${utcDate(date).replaceAll('-', '/')} ${utcTime(date)}`, + `Load Avg: ${load}, ${load}, ${load} `, + 'CPU usage: 50.0% user, 50.0% sys, 0.0% idle ' + ].join('\n') + }) + .join('\n') + .concat('\n') +} + +/** + * One `log show --style compact` line of a touch, in UTC. A touch reaches the app. A latent + * touch goes to the latent system gesture destinations of the app scene first. + */ +export function touchLine(at, latent = false) { + const date = new Date(at) + const destinations = latent ? 'sceneID:com.swipeabledemo.app-default' : '' + return ( + `${utcDate(date)} ${utcTime(date)}.${pad(date.getUTCMilliseconds(), 3)} Df backboardd[5005:e726] ` + + '[com.apple.BackBoard:TouchEvents] init: \n` + ) +} + +/** + * Attempt 1 of a shard that ends like the swipeable shard of run 39273407706: the first case + * passes, then its afterEach relaunch times out in the touch readiness gate. The worker + * quarantines the session in that hook and ends. Mocha starts no later case. `load` is the + * 1-minute load average of each `top` sample inside the hook window. + */ +export function readinessTimeoutAttempt(shard, context, { start, load, file, pid = 41 }) { + const at = (seconds) => new Date(start + seconds * 1000).toISOString() + const common = { pid, worker: '0-0' } + const [first] = shard.cases + const hook = { parent: 'suite', title: `"after each" hook: wrappedHook for "${first.title}"` } + const runnable = `${hook.parent} ${hook.title}` + const metadata = { ...common, ...first, file, attempt: 0 } + const attempt = { + receipt: { + version: 1, + sha: context.sha, + runId: context.runId, + runAttempt: context.runAttempt, + id: shard.id, + spec: shard.spec, + sourceHash: shard.sourceHash, + attempt: 1, + retryReason: null, + wdioOutcome: 'failure', + maestroOutcome: 'skipped' + }, + workers: [ + [ + { ...common, timestamp: at(0), event: 'worker-start', specs: [file] }, + { ...common, timestamp: at(60), event: 'warm-up', ok: true, drags: 2, duration: 57274 }, + { ...metadata, timestamp: at(61), event: 'test-start' }, + { + ...metadata, + timestamp: at(90), + event: 'test-result', + status: 'passed', + duration: 29000, + retries: { attempts: 0, limit: 0 } + }, + { + ...common, + ...hook, + timestamp: at(127), + event: 'hook-failure', + hook: 'afterEach', + file, + duration: 36899, + error: { + name: 'Error', + message: `${READINESS_FAILURE} after 0 drag(s) in 30128 ms: the list screen seekbar did not appear` + } + }, + { + ...common, + timestamp: at(127.1), + event: 'worker-end', + result: 2, + specs: [file], + quarantine: { reason: `unresolved request when "${runnable}" ended`, runnable } + } + ] + ], + simulatorLog: touchLine(start + 70000), + hostLoad: topSamples([95, 105, 115, 125].map((second) => [start + second * 1000, load])) + } + attempt.health = classifyAttempt(shard, attempt, context) + return attempt +} diff --git a/e2e/helpers/host-health.mjs b/e2e/helpers/host-health.mjs index c5a37a6..7925c3e 100644 --- a/e2e/helpers/host-health.mjs +++ b/e2e/helpers/host-health.mjs @@ -1,9 +1,17 @@ +import assert from 'node:assert/strict' import { appendFile, readdir, readFile, stat, writeFile } from 'node:fs/promises' import path from 'node:path' import { pathToFileURL } from 'node:url' -import { artifactName, loadShardBundle, validateShard } from './ios-shards.mjs' +import { + artifactName, + loadAttempt, + loadShardBundle, + validateShard, + validateShardEvidence +} from './ios-shards.mjs' -// Classifies the CI host only. It never changes a shard or gate result. +// Classifies the CI host. Host trouble never passes a case. It only permits the one retry +// of a shard that `retryReason` allows. const APP_SCENE = 'sceneID:com.swipeabledemo.app' const LOG_FILE = 'simulator-log.txt' const LOAD_FILE = 'host-load.txt' @@ -11,6 +19,38 @@ const RESULTS = 'ios-results' const EVIDENCE = 'ios-evidence' export const HEALTH_FILE = 'host-health.json' +/** The start of each error message of the touch readiness gate (`failure` in touch-readiness.ts). */ +export const READINESS_FAILURE = 'Touch readiness gate failed' + +/** + * The reasons of the touch readiness gate that a stalled host explains: the gate waited for + * the list screen or for a thumb move until its phase timed out. The other reasons are app + * states (the app left the list screen, the thumb did not return to rest), so they stay real + * failures. + */ +export const READINESS_TIMEOUTS = Object.freeze([ + 'the list screen seekbar did not appear', + 'the seekbar thumb never moved, so touches do not reach the app' +]) + +/** True for the error message of a touch readiness timeout that host trouble can explain. */ +export function isReadinessTimeout(message) { + const reason = String(message).match( + new RegExp(`^${READINESS_FAILURE} after \\d+ drag\\(s\\) in \\d+ ms: (.*)$`) + )?.[1] + return READINESS_TIMEOUTS.includes(reason) +} + +/** + * The 1-minute load average from which the host alone explains a touch readiness timeout. + * The runner has 3 vCPUs, so at load L a runnable thread gets about 3/L of one CPU. A relaunch + * that needs 1 CPU second on an idle host then needs about L/3 seconds, and from load 90 on it + * misses the 30-second phase of the readiness gate. A host that keeps up has a load near 3. + * The overload in the stored shard evidence of runs 39053954618, 39128353788 and 39273407706 + * never fell below 136 in 168 samples, in passed and in failed shards. + */ +export const HIGH_LOAD = 100 + /** * backboardd logs each XCTest touch sequence with the sender ID of its virtual digitizer. * A gesture that it routes to latent system gesture destinations for the app scene goes @@ -53,6 +93,30 @@ export function parseHostLoad(text) { } } +/** + * Each `top -l` sample with its time and 1-minute load average. `top` prints a time without + * a zone before the load. CI runs it with TZ=UTC, so the time is UTC in any zone of the + * reader. A sample without a time has none. + */ +export function parseLoadSamples(text) { + const samples = [] + let at = Number.NaN + for (const line of text.split('\n')) { + const time = line.match(/^(\d{4})\/(\d{2})\/(\d{2}) (\d{2}):(\d{2}):(\d{2})\s*$/) + if (time) { + const [year, month, day, hour, minute, second] = time.slice(1).map(Number) + at = Date.UTC(year, month - 1, day, hour, minute, second) + continue + } + const load = line.match(/^Load Avg: ([0-9.]+),/) + if (load) { + samples.push({ at, load: Number(load[1]) }) + at = Number.NaN + } + } + return samples +} + async function optionalText(filename) { try { return await readFile(filename, 'utf8') @@ -63,14 +127,15 @@ async function optionalText(filename) { } /** - * `log show --style compact` prints the local time of the shard host without a zone. - * The report step runs on the same host, so the local zone of this process applies. + * `log show --style compact` prints a time without a zone. CI runs it with `--timezone UTC`, + * so the time is UTC. The gate classifies again on another host, so the zone of the reader + * must not apply. */ export function logTime(stamp) { const parts = stamp.match(/^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\.(\d{3})$/) if (!parts) return NaN const [year, month, day, hour, minute, second, ms] = parts.slice(1).map(Number) - return new Date(year, month - 1, day, hour, minute, second, ms).getTime() + return Date.UTC(year, month - 1, day, hour, minute, second, ms) } /** Reads the NDJSON worker results. A truncated or invalid line is ignored. */ @@ -89,10 +154,17 @@ export function parseResults(text) { return records } +/** True for a hook that times out in the touch readiness gate, for example in a relaunch. */ +const readinessFailure = (record) => + record.event === 'hook-failure' && + typeof record.error?.message === 'string' && + isReadinessTimeout(record.error.message) + /** * One [start, end] window for each failure in the order of the records: a failed test, * a failed hook and a failed warm-up. A test without its result fails. Its window ends * at the next record. Without a next record, the window has no end and no gesture. + * The window of a touch readiness hook failure has `readinessTimeout: true`. */ export function failureWindows(records) { const windows = [] @@ -126,7 +198,12 @@ export function failureWindows(records) { const start = open ? time(open) : previous && time(previous) const title = open ? (open.fullTitle ?? open.title) : (record.hook ?? record.event) open = undefined - windows.push({ title, start, end: time(record) }) + windows.push({ + title, + start, + end: time(record), + ...(readinessFailure(record) && { readinessTimeout: true }) + }) } else if (record.event === 'worker-end') { close(time(record)) } else if (record.event === 'worker-start') { @@ -142,9 +219,13 @@ export function failureWindows(records) { const failedCase = (record) => record.event === 'test-result' && record.status === 'failed' && !record.pending && !record.skipped +/** True for a failure that host trouble can explain: a failed case or a readiness hook failure. */ +const explainable = (record) => failedCase(record) || readinessFailure(record) + /** - * True when the quarantine of the worker-end record starts in the case of this test-result. - * Its runnable is the Mocha full title: the titles of the suites, then the case title. + * True when the quarantine of the worker-end record starts in the runnable of this record: + * a case or a hook. Its runnable is the Mocha full title: the titles of the suites, then the + * title of the case or hook. */ function quarantinedIn(quarantine, record) { const runnable = quarantine?.runnable @@ -155,27 +236,59 @@ function quarantinedIn(quarantine, record) { } /** - * True when the failed cases alone cause the worker result. WDIO gives `after` the Mocha - * failure count: one for each failed case and for each failed hook. A hook that fails in - * its body has a hook record. A session quarantine fails each later hook before the WDIO + * True when the explainable failures alone cause the worker result. WDIO gives `after` the + * Mocha failure count: one for each failed case and for each failed hook. A hook that fails + * in its body has a hook record. A session quarantine fails each later hook before the WDIO * hook wrapper runs, so these hooks have no record (command-scope.ts `wrap`). A larger - * count is part of the failed cases only when the worker ends directly after the last - * failed case, and its worker-end record shows a quarantine that starts in that case. + * count is part of the failures only when the worker ends directly after the last failure, + * and its worker-end record shows a quarantine that starts in that failure. */ -function failedCasesCauseResult(records) { +function failuresCauseResult(records) { const end = records.at(-1) if (end?.event !== 'worker-end' || !Number.isInteger(end.result)) return false - const failed = records.filter(failedCase).length + const failed = records.filter(explainable).length if (failed === 0) return false if (end.result === failed) return true const last = records.at(-2) - return end.result > failed && failedCase(last ?? {}) && quarantinedIn(end.quarantine, last) + return end.result > failed && explainable(last ?? {}) && quarantinedIn(end.quarantine, last) } /** - * The same evidence with each failed case passed, and with the step and worker outcomes - * that a failed case causes. CI runs the Maestro step only after a successful WDIO step, - * so a failed WDIO step skips it. Nothing else changes. + * Passed records for each case that never started, when a touch readiness hook failure ends + * the worker. Mocha starts no later case of the suite after a failed hook. The cases that + * started must be the first cases of the plan, in its order: Mocha runs the cases of these + * specs in plan order. A gap, such as from a grep filter, is not part of the timeout. Any + * other case that did not start, or that started and did not finish, stays a gate rejection. + */ +function unstartedCasesPassed(shard, records) { + const hook = records.at(-2) + if (records.at(-1)?.event !== 'worker-end' || !readinessFailure(hook ?? {})) return [] + const key = (record) => JSON.stringify([record.parent, record.title]) + const started = records.filter((record) => record.event === 'test-start').map(key) + const first = shard.cases.slice(0, started.length).map(key) + if (started.some((entry, index) => entry !== first[index])) return [] + const { pid, worker } = records[0] ?? {} + return shard.cases.slice(started.length).flatMap((entry) => { + const metadata = { timestamp: hook.timestamp, pid, worker, ...entry, file: hook.file } + return [ + { ...metadata, event: 'test-start', attempt: 0 }, + { + ...metadata, + event: 'test-result', + attempt: 0, + status: 'passed', + duration: 0, + retries: { attempts: 0, limit: 0 } + } + ] + }) +} + +/** + * The same evidence with each explainable failure passed, and with the step and worker + * outcomes that it causes. A failed case passes. A touch readiness hook failure goes, and + * when it ends the worker, each case that never started passes. CI runs the Maestro step + * only after a successful WDIO step, so a failed WDIO step skips it. Nothing else changes. */ function withFailedCasesPassed(shard, { receipt, workers }) { const wdioFailed = receipt.wdioOutcome === 'failure' @@ -187,25 +300,29 @@ function withFailedCasesPassed(shard, { receipt, workers }) { ...(maestroSkipped && { maestroOutcome: 'success' }) }, workers: workers.map((records) => { - const causedResult = failedCasesCauseResult(records) - return records.map((record) => { - if (failedCase(record)) return { ...record, status: 'passed', error: null } - if (record.event === 'worker-end' && causedResult) return { ...record, result: 0 } - return record + const causedResult = failuresCauseResult(records) + const unstarted = unstartedCasesPassed(shard, records) + return records.flatMap((record) => { + if (failedCase(record)) return [{ ...record, status: 'passed', error: null }] + if (readinessFailure(record)) return [] + if (record.event === 'worker-end') + return [...unstarted, causedResult ? { ...record, result: 0 } : record] + return [record] }) }) } } /** - * The gate checks for one shard. `onlyFailedCases` is true only when the gate rejects the - * shard for its failed cases and for nothing else. Only then can host trouble explain it. + * The gate checks for one attempt. `failedCases` counts the failed cases and the touch + * readiness hook failures. `onlyFailedCases` is true only when the gate rejects the attempt + * for these failures and for nothing else. Only then can host trouble explain it. */ export function gateCheck(shard, bundle, context) { try { return { passed: validateShard(shard, bundle, context), failedCases: 0, onlyFailedCases: false } } catch (error) { - const failedCases = bundle.workers.flat().filter(failedCase).length + const failedCases = bundle.workers.flat().filter(explainable).length let onlyFailedCases = false if (failedCases > 0) { try { @@ -219,9 +336,15 @@ export function gateCheck(shard, bundle, context) { } } +/** True when each load sample in the window is at least HIGH_LOAD, and it has a sample. */ +const highLoad = (failure) => failure.minLoadAverage !== null && failure.minLoadAverage >= HIGH_LOAD + /** * `gate` is the result of `gateCheck` for the shard. Without it, the shard is never host - * trouble. + * trouble. A failed case needs a latent-routed gesture in its window. A touch readiness + * timeout needs high load in its window instead: at least one `top` sample, and each sample + * at HIGH_LOAD or more. Latent-routed gestures in the window of a timeout do not count, and + * a timeout needs no simulator log. */ export function classify({ simulatorLog, hostLoad, results, gate }) { const missing = [] @@ -236,43 +359,103 @@ export function classify({ simulatorLog, hostLoad, results, gate }) { hostLoad === undefined ? { loadSamples: 0, maxLoadAverage: null, minIdlePercent: null } : parseHostLoad(hostLoad) + const samples = hostLoad === undefined ? [] : parseLoadSamples(hostLoad) const latent = routing.latentRoutedAt.map(logTime) const inside = (window, at) => at >= window.start && at <= window.end const failures = (results === undefined ? [] : failureWindows(parseResults(results))).map( - (window) => ({ - ...window, - latentRoutedGestures: latent.filter((at) => inside(window, at)).length - }) + (window) => { + const failure = { + ...window, + latentRoutedGestures: latent.filter((at) => inside(window, at)).length + } + if (!window.readinessTimeout) return failure + const loads = samples.filter(({ at }) => inside(window, at)).map((sample) => sample.load) + return { + ...failure, + loadSamples: loads.length, + minLoadAverage: loads.length > 0 ? Math.min(...loads) : null + } + } ) - const failuresWithLatent = failures.filter((window) => window.latentRoutedGestures > 0).length - // Host trouble explains a shard only when the gate rejects it for failed cases alone, and - // each failed case has a latent-routed gesture. + // Only the windows of failed cases use gestures. + const cases = failures.filter((failure) => !failure.readinessTimeout) + const failuresWithLatent = cases.filter((window) => window.latentRoutedGestures > 0).length + const failuresExplained = failures.filter((failure) => + failure.readinessTimeout ? highLoad(failure) : failure.latentRoutedGestures > 0 + ).length + // Host trouble explains a shard only when the gate rejects it for these failures alone, + // and host trouble explains each of them. const onlyFailedCases = gate?.onlyFailedCases === true && gate.failedCases === failures.length + // Without gestures, only readiness timeouts can be classified: they use load samples. + const needsGestures = failures.length === 0 || cases.length > 0 let status = 'healthy' - if (routing.gestures === 0) status = 'unknown' + if (routing.gestures === 0 && needsGestures) status = 'unknown' else if (results === undefined && routing.latentRoutedGestures > 0) status = 'unknown' - else if (onlyFailedCases && failures.length > 0 && failuresWithLatent === failures.length) + else if (onlyFailedCases && failures.length > 0 && failuresExplained === failures.length) status = 'trouble' - else if (failuresWithLatent > 0) status = 'mixed' + else if (failuresExplained > 0) status = 'mixed' + const time = (at) => (Number.isFinite(at) ? new Date(at).toISOString() : null) return { - version: 3, + version: 4, status, onlyFailedCases, ...routing, - latentRoutedInFailures: latent.filter((at) => failures.some((w) => inside(w, at))).length, + latentRoutedInFailures: latent.filter((at) => cases.some((w) => inside(w, at))).length, failures: failures.length, failuresWithLatent, + failuresExplained, failedTests: failures.map(({ title, start, end, latentRoutedGestures }) => ({ title, - start: Number.isFinite(start) ? new Date(start).toISOString() : null, - end: Number.isFinite(end) ? new Date(end).toISOString() : null, + start: time(start), + end: time(end), latentRoutedGestures })), + readinessTimeouts: failures + .filter((failure) => failure.readinessTimeout) + .map((failure) => ({ + title: failure.title, + start: time(failure.start), + end: time(failure.end), + loadSamples: failure.loadSamples, + minLoadAverage: failure.minLoadAverage, + highLoad: highLoad(failure) + })), ...load, missing } } +/** The host class of one attempt from its raw evidence, the way the gate derives it again. */ +export function classifyAttempt(shard, attempt, context) { + const records = attempt.workers.flat() + return classify({ + simulatorLog: attempt.simulatorLog, + hostLoad: attempt.hostLoad, + results: + records.length > 0 + ? `${records.map((record) => JSON.stringify(record)).join('\n')}\n` + : undefined, + gate: gateCheck(shard, attempt, context) + }) +} + +/** + * The reason for the one retry of a shard after this attempt. It throws when the attempt + * cannot have a retry. The attempt must be attempt 1, its WDIO step must fail, and its + * Maestro step must not run. The classifier must find host trouble in its raw evidence, + * and its own host-health.json must say host trouble too. The WDIO step of CI and the gate + * both use this check. + */ +export function retryReason(shard, attempt, context) { + assert.equal(attempt.receipt?.attempt, 1, 'Only attempt 1 can have a retry') + assert.equal(attempt.receipt.wdioOutcome, 'failure', 'A retry needs a failed WDIO step') + assert.equal(attempt.receipt.maestroOutcome, 'skipped', 'A Maestro step never gets a retry') + assert.equal(attempt.health?.status, 'trouble', 'A retry needs host trouble in host-health.json') + const health = classifyAttempt(shard, attempt, context) + assert.equal(health.status, 'trouble', `A retry needs host trouble: ${describeHealth(health)}`) + return describeHealth(health) +} + async function optionalResults(directory) { let names try { @@ -287,16 +470,16 @@ async function optionalResults(directory) { return texts.join('\n') } -/** The gate checks for one shard artifact. Evidence that the gate cannot read has none. */ +/** The gate checks for one attempt directory. Evidence that the gate cannot read has none. */ async function optionalGateCheck(manifest, shard, root) { try { - return gateCheck(shard, await loadShardBundle(root), manifest) + return gateCheck(shard, await loadAttempt(root), manifest) } catch { return } } -/** Writes host-health.json for one shard artifact: its receipt, results and evidence. */ +/** Writes host-health.json for one attempt directory: its receipt, results and evidence. */ export async function report(manifest, id, root) { const shard = manifest.shards.find((entry) => entry.id === id) const directory = path.join(root, EVIDENCE) @@ -313,12 +496,24 @@ export async function report(manifest, id, root) { export function describeHealth(health) { if (!health) return 'host unknown: no host-health.json' const load = health.maxLoadAverage ?? 'n/a' - const gestures = `${health.latentRoutedInFailures} of ${health.latentRoutedGestures} latent-routed gestures in failures` + const timeouts = health.readinessTimeouts ?? [] + const causes = + timeouts.length > 0 + ? `latent-routed gestures or a touch readiness timeout at load ${HIGH_LOAD} or more` + : 'latent-routed gestures' + const highLoads = timeouts.filter((timeout) => timeout.highLoad).length + const gestures = + `${health.latentRoutedInFailures} of ${health.latentRoutedGestures} latent-routed gestures in failures` + + (timeouts.length > 0 + ? `, ${highLoads} of ${timeouts.length} touch readiness timeouts at load ${HIGH_LOAD} or more` + : '') if (health.status === 'trouble') - return `host trouble: all ${health.failures} failures have latent-routed gestures (${gestures}), load ${load}` + return `host trouble: all ${health.failures} failures have ${causes} (${gestures}), load ${load}` if (health.status === 'mixed') { const other = health.onlyFailedCases ? '' : ', other gate checks fail too' - return `host mixed: ${health.failuresWithLatent} of ${health.failures} failures have latent-routed gestures${other} (${gestures}), load ${load}` + // Version 3 has no failuresExplained. Its failures with latent gestures are the explained ones. + const explained = health.failuresExplained ?? health.failuresWithLatent + return `host mixed: ${explained} of ${health.failures} failures have ${causes}${other} (${gestures}), load ${load}` } if (health.status === 'healthy') return `host healthy: ${gestures}` if (health.missing?.includes(LOG_FILE)) return 'host unknown: no simulator log' @@ -389,11 +584,11 @@ function failedConditions(shard, bundle, context) { return conditions } -/** The gate's own checks for one shard. The gate also needs the plan, the build and every shard. */ +/** The gate's own checks for one attempt. The gate also needs the plan, the build and every shard. */ async function shardResult(manifest, shard, root) { let bundle try { - bundle = await loadShardBundle(root) + bundle = await loadAttempt(root) } catch (error) { if (error?.code === 'ENOENT') return { result: `rejected: missing ${path.relative(root, error.path)}` } @@ -434,25 +629,66 @@ async function exists(filename) { } } -/** One row for each planned shard, in plan order, with its gate checks and host class. */ +/** The attempt directories of one shard artifact, in order: attempt-1, attempt-2 and so on. */ +async function attemptDirectories(root) { + const number = (name) => Number(name.slice('attempt-'.length)) + return (await readdir(root)) + .filter((name) => /^attempt-[1-9][0-9]*$/.test(name)) + .sort((a, b) => number(a) - number(b)) +} + +/** The first gate message for the shard and all its attempts, or nothing when the gate accepts it. */ +async function shardRejection(manifest, shard, root) { + try { + validateShardEvidence(shard, await loadShardBundle(root), manifest) + } catch (error) { + return firstLine(error) + } +} + +function summaryRow(id, result, health, check) { + return { + id, + result, + host: describeHealth(checkedHealth(health, check)), + failures: health?.failures ?? null, + latentRoutedInFailures: health?.latentRoutedInFailures ?? null, + latentRoutedGestures: health?.latentRoutedGestures ?? null, + gestures: health?.gestures ?? null, + maxLoadAverage: health?.maxLoadAverage ?? null, + minIdlePercent: health?.minIdlePercent ?? null + } +} + +/** + * One row for each attempt of each planned shard, in plan order, with its gate checks and host + * class. The last row of a shard with a retry also says whether the gate accepts the retry. + */ export async function summarize(manifest, directory) { const rows = [] for (const shard of manifest.shards) { const root = path.join(directory, artifactName(manifest, shard.id)) - const found = await exists(root) - const health = found ? await optionalJson(path.join(root, EVIDENCE, HEALTH_FILE)) : undefined - const { result, check } = found ? await shardResult(manifest, shard, root) : {} - rows.push({ - id: shard.id, - result: result ?? 'no evidence', - host: describeHealth(checkedHealth(health, check)), - failures: health?.failures ?? null, - latentRoutedInFailures: health?.latentRoutedInFailures ?? null, - latentRoutedGestures: health?.latentRoutedGestures ?? null, - gestures: health?.gestures ?? null, - maxLoadAverage: health?.maxLoadAverage ?? null, - minIdlePercent: health?.minIdlePercent ?? null - }) + if (!(await exists(root))) { + rows.push(summaryRow(shard.id, 'no evidence')) + continue + } + const attempts = await attemptDirectories(root) + if (attempts.length === 0) { + rows.push(summaryRow(shard.id, 'rejected: missing attempt-1')) + continue + } + for (const [index, name] of attempts.entries()) { + const attemptRoot = path.join(root, name) + const health = await optionalJson(path.join(attemptRoot, EVIDENCE, HEALTH_FILE)) + const { result, check } = await shardResult(manifest, shard, attemptRoot) + const id = attempts.length > 1 ? `${shard.id} (attempt ${index + 1})` : shard.id + rows.push(summaryRow(id, result, health, check)) + } + const last = rows.at(-1) + const rejection = await shardRejection(manifest, shard, root) + if (attempts.length > 1) + last.result += rejection ? `; retry rejected: ${rejection}` : '; retry accepted' + else if (rejection && last.result.startsWith('passed')) last.result = `rejected: ${rejection}` } return rows } @@ -465,9 +701,11 @@ export function summaryMarkdown(rows) { return [ '### iOS shard host health', '', - 'The host class does not change the gate result.', + 'Host trouble never passes a case. It only permits one retry of a shard.', "Gate checks runs the gate's checks for one shard. The gate also needs the plan, the build and every shard.", - 'Host trouble needs failed cases as the only gate rejections, each with a latent-routed gesture inside its window.', + 'Host trouble needs failed cases and touch readiness hook timeouts as the only gate rejections.', + `Each failed case needs a latent-routed gesture in its window. Each timeout needs load ${HIGH_LOAD} or more in its window.`, + 'A shard with a retry has one row for each attempt. Its last row says whether the gate accepts the retry.', '', '| Shard | Gate checks | Host | Failures | Latent in failures | Latent total | Gestures | Max load | Min idle % |', '| --- | --- | --- | --- | --- | --- | --- | --- | --- |', @@ -498,6 +736,19 @@ async function main([command, ...args]) { const manifest = JSON.parse(await readFile(args[0], 'utf8')) const health = await report(manifest, args[1], args[2]) console.log(describeHealth(health)) + } else if (command === 'retry' && args.length === 3) { + // Writes host-health.json like `report`. Exit code 0 permits the one retry of the shard. + const [planFile, id, root] = args + const manifest = JSON.parse(await readFile(planFile, 'utf8')) + console.log(describeHealth(await report(manifest, id, root))) + try { + const shard = manifest.shards.find((entry) => entry.id === id) + assert.ok(shard, 'Unknown shard') + console.log(`Retry: ${retryReason(shard, await loadAttempt(root, true), manifest)}`) + } catch (error) { + console.log(`No retry: ${firstLine(error)}`) + process.exitCode = 1 + } } else if (command === 'summary' && args.length === 2) { const manifest = JSON.parse(await readFile(args[0], 'utf8')) const markdown = summaryMarkdown(await summarize(manifest, args[1])) @@ -505,7 +756,7 @@ async function main([command, ...args]) { if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, markdown) } else { throw new Error( - 'Expected: report | summary ' + 'Expected: report | retry | summary ' ) } } diff --git a/e2e/helpers/host-health.test.mjs b/e2e/helpers/host-health.test.mjs index c30e34c..186f80a 100644 --- a/e2e/helpers/host-health.test.mjs +++ b/e2e/helpers/host-health.test.mjs @@ -1,20 +1,29 @@ import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import path from 'node:path' import test from 'node:test' -import { fileURLToPath } from 'node:url' +import { fileURLToPath, pathToFileURL } from 'node:url' +import { readinessTimeoutAttempt, topSamples, touchLine } from './fixtures/host-trouble.mjs' import { classify, + classifyAttempt, describeHealth, failureWindows, gateCheck, HEALTH_FILE, + HIGH_LOAD, + isReadinessTimeout, logTime, parseHostLoad, + parseLoadSamples, parseResults, parseSimulatorLog, + READINESS_FAILURE, + READINESS_TIMEOUTS, report, + retryReason, summarize, summaryMarkdown } from './host-health.mjs' @@ -26,8 +35,8 @@ import { validateAggregate } from './ios-shards.mjs' -// The CI host runs in UTC. `log show` prints its local time without a zone. -process.env.TZ = 'UTC' +// CI prints each evidence time in UTC, without a zone. These tests run in the zone of this +// machine, and one test classifies the same evidence in other zones too. // Trimmed from the CI shard evidence of run 39053954618 (swipeable). const fixtures = path.join(path.dirname(fileURLToPath(import.meta.url)), 'fixtures', 'host-health') @@ -128,13 +137,16 @@ test('max load uses every sample, min idle skips the first top sample', () => { }) }) -test('the compact log time is the local time of the host', () => { - assert.equal(logTime('2026-10-17 18:04:07.338'), Date.parse('2026-10-17T18:04:07.338Z')) - process.env.TZ = 'Asia/Tokyo' +test('the compact log time is UTC in any zone of the reader', () => { + const zone = process.env.TZ try { - assert.equal(logTime('2026-10-17 18:04:07.338'), Date.parse('2026-10-17T09:04:07.338Z')) + for (const reader of ['UTC', 'Asia/Tokyo', 'America/Los_Angeles']) { + process.env.TZ = reader + assert.equal(logTime('2026-10-17 18:04:07.338'), Date.parse('2026-10-17T18:04:07.338Z')) + } } finally { - process.env.TZ = 'UTC' + if (zone === undefined) delete process.env.TZ + else process.env.TZ = zone } assert.ok(Number.isNaN(logTime('Timestamp Ty'))) }) @@ -345,6 +357,8 @@ function passingBundle(shard) { id: shard.id, spec: shard.spec, sourceHash: shard.sourceHash, + attempt: 1, + retryReason: null, wdioOutcome: 'success', maestroOutcome: shard.maestro ? 'success' : 'skipped' }, @@ -352,9 +366,10 @@ function passingBundle(shard) { } } -async function writeShard(directory, shard, bundle, health) { - const { receipt, workers } = bundle ?? {} - const root = path.join(directory, artifactName(context, shard.id)) +/** Writes one attempt directory of a shard artifact, and its evidence files. */ +async function writeShard(directory, shard, bundle, health, attempt = 1) { + const { receipt, workers, simulatorLog, hostLoad } = bundle ?? {} + const root = path.join(directory, artifactName(context, shard.id), `attempt-${attempt}`) await mkdir(path.join(root, 'ios-evidence'), { recursive: true }) if (receipt) await writeFile(path.join(root, 'ios-shard-receipt.json'), JSON.stringify(receipt)) if (workers) { @@ -362,6 +377,9 @@ async function writeShard(directory, shard, bundle, health) { await writeFile(path.join(root, 'ios-results', 'worker-42-1.ndjson'), ndjson(...workers[0])) } if (health) await writeFile(path.join(root, 'ios-evidence', HEALTH_FILE), JSON.stringify(health)) + if (simulatorLog) + await writeFile(path.join(root, 'ios-evidence', 'simulator-log.txt'), simulatorLog) + if (hostLoad) await writeFile(path.join(root, 'ios-evidence', 'host-load.txt'), hostLoad) } const shardById = (id) => manifest.shards.find((shard) => shard.id === id) @@ -685,14 +703,14 @@ test('report writes host-health.json next to the evidence, even without inputs', await temporary(async (directory) => { const shard = shardById('swipeable') await writeShard(directory, shard, failingBundle()) - const root = path.join(directory, artifactName(context, shard.id)) + const root = path.join(directory, artifactName(context, shard.id), 'attempt-1') const evidence = path.join(root, 'ios-evidence') await writeFile(path.join(evidence, 'simulator-log.txt'), simulatorLog + failedTestLog) await copyFile(path.join(fixtures, 'host-load.txt'), path.join(evidence, 'host-load.txt')) const health = await report(manifest, shard.id, root) assert.deepEqual(JSON.parse(await readFile(path.join(evidence, HEALTH_FILE), 'utf8')), health) assert.deepEqual(health, { - version: 3, + version: 4, status: 'trouble', onlyFailedCases: true, gestures: 3, @@ -701,6 +719,7 @@ test('report writes host-health.json next to the evidence, even without inputs', latentRoutedInFailures: 1, failures: 1, failuresWithLatent: 1, + failuresExplained: 1, failedTests: [ { title: 'suite case 0', @@ -709,6 +728,7 @@ test('report writes host-health.json next to the evidence, even without inputs', latentRoutedGestures: 1 } ], + readinessTimeouts: [], loadSamples: 3, maxLoadAverage: 512.16, minIdlePercent: 0.4, @@ -873,9 +893,9 @@ for (const [reason, [mutate, cell]] of Object.entries(gateRejections)) { const bundle = passingBundle(shard) mutate(bundle) const bundles = Object.fromEntries( - manifest.shards.map((entry) => [entry.id, passingBundle(entry)]) + manifest.shards.map((entry) => [entry.id, { attempts: [passingBundle(entry)] }]) ) - bundles[shard.id] = bundle + bundles[shard.id] = { attempts: [bundle] } const gate = { ...context, planResult: 'success', @@ -904,7 +924,10 @@ test('the summary passes a shard only when the gate accepts its evidence', async buildResult: 'success', shardsResult: 'success' } - assert.equal(validateAggregate(manifest, bundles, gate).passed, 62) + const shardBundles = Object.fromEntries( + Object.entries(bundles).map(([id, bundle]) => [id, { attempts: [bundle] }]) + ) + assert.equal(validateAggregate(manifest, shardBundles, gate).passed, 62) await temporary(async (directory) => { for (const shard of manifest.shards) await writeShard(directory, shard, bundles[shard.id]) const rows = await summarize(manifest, directory) @@ -920,3 +943,446 @@ test('the summary passes a shard only when the gate accepts its evidence', async ) }) }) + +test('each top sample has its UTC time and its 1-minute load average', () => { + assert.deepEqual(parseLoadSamples(hostLoad), [ + { at: Date.parse('2026-10-17T18:01:14Z'), load: 512.16 }, + { at: Date.parse('2026-10-17T18:01:48Z'), load: 358.33 }, + { at: Date.parse('2026-10-17T18:02:02Z'), load: 348.67 } + ]) + assert.ok(Number.isNaN(parseLoadSamples('Load Avg: 1.00, 1.00, 1.00\n')[0].at)) +}) + +// Synthetic: attempt 1 of the swipeable shard. Its first case passes at 17:51:30, then its +// afterEach relaunch times out at 17:52:07. Each `top` sample in that window has one load. +const timeoutStart = Date.parse('2026-10-17T17:50:00.000Z') +const timeoutAttempt = (load) => + readinessTimeoutAttempt(shardById('swipeable'), context, { + start: timeoutStart, + load, + file: '/checkout/e2e/specs/swipeable.spec.ts' + }) + +test('a touch readiness timeout in a relaunch under high load is host trouble', () => { + const attempt = timeoutAttempt(HIGH_LOAD) + assert.deepEqual( + { ...gateCheck(shardById('swipeable'), attempt, context), error: undefined }, + { error: undefined, failedCases: 1, onlyFailedCases: true } + ) + const health = attempt.health + assert.equal(health.status, 'trouble') + assert.equal(health.failuresExplained, 1) + assert.deepEqual(health.readinessTimeouts, [ + { + title: 'afterEach', + start: '2026-10-17T17:51:30.000Z', + end: '2026-10-17T17:52:07.000Z', + loadSamples: 4, + minLoadAverage: HIGH_LOAD, + highLoad: true + } + ]) + assert.equal( + describeHealth(health), + 'host trouble: all 1 failures have latent-routed gestures or a touch readiness timeout at ' + + 'load 100 or more (0 of 0 latent-routed gestures in failures, 1 of 1 touch readiness ' + + 'timeouts at load 100 or more), load 100' + ) +}) + +test('the same touch readiness timeout under low load is healthy', () => { + const health = timeoutAttempt(HIGH_LOAD - 1).health + assert.equal(health.status, 'healthy') + assert.equal(health.failuresExplained, 0) + assert.equal(health.readinessTimeouts[0].highLoad, false) + assert.equal( + describeHealth(health), + 'host healthy: 0 of 0 latent-routed gestures in failures, 0 of 1 touch readiness timeouts ' + + 'at load 100 or more' + ) +}) + +test('a touch readiness timeout needs each load sample in its window at high load', () => { + const attempt = timeoutAttempt(300) + const at = (seconds) => timeoutStart + seconds * 1000 + // One sample below the threshold, then no sample in the window at all. + for (const samples of [ + [ + [at(95), 300], + [at(105), HIGH_LOAD - 1], + [at(125), 300] + ], + [ + [at(60), 300], + [at(128), 300] + ] + ]) { + const health = classifyAttempt( + shardById('swipeable'), + { ...attempt, hostLoad: topSamples(samples) }, + context + ) + assert.equal(health.status, 'healthy') + } + const noLoad = classifyAttempt( + shardById('swipeable'), + { ...attempt, hostLoad: undefined }, + context + ) + assert.equal(noLoad.status, 'healthy') + assert.deepEqual(noLoad.missing, ['host-load.txt']) +}) + +test('a touch readiness timeout inside a case body needs a latent-routed gesture, not load', () => { + const attempt = timeoutAttempt(300) + const records = attempt.workers[0] + const [hook] = records.splice(4, 1) + Object.assign(records[3], { status: 'failed', error: hook.error }) + records.at(-1).result = 1 + const health = classifyAttempt(shardById('swipeable'), attempt, context) + assert.deepEqual(health.readinessTimeouts, []) + assert.notEqual(health.status, 'trouble') +}) + +const readinessMessage = (reason) => `${READINESS_FAILURE} after 3 drag(s) in 30128 ms: ${reason}` +const appStateReasons = [ + 'the app left the list screen before a drag moved the seekbar thumb', + 'the seekbar thumb did not return to rest' +] + +test('only the two timeouts of the touch readiness gate can be host trouble', () => { + for (const reason of READINESS_TIMEOUTS) + assert.equal(isReadinessTimeout(readinessMessage(reason)), true) + for (const reason of appStateReasons) + assert.equal(isReadinessTimeout(readinessMessage(reason)), false) + assert.equal(isReadinessTimeout(`${readinessMessage(READINESS_TIMEOUTS[0])}, then more`), false) + assert.equal(isReadinessTimeout(READINESS_TIMEOUTS[0]), false) + assert.equal(isReadinessTimeout(undefined), false) +}) + +test('a touch readiness failure for an app state stays a real failure under high load', () => { + const shard = shardById('swipeable') + for (const reason of appStateReasons) { + const attempt = timeoutAttempt(300) + attempt.workers[0][4].error.message = readinessMessage(reason) + assert.equal(gateCheck(shard, attempt, context).onlyFailedCases, false) + const health = classifyAttempt(shard, attempt, context) + assert.equal(health.status, 'healthy') + assert.deepEqual(health.readinessTimeouts, []) + assert.throws( + () => retryReason(shard, { ...attempt, health }, context), + /A retry needs host trouble/ + ) + } + const lostTouches = timeoutAttempt(300) + lostTouches.workers[0][4].error.message = readinessMessage(READINESS_TIMEOUTS[1]) + assert.equal(classifyAttempt(shard, lostTouches, context).status, 'trouble') +}) + +test('a touch readiness timeout under high load needs no simulator log and no gesture', () => { + const shard = shardById('swipeable') + const attempt = timeoutAttempt(300) + for (const simulatorLog of [undefined, 'Timestamp Ty Process[PID:TID]\n']) { + const health = classifyAttempt(shard, { ...attempt, simulatorLog }, context) + assert.equal(health.gestures, 0) + assert.equal(health.status, 'trouble') + } + // A failed case needs gesture evidence, so without the simulator log the attempt is unknown. + const withCase = timeoutAttempt(300) + Object.assign(withCase.workers[0][3], { + status: 'failed', + error: { name: 'Error', message: 'row did not open' } + }) + const health = classifyAttempt(shard, { ...withCase, simulatorLog: undefined }, context) + assert.equal(health.status, 'unknown') + assert.equal(describeHealth(health), 'host unknown: no simulator log') +}) + +test('a latent-routed gesture in a touch readiness timeout window does not count', () => { + const shard = shardById('swipeable') + for (const [load, status] of [ + [20, 'healthy'], + [300, 'trouble'] + ]) { + const attempt = timeoutAttempt(load) + attempt.simulatorLog += touchLine(timeoutStart + 110000, true) + const health = classifyAttempt(shard, attempt, context) + assert.equal(health.status, status) + assert.equal(health.latentRoutedGestures, 1) + assert.equal(health.latentRoutedInFailures, 0) + assert.equal(health.failuresWithLatent, 0) + assert.match(describeHealth(health), /\b0 of 1 latent-routed gestures in failures, /) + } +}) + +test('host mixed counts the failures that host trouble explains', () => { + const attempt = timeoutAttempt(300) + Object.assign(attempt.workers[0][3], { + status: 'failed', + error: { name: 'Error', message: 'row did not open' } + }) + const health = classifyAttempt(shardById('swipeable'), attempt, context) + assert.equal(health.status, 'mixed') + assert.equal(health.failuresExplained, 1) + assert.equal( + describeHealth(health), + 'host mixed: 1 of 2 failures have latent-routed gestures or a touch readiness timeout at ' + + 'load 100 or more (0 of 0 latent-routed gestures in failures, 1 of 1 touch readiness ' + + 'timeouts at load 100 or more), load 300' + ) +}) + +// Classifies CI evidence in a new process with the given zone. The evidence prints UTC +// times without a zone, so each zone of the reader must give the same host class. +const zoneProbe = ` +import { readFileSync } from 'node:fs' +import { classify } from ${JSON.stringify(new URL('./host-health.mjs', import.meta.url).href)} +const read = (name) => readFileSync(new URL(name, ${JSON.stringify(pathToFileURL(`${fixtures}/`).href)}), 'utf8') +const gate = { onlyFailedCases: true, failedCases: 1 } +// Four top samples inside the hook window of run 39273407706, as CI prints them. +const hookWindow = ['15:27:40', '15:27:50', '15:28:00', '15:28:10'] + .map((time) => 'Processes: 800 total\\n2026/10/19 ' + time + '\\nLoad Avg: 300.00, 300.00, 300.00 \\n') + .join('') +process.stdout.write(JSON.stringify([ + classify({ + simulatorLog: read('simulator-log.txt') + read('simulator-log-failed-test.txt'), + hostLoad: read('host-load.txt'), + results: read('ios-results.ndjson'), + gate + }), + classify({ + simulatorLog: read('run-39273407706/swipeable-simulator-log.txt'), + hostLoad: read('run-39273407706/swipeable-host-load.txt') + hookWindow, + results: read('run-39273407706/swipeable-results.ndjson'), + gate + }) +])) +` + +test('the host class of the same evidence is the same in each zone of the reader', () => { + const classes = {} + for (const zone of ['UTC', 'Europe/Kyiv', 'America/Los_Angeles']) { + const probe = spawnSync(process.execPath, ['--input-type=module', '--eval', zoneProbe], { + env: { ...process.env, TZ: zone }, + encoding: 'utf8' + }) + assert.equal(probe.status, 0, probe.stderr) + classes[zone] = JSON.parse(probe.stdout) + } + // A latent-routed gesture in a failed case, then a relaunch timeout at high load. + assert.deepEqual( + classes.UTC.map((health) => [health.status, health.failuresExplained]), + [ + ['trouble', 1], + ['trouble', 1] + ] + ) + assert.deepEqual(classes['Europe/Kyiv'], classes.UTC) + assert.deepEqual(classes['America/Los_Angeles'], classes.UTC) +}) + +// Trimmed from the CI shard evidence of run 39273407706 (swipeable). Its 8 cases passed. +// Then the afterEach relaunch timed out in the touch readiness gate, and the worker +// quarantined the session in that hook. Mocha started none of the other 17 cases. The +// host load of the run has samples from 15:16:55 to 15:20:01 only: the WDIO step then +// sampled its first two minutes. The worker started at 15:21:21. +const run3 = path.join(fixtures, 'run-39273407706') +const run3Context = { + sha: 'd06730521c230f685bc09a9980ca92d957e47546', + runId: '39273407706', + runAttempt: '1' +} +const run3Results = await readFile(path.join(run3, 'swipeable-results.ndjson'), 'utf8') +const run3Evidence = { + receipt: JSON.parse(await readFile(path.join(run3, 'swipeable-receipt.json'), 'utf8')), + simulatorLog: await readFile(path.join(run3, 'swipeable-simulator-log.txt'), 'utf8'), + hostLoad: await readFile(path.join(run3, 'swipeable-host-load.txt'), 'utf8') +} +// The 8 cases that ran, then 17 cases that never started. +const run3Manifest = buildManifest( + Object.fromEntries( + Object.entries(EXPECTED_COUNTS).map(([spec, count]) => { + if (spec === 'specs/swipeable.spec.ts') + return [ + spec, + `${registeredCases(run3Results)}\ndescribe("Later tests", () => { for (let i = 0; i < 17; i++) it("case " + i, () => {}) })` + ] + return [ + spec, + `describe("suite", () => { for (let i = 0; i < ${count}; i++) it("case " + i, () => {}) })` + ] + }) + ), + run3Context +) +const run3Shard = run3Manifest.shards.find((shard) => shard.id === 'swipeable') +// Synthetic: four top samples inside the window of the hook, from 15:27:35.902 to 15:28:12.857. +const hookWindowLoad = (load) => + topSamples( + ['15:27:40', '15:27:50', '15:28:00', '15:28:10'].map((time) => [ + Date.parse(`2026-10-19T${time}Z`), + load + ]) + ) + +function run3Attempt(hostLoadText) { + const attempt = { + receipt: { + ...run3Evidence.receipt, + sourceHash: run3Shard.sourceHash, + attempt: 1, + retryReason: null + }, + workers: [parseRecords(run3Results)], + simulatorLog: run3Evidence.simulatorLog, + hostLoad: hostLoadText + } + return { ...attempt, health: classifyAttempt(run3Shard, attempt, run3Context) } +} + +test('run 39273407706: the relaunch timeout, its quarantine and the unstarted cases are host trouble at high load', () => { + const attempt = run3Attempt(run3Evidence.hostLoad + hookWindowLoad(300)) + const records = attempt.workers[0] + assert.equal(records.filter((record) => record.event === 'test-start').length, 8) + assert.equal(records.at(-1).result, 2) + assert.deepEqual( + { ...gateCheck(run3Shard, attempt, run3Context), error: undefined }, + { error: undefined, failedCases: 1, onlyFailedCases: true } + ) + assert.equal(attempt.health.status, 'trouble') + assert.equal(attempt.health.latentRoutedGestures, 1) + assert.equal(attempt.health.latentRoutedInFailures, 0) + assert.equal( + retryReason(run3Shard, attempt, run3Context), + 'host trouble: all 1 failures have latent-routed gestures or a touch readiness timeout at ' + + 'load 100 or more (0 of 1 latent-routed gestures in failures, 1 of 1 touch readiness ' + + 'timeouts at load 100 or more), load 385.07' + ) +}) + +test('run 39273407706: the same relaunch timeout is healthy at low load or without a sample in its window', () => { + for (const hostLoadText of [ + run3Evidence.hostLoad + hookWindowLoad(20), + // The run itself: the load samples end before the worker starts. + run3Evidence.hostLoad + ]) { + const attempt = run3Attempt(hostLoadText) + assert.equal(attempt.health.status, 'healthy') + assert.match(describeHealth(attempt.health), /^host healthy: 0 of 1 latent-routed gestures/) + assert.throws(() => retryReason(run3Shard, attempt, run3Context), /A retry needs host trouble/) + } +}) + +// Each of these is a rejection that the relaunch timeout of the run does not cause. +const timeoutRejections = { + 'a case after the timeout hook': (records) => { + const [start, result] = records.slice(2, 4).map((record) => ({ + ...record, + ...run3Shard.cases[8], + timestamp: '2026-10-19T15:28:12.900Z' + })) + records.splice(-1, 0, start, result) + }, + 'an unfinished case before the timeout hook': (records) => { + records.splice(-3, 1) + }, + 'a gap in the started cases, as from a grep filter': (records) => { + records.splice(4, 2) + }, + 'a skipped case': (records) => { + records[3].status = 'skipped' + }, + 'another hook failure': (records) => { + records.splice(4, 0, { ...records.at(-2), error: { name: 'Error', message: 'other' } }) + }, + 'a failed warm-up': (records) => { + records[1].ok = false + }, + 'a quarantine that starts in another runnable': (records) => { + records.at(-1).quarantine.runnable = 'Swipeable E2E Tests "after all" hook' + }, + 'a worker that did not finish': (records) => { + records.pop() + } +} + +for (const [reason, mutate] of Object.entries(timeoutRejections)) { + test(`run 39273407706: a relaunch timeout at high load with another rejection is not host trouble: ${reason}`, () => { + const attempt = run3Attempt(run3Evidence.hostLoad + hookWindowLoad(300)) + mutate(attempt.workers[0]) + const check = gateCheck(run3Shard, attempt, run3Context) + assert.equal(check.passed, undefined) + assert.equal(check.onlyFailedCases, false) + const health = classifyAttempt(run3Shard, attempt, run3Context) + assert.notEqual(health.status, 'trouble') + }) +} + +for (const [reason, mutate] of Object.entries({ + 'a Maestro failure': (receipt) => { + receipt.maestroOutcome = 'failure' + }, + 'a cancelled WDIO step': (receipt) => { + receipt.wdioOutcome = 'cancelled' + } +})) { + test(`run 39273407706: a relaunch timeout at high load with ${reason} is not host trouble`, () => { + const attempt = run3Attempt(run3Evidence.hostLoad + hookWindowLoad(300)) + mutate(attempt.receipt) + assert.notEqual(classifyAttempt(run3Shard, attempt, run3Context).status, 'trouble') + assert.throws(() => retryReason(run3Shard, attempt, run3Context)) + }) +} + +test('a retry follows attempt 1 only, with host trouble in host-health.json and in the evidence', () => { + const attempt = run3Attempt(run3Evidence.hostLoad + hookWindowLoad(300)) + assert.match(retryReason(run3Shard, attempt, run3Context), /^host trouble: /) + const retry = (change) => () => { + const value = structuredClone(attempt) + change(value) + retryReason(run3Shard, value, run3Context) + } + assert.throws( + retry((value) => { + value.receipt.attempt = 2 + }), + /Only attempt 1 can have a retry/ + ) + assert.throws( + retry((value) => { + delete value.health + }), + /A retry needs host trouble in host-health.json/ + ) + assert.throws( + retry((value) => { + value.hostLoad = run3Evidence.hostLoad + }), + /A retry needs host trouble: host healthy/ + ) +}) + +test('the summary has one row for each attempt and gives the gate verdict on the retry', async () => { + const shard = shardById('swipeable') + for (const [load, verdict] of [ + [300, '; retry accepted'], + [20, '; retry rejected: A retry needs host trouble in host-health.json'] + ]) { + await temporary(async (directory) => { + const first = timeoutAttempt(load) + const second = passingBundle(shard) + second.receipt.attempt = 2 + second.receipt.retryReason = describeHealth(first.health) + await writeShard(directory, shard, first, first.health, 1) + await writeShard(directory, shard, second, undefined, 2) + const rows = await summarize({ ...manifest, shards: [shard] }, directory) + assert.deepEqual( + rows.map(({ id }) => id), + ['swipeable (attempt 1)', 'swipeable (attempt 2)'] + ) + assert.match(rows[0].result, /^rejected: wdio failure, .*1 failed or skipped hook/) + assert.equal(rows[0].host, describeHealth(first.health)) + assert.equal(rows[1].result, `passed (25 cases)${verdict}`) + }) + } +}) diff --git a/e2e/helpers/ios-shards.mjs b/e2e/helpers/ios-shards.mjs index 9a57043..d592ad9 100644 --- a/e2e/helpers/ios-shards.mjs +++ b/e2e/helpers/ios-shards.mjs @@ -5,6 +5,9 @@ import { createRequire } from 'node:module' import path from 'node:path' import { fileURLToPath, pathToFileURL } from 'node:url' import vm from 'node:vm' +// host-health.mjs imports this module too. Each module uses the exports of the other only +// inside its functions, so the import cycle is safe. +import { retryReason } from './host-health.mjs' const require = createRequire(import.meta.url) const e2eDirectory = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') @@ -17,6 +20,11 @@ export const EXPECTED_COUNTS = Object.freeze({ }) const TOTAL = 62 const identityKeys = ['sha', 'runId', 'runAttempt'] +/** One attempt of a shard, and at most one retry after host trouble (docs/releasing.md). */ +export const MAX_ATTEMPTS = 2 +/** Each attempt directory `attempt-N/` of a shard artifact has its own receipt. */ +export const RECEIPT_FILE = 'ios-shard-receipt.json' +const attemptPattern = /^attempt-([1-9][0-9]*)\// function identity(value) { assert.match(value.sha, /^[a-f0-9]{40}$/, 'full checkout SHA required') @@ -259,7 +267,7 @@ export function artifactName(context, id) { return ['ios-shard', context.sha, context.runId, context.runAttempt, id].join('-') } -/** The gate checks for one shard: its receipt, its step outcomes and its worker evidence. */ +/** The gate checks for one attempt: its receipt, its step outcomes and its worker evidence. */ export function validateShard(shard, { receipt, workers }, context) { assert.equal(receipt.version, 1) for (const key of identityKeys) assert.equal(receipt[key], context[key], 'Stale receipt: ' + key) @@ -275,17 +283,109 @@ export function validateShard(shard, { receipt, workers }, context) { return validateRecords(shard, workers[0]) } -/** Reads one shard artifact the way the gate does. */ -export async function loadShardBundle(root) { - const receipt = JSON.parse(await readFile(path.join(root, 'ios-shard-receipt.json'), 'utf8')) - const results = path.join(root, 'ios-results') - const files = await readdir(results, { withFileTypes: true }) - assert.equal(files.length, 1, 'Missing/duplicate worker artifacts') - assert.ok(files[0].isFile() && files[0].name.endsWith('.ndjson'), 'Invalid worker artifact') +/** + * The gate checks for one shard and its attempts. Either one attempt passes, or attempt 1 + * fails, `retryReason` permits a retry after it, and attempt 2 passes. The gate classifies + * attempt 1 again from its raw evidence. Host trouble never passes a case. + */ +export function validateShardEvidence(shard, { attempts }, context) { + assert.ok(Array.isArray(attempts) && attempts.length > 0, 'Missing attempt evidence') + assert.ok(attempts.length <= MAX_ATTEMPTS, 'More than one retry') + for (const [index, attempt] of attempts.entries()) { + assert.equal(attempt.receipt?.attempt, index + 1, 'Attempt receipt out of order') + } + const last = attempts.at(-1) + const passed = validateShard(shard, last, context) + if (attempts.length === 1) { + assert.equal(last.receipt.retryReason, null, 'Retry reason without a retry') + return passed + } + const [first] = attempts + assert.equal(first.receipt.retryReason, null, 'Retry reason on attempt 1') + assert.equal( + last.receipt.retryReason, + retryReason(shard, first, context), + 'Retry reason differs from the attempt 1 evidence' + ) + const ended = Date.parse(first.workers.flat().at(-1)?.timestamp) + assert.ok( + Date.parse(last.workers[0][0].timestamp) > ended, + 'Attempt 2 started before attempt 1 ended' + ) + return passed +} + +/** + * Reads one attempt the way the gate does. `names` lists its files by relative path, and + * `read` returns the text of one. With `logs`, the attempt also has its simulator log and + * host load: the gate classifies an attempt again before it accepts a retry after it. + */ +async function readAttempt(names, read, logs) { + const optional = async (name) => (names.includes(name) ? read(name) : undefined) + const receipt = JSON.parse(await read(RECEIPT_FILE)) + const workers = [] + for (const name of names.filter((entry) => entry.startsWith('ios-results/')).sort()) { + workers.push(parseRecords(await read(name))) + } + // A triage file. A retry check fails closed without it. + let health + try { + health = JSON.parse(await optional('ios-evidence/host-health.json')) + } catch { + health = undefined + } return { receipt, - workers: [parseRecords(await readFile(path.join(results, files[0].name), 'utf8'))] + workers, + health, + ...(logs && { + simulatorLog: await optional('ios-evidence/simulator-log.txt'), + hostLoad: await optional('ios-evidence/host-load.txt') + }) + } +} + +/** + * Reads one shard artifact the way the gate does: each attempt directory in order. `names` + * lists every file of the artifact by its relative path, and `read` returns the text of one. + * Every file must be in attempt-1 to attempt-N, with no gap. + */ +export async function shardBundle(names, read) { + const numbers = names.map((name) => Number(name.match(attemptPattern)?.[1])) + assert.ok(numbers.every(Number.isInteger), 'Shard evidence outside an attempt directory') + const count = Math.max(0, ...numbers) + const attempts = [] + for (let attempt = 1; attempt <= count; attempt++) { + const directory = `attempt-${attempt}/` + const files = names + .filter((name) => name.startsWith(directory)) + .map((name) => name.slice(directory.length)) + assert.ok(files.length > 0, `Missing attempt directory: ${directory}`) + attempts.push(await readAttempt(files, (name) => read(directory + name), attempt < count)) + } + return { attempts } +} + +/** Every file below a directory, by its relative path with `/` separators. */ +async function listFiles(root, relative = '') { + const names = [] + for (const entry of await readdir(path.join(root, relative), { withFileTypes: true })) { + const name = relative ? `${relative}/${entry.name}` : entry.name + if (entry.isDirectory()) names.push(...(await listFiles(root, name))) + else names.push(name) } + return names +} + +/** Reads one attempt directory the way the gate does. */ +export async function loadAttempt(directory, logs = false) { + const read = (name) => readFile(path.join(directory, name), 'utf8') + return readAttempt(await listFiles(directory), read, logs) +} + +/** Reads one shard artifact the way the gate does. */ +export async function loadShardBundle(root) { + return shardBundle(await listFiles(root), (name) => readFile(path.join(root, name), 'utf8')) } export function validateAggregate(manifest, bundles, context) { @@ -301,9 +401,20 @@ export function validateAggregate(manifest, bundles, context) { 'Missing/extra shard artifacts' ) let passed = 0 - for (const shard of manifest.shards) passed += validateShard(shard, bundles[shard.id], context) + const retried = {} + for (const shard of manifest.shards) { + const { attempts } = bundles[shard.id] + passed += validateShardEvidence(shard, bundles[shard.id], context) + if (attempts.length > 1) retried[shard.id] = attempts.at(-1).receipt.retryReason + } assert.equal(passed, TOTAL) - return { ...identity(context), passed, shards: manifest.shards.length, maestro: 'success' } + return { + ...identity(context), + passed, + shards: manifest.shards.length, + maestro: 'success', + retried + } } function environmentIdentity() { @@ -347,17 +458,24 @@ async function main([command, ...args]) { await verifiedPlan(args[0], context) } else if (command === 'receipt') { assert.equal(args.length, 3) - const [id, planFile, filename] = args + const [id, planFile, directory] = args const manifest = validateManifest(JSON.parse(await readFile(planFile, 'utf8'))) for (const key of identityKeys) assert.equal(manifest[key], context[key]) const shard = manifest.shards.find((entry) => entry.id === id) assert.ok(shard, 'Unknown shard') - await save(filename, { + const attempt = Number(path.basename(directory).match(/^attempt-([1-9][0-9]*)$/)?.[1]) + assert.ok(attempt >= 1 && attempt <= MAX_ATTEMPTS, 'Expected an attempt directory') + // The receipt of a retry records why it ran: the host class of attempt 1. + const previous = path.join(path.dirname(directory), `attempt-${attempt - 1}`) + await save(path.join(directory, RECEIPT_FILE), { version: 1, ...context, id, spec: shard.spec, sourceHash: shard.sourceHash, + attempt, + retryReason: + attempt === 1 ? null : retryReason(shard, await loadAttempt(previous, true), manifest), wdioOutcome: process.env.IOS_WDIO_OUTCOME, maestroOutcome: process.env.IOS_MAESTRO_OUTCOME }) diff --git a/e2e/helpers/ios-shards.test.mjs b/e2e/helpers/ios-shards.test.mjs index 22b4443..106c889 100644 --- a/e2e/helpers/ios-shards.test.mjs +++ b/e2e/helpers/ios-shards.test.mjs @@ -1,5 +1,7 @@ import assert from 'node:assert/strict' import test from 'node:test' +import { readinessTimeoutAttempt } from './fixtures/host-trouble.mjs' +import { describeHealth } from './host-health.mjs' import { artifactName, buildManifest, @@ -7,6 +9,7 @@ import { discoverSources, EXPECTED_COUNTS, parseRecords, + shardBundle, validateAggregate, validateManifest, validateRecords @@ -55,18 +58,24 @@ function fixture() { return [ shard.id, { - receipt: { - version: 1, - sha: context.sha, - runId: context.runId, - runAttempt: context.runAttempt, - id: shard.id, - spec: shard.spec, - sourceHash: shard.sourceHash, - wdioOutcome: 'success', - maestroOutcome: shard.maestro ? 'success' : 'skipped' - }, - workers: [records] + attempts: [ + { + receipt: { + version: 1, + sha: context.sha, + runId: context.runId, + runAttempt: context.runAttempt, + id: shard.id, + spec: shard.spec, + sourceHash: shard.sourceHash, + attempt: 1, + retryReason: null, + wdioOutcome: 'success', + maestroOutcome: shard.maestro ? 'success' : 'skipped' + }, + workers: [records] + } + ] } ] }) @@ -74,6 +83,9 @@ function fixture() { return { plan, bundles, run: { ...context } } } +/** The only attempt of a shard in `fixture`. */ +const only = (bundle) => bundle.attempts[0] + test('real checkout discovers every eligible spec and all 62 cases without executing bodies', async () => { const actual = buildManifest(await discoverSources(), context) assert.deepEqual( @@ -155,7 +167,8 @@ test('healthy aggregate requires all 62 cases and Maestro exactly once', () => { runAttempt: '1', passed: 62, shards: 5, - maestro: 'success' + maestro: 'success', + retried: {} }) }) @@ -167,43 +180,43 @@ const mutations = { bundles.extra = bundles.rtl }, 'missing receipt': ({ bundles }) => { - delete bundles.rtl.receipt + delete only(bundles.rtl).receipt }, 'wrong receipt SHA': ({ bundles }) => { - bundles.rtl.receipt.sha = 'b'.repeat(40) + only(bundles.rtl).receipt.sha = 'b'.repeat(40) }, 'stale run': ({ bundles }) => { - bundles.rtl.receipt.runId = '122' + only(bundles.rtl).receipt.runId = '122' }, 'stale attempt': ({ bundles }) => { - bundles.rtl.receipt.runAttempt = '2' + only(bundles.rtl).receipt.runAttempt = '2' }, 'wrong shard': ({ bundles }) => { - bundles.rtl.receipt.id = 'swipeable' + only(bundles.rtl).receipt.id = 'swipeable' }, 'wrong spec receipt': ({ bundles }) => { - bundles.rtl.receipt.spec = 'specs/swipeable.spec.ts' + only(bundles.rtl).receipt.spec = 'specs/swipeable.spec.ts' }, 'wrong source digest': ({ bundles }) => { - bundles.rtl.receipt.sourceHash = 'b'.repeat(64) + only(bundles.rtl).receipt.sourceHash = 'b'.repeat(64) }, 'failed WDIO': ({ bundles }) => { - bundles.rtl.receipt.wdioOutcome = 'failure' + only(bundles.rtl).receipt.wdioOutcome = 'failure' }, 'cancelled WDIO': ({ bundles }) => { - bundles.rtl.receipt.wdioOutcome = 'cancelled' + only(bundles.rtl).receipt.wdioOutcome = 'cancelled' }, 'skipped WDIO': ({ bundles }) => { - bundles.rtl.receipt.wdioOutcome = 'skipped' + only(bundles.rtl).receipt.wdioOutcome = 'skipped' }, 'failed Maestro': ({ bundles }) => { - bundles.swipeable.receipt.maestroOutcome = 'failure' + only(bundles.swipeable).receipt.maestroOutcome = 'failure' }, 'missing Maestro': ({ bundles }) => { - delete bundles.swipeable.receipt.maestroOutcome + delete only(bundles.swipeable).receipt.maestroOutcome }, 'Maestro on another shard': ({ bundles }) => { - bundles.rtl.receipt.maestroOutcome = 'success' + only(bundles.rtl).receipt.maestroOutcome = 'success' }, 'failed shard job': ({ run }) => { run.shardsResult = 'failure' @@ -233,10 +246,30 @@ const mutations = { plan.shards[1].cases[1] = plan.shards[1].cases[0] }, 'missing worker file': ({ bundles }) => { - bundles.rtl.workers = [] + only(bundles.rtl).workers = [] }, 'duplicate worker file': ({ bundles }) => { - bundles.rtl.workers.push(bundles.rtl.workers[0]) + only(bundles.rtl).workers.push(only(bundles.rtl).workers[0]) + }, + 'missing attempt': ({ bundles }) => { + bundles.rtl.attempts = [] + }, + 'attempt receipt without its attempt number': ({ bundles }) => { + delete only(bundles.rtl).receipt.attempt + }, + 'attempt 2 without attempt 1': ({ bundles }) => { + only(bundles.rtl).receipt.attempt = 2 + }, + 'retry reason without a retry': ({ bundles }) => { + only(bundles.rtl).receipt.retryReason = 'host trouble: all 1 failures' + }, + 'missing retry reason field': ({ bundles }) => { + delete only(bundles.rtl).receipt.retryReason + }, + 'a green attempt repeated as its own retry': ({ bundles }) => { + const second = structuredClone(only(bundles.rtl)) + second.receipt.attempt = 2 + bundles.rtl.attempts.push(second) } } for (const [name, mutate] of Object.entries(mutations)) @@ -336,13 +369,13 @@ const recordMutations = { for (const [name, mutate] of Object.entries(recordMutations)) test('evidence rejects ' + name, () => { const value = fixture() - mutate(value.bundles.rtl.workers[0]) + mutate(only(value.bundles.rtl).workers[0]) assert.throws(() => validateAggregate(value.plan, value.bundles, value.run)) }) test('NDJSON round-trip accepts actual optional fullTitle omission', () => { const { plan, bundles } = fixture() - const records = bundles.rtl.workers[0] + const records = only(bundles.rtl).workers[0] for (const record of records) delete record.fullTitle assert.equal( validateRecords(plan.shards[1], parseRecords(records.map(JSON.stringify).join('\n') + '\n')), @@ -351,9 +384,9 @@ test('NDJSON round-trip accepts actual optional fullTitle omission', () => { }) test('the gate reads the worker-end quarantine field as evidence only', () => { const { plan, bundles, run } = fixture() - for (const bundle of Object.values(bundles)) bundle.workers[0].at(-1).quarantine = null + for (const bundle of Object.values(bundles)) only(bundle).workers[0].at(-1).quarantine = null assert.equal(validateAggregate(plan, bundles, run).passed, 62) - const records = bundles.rtl.workers[0] + const records = only(bundles.rtl).workers[0] records.at(-1).quarantine = { reason: 'unresolved request when "suite case 0" ended', runnable: 'suite case 0' @@ -381,3 +414,204 @@ test('artifact names distinguish shard, SHA, run and attempt', () => { assert.throws(() => artifactName({ ...context, sha: 'short' }, 'rtl')) assert.throws(() => validateManifest({ ...manifest, runAttempt: '' })) }) + +// Attempt 1 ends at 11:02:07 in a touch readiness timeout. The fixture records of +// attempt 2 are at 12:00:00. +const attemptStart = Date.parse('2026-10-16T11:00:00.000Z') + +/** + * A swipeable shard with a retry: attempt 1 times out in a relaunch at the given load, + * and attempt 2 is the green fixture attempt. Its receipt gives the attempt 1 host class. + */ +function retried(load = 300) { + const value = fixture() + const shard = value.plan.shards.find((entry) => entry.id === 'swipeable') + const first = readinessTimeoutAttempt(shard, context, { + start: attemptStart, + load, + file: `/checkout/e2e/${shard.spec}` + }) + const second = only(value.bundles.swipeable) + second.receipt.attempt = 2 + second.receipt.retryReason = describeHealth(first.health) + value.bundles.swipeable.attempts = [first, second] + return value +} + +test('aggregate accepts one retry after host trouble in attempt 1 and a green attempt 2', () => { + const { plan, bundles, run } = retried() + const [first] = bundles.swipeable.attempts + assert.equal(first.health.status, 'trouble') + const result = validateAggregate(plan, bundles, run) + assert.equal(result.passed, 62) + assert.deepEqual(result.retried, { swipeable: describeHealth(first.health) }) + assert.match(result.retried.swipeable, /^host trouble: .*1 of 1 touch readiness timeouts/) +}) + +const noChange = () => {} +// Each entry gives the attempt 1 load, a change to the attempts, and the gate message. +const retryRejections = { + 'attempt 1 healthy at low load': [ + 20, + noChange, + /^A retry needs host trouble in host-health\.json/ + ], + 'host trouble only in host-health.json, not in the raw evidence': [ + 20, + ([first, second]) => { + first.health = retried().bundles.swipeable.attempts[0].health + second.receipt.retryReason = describeHealth(first.health) + }, + /^A retry needs host trouble: host healthy: .*0 of 1 touch readiness timeouts/ + ], + 'attempt 1 without host-health.json': [ + 300, + ([first]) => { + delete first.health + }, + /^A retry needs host trouble in host-health\.json/ + ], + 'attempt 1 without its host load': [ + 300, + ([first]) => { + delete first.hostLoad + }, + /^A retry needs host trouble: host healthy/ + ], + 'attempt 1 with a Maestro failure': [ + 300, + ([first]) => { + first.receipt.maestroOutcome = 'failure' + }, + /^A Maestro step never gets a retry/ + ], + 'attempt 1 cancelled': [ + 300, + ([first]) => { + first.receipt.wdioOutcome = 'cancelled' + }, + /^A retry needs a failed WDIO step/ + ], + 'attempt 1 with a retry reason': [ + 300, + ([first]) => { + first.receipt.retryReason = 'host trouble' + }, + /^Retry reason on attempt 1/ + ], + 'retry reason that differs from the attempt 1 evidence': [ + 300, + ([, second]) => { + second.receipt.retryReason = 'host trouble: something else' + }, + /^Retry reason differs from the attempt 1 evidence/ + ], + 'attempt 2 without a retry reason': [ + 300, + ([, second]) => { + second.receipt.retryReason = null + }, + /^Retry reason differs from the attempt 1 evidence/ + ], + 'attempt 2 incomplete': [ + 300, + ([, second]) => { + second.workers[0].splice(2, 2) + }, + /^Missing\/duplicate\/unstarted cases/ + ], + 'attempt 2 with a failed case': [ + 300, + ([, second]) => { + second.workers[0][3].status = 'failed' + }, + /^Failed\/skipped case/ + ], + 'attempt 2 with a retried case': [ + 300, + ([, second]) => { + second.workers[0][2].attempt = 1 + }, + /^Retried case/ + ], + 'attempt 2 with Mocha retry attempts': [ + 300, + ([, second]) => { + second.workers[0][3].retries.attempts = 1 + }, + /^Retry attempts missing\/nonzero/ + ], + 'attempt 2 with a failed Maestro step': [ + 300, + ([, second]) => { + second.receipt.maestroOutcome = 'failure' + }, + /^Maestro outcome mismatch/ + ], + 'attempt 2 that starts before attempt 1 ends': [ + 300, + ([, second]) => { + for (const record of second.workers[0]) record.timestamp = '2026-10-16T11:01:00.000Z' + }, + /^Attempt 2 started before attempt 1 ended/ + ], + 'three attempts': [ + 300, + (attempts) => { + const third = structuredClone(attempts[1]) + third.receipt.attempt = 3 + attempts.push(third) + }, + /^More than one retry/ + ], + 'attempts out of order': [ + 300, + (attempts) => { + attempts.reverse() + }, + /^Attempt receipt out of order/ + ] +} +for (const [name, [load, mutate, message]] of Object.entries(retryRejections)) + test(`aggregate rejects a retry: ${name}`, () => { + const { plan, bundles, run } = retried(load) + mutate(bundles.swipeable.attempts) + assert.throws( + () => validateAggregate(plan, bundles, run), + (error) => message.test(error.message.split('\n')[0]) + ) + }) + +test('the gate reads each attempt directory of a shard artifact in order', async () => { + const { bundles } = retried() + const [first, second] = bundles.swipeable.attempts + const ndjsonText = (records) => `${records.map((record) => JSON.stringify(record)).join('\n')}\n` + const files = { + 'attempt-1/ios-shard-receipt.json': JSON.stringify(first.receipt), + 'attempt-1/ios-results/worker-41-1.ndjson': ndjsonText(first.workers[0]), + 'attempt-1/ios-evidence/host-health.json': JSON.stringify(first.health), + 'attempt-1/ios-evidence/simulator-log.txt': first.simulatorLog, + 'attempt-1/ios-evidence/host-load.txt': first.hostLoad, + 'attempt-1/ios-4723/wdio-appium.log': 'appium\n', + 'attempt-2/ios-shard-receipt.json': JSON.stringify(second.receipt), + 'attempt-2/ios-results/worker-42-2.ndjson': ndjsonText(second.workers[0]), + 'attempt-2/ios-evidence/simulator-log.txt': 'not read for the last attempt\n' + } + const read = (name) => { + assert.ok(name in files, `read of a missing file: ${name}`) + return files[name] + } + const bundle = await shardBundle(Object.keys(files), read) + assert.deepEqual(bundle.attempts, [first, { ...second, health: undefined }]) + assert.equal('simulatorLog' in bundle.attempts[1], false) + const attempt1 = Object.keys(files).filter((name) => name.startsWith('attempt-1/')) + await assert.rejects( + shardBundle([...Object.keys(files), 'ios-shard-receipt.json'], read), + /Shard evidence outside an attempt directory/ + ) + await assert.rejects( + shardBundle([...attempt1, 'attempt-3/ios-shard-receipt.json'], read), + /Missing attempt directory: attempt-2\// + ) + assert.deepEqual(await shardBundle([], read), { attempts: [] }) +}) diff --git a/e2e/helpers/touch-readiness.test.mjs b/e2e/helpers/touch-readiness.test.mjs index e0f90b6..8d36f51 100644 --- a/e2e/helpers/touch-readiness.test.mjs +++ b/e2e/helpers/touch-readiness.test.mjs @@ -1,5 +1,6 @@ import assert from 'node:assert/strict' import { test } from 'node:test' +import { isReadinessTimeout, READINESS_FAILURE, READINESS_TIMEOUTS } from './host-health.mjs' import readiness from './touch-readiness.ts' const { relaunchApp, waitForTouchReadiness } = readiness @@ -160,6 +161,32 @@ test('touch readiness gate without a device or transport', async (t) => { }) }) + await t.test('the host classifier counts only the two timeouts of the gate', async () => { + // Each gate failure, and whether a stalled host can explain it. + const failures = [ + [{ displayed: () => false }, true], + [{ lost: () => true }, true], + [{ lost: () => true, displayed: (state) => state.drags.length === 0 }, false], + [{ lost: (n) => n >= 2 }, false] + ] + const reasons = [] + for (const [options, timeout] of failures) { + await withFixture(options, async () => { + await assert.rejects(waitForTouchReadiness(fast), (error) => { + reasons.push(error.message.slice(error.message.indexOf(' ms: ') + 5)) + assert.ok(error.message.startsWith(`${READINESS_FAILURE} after `)) + assert.equal(isReadinessTimeout(error.message), timeout, error.message) + return true + }) + }) + } + assert.deepEqual(reasons.slice(0, 2), READINESS_TIMEOUTS) + assert.deepEqual(reasons.slice(2), [ + 'the app left the list screen before a drag moved the seekbar thumb', + 'the seekbar thumb did not return to rest' + ]) + }) + await t.test('a thumb outside its pan area is dragged through the pan area', async () => { // RTL: the thumb starts at the far track end and the rest offset pushes it past. const track = { x: 104, y: 300, width: 200, height: 48 } diff --git a/e2e/wdio.ios.conf.ts b/e2e/wdio.ios.conf.ts index 49c76af..80c4e59 100644 --- a/e2e/wdio.ios.conf.ts +++ b/e2e/wdio.ios.conf.ts @@ -6,7 +6,9 @@ import { type TouchReadiness, waitForTouchReadiness } from './helpers/touch-read const appiumPort = Number(process.env.APPIUM_PORT || 4723) -const resultsDirectory = path.resolve(__dirname, 'logs', 'ios-results') +// CI runs each attempt of a shard in its own directory, e2e/logs/attempt-N. Local runs use e2e/logs. +const logsDirectory = process.env.IOS_ATTEMPT_DIR || path.resolve(__dirname, 'logs') +const resultsDirectory = path.join(logsDirectory, 'ios-results') const resultsFile = path.join(resultsDirectory, `worker-${process.pid}-${Date.now()}.ndjson`) function failureDetails(error: unknown) { @@ -92,7 +94,7 @@ export const config: WebdriverIO.Config = { [ 'appium', { - logPath: path.resolve(__dirname, 'logs', `ios-${appiumPort}`), + logPath: path.join(logsDirectory, `ios-${appiumPort}`), args: { address: '127.0.0.1', port: appiumPort } } ] diff --git a/scripts/release-readiness.mjs b/scripts/release-readiness.mjs index 7f6198d..39ff76b 100644 --- a/scripts/release-readiness.mjs +++ b/scripts/release-readiness.mjs @@ -9,7 +9,7 @@ import { artifactName, buildManifest, discoverSources, - parseRecords, + shardBundle, validateAggregate } from '../e2e/helpers/ios-shards.mjs' @@ -123,12 +123,14 @@ export function validateReadiness(input) { assert.equal(manifest.sha, input.sourceSha) assert.equal(manifest.runId, String(run.id)) assert.equal(manifest.runAttempt, String(run.run_attempt)) - validateAggregate(manifest, bundles, { + // A shard passes with one attempt, or with one retry after host trouble in attempt 1. + // The aggregate classifies attempt 1 again from its raw evidence. + const ios = validateAggregate(manifest, bundles, { sha: input.sourceSha, runId: String(run.id), runAttempt: String(run.run_attempt), planResult: 'success', buildResult: 'success', shardsResult: 'success' }) for (const bundle of Object.values(bundles)) { - for (const record of bundle.workers[0]) { + for (const record of bundle.attempts.flatMap((attempt) => attempt.workers.flat())) { const time = Date.parse(record.timestamp) assert.ok(time >= Date.parse(run.run_started_at) && time <= Date.parse(run.updated_at), 'Worker evidence is outside this run') } @@ -136,7 +138,7 @@ export function validateReadiness(input) { validateAndroidLog(androidLog, manifest) const packageIntegrity = validatePackage(report, bytes, input.version, input.sourceSha, packedPackage) return { schemaVersion: 1, sourceSha: input.sourceSha, version: input.version, ...release, - packageIntegrity, ciRuns: [run.id] } + packageIntegrity, ciRuns: [run.id], iosRetries: ios.retried } } async function main(args) { @@ -215,12 +217,7 @@ async function main(args) { const bundles = {} for (const shard of manifest.shards) { const zip = await archive(artifactName(context, shard.id)) - bundles[shard.id] = { - receipt: zip.json('ios-shard-receipt.json'), - workers: zip.names.filter((name) => name.startsWith('ios-results/')).map( - (name) => parseRecords(zip.read(name).toString('utf8')) - ) - } + bundles[shard.id] = await shardBundle(zip.names, (name) => zip.read(name).toString('utf8')) } const androidJob = jobs.find((job) => job.name === 'e2e-android') const androidLog = await (await request('/actions/jobs/' + androidJob.id + '/logs')).text() diff --git a/scripts/release-readiness.test.mjs b/scripts/release-readiness.test.mjs index fa545f3..e0bf72b 100644 --- a/scripts/release-readiness.test.mjs +++ b/scripts/release-readiness.test.mjs @@ -5,6 +5,8 @@ import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSy import { tmpdir } from 'node:os' import path from 'node:path' import test from 'node:test' +import { readinessTimeoutAttempt } from '../e2e/helpers/fixtures/host-trouble.mjs' +import { describeHealth } from '../e2e/helpers/host-health.mjs' import { buildManifest, EXPECTED_COUNTS } from '../e2e/helpers/ios-shards.mjs' import { REQUIRED_JOBS, validateReadiness, validateSource } from './release-readiness.mjs' @@ -42,8 +44,9 @@ function fixture(version = '1.0.0-rc.0', runIds = [101]) { records.push({ ...common, ...entry, file, attempt: 0, event: 'test-result', status: 'passed', duration: 1, retries: { attempts: 0, limit: 0 } }) } records.push({ ...common, event: 'worker-end', specs: [file], result: 0 }) - return [shard.id, { receipt: { version: 1, ...context, id: shard.id, spec: shard.spec, - sourceHash: shard.sourceHash, wdioOutcome: 'success', maestroOutcome: shard.maestro ? 'success' : 'skipped' }, workers: [records] }] + return [shard.id, { attempts: [{ receipt: { version: 1, ...context, id: shard.id, spec: shard.spec, + sourceHash: shard.sourceHash, attempt: 1, retryReason: null, wdioOutcome: 'success', + maestroOutcome: shard.maestro ? 'success' : 'skipped' }, workers: [records] }] }] })) const androidLog = manifest.shards.map((shard, index) => shard.cases.map((entry) => '[app Android #0-' + index + '] \u2713 ' + entry.title).join('\n') + @@ -109,11 +112,13 @@ const mutations = { 'Maestro skipped': (v) => { v.runs[0].jobs.find((j) => j.name === 'iOS / swipeable').steps[1].conclusion = 'skipped' }, 'metadata-only evidence': (v) => { v.runs[0].bundles = {} }, 'stale inventory': (v) => { v.runs[0].manifest.shards[0].sourceHash = 'b'.repeat(64) }, - 'missing iOS worker': (v) => { v.runs[0].bundles.rtl.workers = [] }, - 'missing iOS case': (v) => { v.runs[0].bundles.swipeable.workers[0].splice(2, 2) }, - 'failed iOS case': (v) => { v.runs[0].bundles['two-sided'].workers[0][3].status = 'failed' }, - 'retried iOS case': (v) => { v.runs[0].bundles.rtl.workers[0][3].retries.attempts = 1 }, - 'replayed old records': (v) => { v.runs[0].bundles.rtl.workers[0][0].timestamp = '2026-10-15T11:00:00Z' }, + 'missing iOS worker': (v) => { v.runs[0].bundles.rtl.attempts[0].workers = [] }, + 'missing iOS case': (v) => { v.runs[0].bundles.swipeable.attempts[0].workers[0].splice(2, 2) }, + 'failed iOS case': (v) => { v.runs[0].bundles['two-sided'].attempts[0].workers[0][3].status = 'failed' }, + 'retried iOS case': (v) => { v.runs[0].bundles.rtl.attempts[0].workers[0][3].retries.attempts = 1 }, + 'replayed old records': (v) => { v.runs[0].bundles.rtl.attempts[0].workers[0][0].timestamp = '2026-10-15T11:00:00Z' }, + 'missing iOS attempt': (v) => { v.runs[0].bundles.rtl.attempts = [] }, + 'iOS retry reason without a retry': (v) => { v.runs[0].bundles.rtl.attempts[0].receipt.retryReason = 'host trouble' }, 'Android summary only': (v) => { v.runs[0].androidLog = 'Spec Files: 5 passed, 5 total\n' }, 'Android missing case': (v) => { v.runs[0].androidLog = v.runs[0].androidLog.replace('[app Android #0-0] \u2713 case 0', '') }, 'Android duplicate output': (v) => { v.runs[0].androidLog += v.runs[0].androidLog }, @@ -139,10 +144,70 @@ test('accepts exact case multisets despite nested-suite display order, timestamp test('accepts the worker-end quarantine field of current iOS evidence', () => { const value = fixture() for (const run of value.runs) { - for (const bundle of Object.values(run.bundles)) bundle.workers[0].at(-1).quarantine = null + for (const bundle of Object.values(run.bundles)) bundle.attempts[0].workers[0].at(-1).quarantine = null } assert.equal(validateReadiness(value).version, '1.0.0-rc.0') }) + +// Synthetic: attempt 1 of the swipeable shard ends at 10:32:07 in a relaunch timeout of the +// touch readiness gate, at the given load. Attempt 2 is the green fixture attempt at 11:00. +function retriedFixture(load = 300) { + const value = fixture() + const [evidence] = value.runs + const shard = evidence.manifest.shards.find((entry) => entry.id === 'swipeable') + const context = { sha, runId: String(evidence.run.id), runAttempt: '1' } + const first = readinessTimeoutAttempt(shard, context, { + start: Date.parse('2026-10-16T10:30:00Z'), load, pid: 99, file: '/checkout/e2e/' + shard.spec + }) + const second = evidence.bundles.swipeable.attempts[0] + Object.assign(second.receipt, { attempt: 2, retryReason: describeHealth(first.health) }) + evidence.bundles.swipeable.attempts = [first, second] + return value +} +test('accepts one retry of a shard after host trouble, and the report records why', () => { + const value = retriedFixture() + const reason = describeHealth(value.runs[0].bundles.swipeable.attempts[0].health) + assert.match(reason, /^host trouble: /) + assert.deepEqual(validateReadiness(value).iosRetries, { swipeable: reason }) + assert.deepEqual(validateReadiness(fixture()).iosRetries, {}) +}) +// Each entry builds a retried run, and the gate rejects it with this message. +const retryMutations = { + 'healthy attempt 1 at low load': [() => retriedFixture(20), /^A retry needs host trouble in host-health\.json/], + 'host trouble only in host-health.json and the receipt': [() => { + const value = retriedFixture(20) + const [first, second] = value.runs[0].bundles.swipeable.attempts + first.health = retriedFixture().runs[0].bundles.swipeable.attempts[0].health + second.receipt.retryReason = describeHealth(first.health) + return value + }, /^A retry needs host trouble: host healthy/], + 'attempt 2 incomplete': [() => { + const value = retriedFixture() + value.runs[0].bundles.swipeable.attempts[1].workers[0].splice(2, 2) + return value + }, /^Missing\/duplicate\/unstarted cases/], + 'attempt 2 with a retried case': [() => { + const value = retriedFixture() + value.runs[0].bundles.swipeable.attempts[1].workers[0][3].retries.attempts = 1 + return value + }, /^Retry attempts missing\/nonzero/], + 'three attempts': [() => { + const value = retriedFixture() + const { attempts } = value.runs[0].bundles.swipeable + attempts.push({ ...structuredClone(attempts[1]), receipt: { ...attempts[1].receipt, attempt: 3 } }) + return value + }, /^More than one retry/], + 'attempt 1 records outside the run': [() => { + const value = retriedFixture() + value.runs[0].bundles.swipeable.attempts[0].workers[0][0].timestamp = '2026-10-16T09:59:00Z' + return value + }, /^Worker evidence is outside this run/] +} +for (const [name, [build, message]] of Object.entries(retryMutations)) { + test('fails closed on an iOS retry: ' + name, () => { + assert.throws(() => validateReadiness(build()), (error) => message.test(error.message.split('\n')[0])) + }) +} for (const [name, mutate] of Object.entries(mutations)) { test('fails closed: ' + name, () => { const value = fixture()