ci(ios): retry a shard once after host trouble

.github/workflows/ci.yml

@@ -384,9 +384,13 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJSON(needs.ios-plan.outputs.matrix) }}
     runs-on: macos-26
-    timeout-minutes: 45
+    # Two attempts: the first, and one retry after host trouble (docs/releasing.md).
+    timeout-minutes: 70
     env:
       DEVELOPER_DIR: /Applications/Xcode_26.6.app/Contents/Developer
+      # The simulator log of one attempt: the app, the touch path and WebDriverAgent.
+      IOS_LOG_PREDICATE: >-
+        subsystem == "com.swipeabledemo.app" OR process == "SwipeableDemo" OR process == "backboardd" OR process == "SpringBoard" OR process == "WebDriverAgentRunner-Runner" OR process == "testmanagerd" OR eventMessage CONTAINS[c] "WebDriverAgent"
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
 
@@ -477,40 +481,91 @@ jobs:
             CODE_SIGNING_ALLOWED=NO \
             -quiet
 
+      # A shard gets at most one retry, in this step, and only after host trouble in attempt 1
+      # (docs/releasing.md). Each attempt keeps its own evidence in e2e/logs/attempt-N.
       - name: Run E2E tests
         id: wdio
         env:
-          E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e/logs/ios-evidence
           IOS_DEVICE_NAME: ${{ steps.sim.outputs.device_name }}
           IOS_UDID: ${{ steps.sim.outputs.udid }}
           IOS_APP_PATH: ${{ github.workspace }}/example/ios/build/Build/Products/Release-iphonesimulator/SwipeableDemo.app
           # Run the build-for-testing output through XCTest, not as a standalone app.
           IOS_WDA_DERIVED_DATA_PATH: ${{ github.workspace }}/wda-build
           IOS_GREP: ${{ inputs.ios-grep }}
           IOS_VIDEO: ${{ inputs.ios-video }}
+          IOS_SHARD: ${{ matrix.id }}
+          IOS_SPEC: ${{ matrix.spec }}
         run: |
-          # The simulator can stall early in this step. Keep its logs from this point on,
-          # and sample host load for the first two minutes, to explain a lost touch.
-          echo "IOS_TEST_START=$(date '+%Y-%m-%d %H:%M:%S')" >> "$GITHUB_ENV"
-          mkdir -p "$E2E_ARTIFACT_DIR"
-          top -l 24 -s 5 -o cpu -n 15 > "$E2E_ARTIFACT_DIR/host-load.txt" 2>&1 &
-          TOP_PID=$!
+          TOP_PID=
           VIDEO_PID=
-          if [ "$IOS_VIDEO" = true ]; then
-            mkdir -p "$RUNNER_TEMP/ios-video"
-            xcrun simctl io "$IOS_UDID" recordVideo --codec h264 --force "$RUNNER_TEMP/ios-video/simulator.mp4" &
-            VIDEO_PID=$!
-          fi
-          # SIGINT lets simctl finish the movie file.
-          trap 'kill "$TOP_PID" 2>/dev/null || true; [ -n "$VIDEO_PID" ] && kill -INT "$VIDEO_PID" 2>/dev/null && wait "$VIDEO_PID" || true' EXIT
-          cd e2e
+          stop_samplers() {
+            # Wait for top to exit, so that host-load.txt is complete before a reader opens it.
+            if [ -n "$TOP_PID" ]; then kill "$TOP_PID" 2>/dev/null && wait "$TOP_PID" || true; fi
+            # SIGINT lets simctl finish the movie file.
+            if [ -n "$VIDEO_PID" ]; then kill -INT "$VIDEO_PID" 2>/dev/null && wait "$VIDEO_PID" || true; fi
+            TOP_PID=
+            VIDEO_PID=
+          }
+          trap stop_samplers EXIT
+          GREP_ARGS=()
           if [ -n "$IOS_GREP" ]; then
             # Mocha reads grep as a regular expression. Match the text of the input only.
-            GREP_PATTERN=$(node helpers/test-filter.mjs "$IOS_GREP")
-            npm run test:ios -- --spec "${{ matrix.spec }}" --maxInstances 1 --mochaOpts.grep "$GREP_PATTERN"
-          else
-            npm run test:ios -- --spec "${{ matrix.spec }}" --maxInstances 1
+            GREP_PATTERN=$(node e2e/helpers/test-filter.mjs "$IOS_GREP")
+            GREP_ARGS=(--mochaOpts.grep "$GREP_PATTERN")
+          fi
+          # The simulator can stall in an attempt, and a relaunch can time out under load.
+          # Keep the simulator log from the attempt start on, and sample host load for the
+          # whole attempt, to explain a lost touch or a slow relaunch. Each evidence time is
+          # UTC, because the gate classifies the evidence again on another host. The host
+          # class reads only the load and CPU lines of top, so top prints no process list.
+          run_attempt() {
+            export IOS_ATTEMPT_DIR="$GITHUB_WORKSPACE/e2e/logs/attempt-$1"
+            export E2E_ARTIFACT_DIR="$IOS_ATTEMPT_DIR/ios-evidence"
+            IOS_TEST_START=$(TZ=UTC date '+%Y-%m-%d %H:%M:%S+0000')
+            printf 'IOS_ATTEMPT=%s\nIOS_TEST_START=%s\n' "$1" "$IOS_TEST_START" >> "$GITHUB_ENV"
+            mkdir -p "$E2E_ARTIFACT_DIR"
+            TZ=UTC top -l 0 -s 5 -n 0 > "$E2E_ARTIFACT_DIR/host-load.txt" 2>&1 &
+            TOP_PID=$!
+            if [ "$IOS_VIDEO" = true ]; then
+              mkdir -p "$RUNNER_TEMP/ios-video"
+              xcrun simctl io "$IOS_UDID" recordVideo --codec h264 --force "$RUNNER_TEMP/ios-video/attempt-$1.mp4" &
+              VIDEO_PID=$!
+            fi
+            local status=0
+            (cd e2e && npm run test:ios -- --spec "$IOS_SPEC" --maxInstances 1 "${GREP_ARGS[@]}") || status=$?
+            stop_samplers
+            return "$status"
+          }
+          if run_attempt 1; then exit 0; fi
+          # A filtered run is diagnosis only, and the gate rejects it. It never gets a retry.
+          if [ -n "$IOS_GREP" ]; then exit 1; fi
+          # The retry check needs the receipt and the simulator log of attempt 1.
+          # Maestro runs after this step, so it did not run for attempt 1.
+          ATTEMPT_1="$GITHUB_WORKSPACE/e2e/logs/attempt-1"
+          IOS_WDIO_OUTCOME=failure IOS_MAESTRO_OUTCOME=skipped \
+            node e2e/helpers/ios-shards.mjs receipt "$IOS_SHARD" ios-plan/manifest.json "$ATTEMPT_1"
+          xcrun simctl spawn "$IOS_UDID" log show --timezone UTC --start "$IOS_TEST_START" --style compact \
+            --predicate "$IOS_LOG_PREDICATE" > "$ATTEMPT_1/ios-evidence/simulator-log.txt" 2>&1 || true
+          if ! node e2e/helpers/host-health.mjs retry ios-plan/manifest.json "$IOS_SHARD" "$ATTEMPT_1"; then
+            exit 1
           fi
+          # A new boot ends the app, WebDriverAgent and the stalled SpringBoard of attempt 1.
+          # The new WDIO run then starts a new Appium server and session.
+          xcrun simctl shutdown "$IOS_UDID" || true
+          xcrun simctl bootstatus "$IOS_UDID" -b
+          run_attempt 2
+
+      # The WDIO step stops its samplers. If the runner ends that step without its EXIT trap,
+      # for example on a cancel, this step stops them before a later step reads the evidence.
+      - name: Stop host samplers
+        if: always()
+        run: |
+          pkill -f '^top -l 0 -s 5 -n 0$' || true
+          pkill -INT -f 'simctl io .* recordVideo' || true
+          for _ in $(seq 30); do
+            pgrep -f '^top -l 0 -s 5 -n 0$|simctl io .* recordVideo' > /dev/null || break
+            sleep 1
+          done
 
       - name: Upload simulator video
         if: always() && inputs.ios-video
@@ -544,29 +599,36 @@ jobs:
           export PATH="$PATH:$HOME/.maestro/bin"
           maestro test e2e/maestro/seekbar-regression.yaml
 
+      # These steps record the last attempt. The WDIO step records an attempt before its retry.
       - name: Record shard step outcomes
         if: always()
         env:
           IOS_WDIO_OUTCOME: ${{ steps.wdio.outcome }}
           IOS_MAESTRO_OUTCOME: ${{ steps.maestro.outcome }}
-        run: >-
-          node e2e/helpers/ios-shards.mjs receipt "${{ matrix.id }}"
-          ios-plan/manifest.json e2e/logs/ios-shard-receipt.json
+        run: |
+          ATTEMPT_DIR="e2e/logs/attempt-${IOS_ATTEMPT:-1}"
+          if [ ! -e "$ATTEMPT_DIR/ios-shard-receipt.json" ]; then
+            node e2e/helpers/ios-shards.mjs receipt "${{ matrix.id }}" ios-plan/manifest.json "$ATTEMPT_DIR"
+          fi
 
       - name: Capture simulator logs
         if: always()
         run: |
-          if [ -n "${IOS_TEST_START:-}" ]; then WINDOW=(--start "$IOS_TEST_START"); else WINDOW=(--last 10m); fi
-          mkdir -p e2e/logs/ios-evidence
-          xcrun simctl spawn "${{ steps.sim.outputs.udid }}" log show "${WINDOW[@]}" --style compact \
-            --predicate 'subsystem == "com.swipeabledemo.app" OR process == "SwipeableDemo" OR process == "backboardd" OR process == "SpringBoard" OR process == "WebDriverAgentRunner-Runner" OR process == "testmanagerd" OR eventMessage CONTAINS[c] "WebDriverAgent"' \
-            > e2e/logs/ios-evidence/simulator-log.txt 2>&1 || true
+          EVIDENCE="e2e/logs/attempt-${IOS_ATTEMPT:-1}/ios-evidence"
+          mkdir -p "$EVIDENCE"
+          if [ ! -e "$EVIDENCE/simulator-log.txt" ]; then
+            if [ -n "${IOS_TEST_START:-}" ]; then WINDOW=(--start "$IOS_TEST_START"); else WINDOW=(--last 10m); fi
+            xcrun simctl spawn "${{ steps.sim.outputs.udid }}" log show --timezone UTC "${WINDOW[@]}" \
+              --style compact --predicate "$IOS_LOG_PREDICATE" > "$EVIDENCE/simulator-log.txt" 2>&1 || true
+          fi
 
-      # Classifies the host for triage. It never changes the shard or gate result.
+      # Classifies the host for triage. Only the WDIO step uses a host class: for its one retry.
       - name: Classify host health
         if: always()
         continue-on-error: true
-        run: node e2e/helpers/host-health.mjs report ios-plan/manifest.json "${{ matrix.id }}" e2e/logs
+        run: >-
+          node e2e/helpers/host-health.mjs report ios-plan/manifest.json "${{ matrix.id }}"
+          "e2e/logs/attempt-${IOS_ATTEMPT:-1}"
 
       - name: Upload Appium logs and required shard evidence
         if: always()

docs/releasing.md

@@ -22,8 +22,11 @@ checks cannot replace a successful run of the prepared source.
    publishing workflow never bumps versions or commits source changes.
 4. Run full CI once as a fresh `workflow_dispatch` run on this exact `main` SHA
    with both mobile suites enabled. The run must completely succeed on attempt 1.
-   A failed-jobs rerun is not release evidence. Do not change assertions, retries,
-   the 120-second case deadlines, or the 45-minute mobile job deadlines.
+   A failed-jobs rerun is not release evidence. Do not change assertions, Mocha
+   retries, the 120-second case deadlines, or the mobile job deadlines (45 minutes,
+   and 70 minutes for an iOS shard, which can run two attempts).
+   The only retry is the one [shard retry after host trouble](#inconclusive-host-runs)
+   inside an iOS shard job.
    If the [physical iPhone check](#physical-iphone-check) applies, do it on the
    same SHA.
 
@@ -68,42 +71,87 @@ The Prepare step 4 CI run is the only check that the tools still work with an ov
 
 ## Inconclusive host runs
 
-Each iOS shard writes `ios-evidence/host-health.json`. It compares the simulator log
-with the NDJSON results in `ios-results/`. Each failure has a window: a failed test
-from its start to its result, a failed hook, or a failed warm-up. A test without a
-result fails, and its window ends at the next record. A gesture counts in the window
-that contains its start. The `e2e-ios` job summary
-shows the gate checks and one host class for each shard. The gate checks column
-uses the same checks as the gate for that shard: the step outcomes, the warm-up
-and the full case inventory. It lists each condition that fails.
-
-- `host trouble`: the gate rejects the shard only for failed cases, and the window
-  of each failed case contains at least one gesture that went to latent system
-  gesture destinations of the app scene. The simulator sent these gestures to
-  SpringBoard first, and the app did not get them. A Maestro step that a failed WDIO
-  step skipped is part of the failed cases. The worker result is part of them only when
-  it is one Mocha failure for each failed case, or when the worker ends directly after a
-  failed last case and its worker-end record shows a quarantine that started in that
-  case. A failed Maestro step or warm-up, a skipped, pending or unfinished case, a
-  failed hook, a failed worker or a missing case is a real failure, and the shard is
-  never `host trouble`.
-- `host mixed`: some failure windows contain latent-routed gestures, and other
-  failure windows do not, or the gate also rejects the shard for another reason.
-  The failures without them, and the other rejections, are real failures.
-- `host healthy`: no failure window contains a latent-routed gesture, or the shard
-  has no failure. A latent-routed gesture during a passed warm-up or test does not count.
-- `host unknown`: the shard has no usable simulator log, or it has latent-routed
-  gestures and no test results.
-
-The summary shows the latent-routed gestures in failure windows and in total.
-The host class never changes the `e2e-ios` result. A run with a failed case fails.
-
-1. If each failed shard of a failed run shows `host trouble`, the run is
-   "inconclusive (host)". It is not release evidence. One failure without a
-   latent-routed gesture in its window makes the run a real failure.
-2. Never retry individual tests or failed jobs.
+Each iOS shard job runs its spec in an attempt directory: `attempt-1/`, and
+`attempt-2/` after a retry. Each attempt has its own receipt, NDJSON results in
+`ios-results/`, Appium log, simulator log, host load and `ios-evidence/host-health.json`.
+The host load has a `top` sample every 5 seconds for the whole attempt. The simulator
+log and the host load give their times in UTC, so the gate reads them the same way on
+any host.
+
+`host-health.json` compares the simulator log and the host load with the NDJSON
+results. Each failure has a window: a failed test from its start to its result, a
+failed hook, or a failed warm-up. A test without a result fails, and its window ends
+at the next record. A gesture counts in the window that contains its start. A load
+sample counts in the window that contains its time. The `e2e-ios` job summary shows
+the gate checks and one host class for each attempt. The gate checks column uses the
+same checks as the gate for that attempt: the step outcomes, the warm-up and the full
+case inventory. It lists each condition that fails.
+
+- `host trouble`: the gate rejects the attempt only for failed cases and touch
+  readiness timeouts, and host trouble explains each of them.
+  - A failed case needs at least one gesture in its window that went to latent
+    system gesture destinations of the app scene. The simulator sent these
+    gestures to SpringBoard first, and the app did not get them.
+  - A touch readiness timeout is a hook that fails in the touch readiness gate
+    because the list screen seekbar did not appear, or because the seekbar thumb
+    never moved. An example is the relaunch in `afterEach`. If the app left the list
+    screen, or the thumb did not return to rest, the hook failure is a real failure.
+    The window of a timeout needs at least one load sample, and each sample in it
+    needs a 1-minute load average of 100 or more. It needs no simulator log, and a
+    latent-routed gesture in its window does not count.
+    The runner has 3 vCPUs. At load 100, a relaunch that needs 1 CPU second on an
+    idle host needs about 33 seconds. That is more than the 30-second phase of the
+    readiness gate. A host that keeps up has a load near 3. In the stored shard
+    evidence of three runs, the overload never fell below load 136.
+
+  These effects of the failures are part of them:
+  - A Maestro step that a failed WDIO step skipped.
+  - A worker result of one Mocha failure for each failure.
+  - A larger worker result, when the worker ends directly after the last failure
+    and its worker-end record shows a quarantine that started in that failure.
+  - The cases that never started, when a touch readiness timeout ends the worker
+    and the cases that started are the first cases of the plan. Mocha starts no
+    later case of the suite after a failed hook.
+
+  A failed Maestro step or warm-up, a skipped, pending or unfinished case, another
+  failed hook, a failed worker or any other missing case is a real failure. Then
+  the attempt is never `host trouble`.
+- `host mixed`: host trouble explains some failure windows and not others, or the
+  gate also rejects the attempt for another reason. The other failures and
+  rejections are real failures.
+- `host healthy`: host trouble explains no failure window, or the attempt has no
+  failure. A latent-routed gesture during a passed warm-up or test does not count.
+- `host unknown`: the attempt has no usable simulator log and a failure other than
+  a touch readiness timeout, or it has latent-routed gestures and no test results.
+
+The summary shows the latent-routed gestures in the windows of failed cases and in
+total. Host trouble never makes a case pass. It only permits one retry of a shard:
+
+1. If attempt 1 fails, the `Run E2E tests` step classifies it. The step retries the
+   shard once, in the same job, only when all of these are true:
+   - The run has no `ios-grep` filter. A filtered run never gets a retry.
+   - The WDIO run of attempt 1 failed, and Maestro did not run.
+   - The classifier finds `host trouble` in the raw evidence of attempt 1.
+   - The `host-health.json` of attempt 1 says `host trouble`.
+2. Before attempt 2, the step boots the simulator again. Attempt 2 starts a new
+   Appium server, WebDriverAgent session and app.
+3. Attempt 2 must pass every case of the shard, with no Mocha retry. Maestro runs
+   once, after the WDIO run that passed. A Maestro failure never gets a retry.
+4. The receipt of attempt 2 records `attempt: 2` and its `retryReason`: the host
+   class of attempt 1. The shard artifact keeps both attempts.
+5. A shard never gets a second retry. If attempt 2 fails, the shard fails.
+
+The iOS shard job deadline is 70 minutes, so that two attempts fit. A job that
+reaches its deadline fails.
+
+1. If the last attempt of each failed shard shows `host trouble`, the run is
+   "inconclusive (host)". It is not release evidence. One failure that host trouble
+   does not explain makes the run a real failure.
+2. Never rerun failed jobs, and never retry individual tests. The shard retry above
+   is the only retry.
 3. Dispatch a fresh full run only after an inconclusive (host) run.
-4. Release evidence is a fully green full run.
+4. Release evidence is a fully green full run. A shard that passes in its retry is
+   green.
 
 ## Evidence interface
 
@@ -130,10 +178,18 @@ this artifact in their clean consumer builds; publishing uses that same tarball,
 not a new local build or pack.
 
 iOS uses the existing `ios-plan-<sha>-<run>-1` and
-`ios-shard-<sha>-<run>-1-<shard>` artifacts. The gate rediscovers the maintained
+`ios-shard-<sha>-<run>-1-<shard>` artifacts. Each shard artifact has `attempt-1/`,
+and `attempt-2/` after a retry. The gate rediscovers the maintained
 inventory from the prepared checkout, compares source hashes and case identities,
-then validates raw NDJSON, worker completion, receipts, zero retries/skips,
-all 62 cases and Maestro. Records must belong to the run's time interval.
+then validates raw NDJSON, worker completion, receipts, zero Mocha retries and
+skips, all 62 cases and Maestro. A shard passes with one green attempt, or with a
+green attempt 2 after a [retry](#inconclusive-host-runs). For a retry, the gate
+classifies attempt 1 again from its raw NDJSON, host load and simulator log. Both
+that verdict and the stored `host-health.json` must say `host trouble`, and the
+receipt of attempt 2 must give the same retry reason. A third attempt, a retry
+after an attempt 1 that host trouble does not explain, and an incomplete or retried
+attempt 2 fail closed. The report lists each retried shard and its reason in
+`iosRetries`. Records of every attempt must belong to the run's time interval.
 Android uses the actual `e2e-android` job log: five completed spec workers, the
 complete WDIO summary and every passing case title (including duplicate counts) are required.
 A success badge, hand-entered count or summary without case output cannot pass.