flaky

#!/bin/bash
set -euo pipefail

# flaky — report-only flaky-test detector for Where's test suite.
#
# Flakiness rarely shows up in a single test pass, so this exercises the tests
# along two independent dimensions and cross-references the results:
#
#   Phase 1 (suite):      run the whole `Stuff-iOS-Tests` scheme N times and
#                         record per-test pass/fail across the runs. Any test
#                         that fails in at least one run is a "suspect" (it
#                         either raced with a sibling bundle or is genuinely
#                         non-deterministic).
#   Phase 2 (tight loop): re-run *only* the suspects, each on its own via
#                         `-only-testing` with `-test-iterations M`
#                         (a fresh process per iteration by default), to get a
#                         reliable in-isolation flake count.
#
# A test is reported as flaky only when it produced *both* passes and failures
# across everything we observed — a test that fails every time isn't flaky,
# it's broken, and a test that passes every time isn't interesting. The counts
# and a flake rate land in `FLAKY_TESTS.md` at the repo root (a separate weekly
# Cursor automation runs this script to keep that file current).
#
# Report only: like `./profile`, flaky/failing tests never make this script
# exit non-zero — only a genuine *build* failure does. It builds once
# (`build-for-testing`) and reuses that build for every run via
# `test-without-building`, and it runs `tuist generate --no-open` first so it
# never steals focus by opening Xcode.
#
# macOS only (Tuist + iOS Simulator), matching CI on the xcode-27 image.

SUITE_RUNS=10          # Phase 1: how many times to run the full suite
ITERATIONS=50          # Phase 2: how many times to tight-loop each suspect
DEVICE="iPhone 17"
OS="27.0"
SCHEME="Stuff-iOS-Tests"
RELAUNCH="YES"         # fresh process per tight-loop iteration (-test-repetition-relaunch-enabled)
UPDATE=true            # write FLAKY_TESTS.md (disable with --no-update)
TOP=1000000            # cap on how many flaky tests to list in the report/doc

usage() {
    cat <<'USAGE'
Usage: ./flaky [options]

Detects flaky tests by running the suite repeatedly, then tight-looping the
tests that ever failed. Writes the findings (with flake counts) to
FLAKY_TESTS.md. Report only — it never fails on flaky tests.

Options:
  --suite-runs N       Times to run the whole suite in phase 1 (default: 10)
  --iterations M       Times to tight-loop each suspect in phase 2 (default: 50)
  --device NAME        Simulator device name (default: "iPhone 17")
  --os VERSION         Simulator iOS version (default: "27.0")
  --scheme NAME        Scheme to test (default: "Stuff-iOS-Tests")
  --relaunch YES|NO    New process per tight-loop iteration (default: YES)
  --no-update          Print the report but don't write FLAKY_TESTS.md
  --top N              List at most N flaky tests (default: all)
  -h, --help           Show this help

Examples:
  ./flaky
  ./flaky --suite-runs 3 --iterations 20
  ./flaky --no-update --top 10
USAGE
}

while [ $# -gt 0 ]; do
    case "$1" in
        --suite-runs) shift; SUITE_RUNS="${1:?--suite-runs requires a value}" ;;
        --iterations) shift; ITERATIONS="${1:?--iterations requires a value}" ;;
        --device) shift; DEVICE="${1:?--device requires a value}" ;;
        --os) shift; OS="${1:?--os requires a value}" ;;
        --scheme) shift; SCHEME="${1:?--scheme requires a value}" ;;
        --relaunch) shift; RELAUNCH="${1:?--relaunch requires YES or NO}" ;;
        --no-update) UPDATE=false ;;
        --top) shift; TOP="${1:?--top requires a value}" ;;
        -h|--help) usage; exit 0 ;;
        *) echo "error: unknown option '$1' (see ./flaky --help)" >&2; exit 1 ;;
    esac
    shift
done

cd "$(dirname "$0")"

WORKSPACE="Stuff.xcworkspace"
# Boot the target up front and address it by UDID. This matters most here: a
# name-based `simctl`/destination can hit a same-named device on another
# runtime, and a device shared with another checkout can be installed to or
# erased mid-run — either way the `Application failed preflight checks (Busy)` /
# `Mach error -308` launch failures look exactly like the flakes this script
# hunts. `./simulator` hands back this checkout's own device (creating it the
# first time), which rules both out.
DESTINATION="platform=iOS Simulator,id=$(./simulator --device "$DEVICE" --os "$OS")"

# Under Xcode's DerivedData rather than `$TMPDIR`, because the warm rebuild
# below depends on this surviving between runs: macOS sweeps unread files from
# `/var/folders/…/T` after ~3 days, taking package manifests with them, so a
# "warm" re-run would go cold or fail outright. Not in the checkout either —
# the repo's tree-walking scripts would descend into the vendored package
# sources. Suffixed per checkout; nothing prunes it, so delete by hand.
WORKDIR="$HOME/Library/Developer/Xcode/DerivedData/where-flaky-$(basename "$PWD")"
DERIVED="$WORKDIR/DerivedData"
BUILD_LOG="$WORKDIR/build.log"
SUITE_DIR="$WORKDIR/suite"
TIGHT_DIR="$WORKDIR/tight"
SUSPECTS="$WORKDIR/suspects.txt"
SUITE_COUNTS="$WORKDIR/suite_counts.json"
# The suite/tight run outputs are per-invocation; clear them so a re-run never
# mixes in stale results. DerivedData is deliberately kept for a warm rebuild.
rm -rf "$SUITE_DIR" "$TIGHT_DIR" "$SUSPECTS" "$SUITE_COUNTS"
mkdir -p "$WORKDIR" "$SUITE_DIR" "$TIGHT_DIR"

rule() { printf '%s\n' "============================================================"; }

echo "==> Regenerating project (tuist generate --no-open)"
mise exec -- tuist generate --no-open >/dev/null

# Xcode does not expand the scheme's $(BUILT_PRODUCTS_DIR) environment value
# for command-line test runs. Resolve the concrete package-resource location,
# matching ./test and ./profile, so Bundle.module works in every test process.
TEST_RUN_ENV=()
BUILT_PRODUCTS_DIR="$(xcodebuild -showBuildSettings \
    -workspace "$WORKSPACE" \
    -scheme "$SCHEME" \
    -destination "$DESTINATION" \
    -derivedDataPath "$DERIVED" 2>/dev/null \
    | awk -F' = ' '/ BUILT_PRODUCTS_DIR = /{print $2; exit}')"
if [ -n "$BUILT_PRODUCTS_DIR" ]; then
    TEST_RUN_ENV+=("TEST_RUNNER_PACKAGE_RESOURCE_BUNDLE_PATH=$BUILT_PRODUCTS_DIR")
else
    echo "warning: could not resolve BUILT_PRODUCTS_DIR; package resource" \
        "bundle lookups will fall back to the linker's placement" >&2
fi

echo "==> Building for testing ($SCHEME) on $DEVICE / iOS $OS"
set +e
xcodebuild build-for-testing \
    -workspace "$WORKSPACE" \
    -scheme "$SCHEME" \
    -destination "$DESTINATION" \
    -derivedDataPath "$DERIVED" \
    >"$BUILD_LOG" 2>&1
build_status=$?
set -e
if [ "$build_status" -ne 0 ]; then
    echo "error: build failed (exit $build_status). Tail of $BUILD_LOG:" >&2
    tail -n 40 "$BUILD_LOG" >&2
    exit "$build_status"
fi

# ---------------------------------------------------------------------------
# Phase 1 — run the whole suite SUITE_RUNS times.
# ---------------------------------------------------------------------------
echo
echo "==> Phase 1: running the full suite $SUITE_RUNS time(s)"
for i in $(seq 1 "$SUITE_RUNS"); do
    rb="$SUITE_DIR/run_$i.xcresult"
    log="$SUITE_DIR/run_$i.log"
    rm -rf "$rb"
    echo "  - suite run $i/$SUITE_RUNS"
    # Test failures are the whole point here, so don't let a non-zero exit abort
    # the loop — we read the truth out of the .xcresult regardless.
    set +e
    env ${TEST_RUN_ENV[@]+"${TEST_RUN_ENV[@]}"} \
        xcodebuild test-without-building \
        -workspace "$WORKSPACE" \
        -scheme "$SCHEME" \
        -destination "$DESTINATION" \
        -derivedDataPath "$DERIVED" \
        -resultBundlePath "$rb" \
        -collect-test-diagnostics never \
        >"$log" 2>&1
    set -e
    xcrun xcresulttool get test-results tests --path "$rb" >"$SUITE_DIR/run_$i.json" 2>>"$log" \
        || echo "  warning: could not read results for suite run $i (see $log)" >&2
done

echo
echo "==> Analyzing suite runs"
# Build the suspect set (tests that failed in >=1 suite run) and per-test suite
# counts. Best-effort parse: warn and continue if the xcresult schema shifts.
python3 Tools/flaky_results.py analyze-suite \
    --suite-dir "$SUITE_DIR" --suspects "$SUSPECTS" --counts "$SUITE_COUNTS"

# `grep -c` prints 0 *and* exits non-zero on no match, which would trip the
# `|| ...` fallback into appending a second count; count non-blank lines with
# awk instead so an empty/absent suspects file cleanly yields 0.
SUSPECT_COUNT=$(awk 'NF' "$SUSPECTS" 2>/dev/null | wc -l | tr -d '[:space:]')
SUSPECT_COUNT=${SUSPECT_COUNT:-0}

# ---------------------------------------------------------------------------
# Phase 2 — tight-loop each suspect in isolation.
# ---------------------------------------------------------------------------
if [ "$SUSPECT_COUNT" -gt 0 ]; then
    echo
    echo "==> Phase 2: tight-looping $SUSPECT_COUNT suspect(s) $ITERATIONS time(s) each"
    i=0
    while IFS= read -r id; do
        [ -z "$id" ] && continue
        i=$((i + 1))
        rb="$TIGHT_DIR/tight_$i.xcresult"
        log="$TIGHT_DIR/tight_$i.log"
        rm -rf "$rb"
        echo "  - [$i/$SUSPECT_COUNT] $id"
        printf '%s\n' "$id" >"$TIGHT_DIR/tight_$i.id"
        set +e
        env ${TEST_RUN_ENV[@]+"${TEST_RUN_ENV[@]}"} \
            xcodebuild test-without-building \
            -workspace "$WORKSPACE" \
            -scheme "$SCHEME" \
            -destination "$DESTINATION" \
            -derivedDataPath "$DERIVED" \
            -only-testing:"$id" \
            -test-iterations "$ITERATIONS" \
            -test-repetition-relaunch-enabled "$RELAUNCH" \
            -resultBundlePath "$rb" \
            -collect-test-diagnostics never \
            >"$log" 2>&1
        set -e
        xcrun xcresulttool get test-results tests --path "$rb" \
            >"$TIGHT_DIR/tight_$i.tests.json" 2>>"$log" || true
        xcrun xcresulttool get test-results summary --path "$rb" \
            >"$TIGHT_DIR/tight_$i.summary.json" 2>>"$log" || true
    done <"$SUSPECTS"
else
    echo
    echo "==> Phase 2: skipped (no suspects from phase 1)"
fi

# ---------------------------------------------------------------------------
# Aggregate + report + write FLAKY_TESTS.md.
# ---------------------------------------------------------------------------
echo
rule
echo "FLAKY TEST REPORT"
rule
report_args=()
[ "$UPDATE" = true ] && report_args+=(--update)
python3 Tools/flaky_results.py report \
    --counts "$SUITE_COUNTS" --tight-dir "$TIGHT_DIR" \
    --suite-runs "$SUITE_RUNS" --iterations "$ITERATIONS" --top "$TOP" \
    --device "$DEVICE" --os "$OS" --relaunch "$RELAUNCH" \
    ${report_args[@]+"${report_args[@]}"}

echo
echo "Logs and result bundles: $WORKDIR"