Replace the website with a daily cross-environment test dashboard built around promotion drift

.github/workflows/dashboard.yaml

@@ -0,0 +1,164 @@
+# Runs the full validation suite against every environment daily, then builds
+# and deploys the dashboard website to GitHub Pages.
+name: Test dashboard
+
+on:
+  schedule:
+    - cron: '30 6 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write   # the deploy action pushes to the gh-pages branch
+  issues: read      # the GitHub issue tests read this repo's issues
+
+concurrency:
+  group: dashboard-deploy
+  # Only a manual dispatch supersedes what is already running: you press Run
+  # workflow because you want fresh output now, and the run it cancels would have
+  # produced the same two files, so nothing is lost by killing it. A scheduled run
+  # never cancels anything — a cron run must not be able to kill a slower one that
+  # is mid-flight, because that is the run that appends the day's history line —
+  # and neither does a push, which queues behind whatever is deploying rather than
+  # racing it.
+  cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' }}
+
+jobs:
+  test-and-deploy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 350
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Run the test suite against every target
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          # The Google Sheet IDs are secrets: they must never be checked in or
+          # appear in anything this workflow publishes.
+          BABEL_VALIDATION_SHEET_ID: ${{ secrets.BABEL_VALIDATION_SHEET_ID }}
+          BABEL_VALIDATION_BLOCKLIST_SHEET_ID: ${{ secrets.BABEL_VALIDATION_BLOCKLIST_SHEET_ID }}
+        # A nonzero pytest exit (failing tests) is normal here: the report is
+        # the artifact. `timeout 45m` caps the damage from a hung or down
+        # environment, whose per-test timeouts would otherwise add up.
+        # ponytail: sequential loop, and ~25 of the run's ~26 minutes. Splitting it
+        # into a matrix job with artifact merging is #122; the 6h job limit is not
+        # the reason to do it, the feedback loop is.
+        # The explicit `tests` path is required: without a path argument,
+        # pytest-xdist workers do not load tests/conftest.py early enough to
+        # know --target/--report-jsonl, and every worker dies at argparse.
+        run: |
+          mkdir -p raw
+          # Ask the report generator which targets exist rather than keeping a
+          # second list here. The two silently disagreed once already: adding
+          # [test-redis] to targets.ini put it in every table on the site — as a
+          # permanently unreachable environment, sorted after prod, and as an
+          # always-empty column in every ?sig= signature — because read_targets
+          # takes every section but localhost while this loop did not run it.
+          targets=$(uv run python -c "from src.babel_validation.tools.generate_report import read_targets; print(' '.join(read_targets('tests/targets.ini')[0]))")
+          echo "Running against: $targets"
+          broken=""
+          for t in $targets; do
+            # `|| status=$?` rather than a bare call: steps run under `bash -e`,
+            # so an unguarded nonzero exit would abort the loop at the first
+            # target with a failing test — which is most of them.
+            status=0
+            timeout 45m uv run pytest tests --target "$t" -n 8 -m "not unit" \
+              --report-jsonl "raw/$t.jsonl" || status=$?
+            case "$status" in
+              0|1)
+                # 0 = everything passed, 1 = tests failed. Failing tests are the
+                # artifact this workflow exists to publish, so neither is a
+                # problem with the run.
+                #
+                # But the command is `uv run pytest`, and uv exits 1 for its own
+                # failures too — a failed dependency sync, a resolution error —
+                # before pytest ever starts. That target then has no records, the
+                # report renders it as unreachable during the run, and the job
+                # goes green: exactly the silent partial failure this triage
+                # exists to catch. pytest_configure truncates raw/$t.jsonl before
+                # collection, so an empty file means pytest never got that far.
+                if [ ! -s "raw/$t.jsonl" ]; then
+                  echo "::error title=Target run broke::pytest for $t exited $status without writing any results"
+                  broken="$broken $t"
+                fi
+                ;;
+              124)
+                # `timeout` killed it: the environment hung. That is a finding
+                # about the deployment, not about us, and the report already
+                # renders the target as unreachable — so say it loudly and carry
+                # on rather than going red every day an environment is down.
+                echo "::warning title=Target timed out::$t exceeded 45m; it has no results in this report"
+                ;;
+              *)
+                # 2 interrupted (a collection error — a missing sheet ID lands
+                # here), 3 internal error, 4 usage error, 5 nothing collected.
+                # None of these are findings about a deployment: they mean the
+                # run did not do what its green tick suggests.
+                echo "::error title=Target run broke::pytest for $t exited $status; no usable results"
+                broken="$broken $t"
+                ;;
+            esac
+          done
+          # Recorded, not raised here: the report is still worth generating and
+          # publishing for the targets that did work. The last step turns this
+          # into a failed job, after the deploy.
+          echo "BROKEN_TARGETS=$broken" >> "$GITHUB_ENV"
+
+      - name: Fetch the previous run history
+        # The published file is the source of truth: the deploy action
+        # force-pushes gh-pages as a single commit, so the branch is not an
+        # append log. The Pages CDN caches for ~10 minutes, so a manual
+        # dispatch right after a deploy could drop one history line — fine at
+        # a daily cadence.
+        run: |
+          curl -fsSL -o old_history.jsonl \
+            https://translatorsri.github.io/babel-validation/data/history.jsonl \
+            || : > old_history.jsonl
+
+      - name: Generate the report
+        run: |
+          uv run python -m src.babel_validation.tools.generate_report \
+            --raw-dir raw --targets-ini tests/targets.ini \
+            --history-in old_history.jsonl --out-dir website/public/data
+
+      # Node 24 for npm 11 — see the note in tests.yaml. Without this the job
+      # gets the runner's default Node and fails at npm ci, four hours into the
+      # run, after every test has already been paid for.
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: npm
+          cache-dependency-path: website/package-lock.json
+
+      - name: Build the website
+        working-directory: website
+        run: |
+          npm ci
+          npm run build
+          touch dist/.nojekyll
+
+      - name: Deploy to GitHub Pages
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: website/dist
+
+      - name: Fail if any target's run broke
+        # Last, so the report for the targets that worked is already published.
+        # `all_targets_unreachable` in generate_report already stops a run where
+        # *everything* died; this catches the partial case, which otherwise
+        # renders as "environment unreachable" and reads as someone else's fault.
+        if: env.BROKEN_TARGETS != ''
+        run: |
+          echo "These targets produced no usable results:${BROKEN_TARGETS}" >&2
+          echo "See the annotations above for each one's pytest exit code." >&2
+          exit 1
+
+      - name: Upload raw outcomes for debugging
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: raw-jsonl
+          path: raw/
+          retention-days: 14

.github/workflows/tests.yaml

@@ -35,3 +35,27 @@ jobs:
       # secret instead.
       - name: Run unit tests
         run: uv run pytest -m unit -v
+
+  website-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # Node 24 for npm 11: the lockfile npm 11 writes omits `optional: true` on
+      # platform-specific packages, and npm 10 then refuses to install it
+      # (EBADPLATFORM on lightningcss-android-arm64). Keep this in step with the
+      # same block in dashboard.yaml.
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 24
+          cache: npm
+          cache-dependency-path: website/package-lock.json
+
+      # The dashboard's URL round-tripping, filtering and pagination: logic the
+      # Python tests cannot see, and where a shared link quietly losing its page
+      # looks like nothing is wrong.
+      - name: Run the website unit tests
+        working-directory: website
+        run: |
+          npm ci
+          npm test