Replace the GitHub Pages site with a daily cross-environment test dashboard

.github/workflows/dashboard.yaml

@@ -0,0 +1,83 @@
+# Runs the full validation suite against every environment daily, then builds
+# and deploys the dashboard website to GitHub Pages.
+name: Test dashboard
+
+on:
+  schedule:
+    - cron: '30 6 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: write   # the deploy action pushes to the gh-pages branch
+  issues: read      # the GitHub issue tests read this repo's issues
+
+concurrency: dashboard-deploy
+
+jobs:
+  test-and-deploy:
+    runs-on: ubuntu-latest
+    timeout-minutes: 350
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Run the test suite against every target
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          # The Google Sheet IDs are secrets: they must never be checked in or
+          # appear in anything this workflow publishes.
+          BABEL_VALIDATION_SHEET_ID: ${{ secrets.BABEL_VALIDATION_SHEET_ID }}
+          BABEL_VALIDATION_BLOCKLIST_SHEET_ID: ${{ secrets.BABEL_VALIDATION_BLOCKLIST_SHEET_ID }}
+        # A nonzero pytest exit (failing tests) is normal here: the report is
+        # the artifact. `timeout 45m` caps the damage from a hung or down
+        # environment, whose per-test timeouts would otherwise add up.
+        # ponytail: sequential loop; split into a matrix job with artifact
+        # merging if total runtime ever approaches the 6h job limit.
+        # The explicit `tests` path is required: without a path argument,
+        # pytest-xdist workers do not load tests/conftest.py early enough to
+        # know --target/--report-jsonl, and every worker dies at argparse.
+        run: |
+          mkdir -p raw
+          for t in prod test ci ci-es dev exp; do
+            timeout 45m uv run pytest tests --target "$t" -n 8 -m "not unit" \
+              --report-jsonl "raw/$t.jsonl" || echo "target $t exited $?"
+          done
+
+      - name: Fetch the previous run history
+        # The published file is the source of truth: the deploy action
+        # force-pushes gh-pages as a single commit, so the branch is not an
+        # append log. The Pages CDN caches for ~10 minutes, so a manual
+        # dispatch right after a deploy could drop one history line — fine at
+        # a daily cadence.
+        run: |
+          curl -fsSL -o old_history.jsonl \
+            https://translatorsri.github.io/babel-validation/data/history.jsonl \
+            || : > old_history.jsonl
+
+      - name: Generate the report
+        run: |
+          uv run python -m src.babel_validation.tools.generate_report \
+            --raw-dir raw --targets-ini tests/targets.ini \
+            --history-in old_history.jsonl --out-dir website/public/data
+
+      - name: Build the website
+        working-directory: website
+        run: |
+          npm ci
+          npm run build
+          touch dist/.nojekyll
+
+      - name: Deploy to GitHub Pages
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: website/dist
+
+      - name: Upload raw outcomes for debugging
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: raw-jsonl
+          path: raw/
+          retention-days: 14

.github/workflows/tests.yaml

@@ -35,3 +35,23 @@ jobs:
       # secret instead.
       - name: Run unit tests
         run: uv run pytest -m unit -v
+
+  website-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: website/package-lock.json
+
+      # The dashboard's URL round-tripping, filtering and pagination: logic the
+      # Python tests cannot see, and where a shared link quietly losing its page
+      # looks like nothing is wrong.
+      - name: Run the website unit tests
+        working-directory: website
+        run: |
+          npm ci
+          npm test

CLAUDE.md

@@ -13,14 +13,19 @@ Babel Validation is a test suite and web tools for validating outputs from [Babe
 Requires [uv](https://docs.astral.sh/uv/getting-started/installation/). Run from repo root:
 
 ```bash
+# With -n (xdist), always pass the tests path explicitly (`pytest tests -n 8 ...`):
+# without it, workers do not load tests/conftest.py early enough to know the
+# custom options, and every worker dies at argparse — or collects nothing.
 pytest --target dev                    # Run all tests against dev environment (default if no --target)
 pytest --target prod                   # Run against production
 pytest --target dev --target prod      # Run against multiple targets
 pytest --target all                    # Run against all targets in targets.ini
 pytest --category "Unit Tests"         # Filter by Google Sheet category
 pytest --category-exclude "Slow"       # Exclude a category
 pytest tests/nodenorm/test_nodenorm_from_gsheet.py  # Run a specific test file
-pytest tests/nodenorm/test_nodenorm_from_gsheet.py -k "row=42"  # Run a specific test row
+# Run a specific test row: -k rejects '=' in its expression, so use the full node ID
+# (the target name's position in the parametrize id varies; ask pytest with --collect-only -q)
+pytest "tests/nodenorm/test_nodenorm_from_gsheet.py::test_normalization[test_nodenorm_from_gsheet.test_row:row=42-dev]"
 ```
 
 ### Code Formatting
@@ -33,20 +38,24 @@ Note that the repository is *not* currently black-clean — `black --check tests
 ~30 files it would reformat. Running `black` across the tree would bury a real change in
 unrelated churn, so format only the files you touch, or match the surrounding style.
 
-### Vue Website (website-vue3-vite/)
+### Dashboard Website (website/)
 
 ```bash
-cd website-vue3-vite && npm install && npm run dev     # Dev server
-npm run build     # Production build
-npm run lint      # ESLint + auto-fix
-npm run test:unit # Vitest unit tests
+cd website && npm install && npm run dev   # Dev server at localhost:4321/babel-validation/
+npm run build                              # astro check + production build
+npm test                                   # vitest: the Vue components' URL/filter/pagination logic
+npm run fetch-data                         # download the published report.json/history.jsonl
 ```
 
-### Astro Documentation Site (website/)
+The dashboard fetches `data/report.json` and `data/history.jsonl`, which are gitignored.
+`npm run fetch-data` downloads the live site's copies into `website/public/data/` — the
+quickest way to get real data for frontend work. To make them from scratch instead, run
+`pytest --report-jsonl` plus `uv run python -m src.babel_validation.tools.generate_report`
+(see README).
 
-```bash
-cd website && npm install && npm run dev   # Dev server at localhost:4321
-```
+Beware: the root `.gitignore`'s Python-template `lib/` pattern matches *any* directory named
+`lib`, including under `website/src/` — a file there builds locally but never reaches CI.
+Check `git status` shows new frontend files as tracked.
 
 ## Architecture
 
@@ -65,7 +74,11 @@ The core of this project. Tests validate NodeNorm and NameRes services across mu
 
 **Target system:** `tests/targets.ini` defines endpoints for each environment (dev, prod, test, ci, exp, localhost). Tests use `target_info` fixture to get URLs. The `conftest.py` parametrizes tests across targets via `--target` CLI option; default is `dev`.
 
-**Google Sheet integration:** ~2000+ test cases are pulled from a [shared Google Sheet](https://docs.google.com/spreadsheets/d/11zebx8Qs1Tc3ShQR9nh4HRW8QSoo8k65w_xIaftN0no/). `src/babel_validation/sources/google_sheets/google_sheet_test_cases.py` fetches and parses these into `TestRow` dataclasses. Rows marked as not expected to pass are wrapped with `pytest.mark.xfail(strict=True)`. Tests are parametrized by row, with IDs like `gsheet:row=42`.
+**Google Sheet integration:** ~2000+ test cases are pulled from the shared Babel Validation
+Google Sheet. Its ID comes from the `BABEL_VALIDATION_SHEET_ID` environment variable (`.env`
+locally, a repository secret in Actions) and is deliberately not checked in.
+`src/babel_validation/sources/google_sheets/google_sheet_test_cases.py` fetches and parses
+the rows into `TestRow` dataclasses. Rows marked as not expected to pass are wrapped with `pytest.mark.xfail(strict=True)`. Tests are parametrized by row, with IDs like `gsheet:row=42`.
 
 **Category filtering:** Google Sheet rows have a Category column. The `test_category` fixture (from conftest.py) returns a callable that tests use to `pytest.skip()` rows not matching `--category`/`--category-exclude` filters.
 
@@ -74,10 +87,15 @@ The core of this project. Tests validate NodeNorm and NameRes services across mu
 - `tests/nameres/` — NameRes tests (label lookup, autocomplete, Biolink type filtering, blocklist, taxon_specific flag)
 - `tests/nodenorm/by_issue/` — Per-issue regression tests for NodeNorm (hand-written)
 
-### Web Applications
+### Dashboard Website
 
-- **`website-vue3-vite/`** — Active Vue 3 + Vite app that fetches test cases from the same Google Sheet and runs them against multiple endpoints in the browser
-- **`website/`** — Newer Astro-based site deployed to GitHub Pages with prefix comparator and autocomplete tools
+- **`website/`** — Astro + Vue site deployed to GitHub Pages
+  (https://translatorsri.github.io/babel-validation/). Renders `report.json` (per-target
+  `/status` cards plus a tests-by-environment matrix) and `history.jsonl` (one summary line
+  per run). Regenerated daily by `.github/workflows/dashboard.yaml`: pytest per target with
+  `--report-jsonl` (a `pytest_runtest_logreport` hook in `tests/conftest.py`), then
+  `src/babel_validation/tools/generate_report.py` aggregates the raw outcomes, fetches each
+  target's `/status`, and writes both data files into `website/public/data/`.
 - **`scala-validation/`** — Legacy, unmaintained
 
 ## Untrusted Input
@@ -125,6 +143,29 @@ allowlist *before* the call, or the value reaches the GitHub API as a URL path.
 truncated part of it. The same goes for missing credentials: the GitHub issue tests *skip* without
 a token, so a green run may have tested nothing.
 
+**The dashboard publishes untrusted text on a public website.**
+`src/babel_validation/tools/generate_report.py` is the choke point between the raw pytest
+outcomes and `report.json`: it repr-escapes and truncates all text, passes `/status`
+responses through a key whitelist, only emits issue ids and source URLs that match the
+`targets.ini` `Repositories` allowlist, and withholds blocklist test details entirely (that
+sheet may not be public — do not add `ids=` to the blocklist parametrize or link to it).
+The Vue components must render report values with `{{ }}` interpolation only — never
+`v-html` — and construct links from validated parts (allowlisted `org/repo#N`,
+`targets.ini` URLs), never verbatim from report text.
+
+**Never leak the Google Sheet ID or the GitHub token.** The report, the website, and any
+Git commit must not contain the test-case sheet's ID or a link to either sheet — casual
+observers of the public site must not be able to find them, and the ID is the capability
+that grants access (the sheets are shared as "anyone with the link", because the CSV fetch
+is unauthenticated). The IDs live only in the `BABEL_VALIDATION_SHEET_ID` and
+`BABEL_VALIDATION_BLOCKLIST_SHEET_ID` environment variables (`.env` locally — gitignored —
+and repository secrets in Actions), resolved through
+`src/babel_validation/sources/google_sheets/resolve_sheet_id()`. Refer to it as the
+"Babel Validation Google Sheet"; sheet *content* (row numbers, queried/expected CURIEs and
+labels, category, source — often a GitHub issue link) is fine to publish once it passes the
+generator's validation. The sheet is expected to be fully replaced by the GitHub issue
+system over the next few months, at which point it can be removed from this repo entirely.
+
 **Caches belong in `cache_dir()`** (`src/babel_validation/core/__init__.py`), a 0700 directory
 under the user's home — never a fixed name in the shared temp directory. On a CI runner or a
 shared machine anyone can pre-create such a file, and the issue cache decides what a later run

README.md

@@ -9,8 +9,11 @@ underlying data used for the Translator
 ## PyTest
 
 The best tests in this repository are Python tests stored in the [`./tests`](./tests/) folder.
-This includes both unit tests as well as "Google Sheet"-based tests, which uses
-a [shared Google Sheet](https://docs.google.com/spreadsheets/d/11zebx8Qs1Tc3ShQR9nh4HRW8QSoo8k65w_xIaftN0no/edit?gid=0#gid=0) containing facts that we can use to test a NodeNorm instance.
+This includes both unit tests as well as "Google Sheet"-based tests, which use the shared
+Babel Validation Google Sheet containing facts that we can use to test a NodeNorm instance.
+The sheet's ID is deliberately not checked in: set `BABEL_VALIDATION_SHEET_ID` in `.env`
+(ask a maintainer for the ID; in GitHub Actions it comes from a repository secret of the
+same name).
 
 To run these tests, you need to [install `uv`](https://docs.astral.sh/uv/getting-started/installation/).
 You can then use `uv` to run the tests. The file [`tests/targets.ini`](./tests/targets.ini) allows you to
@@ -126,22 +129,41 @@ $ curl -s -H "Authorization: Bearer $GITHUB_TOKEN" https://api.github.com/rate_l
 The Jupyter Notebook in `log-analysis/` contains some basic analysis of the
 logs from NodeNorm (and, someday, NameRes) instances.
 
-## The Babel Validator Vue Application
+## The dashboard website
 
-The easiest way to validate Babel results on NodeNorm is by running the
-Vue app.
+The Astro site in `website/` is deployed to https://translatorsri.github.io/babel-validation/.
+It shows the results of running this test suite against every environment in
+`tests/targets.ini`, alongside each environment's `/status` information (Babel version,
+database sizes, NameRes latency). Because test expectations are pinned to the environment
+where a new Babel version lands first, environments are not expected to all be green — the
+dashboard's purpose is to show *which* issues are visible in *which* environment.
+
+The `.github/workflows/dashboard.yaml` workflow regenerates and deploys it daily (or on
+manual dispatch): it runs pytest per target with `--report-jsonl`, turns the raw outcomes
+and `/status` responses into `report.json` and `history.jsonl` with
+`src.babel_validation.tools.generate_report`, and publishes the built site to the
+`gh-pages` branch.
+
+The Vue components' client-side logic (URL round-tripping, filtering, pagination) has vitest
+tests in `website/test/`, run by `npm test` and by the Tests workflow.
+
+To work on the site against the data the live dashboard is showing, download the published
+`report.json` and `history.jsonl` instead of generating them (both land in
+`website/public/data/`, which is gitignored):
 
 ```shell
-$ cd website-vue3-vite
-$ npm install
-$ npm run dev
+$ cd website && npm install && npm run fetch-data && npm run dev
 ```
 
-This will start a local web application and report the URL for accessing it. This website
-retrieves tests from [a Google Sheet document](https://docs.google.com/spreadsheets/d/11zebx8Qs1Tc3ShQR9nh4HRW8QSoo8k65w_xIaftN0no/edit?usp=sharing)
-and displays their results across multiple NodeNorm (and, someday, NameRes) endpoints.
+To regenerate it locally against a couple of environments:
 
-A new website is in development at `website/` and is currently deployed to https://translatorsri.github.io/babel-validation/.
+```shell
+$ uv run pytest tests/nodenorm/test_nodenorm_from_gsheet.py tests/nameres/test_nameres_from_gsheet.py \
+      --target dev --target prod -n 8 --report-jsonl raw/local.jsonl
+$ uv run python -m src.babel_validation.tools.generate_report --raw-dir raw \
+      --targets-ini tests/targets.ini --out-dir website/public/data
+$ cd website && npm install && npm run dev
+```
 
 ## The Babel Validator in Scala
 

src/babel_validation/sources/google_sheets/__init__.py

@@ -0,0 +1,31 @@
+# Google Sheet IDs are deliberately not checked in: for an unauthenticated CSV
+# export the ID is the capability that grants access, so each sheet's ID lives
+# in an environment variable (via .env locally, a repository secret in GitHub
+# Actions) and must never appear in the code, the Git history, or anything we
+# publish.
+import os
+import re
+
+import dotenv
+
+_SHEET_ID_RE = re.compile(r"[A-Za-z0-9_-]{20,}")
+
+
+def resolve_sheet_id(env_var, sheet_id=None):
+    """
+    Return a validated Google Sheet ID: the one passed in, or the value of the
+    named environment variable (loading .env first). A missing or implausible
+    value fails loudly — the ID goes into a URL path, and the format check also
+    catches quoting mistakes in .env.
+    """
+    if sheet_id is None:
+        dotenv.load_dotenv()
+        sheet_id = os.environ.get(env_var)
+        if not sheet_id:
+            raise RuntimeError(
+                f"No Google Sheet ID: set {env_var} (e.g. in .env). "
+                "Ask a maintainer for the ID."
+            )
+    if not _SHEET_ID_RE.fullmatch(sheet_id):
+        raise RuntimeError(f"{env_var} does not look like a Google Sheet ID.")
+    return sheet_id