From 13b555b496a492dd47166ed8a4c36c696e71cc86 Mon Sep 17 00:00:00 2001 From: Steven Roberts Date: Sat, 8 Aug 2026 15:24:48 -0700 Subject: [PATCH] feat: add server status lights to Computing Hardware page Adds an at-a-glance status table for raven, gannet, and klone at the top of the Computing Hardware page. The three hosts are not reachable from the same place, which shapes the design: raven not in public DNS; visible only from inside the UW network gannet public HTTPS klone public, but SSH only -- no web port to probe, and browsers refuse to connect to port 22 So no purely client-side check can cover all three. Instead two probers write status/.json to an orphan `server-status` branch: internal cron inside the UW network; the only source that sees raven external scheduled GitHub Action; covers the two public hosts so a dead internal prober does not take out all three lights The page fetches both from raw.githubusercontent.com (which sends access-control-allow-origin: *) and takes the most recent fresh reading per host. Readings older than 30 minutes render as "unknown" rather than as a stale green light, so a dead prober is visible instead of silently reporting everything up. The status branch is rewritten as a single root commit each run; at one check every 10 minutes an append-only branch would add ~50k commits a year to a repo everyone clones. Concurrent probers are handled with --force-with-lease, so a race is rejected and retried rather than clobbering the other prober's file. The in-network cron still needs to be set up on gannet before raven reports; see scripts/README.md. Co-Authored-By: Claude Opus 5 --- .github/workflows/server-status.yml | 39 ++++++++ docs/Computing-Hardware.md | 30 ++++++ docs/javascripts/server-status.js | 141 ++++++++++++++++++++++++++++ docs/stylesheets/server-status.css | 49 ++++++++++ mkdocs.yml | 6 ++ scripts/README.md | 106 +++++++++++++++++++++ scripts/check_servers.py | 135 ++++++++++++++++++++++++++ scripts/publish_status.sh | 111 ++++++++++++++++++++++ 8 files changed, 617 insertions(+) create mode 100644 .github/workflows/server-status.yml create mode 100644 docs/javascripts/server-status.js create mode 100644 docs/stylesheets/server-status.css create mode 100644 scripts/README.md create mode 100755 scripts/check_servers.py create mode 100755 scripts/publish_status.sh diff --git a/.github/workflows/server-status.yml b/.github/workflows/server-status.yml new file mode 100644 index 00000000..c74b2e4f --- /dev/null +++ b/.github/workflows/server-status.yml @@ -0,0 +1,39 @@ +# Outside-the-network half of the server status lights on the Computing +# Hardware page. +# +# This runner can reach gannet (public HTTPS) and klone (public SSH), but NOT +# raven, which is not in public DNS. Raven is covered by the cron job inside +# the UW network -- see scripts/README.md. Running both means a dead internal +# prober does not take all three lights out at once. +# +# GitHub's scheduled workflows are best-effort: they are delayed under load and +# are disabled automatically after 60 days without repo activity. That is why +# the in-network cron is the primary prober and this is the backstop. +name: server-status + +on: + schedule: + - cron: "*/15 * * * *" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: server-status + cancel-in-progress: false + +jobs: + probe: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Probe public hosts and publish status + run: | + ./scripts/publish_status.sh \ + --profile external \ + --repo "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" \ + --workdir "${RUNNER_TEMP}/server-status" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/docs/Computing-Hardware.md b/docs/Computing-Hardware.md index af4fe55c..de500923 100644 --- a/docs/Computing-Hardware.md +++ b/docs/Computing-Hardware.md @@ -6,6 +6,36 @@ Below is a list of computing resources we have available, as well as some links --- +## Server Status + + + + + + + + + + + + + + + + + + + + + + +
ServerStatusLast checked
raven.fish.washington.educhecking…—
gannet.fish.washington.educhecking…—
klone.hyak.uw.educhecking…—
+ +Checks run every 10–15 minutes. Hover a status for details. A reading older than 30 minutes is shown as `unknown` rather than as a stale green light. + +!!! note + A green light only means the machine answered on the network. It does not mean Slurm is healthy, that there is free disk space, or that your jobs are running. For Hyak-wide outages and maintenance, check [UW-IT Hyak status](https://status.uw.edu/). + ## Accounts You will need accounts with the following services in order to minimally function in the Roberts Lab: diff --git a/docs/javascripts/server-status.js b/docs/javascripts/server-status.js new file mode 100644 index 00000000..37f74527 --- /dev/null +++ b/docs/javascripts/server-status.js @@ -0,0 +1,141 @@ +/* + * Server status lights for the Computing Hardware page. + * + * Reads JSON written by the probers in scripts/ (see scripts/README.md) and + * fills in the status table. Two sources are merged: + * + * internal.json - cron on a machine inside the UW network. Only source that + * can see raven, which is not in public DNS. + * external.json - scheduled GitHub Action. Sees gannet and klone from the + * public internet. Acts as a backstop if the internal + * prober host is itself down. + * + * For each host the most recent fresh reading wins. Anything older than + * STALE_MS is reported as unknown rather than shown as a stale green light. + */ +(function () { + "use strict"; + + var STALE_MS = 30 * 60 * 1000; // cron runs every ~10 min; 30 min = missed 3 + var SOURCES = ["internal.json", "external.json"]; + + function fetchSource(base, name) { + // Cache-bust: raw.githubusercontent.com serves with max-age=300. + var url = base + "/" + name + "?t=" + Date.now(); + return fetch(url, { cache: "no-store" }) + .then(function (res) { + return res.ok ? res.json() : null; + }) + .catch(function () { + return null; // source may not exist yet, or network is down + }); + } + + // Flatten both documents into { host: bestReading }. + function mergeReadings(docs) { + var best = {}; + docs.forEach(function (doc) { + if (!doc || !doc.hosts) return; + var checkedAt = Date.parse(doc.checked); + if (isNaN(checkedAt)) return; + Object.keys(doc.hosts).forEach(function (host) { + var current = best[host]; + if (current && current.checkedAt >= checkedAt) return; + best[host] = { + up: doc.hosts[host].up === true, + detail: doc.hosts[host].detail || "", + latencyMs: doc.hosts[host].latency_ms, + source: doc.source || "unknown", + checkedAt: checkedAt + }; + }); + }); + return best; + } + + function relativeTime(then, now) { + var mins = Math.round((now - then) / 60000); + if (mins < 1) return "just now"; + if (mins === 1) return "1 min ago"; + if (mins < 60) return mins + " min ago"; + var hrs = Math.round(mins / 60); + if (hrs === 1) return "1 hr ago"; + if (hrs < 24) return hrs + " hr ago"; + var days = Math.round(hrs / 24); + return days === 1 ? "1 day ago" : days + " days ago"; + } + + function paintRow(row, reading, now) { + var stateCell = row.querySelector(".ss-state"); + var timeCell = row.querySelector(".ss-time"); + if (!stateCell || !timeCell) return; + + var cls, label, title; + + if (!reading) { + cls = "ss-unknown"; + label = "unknown"; + title = "No status data available for this host yet."; + timeCell.textContent = "—"; + } else if (now - reading.checkedAt > STALE_MS) { + cls = "ss-unknown"; + label = "unknown"; + title = + "Last reading is stale (" + + relativeTime(reading.checkedAt, now) + + ") — the prober itself may be down."; + timeCell.textContent = relativeTime(reading.checkedAt, now); + } else { + cls = reading.up ? "ss-up" : "ss-down"; + label = reading.up ? "up" : "not responding"; + title = + reading.detail + + (reading.latencyMs != null ? " (" + reading.latencyMs + " ms)" : "") + + " — checked by " + + reading.source + + " prober"; + timeCell.textContent = relativeTime(reading.checkedAt, now); + } + + stateCell.innerHTML = ""; + var dot = document.createElement("span"); + dot.className = "ss-dot " + cls; + var text = document.createElement("span"); + text.className = "ss-label"; + text.textContent = label; // never color alone: the word carries the meaning + stateCell.appendChild(dot); + stateCell.appendChild(text); + stateCell.title = title; + } + + function render() { + var table = document.querySelector("[data-status-base]"); + if (!table) return; + var base = table.getAttribute("data-status-base"); + + Promise.all( + SOURCES.map(function (name) { + return fetchSource(base, name); + }) + ).then(function (docs) { + var readings = mergeReadings(docs); + var now = Date.now(); + var rows = table.querySelectorAll("tr[data-host]"); + Array.prototype.forEach.call(rows, function (row) { + paintRow(row, readings[row.getAttribute("data-host")], now); + }); + }); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", render); + } else { + render(); + } + + // mkdocs-material swaps page content without a reload when instant + // navigation is enabled, so re-render on those transitions too. + if (window.document$ && typeof window.document$.subscribe === "function") { + window.document$.subscribe(render); + } +})(); diff --git a/docs/stylesheets/server-status.css b/docs/stylesheets/server-status.css new file mode 100644 index 00000000..75368a9f --- /dev/null +++ b/docs/stylesheets/server-status.css @@ -0,0 +1,49 @@ +/* Status lights for the Computing Hardware page. */ + +.ss-dot { + display: inline-block; + width: 0.7em; + height: 0.7em; + margin-right: 0.45em; + border-radius: 50%; + vertical-align: baseline; + background: var(--ss-color, #9aa0a6); + box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.12) inset; +} + +.ss-up { + --ss-color: #2e9e4f; +} + +.ss-down { + --ss-color: #d64545; +} + +.ss-unknown { + --ss-color: #9aa0a6; +} + +[data-md-color-scheme="slate"] .ss-up { + --ss-color: #4ad07d; +} + +[data-md-color-scheme="slate"] .ss-down { + --ss-color: #ff6b6b; +} + +[data-md-color-scheme="slate"] .ss-unknown { + --ss-color: #8a8f98; +} + +.ss-label { + font-variant-numeric: tabular-nums; +} + +.ss-state { + white-space: nowrap; +} + +.ss-time { + white-space: nowrap; + opacity: 0.75; +} diff --git a/mkdocs.yml b/mkdocs.yml index 2451df63..e262ffef 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,6 +27,12 @@ theme: toggle: icon: material/toggle-switch name: Switch to light mode +extra_css: + - stylesheets/server-status.css + +extra_javascript: + - javascripts/server-status.js + markdown_extensions: - admonition - pymdownx.details diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..d99cbe73 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,106 @@ +# Server status lights + +These scripts drive the status table at the top of +[Computing Hardware](../docs/Computing-Hardware.md). + +## Why there are two probers + +The three servers are not reachable from the same place: + +| Host | Public DNS | How it is checked | +|---|---|---| +| `raven.fish.washington.edu` | no | TCP 8787 (RStudio Server), falling back to 22 — **UW network only** | +| `gannet.fish.washington.edu` | yes | HTTPS `GET /` | +| `klone.hyak.uw.edu` | yes | TCP 22 — there is no web port to probe | + +Raven is not in public DNS, so nothing outside the UW network can see it. Klone +has no HTTP port at all, and browsers refuse to connect to port 22, so no +purely client-side check can work either. + +So there are two probers writing to the same place: + +- **`internal`** — cron on a machine inside the UW network. The only source + that can see raven. This is the primary. +- **`external`** — the `server-status` GitHub Action. Sees gannet and klone + from the public internet, so a dead internal prober does not take out all + three lights at once. + +Each writes `status/.json` to the orphan `server-status` branch. The +page fetches both from `raw.githubusercontent.com` (which sends +`access-control-allow-origin: *`, so there is no CORS problem) and takes the +most recent fresh reading per host. + +The branch is rewritten as a single root commit on every run. At one check +every 10 minutes an append-only branch would add roughly 50,000 commits a year +to a repo that everyone clones. + +## Files + +- `check_servers.py` — runs the probes, prints or writes the JSON. Stdlib only. +- `publish_status.sh` — runs the checker and pushes the result to the + `server-status` branch. Used by both the cron job and the Action. + +Check without publishing anything: + +```bash +./scripts/check_servers.py --profile internal +``` + +Off the UW network, raven will report `DNS lookup failed` — that is expected, +and is exactly why the internal prober has to run inside. + +## Setting up the in-network cron + +Run this on a machine inside the UW network that is up continuously. Gannet is +the natural choice. + +**1. Give the machine push access.** Generate a deploy key on that host: + +```bash +ssh-keygen -t ed25519 -f ~/.ssh/robertslab_status -C "roberts-lab status bot" -N "" +``` + +Add the public key (`~/.ssh/robertslab_status.pub`) at + as a deploy key **with +write access checked**. A deploy key is scoped to this one repo, so it cannot +be used to touch anything else in the org. + +Then point git at it in `~/.ssh/config`: + +``` +Host github-robertslab + HostName github.com + User git + IdentityFile ~/.ssh/robertslab_status + IdentitiesOnly yes +``` + +**2. Clone the repo** somewhere on that host, e.g. `~/robertslab-resources`. + +**3. Add the cron entry** with `crontab -e`: + +``` +*/10 * * * * ~/robertslab-resources/scripts/publish_status.sh --profile internal --repo github-robertslab:RobertsLab/resources.git >> ~/status-cron.log 2>&1 +``` + +`publish_status.sh` keeps its own scratch clone under +`~/.cache/robertslab-server-status`, so it will not disturb the checkout it is +run from. + +**4. Confirm** that `status/internal.json` appears on the +[`server-status` branch](https://github.com/RobertsLab/resources/tree/server-status/status) +and that the lights on the handbook page go green within a few minutes. + +## Notes and limits + +- A green light means the port answered. It says nothing about Slurm health, + disk space, or whether jobs are running. +- `raw.githubusercontent.com` caches for about 5 minutes, so the page can lag + the actual check by that much on top of the check interval. +- GitHub's scheduled workflows are best-effort: delayed under load, minimum + 5-minute interval, and disabled automatically after 60 days without repo + activity. That is why the in-network cron is primary and the Action is only a + backstop. +- To add a host: add a probe in `check_servers.py` and a `` + row in `docs/Computing-Hardware.md`. The JavaScript matches the two by name + and needs no change. diff --git a/scripts/check_servers.py b/scripts/check_servers.py new file mode 100755 index 00000000..1ef4793d --- /dev/null +++ b/scripts/check_servers.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Probe Roberts Lab servers and emit a status JSON document. + +Usage: + check_servers.py --profile internal [--out FILE] + check_servers.py --profile external [--out FILE] + +Profiles exist because the three hosts are not reachable from the same place: + + raven not in public DNS; only visible from inside the UW network + gannet public HTTPS + klone public, but SSH (22) only -- there is no web port to probe + +"internal" runs on a machine inside the UW network and checks all three. +"external" runs on a GitHub Actions runner and checks the two public hosts, so +that a dead internal prober does not blind us on everything at once. + +A green light means the port answered. It says nothing about whether Slurm is +healthy, disks are full, or anyone's jobs are actually running. + +Stdlib only, Python 3.6+. +""" + +import argparse +import json +import socket +import ssl +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone + +DEFAULT_TIMEOUT = 8.0 + + +def tcp_check(host, port, timeout): + """Plain TCP connect. Used for klone (SSH) and raven (RStudio Server).""" + start = time.monotonic() + try: + # create_connection resolves and tries every address, which matters for + # klone -- it has two A records. + with socket.create_connection((host, port), timeout=timeout): + pass + except (socket.timeout, socket.gaierror, OSError) as exc: + if isinstance(exc, socket.gaierror): + detail = "DNS lookup failed (host is not resolvable from here)" + elif isinstance(exc, socket.timeout): + detail = "TCP {} did not answer within {:.0f}s".format(port, timeout) + else: + detail = "TCP {} refused: {}".format(port, exc.strerror or exc) + return {"up": False, "detail": detail, "latency_ms": None} + elapsed = int((time.monotonic() - start) * 1000) + return {"up": True, "detail": "TCP {} open".format(port), "latency_ms": elapsed} + + +def http_check(url, timeout): + """HTTP(S) check. Any 2xx/3xx counts as up.""" + start = time.monotonic() + request = urllib.request.Request(url, method="HEAD") + request.add_header("User-Agent", "robertslab-handbook-status-check") + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + code = response.status + except urllib.error.HTTPError as exc: + code = exc.code # server answered, just not with success + except ssl.SSLError as exc: + return { + "up": False, + "detail": "TLS error: {}".format(exc), + "latency_ms": None, + } + except (urllib.error.URLError, socket.timeout, OSError) as exc: + reason = getattr(exc, "reason", exc) + return { + "up": False, + "detail": "no response within {:.0f}s ({})".format(timeout, reason), + "latency_ms": None, + } + elapsed = int((time.monotonic() - start) * 1000) + up = 200 <= code < 400 + return { + "up": up, + "detail": "HTTP {}".format(code), + "latency_ms": elapsed if up else None, + } + + +def check_raven(timeout): + """RStudio Server is what people actually want from raven, so probe 8787 + rather than just SSH -- but distinguish 'whole box is down' from 'the box is + up and RStudio isn't'.""" + result = tcp_check("raven.fish.washington.edu", 8787, timeout) + if result["up"]: + return result + ssh = tcp_check("raven.fish.washington.edu", 22, timeout) + if ssh["up"]: + return { + "up": False, + "detail": "SSH is up but RStudio Server (8787) is not answering", + "latency_ms": ssh["latency_ms"], + } + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", required=True, choices=["internal", "external"]) + parser.add_argument("--out", help="write JSON here instead of stdout") + parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT) + args = parser.parse_args() + + hosts = {} + if args.profile == "internal": + hosts["raven"] = check_raven(args.timeout) + hosts["gannet"] = http_check("https://gannet.fish.washington.edu/", args.timeout) + hosts["klone"] = tcp_check("klone.hyak.uw.edu", 22, args.timeout) + + document = { + "checked": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "source": args.profile, + "hosts": hosts, + } + text = json.dumps(document, indent=2, sort_keys=True) + + if args.out: + with open(args.out, "w") as handle: + handle.write(text + "\n") + print("wrote {}".format(args.out), file=sys.stderr) + else: + print(text) + + +if __name__ == "__main__": + main() diff --git a/scripts/publish_status.sh b/scripts/publish_status.sh new file mode 100755 index 00000000..2a2c4e88 --- /dev/null +++ b/scripts/publish_status.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Run the server checks and publish the result to the `server-status` branch. +# Used both by the cron job inside the UW network (--profile internal) and by +# the GitHub Action (--profile external). See scripts/README.md for setup. +# +# The status branch is kept separate from master on purpose: it updates every +# few minutes, and putting that in the docs history would spam the log and +# trigger a full site rebuild on every check. +# +# The branch is rewritten as a single root commit each time rather than +# accumulating history -- at one check every 10 minutes an append-only branch +# would add ~50k commits a year to a repo everyone clones. +# +# Usage: publish_status.sh --profile internal|external [--repo URL] [--workdir DIR] + +set -euo pipefail + +PROFILE="internal" +REPO="git@github.com:RobertsLab/resources.git" +WORKDIR="${HOME}/.cache/robertslab-server-status" +BRANCH="server-status" + +while [ $# -gt 0 ]; do + case "$1" in + --profile) PROFILE="$2"; shift 2 ;; + --repo) REPO="$2"; shift 2 ;; + --workdir) WORKDIR="$2"; shift 2 ;; + -h|--help) sed -n '2,16p' "$0"; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +case "$PROFILE" in + internal|external) ;; + *) echo "usage: $0 --profile internal|external" >&2; exit 2 ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHECKER="${SCRIPT_DIR}/check_servers.py" + +if [ ! -f "$CHECKER" ]; then + echo "cannot find checker at ${CHECKER}" >&2 + exit 1 +fi + +if [ ! -d "${WORKDIR}/.git" ]; then + mkdir -p "$WORKDIR" + git init --quiet "$WORKDIR" + git -C "$WORKDIR" remote add origin "$REPO" +fi + +cd "$WORKDIR" + +git config user.name "roberts-lab-status-bot" +git config user.email "roberts-lab-status-bot@users.noreply.github.com" +git remote set-url origin "$REPO" + +# Every step is checked explicitly: `set -e` does not apply inside a function +# invoked as an `if` condition, so a bare command failing here would otherwise +# be silently skipped and the run would report success. +publish() { + local old_sha="" + + # Get off `fresh` before deleting it, in case a previous run died mid-flight. + git checkout --quiet -B staging >/dev/null 2>&1 || true + git branch -D fresh >/dev/null 2>&1 || true + + # Start from whatever the remote currently has, so the other prober's file is + # carried forward rather than clobbered. + if git fetch --quiet --depth 1 origin "$BRANCH" 2>/dev/null; then + old_sha="$(git rev-parse FETCH_HEAD)" || return 1 + git checkout --quiet -B staging "$old_sha" || return 1 + git reset --quiet --hard "$old_sha" || return 1 + git clean --quiet -fd || return 1 + else + find . -mindepth 1 -maxdepth 1 -not -name .git -exec rm -rf {} + || return 1 + fi + + mkdir -p status || return 1 + python3 "$CHECKER" --profile "$PROFILE" --out "status/${PROFILE}.json" || return 1 + + # Re-orphan so the branch stays at exactly one commit. --orphan keeps the + # working tree and index, so `git add -A` below stages the full contents. + git checkout --quiet --orphan fresh || return 1 + git add -A || return 1 + git commit --quiet -m "status(${PROFILE}): $(date -u +%Y-%m-%dT%H:%M:%SZ)" || return 1 + + if [ -n "$old_sha" ]; then + # A concurrent push from the other prober invalidates the lease, which + # rejects our push instead of silently dropping their update. + git push --quiet \ + --force-with-lease="refs/heads/${BRANCH}:${old_sha}" \ + origin "fresh:refs/heads/${BRANCH}" || return 1 + else + git push --quiet origin "fresh:refs/heads/${BRANCH}" || return 1 + fi + + echo "published status/${PROFILE}.json" >&2 +} + +for attempt in 1 2 3; do + if publish; then + exit 0 + fi + echo "publish attempt ${attempt} failed; retrying" >&2 + sleep 5 +done + +echo "failed to publish status after 3 attempts" >&2 +exit 1