Add content hash to exported artifacts, and PersistentVolumeClaims for k8s hosting

BUILD.md

@@ -53,7 +53,7 @@ This approach will download all the mapping data you need automatically, but onl
 
 That's it!
 
-There are some experimental kubernetes configs in k8s/configs, but they are pretty specific to my own needs at this point.
+There are some experimental kubernetes configs in [k8s/](./k8s/), but they are pretty specific to my own needs at this point. See [k8s/README.md](./k8s/README.md) for how they're generated and deployed.
 
 ### Building Headway from your own OSM extract
 

bin/_compose-file.sh

@@ -16,5 +16,12 @@ if [ -z "${COMPOSE_FILE:-}" ]; then
     fi
 fi
 
+# Artifact names carry a content hash, so the compose file can't spell them out.
 export HEADWAY_OTP_GRAPH_FILE=$(bin/artifacts otp-graph "$CONFIG_DIR")
 export HEADWAY_ELEVATION_FILE=$(bin/artifacts elevation "$CONFIG_DIR")
+export HEADWAY_PMTILES_FILE=$(bin/artifacts pmtiles "$CONFIG_DIR")
+export HEADWAY_VALHALLA_FILE=$(bin/artifacts valhalla "$CONFIG_DIR")
+export HEADWAY_ELASTICSEARCH_FILE=$(bin/artifacts elasticsearch "$CONFIG_DIR")
+export HEADWAY_PLACEHOLDER_FILE=$(bin/artifacts placeholder "$CONFIG_DIR")
+export HEADWAY_TERRAIN_FILE=$(bin/artifacts terrain "$CONFIG_DIR")
+export HEADWAY_LANDCOVER_FILE=$(bin/artifacts landcover "$CONFIG_DIR")

bin/artifacts

@@ -11,7 +11,8 @@ Pin and look up build artifacts. This runs on the build machine: --pin reads
 data/<Area>, and the lookups exist for the local docker compose stack. Anything
 that generates deploy config reads builds/<Area>/artifacts.json directly.
 
-usage: $0 <otp-graph|elevation> <build-config-dir>
+usage: $0 <otp-graph|elevation|pmtiles|valhalla|elasticsearch|placeholder|osm> <build-config-dir>
+       $0 <terrain|landcover> <build-config-dir>
        $0 <files|shared-files> <build-config-dir>
        $0 --pin <build-config-dir>
        $0 --verify <build-config-dir>
@@ -62,60 +63,106 @@ case "$COMMAND" in
 
         manifest="{}"
 
-        function add_if_present() {
-            local key="$1" file="$2"
-            if [ -f "${AREA_DIR}/${file}" ]; then
-                manifest=$(jq --arg key "$key" --arg file "$file" '.[$key] = $file' <<< "$manifest")
+        # Artifacts are named <stem>-<date>-<hash>.<ext>, so a rebuild lands
+        # beside what it replaced rather than overwriting it. Pin the newest;
+        # two builds of the same day are a tie the date can't break.
+        VERSION_RE='([0-9]{4}-[0-9]{2}-[0-9]{2})-[0-9a-f]+'
+
+        # Prints the newest match of <dir>/<prefix>-<version><suffix>, if any.
+        function latest_match() {
+            local dir="$1" prefix="$2" suffix="$3"
+            local path file version newest="" newest_date="" tied=""
+
+            for path in "${dir}/${prefix}-"*"${suffix}"; do
+                [ -f "$path" ] || continue
+                file=$(basename "$path")
+
+                version=${file#"${prefix}-"}
+                version=${version%"$suffix"}
+                if [[ ! $version =~ ^${VERSION_RE}$ ]]; then
+                    echo "Error: unexpected ${prefix} artifact name: ${file}" >&2
+                    exit 1
+                fi
+
+                if [[ "${BASH_REMATCH[1]}" > "$newest_date" ]]; then
+                    newest_date="${BASH_REMATCH[1]}"
+                    newest="$file"
+                    tied=""
+                elif [ "${BASH_REMATCH[1]}" = "$newest_date" ]; then
+                    tied="$file"
+                fi
+            done
+
+            if [ -n "$tied" ]; then
+                {
+                    echo "Error: two ${newest_date} builds of ${prefix} to choose between:"
+                    printf '  %s\n' "$newest" "$tied"
+                    echo "Delete the one you don't want pinned, then run --pin again."
+                } >&2
+                exit 1
             fi
+            echo "$newest"
         }
 
-        add_if_present valhalla      "${HEADWAY_AREA}.valhalla.tar.zst"
-        add_if_present elasticsearch "${HEADWAY_AREA}.elasticsearch.tar.zst"
-        add_if_present placeholder   "${HEADWAY_AREA}.placeholder.tar.zst"
-        add_if_present pmtiles       "${HEADWAY_AREA}.pmtiles"
-        add_if_present osm           "${HEADWAY_AREA}.osm.pbf"
+        function add_latest() {
+            local key="$1" prefix="$2" suffix="$3" file
+            file=$(latest_match "$AREA_DIR" "$prefix" "$suffix")
+            [ -n "$file" ] || return 0
+            manifest=$(jq --arg key "$key" --arg file "$file" '.[$key] = $file' <<< "$manifest")
+        }
+
+        add_latest valhalla      "${HEADWAY_AREA}-valhalla"      .tar.zst
+        add_latest elasticsearch "${HEADWAY_AREA}-elasticsearch" .tar.zst
+        add_latest placeholder   "${HEADWAY_AREA}-placeholder"   .tar.zst
+        add_latest pmtiles       "${HEADWAY_AREA}"               .pmtiles
+        add_latest osm           "${HEADWAY_AREA}"               .osm.pbf
 
         shared="{}"
-        for file in terrain.mbtiles landcover.mbtiles; do
-            if [ -f "${AREA_DIR}/${file}" ]; then
-                shared=$(jq --arg key "${file%.mbtiles}" --arg file "$file" '.[$key] = $file' <<< "$shared")
-            fi
+        for name in terrain landcover; do
+            file=$(latest_match "$AREA_DIR" "$name" .mbtiles)
+            [ -n "$file" ] || continue
+            shared=$(jq --arg key "$name" --arg file "$file" '.[$key] = $file' <<< "$shared")
         done
         if [ "$shared" != "{}" ]; then
             manifest=$(jq --argjson shared "$shared" '.shared = $shared' <<< "$manifest")
         fi
 
-        # Dated names sort oldest to newest, so the last write wins.
-        function add_dated_by_zone() {
-            local key="$1" suffix="$2"
-            local found="{}" path file rest
+        # Like add_latest, but the stem carries a zone name:
+        #   Seattle-puget-sound-graph-2026-08-14-2f1c9ab7.obj.zst
+        function add_by_zone() {
+            local key="$1" kind="$2" suffix="$3"
+            local found="{}" path file zone zones=() seen
 
-            for path in "${TRANSIT_DIR}/${HEADWAY_AREA}-"*"$suffix"; do
-                [ -e "$path" ] || continue
+            for path in "${TRANSIT_DIR}/${HEADWAY_AREA}-"*"-${kind}-"*"$suffix"; do
+                [ -f "$path" ] || continue
                 file=$(basename "$path")
 
-                rest=${file#"${HEADWAY_AREA}-"}
-                rest=${rest%"$suffix"}
-                if [[ ! $rest =~ ^(.+)-([0-9]{4}-[0-9]{2}-[0-9]{2})$ ]]; then
-                    echo "Error: unexpected ${suffix} name: ${file}" >&2
+                if [[ ! $file =~ ^${HEADWAY_AREA}-(.+)-${kind}-${VERSION_RE}${suffix//./\\.}$ ]]; then
+                    echo "Error: unexpected ${kind} artifact name: ${file}" >&2
                     exit 1
                 fi
-                found=$(jq --arg zone "${BASH_REMATCH[1]}" --arg file "transit/$file" '.[$zone] = $file' <<< "$found")
+                zone="${BASH_REMATCH[1]}"
+
+                seen=0
+                for z in "${zones[@]+"${zones[@]}"}"; do
+                    [ "$z" = "$zone" ] && seen=1
+                done
+                [ "$seen" -eq 1 ] && continue
+                zones+=("$zone")
+
+                file=$(latest_match "$TRANSIT_DIR" "${HEADWAY_AREA}-${zone}-${kind}" "$suffix")
+                found=$(jq --arg zone "$zone" --arg file "transit/$file" '.[$zone] = $file' <<< "$found")
             done
 
             if [ "$found" != "{}" ]; then
                 manifest=$(jq --arg key "$key" --argjson found "$found" '.[$key] = $found' <<< "$manifest")
             fi
         }
 
-        add_dated_by_zone otp-graphs .graph.obj.zst
-        add_dated_by_zone gtfs .gtfs.tar.zst
+        add_by_zone otp-graphs graph .obj.zst
+        add_by_zone gtfs gtfs .tar.zst
 
-        elevation=""
-        for path in "${TRANSIT_DIR}/${HEADWAY_AREA}-"*.elevation-tifs.tar.zst; do
-            [ -e "$path" ] || continue
-            elevation=$(basename "$path")
-        done
+        elevation=$(latest_match "$TRANSIT_DIR" "${HEADWAY_AREA}-elevation-tifs" .tar.zst)
         if [ -n "$elevation" ]; then
             manifest=$(jq --arg file "transit/$elevation" '.elevation = $file' <<< "$manifest")
         fi
@@ -159,9 +206,13 @@ case "$COMMAND" in
         # docker compose runs a single OTP, so serve the first zone
         jq -r 'first(."otp-graphs"[]?) // empty' "$MANIFEST"
         ;;
-    elevation)
+    elevation|pmtiles|valhalla|elasticsearch|placeholder|osm)
+        [ -f "$MANIFEST" ] || exit 0
+        jq -r --arg key "$COMMAND" '.[$key] // empty' "$MANIFEST"
+        ;;
+    terrain|landcover)
         [ -f "$MANIFEST" ] || exit 0
-        jq -r '.elevation // empty' "$MANIFEST"
+        jq -r --arg key "$COMMAND" '.shared[$key]? // empty' "$MANIFEST"
         ;;
     files)
         # All artifacts for the deployment except `shared`,

bin/build

@@ -42,7 +42,10 @@ dagger --progress=plain -c "with-area ${HEADWAY_AREA} --countries=${HEADWAY_COUN
 
 TRANSIT_CONFIG_DIR="$CONFIG_DIR/transit"
 if [ -d "$TRANSIT_CONFIG_DIR" ]; then
+    # build-transit calls pin for us, once the transit artifacts have landed too.
     bin/build-transit "$CONFIG_DIR"
+else
+    bin/artifacts --pin "$CONFIG_DIR"
 fi
 
 echo "Next, run: ./bin/revert-fetch-urls && ./bin/k8s/regenerate-all"

bin/k8s/_deploy-lib.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# Shared setup for the deploy scripts: argument parsing, the clean-tree check,
+# and applying a namespace's rendered configs. Sourced, not run.
+
+NAMESPACE=""
+CONFIG_DIR=""
+
+# Reads `<namespace> [--dev]` into NAMESPACE and CONFIG_DIR, and cds to the repo
+# root so the paths the callers use resolve.
+function deploy_lib_parse_args() {
+    local dev=false
+
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            --dev)
+                dev=true
+                shift
+                ;;
+            *)
+                if [ -z "$NAMESPACE" ]; then
+                    NAMESPACE="$1"
+                else
+                    echo "Unknown argument: $1" >&2
+                    exit 1
+                fi
+                shift
+                ;;
+        esac
+    done
+
+    if [ -z "$NAMESPACE" ]; then
+        echo "Usage: $0 <namespace> [--dev]"
+        echo "Examples:"
+        echo "  $0 planet"
+        echo "  $0 seattle --dev"
+        echo "  $0 planet --dev"
+        exit 1
+    fi
+
+    cd "$(git -C "$(dirname "$0")" rev-parse --show-toplevel)"
+
+    if [ "$dev" = true ]; then
+        CONFIG_DIR="k8s/configs/${NAMESPACE}-dev"
+    else
+        CONFIG_DIR="k8s/configs/${NAMESPACE}"
+    fi
+
+    if [ ! -d "$CONFIG_DIR" ]; then
+        echo "no rendered configs at ${CONFIG_DIR} - run bin/k8s/generate first" >&2
+        exit 1
+    fi
+}
+
+# update-fetch-urls dirties the tree and a trap puts it back, which can't tell
+# your edits from its own - so refuse to start from a dirty tree.
+function deploy_lib_require_clean_tree() {
+    local modified_files
+    modified_files="$(git diff --name-only)"
+    if [ -n "$modified_files" ]; then
+        echo "$modified_files"
+        echo "👆 Modified files in local directory. Clean first."
+        echo "git co ."
+        exit 1
+    fi
+}
+
+# The committed configs carry a placeholder artifact host; swap the real one in
+# just long enough to apply, then put it back.
+function deploy_lib_apply() {
+    trap 'bin/revert-fetch-urls' EXIT
+    bin/update-fetch-urls
+
+    (cd "$CONFIG_DIR" && kubectl apply -f . -n "$NAMESPACE")
+}

bin/k8s/_pvc-lib.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Shared helpers for inspecting headway's PersistentVolumeClaims. Sourced, not run.
+
+HEADWAY_PVC_SELECTOR="app.kubernetes.io/part-of=headway"
+
+KUBECTL=()
+
+function pvc_lib_set_namespace() {
+    KUBECTL=(kubectl --namespace "$1")
+}
+
+function pvc_lib_require_deps() {
+    if ! command -v jq > /dev/null; then
+        echo "jq is required (brew install jq)" >&2
+        exit 1
+    fi
+    if ! "${KUBECTL[@]}" version > /dev/null 2>&1; then
+        echo "can't reach the cluster; check your kubectl context and VPN" >&2
+        exit 1
+    fi
+}
+
+# Every claim name spoken for, by a workload's pod template or by a pod that
+# exists right now. The latter matters mid-rollout: the outgoing pod still holds
+# the old claim after no Deployment names it.
+function in_use_claims() {
+    {
+        "${KUBECTL[@]}" get deployments,statefulsets -o json \
+            | jq -r '.items[].spec.template.spec.volumes[]?.persistentVolumeClaim.claimName // empty'
+        "${KUBECTL[@]}" get pods -o json \
+            | jq -r '.items[].spec.volumes[]?.persistentVolumeClaim.claimName // empty'
+    } | sort -u
+}
+
+# name <TAB> capacity <TAB> phase <TAB> created
+function headway_pvcs() {
+    "${KUBECTL[@]}" get pvc -l "$HEADWAY_PVC_SELECTOR" -o json | jq -r '
+        .items[]
+        | [ .metadata.name,
+            (.spec.resources.requests.storage // "?"),
+            (.status.phase // "?"),
+            (.metadata.creationTimestamp // "?") ]
+        | @tsv'
+}

bin/k8s/apply-latest

@@ -0,0 +1,18 @@
+#!/bin/bash -e
+# Apply a namespace's rendered configs and nothing else.
+#
+# Pods restart only where the manifest actually changed - a new artifact URL or
+# volume version moves the pod template, so kubectl rolls exactly those pods and
+# leaves the rest serving. Republishing a container to the same `:latest` tag
+# leaves the manifest untouched and so isn't picked up here; use
+# bin/k8s/rollout-latest for that.
+
+source "$(dirname "${BASH_SOURCE[0]}")/_deploy-lib.sh"
+
+deploy_lib_parse_args "$@"
+deploy_lib_require_clean_tree
+echo "ready to apply"
+
+deploy_lib_apply
+
+watch kubectl get pods -n "$NAMESPACE"