[build] pin artifacts in a committed manifest

.github/workflows/checks.yml

@@ -29,6 +29,15 @@ jobs:
           name: bogota-artifacts-${{ github.sha }}
           path: data/Bogota
           retention-days: 7
+      # The build re-pins artifacts.json to the names it just built, so the
+      # committed manifest is stale for dated artifacts (otp graphs, gtfs,
+      # elevation). Downstream jobs need this copy, not the checked-out one.
+      - name: Upload artifact manifest
+        uses: actions/upload-artifact@v5
+        with:
+          name: bogota-manifest-${{ github.sha }}
+          path: builds/Bogota/artifacts.json
+          retention-days: 7
   build_images:
     name: Build service images
     runs-on: ubuntu-latest
@@ -126,6 +135,11 @@ jobs:
         with:
           name: bogota-artifacts-${{ github.sha }}
           path: data/Bogota
+      - name: Download artifact manifest
+        uses: actions/download-artifact@v6
+        with:
+          name: bogota-manifest-${{ github.sha }}
+          path: builds/Bogota
       - name: Download Docker images
         uses: actions/download-artifact@v6
         with:
@@ -164,17 +178,18 @@ jobs:
         with:
           name: bogota-artifacts-${{ github.sha }}
           path: data/Bogota
+      - name: Download artifact manifest
+        uses: actions/download-artifact@v6
+        with:
+          name: bogota-manifest-${{ github.sha }}
+          path: builds/Bogota
       - name: Download Docker images
         uses: actions/download-artifact@v6
         with:
           name: docker-images-${{ github.sha }}
           path: .
       - name: Load Docker images
         run: gunzip -c docker-images.tar.gz | docker load
-      # The build writes dated transit artifacts; link the latest to the names
-      # docker compose serves them from (as run-integration-tests.sh does).
-      - name: Link transit artifacts
-        run: TRANSIT_DATA_ROOT=./data bin/link-latest-transit builds/Bogota
       - name: Start services
         run: bin/start-services --no-follow-logs builds/Bogota
       - name: Wait for services

BUILD.md

@@ -44,14 +44,12 @@ This approach will download all the mapping data you need automatically, but onl
 1. Pick a metro area from the list above, like "Amsterdam" or "Denver". These values are case-sensitive. In all the examples, replace "Amsterdam" with your metro area of choice.
 2. Configuration is managed per build directory in `builds/<Area>`. Copy a template build directory: `cp -r builds/Bogota builds/Amsterdam`, review and edit `builds/Amsterdam/.env`. Bogota is configured for transit routing, so unless you're setting that up too (step 4), delete the copied `builds/Amsterdam/transit` directory and unset `HEADWAY_ENABLE_TRANSIT_ROUTING`.
 3. Execute `bin/build builds/Amsterdam` to build data artifacts
-4. (Optional) Set up transit routing. Note: This dramatically increases hardware requirements for large metro areas.
+4. (Optional) Set up transit routing. Note: This increases hosting requirements for large metro areas - you'll want at least 4GB RAM extra for a medium sized city's transit service.
    1. Find nearby transit schedules by running `bin/export-nearby-transit-feeds builds/Amsterdam`
-   2. Examine `builds/Amsterdam/transit/gtfs-feeds/Amsterdam.gtfs_feeds.csv` and manually edit it if necessary to curate GTFS feeds. Some may have errors, and many may be useless for your purposes.
+   2. Examine `builds/Amsterdam/transit/gtfs-feeds/amsterdam.gtfs_feeds.csv` and manually edit it if necessary to curate GTFS feeds. Some may have errors, and many may be useless for your purposes.
    3. Build transit routing with `bin/build-transit builds/Amsterdam`
-   4. Transit artifacts are built with dated names, so link the ones to serve: `TRANSIT_DATA_ROOT=./data bin/link-latest-transit builds/Amsterdam`. Re-run this after each `bin/build-transit`.
-5. Start services from the build directory by running: `cd builds/Amsterdam && docker compose -f ../../docker-compose.yaml up -d`. With transit routing, use `../../docker-compose-with-transit.yaml` instead - or `bin/start-services builds/Amsterdam`, which picks the compose file based on `HEADWAY_ENABLE_TRANSIT_ROUTING`.
-6. This will bring up the headway stack with a web frontend on port 8080.
-7. (For https and non-default port use only) reverse-proxy traffic to port 8080.
+5. Run `bin/start-services builds/Amsterdam`. This will bring up the Headway stack with a web frontend on port 8080.
+  1. (Optional for https and non-default port use only) reverse-proxy traffic to port 8080.
 
 That's it!
 
@@ -65,9 +63,17 @@ The process is largely the same as above. After downloading your OSM extract, mo
 
 ## Docker-compose restarts
 
-Because Headway's docker-compose configuration uses init containers to populate a docker volume containing internal data, rebuilding the data for a metro area won't update existing containers. It's necessary to run `docker compose down --volumes` to re-initialize the data in the init containers.
+Rebuilding the data for a metro area won't update existing containers.
 
-When running docker compose commands, make sure to run them from the build directory (e.g., `cd builds/Amsterdam && docker compose -f ../../docker-compose.yaml down --volumes`) so they use the correct `.env` file, or use the integration test helper: `tests/integration/stop-and-remove-services builds/Amsterdam`
+```
+# delete all existing docker data volumes and containers
+bin/stop-and-remove-services builds/Amsterdam
+# start the services again, which will pull in fresh data
+bin/start-services builds/Amsterdam
+
+# or both in a single command
+bin/reset-services builds/Amsterdam
+```
 
 This is necessary whenever you rebuild the data for a metro area, or change which area you're serving data for in the `builds/<area>/.env` file.
 

bin/_compose-file.sh

@@ -15,3 +15,6 @@ if [ -z "${COMPOSE_FILE:-}" ]; then
         COMPOSE_FILE=docker-compose.yaml
     fi
 fi
+
+export HEADWAY_OTP_GRAPH_FILE=$(bin/artifacts otp-graph "$CONFIG_DIR")
+export HEADWAY_ELEVATION_FILE=$(bin/artifacts elevation "$CONFIG_DIR")

bin/artifacts

@@ -0,0 +1,180 @@
+#!/bin/bash
+
+set -e
+
+APP_ROOT=$(git rev-parse --show-toplevel)
+cd "$APP_ROOT"
+
+function usage() {
+  cat << EOF
+Pin and look up build artifacts. This runs on the build machine: --pin reads
+data/<Area>, and the lookups exist for the local docker compose stack. Anything
+that generates deploy config reads builds/<Area>/artifacts.json directly.
+
+usage: $0 <otp-graph|elevation> <build-config-dir>
+       $0 <files|shared-files> <build-config-dir>
+       $0 --pin <build-config-dir>
+       $0 --verify <build-config-dir>
+
+examples: # pin the newest artifacts to builds/<Area>/artifacts.json
+    \$0 --pin builds/Amsterdam
+    # check every pinned artifact is present in data/<Area>
+    \$0 --verify builds/Amsterdam
+    # what the local compose stack mounts
+    \$0 otp-graph builds/planet
+EOF
+}
+
+COMMAND="${1:-}"
+CONFIG_DIR="${2:-}"
+if [ -z "$COMMAND" ] || [ -z "$CONFIG_DIR" ]; then
+    usage
+    exit 1
+fi
+
+if [ ! -d "$CONFIG_DIR" ]; then
+    echo "Error: config dir \"$CONFIG_DIR\" doesn't exist" >&2
+    exit 1
+fi
+
+source bin/_source-env.sh "$CONFIG_DIR"
+
+set -o nounset
+
+MANIFEST="${CONFIG_DIR}/artifacts.json"
+
+if ! command -v jq > /dev/null; then
+    echo "Error: jq is required to read ${MANIFEST}" >&2
+    exit 1
+fi
+
+case "$COMMAND" in
+    --pin)
+        # Generate artifacts.json from what's in data/<Area>, pinning the newest of each dated artifact.
+        AREA_DIR="data/${HEADWAY_AREA}"
+        TRANSIT_DIR="${AREA_DIR}/transit"
+        MANIFEST="${CONFIG_DIR}/artifacts.json"
+
+        if [ ! -d "$AREA_DIR" ]; then
+            echo "Error: nothing built at ${AREA_DIR}" >&2
+            exit 1
+        fi
+
+        manifest="{}"
+
+        function add_if_present() {
+            local key="$1" file="$2"
+            if [ -f "${AREA_DIR}/${file}" ]; then
+                manifest=$(jq --arg key "$key" --arg file "$file" '.[$key] = $file' <<< "$manifest")
+            fi
+        }
+
+        add_if_present valhalla      "${HEADWAY_AREA}.valhalla.tar.zst"
+        add_if_present elasticsearch "${HEADWAY_AREA}.elasticsearch.tar.zst"
+        add_if_present placeholder   "${HEADWAY_AREA}.placeholder.tar.zst"
+        add_if_present pmtiles       "${HEADWAY_AREA}.pmtiles"
+        add_if_present osm           "${HEADWAY_AREA}.osm.pbf"
+
+        shared="{}"
+        for file in terrain.mbtiles landcover.mbtiles; do
+            if [ -f "${AREA_DIR}/${file}" ]; then
+                shared=$(jq --arg key "${file%.mbtiles}" --arg file "$file" '.[$key] = $file' <<< "$shared")
+            fi
+        done
+        if [ "$shared" != "{}" ]; then
+            manifest=$(jq --argjson shared "$shared" '.shared = $shared' <<< "$manifest")
+        fi
+
+        # Dated names sort oldest to newest, so the last write wins.
+        function add_dated_by_zone() {
+            local key="$1" suffix="$2"
+            local found="{}" path file rest
+
+            for path in "${TRANSIT_DIR}/${HEADWAY_AREA}-"*"$suffix"; do
+                [ -e "$path" ] || continue
+                file=$(basename "$path")
+
+                rest=${file#"${HEADWAY_AREA}-"}
+                rest=${rest%"$suffix"}
+                if [[ ! $rest =~ ^(.+)-([0-9]{4}-[0-9]{2}-[0-9]{2})$ ]]; then
+                    echo "Error: unexpected ${suffix} name: ${file}" >&2
+                    exit 1
+                fi
+                found=$(jq --arg zone "${BASH_REMATCH[1]}" --arg file "transit/$file" '.[$zone] = $file' <<< "$found")
+            done
+
+            if [ "$found" != "{}" ]; then
+                manifest=$(jq --arg key "$key" --argjson found "$found" '.[$key] = $found' <<< "$manifest")
+            fi
+        }
+
+        add_dated_by_zone otp-graphs .graph.obj.zst
+        add_dated_by_zone gtfs .gtfs.tar.zst
+
+        elevation=""
+        for path in "${TRANSIT_DIR}/${HEADWAY_AREA}-"*.elevation-tifs.tar.zst; do
+            [ -e "$path" ] || continue
+            elevation=$(basename "$path")
+        done
+        if [ -n "$elevation" ]; then
+            manifest=$(jq --arg file "transit/$elevation" '.elevation = $file' <<< "$manifest")
+        fi
+
+        jq -S . <<< "$manifest" > "$MANIFEST"
+
+        echo "Wrote ${MANIFEST}:" >&2
+        cat "$MANIFEST" >&2
+        ;;
+    --verify)
+        # Ensure all pinned assets are present locally
+        AREA_DIR="data/${HEADWAY_AREA}"
+
+        if [ ! -f "$MANIFEST" ]; then
+            echo "Error: no ${MANIFEST}; run bin/build ${CONFIG_DIR}" >&2
+            exit 1
+        fi
+
+        missing=()
+        while IFS= read -r file; do
+            [ -n "$file" ] || continue
+            [ -f "${AREA_DIR}/${file}" ] || missing+=("$file")
+        done < <(jq -r '.. | strings' "$MANIFEST")
+
+        if [ ${#missing[@]} -gt 0 ]; then
+            {
+                echo "Error: ${MANIFEST} pins ${#missing[@]} artifact(s) missing from ${AREA_DIR}:"
+                printf '  %s\n' "${missing[@]}"
+                echo
+                echo "The pinned names are stale, or the build didn't finish. Either rebuild"
+                echo "(bin/build ${CONFIG_DIR}), or re-pin what's already built"
+                echo "(bin/artifacts --pin ${CONFIG_DIR})."
+            } >&2
+            exit 1
+        fi
+
+        echo "All artifacts pinned by ${MANIFEST} are present in ${AREA_DIR}."
+        ;;
+    otp-graph)
+        [ -f "$MANIFEST" ] || exit 0
+        # docker compose runs a single OTP, so serve the first zone
+        jq -r 'first(."otp-graphs"[]?) // empty' "$MANIFEST"
+        ;;
+    elevation)
+        [ -f "$MANIFEST" ] || exit 0
+        jq -r '.elevation // empty' "$MANIFEST"
+        ;;
+    files)
+        # All artifacts for the deployment except `shared`,
+        # which is excluded because those publish to a different prefix.
+        [ -f "$MANIFEST" ] || exit 0
+        jq -r 'del(.shared) | .. | strings' "$MANIFEST"
+        ;;
+    shared-files)
+        [ -f "$MANIFEST" ] || exit 0
+        jq -r '.shared // {} | .[]' "$MANIFEST"
+        ;;
+    *)
+        usage
+        exit 1
+        ;;
+esac

bin/build

@@ -45,4 +45,4 @@ if [ -d "$TRANSIT_CONFIG_DIR" ]; then
     bin/build-transit "$CONFIG_DIR"
 fi
 
-echo "Next, run: git co . && ./bin/k8s-regenerate-all"
+echo "Next, run: ./bin/revert-fetch-urls && ./bin/k8s-regenerate-all"

bin/build-transit

@@ -42,6 +42,32 @@ fi
 
 TRANSIT_CONFIG_DIR="$CONFIG_DIR/transit"
 
+# Each file in gtfs-feeds/ is built into a zone named after the file, and that
+# name is used verbatim from here on: it keys artifacts.json and it names the OTP
+# deployment, service and config map in k8s. So it has to be a valid k8s object
+# name - cheap to check, expensive to discover after a multi-hour build.
+function lint_zone_names() {
+    set +x
+    local found=0 zone file
+    for file in "${TRANSIT_CONFIG_DIR}"/gtfs-feeds/*.gtfs_feeds.csv; do
+        [ -e "$file" ] || continue
+        found=1
+        zone=$(basename "$file" .gtfs_feeds.csv)
+        # 63 chars is the k8s limit, minus room for the "opentripplanner-" prefix.
+        if [[ ! $zone =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]] || [ ${#zone} -gt 40 ]; then
+            echo "Error: ${file}: zone \"${zone}\" is not a valid k8s object name; rename it to lowercase letters, digits and dashes" >&2
+            exit 1
+        fi
+    done
+
+    if [ "$found" -eq 0 ]; then
+        echo "Error: no *.gtfs_feeds.csv in ${TRANSIT_CONFIG_DIR}/gtfs-feeds - nothing to build" >&2
+        exit 1
+    fi
+    set -x
+}
+lint_zone_names
+
 USE_LOCAL_PBF=""
 INPUT_PBF="${HEADWAY_AREA}.osm.pbf"
 if [[ -f $INPUT_PBF ]]; then
@@ -50,14 +76,17 @@ fi
 
 dagger -c "with-area ${HEADWAY_AREA} ${USE_LOCAL_PBF} | build-transit ${TRANSIT_CONFIG_DIR} ${GTFS_API_KEYS} | export 'data/${HEADWAY_AREA}/transit'"
 
+bin/artifacts --pin "$CONFIG_DIR"
+
 set +x
 cat <<EOS
 
-Transit artifacts written to data/${HEADWAY_AREA}/transit, with dated filenames.
-To serve them locally, link the latest ones:
-    TRANSIT_DATA_ROOT=./data bin/link-latest-transit ${CONFIG_DIR}
+Transit artifacts written to data/${HEADWAY_AREA}/transit and artifacts pinned.
+
+This build rewrote the artifact URLs in place; to put them back:
+    bin/revert-fetch-urls
 
-To publish them to the asset host:
+To publish the artifacts to the asset host:
     bin/publish-data ${CONFIG_DIR} [--host remote.host.com]
 EOS
 

bin/export-nearby-transit-feeds

@@ -31,7 +31,10 @@ source bin/_source-env.sh "$CONFIG_DIR"
 TRANSIT_DIR="${CONFIG_DIR}/transit/gtfs-feeds"
 mkdir -p "$TRANSIT_DIR"
 
-OUTPUT_FILE="${TRANSIT_DIR}/${HEADWAY_AREA}.gtfs_feeds.csv"
+# The zone name is the filename, and it ends up as a k8s object name, so mint it
+# in the spelling those require - see the lint in bin/build-transit.
+ZONE_NAME=$(echo "$HEADWAY_AREA" | sed -E 's/([a-z0-9])([A-Z])/\1-\2/g' | tr '[:upper:]_' '[:lower:]-')
+OUTPUT_FILE="${TRANSIT_DIR}/${ZONE_NAME}.gtfs_feeds.csv"
 
 echo "Finding nearby GTFS feeds for ${HEADWAY_AREA}..."
 echo "Output will be written to: ${OUTPUT_FILE}"