From 3550fb53561b68d4b188012671c67f0ab70b5187 Mon Sep 17 00:00:00 2001 From: Kevin Spurrier Date: Fri, 18 Sep 2026 18:59:13 -0500 Subject: [PATCH 1/2] Podman / CI: add additive Podman build and smoke verification workflow --- .github/workflows/podman-smoke.yml | 113 +++++++++++++++++++++++++++++ PODMAN.md | 76 +++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 .github/workflows/podman-smoke.yml create mode 100644 PODMAN.md diff --git a/.github/workflows/podman-smoke.yml b/.github/workflows/podman-smoke.yml new file mode 100644 index 00000000..48ccc440 --- /dev/null +++ b/.github/workflows/podman-smoke.yml @@ -0,0 +1,113 @@ +# .github/workflows/podman-smoke.yml +# ADDITIVE WORKFLOW: Proves Podman build and smoke verification for ngen-forcing. +# Focuses on Dockerfile.bmi-forcings (primary CI/CD and downstream ngen dependency). +# Leaves existing .github/workflows/cicd.yml and release paths untouched. + +name: Podman Build & Smoke (Additive) + +on: + workflow_dispatch: + inputs: + push_images: + description: "Push test tags (:podman-test) to GHCR" + required: true + type: boolean + default: false + pull_request: + branches: + - development + paths: + - '.github/workflows/podman-smoke.yml' + - 'Dockerfile.bmi-forcings' + - 'pyproject.toml' + - 'NextGen_Forcings_Engine_BMI/**' + - 'Forcing_Extraction_Scripts/**' + - 'ESMF_Mesh_Domain_Configuration_Production/**' + +permissions: + contents: read + packages: write + +jobs: + podman-build-smoke: + name: Podman Build & Smoke (BMI Forcing) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Verify Preinstalled Podman Environment + run: | + podman version + podman info --format 'OS: {{.Host.Distribution.Distribution}} {{.Host.Distribution.Version}} | Storage: {{.Store.GraphDriverName}}' + + - name: Log in to GitHub Container Registry (for base image pull) + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io \ + -u "${{ github.actor }}" --password-stdin + + - name: Build BMI Forcing Image with Podman (Docker Format) + id: build + run: | + set -euo pipefail + ulimit -n 65535 || true + IMAGE_TAG="localhost/ngen-bmi-forcing:podman-test" + + echo "Building ${IMAGE_TAG} using Dockerfile.bmi-forcings with --format docker..." + podman build \ + --ulimit nofile=65535:65535 \ + --build-arg DEPS_IMAGE="ghcr.io/ngwpc/ngen-dependencies-bookworm:latest" \ + --build-arg CI_COMMIT_REF_NAME="${GITHUB_REF_NAME}" \ + --format docker \ + -f Dockerfile.bmi-forcings \ + -t "${IMAGE_TAG}" \ + . + + echo "IMAGE_LOCAL=${IMAGE_TAG}" >> "${GITHUB_ENV}" + + - name: Smoke Probe 1 - Entrypoint & Shell Check + run: | + set -euo pipefail + echo "Verifying entrypoint (/bin/bash) and shell execution..." + podman run --rm "${IMAGE_LOCAL}" -c "echo 'Probe 1 Passed: /bin/bash functional' && /bin/bash --version | head -n1" + + - name: Smoke Probe 2 - Python Environment & Package Import + run: | + set -euo pipefail + echo "Verifying Python packages (NextGen_Forcings_Engine, ewts)..." + podman run --rm "${IMAGE_LOCAL}" -c "python --version && python -c 'import NextGen_Forcings_Engine, ewts; print(\"Probe 2 Passed: NextGen_Forcings_Engine and ewts imported successfully. Version:\", NextGen_Forcings_Engine.__version__)'" + + - name: Smoke Probe 3 - Git Provenance Metadata Check + run: | + set -euo pipefail + echo "Verifying provenance metadata file existence and content..." + podman run --rm "${IMAGE_LOCAL}" -c "cat /ngen-app/ngen-bmi-forcing_git_info.json && test -s /ngen-app/ngen-bmi-forcing_git_info.json" + echo "Probe 3 Passed: Git provenance metadata file verified." + + - name: Inspect Built OCI Image + run: | + podman image inspect "${IMAGE_LOCAL}" \ + --format 'Image ID: {{.Id}} | Digest: {{.Digest}} | Entrypoint: {{.Config.Entrypoint}} | Cmd: {{.Config.Cmd}}' + + - name: Optional GHCR Push (:podman-test only) + if: ${{ inputs.push_images }} + run: | + set -euo pipefail + REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + DEST_BASE="ghcr.io/${REPO_LOWER}/ngen-bmi-forcing" + TAG_TEST="${DEST_BASE}:podman-test" + TAG_SHA="${DEST_BASE}:${{ github.sha }}-podman-test" + + echo "Publishing test verification tags..." + podman tag "${IMAGE_LOCAL}" "${TAG_TEST}" + podman tag "${IMAGE_LOCAL}" "${TAG_SHA}" + + podman push "${TAG_TEST}" + podman push "${TAG_SHA}" + + echo "Pushed ${TAG_TEST} and ${TAG_SHA}" + echo "NOTICE: Production aliases (:latest, branch tags) remain untouched." diff --git a/PODMAN.md b/PODMAN.md new file mode 100644 index 00000000..bbbcdb4d --- /dev/null +++ b/PODMAN.md @@ -0,0 +1,76 @@ +# Podman Support (Additive Path) + +This repository maintains Docker as its primary documented and production CI path. Podman is supported as an **additional** option for local development, rootless execution, and container build/smoke verification. + +Existing Docker workflows (`.github/workflows/cicd.yml`) and production release tags remain untouched and active. + +--- + +## Prerequisites + +Verify that Podman is installed on your workstation: + +```bash +podman version +podman info +``` + +For Ubuntu 24.04+ (Noble) or RHEL 8/9, Podman 4.9+ is recommended. + +--- + +## Registry Authentication + +`Dockerfile.bmi-forcings` builds from `ghcr.io/ngwpc/ngen-dependencies-bookworm:latest`. Ensure you are logged in to GHCR before building: + +```bash +echo "" | podman login ghcr.io -u "" --password-stdin +``` + +--- + +## Building with Podman + +### NextGen BMI Forcing (`Dockerfile.bmi-forcings`) + +Build the image directly using `Dockerfile.bmi-forcings`. Following NOAA-OWP/WRES conventions, use `--format docker` to ensure standard OCI/Docker compatibility: + +```bash +podman build \ + --ulimit nofile=65535:65535 \ + --format docker \ + -f Dockerfile.bmi-forcings \ + -t local/ngen-bmi-forcing:podman-test \ + . +``` + +*Note: The Dockerfile uses BuildKit syntax (`--mount=type=cache`) for pip caching. Modern Podman (via Buildah $\ge$ 1.24) natively resolves cache mounts locally without requiring a Docker daemon. The `--ulimit nofile=65535:65535` flag ensures sufficient file descriptors are available during package installations.* + +--- + +## Smoke Verification + +### 1. Test Entrypoint & Bash Execution +```bash +podman run --rm local/ngen-bmi-forcing:podman-test -c "echo 'Entrypoint functional' && /bin/bash --version" +``` + +### 2. Verify Python Environment & Dependencies +Test that `NextGen_Forcings_Engine` and `ewts` are installed and importable: +```bash +podman run --rm local/ngen-bmi-forcing:podman-test -c "python --version && python -c 'import NextGen_Forcings_Engine, ewts; print(\"Packages healthy:\", NextGen_Forcings_Engine.__version__)'" +``` + +### 3. Verify Provenance Metadata +```bash +podman run --rm local/ngen-bmi-forcing:podman-test -c "cat /ngen-app/ngen-bmi-forcing_git_info.json && test -s /ngen-app/ngen-bmi-forcing_git_info.json" +``` + +--- + +## CI / Automation + +* **Workflow:** `.github/workflows/podman-smoke.yml` +* **Triggers:** Manual (`workflow_dispatch`) and automated checks on pull requests modifying container/source files (`Dockerfile.bmi-forcings`, `pyproject.toml`, `NextGen_Forcings_Engine_BMI/**`, `Forcing_Extraction_Scripts/**`, `ESMF_Mesh_Domain_Configuration_Production/**`). +* **Runner Environment:** Pinned to `ubuntu-24.04`. +* **Registry Policy:** By default, builds remain local to the runner. When `push_images=true` is dispatched, only `:podman-test` and `:-podman-test` tags are published to GHCR under `ghcr.io//ngen-bmi-forcing`. Production aliases (`:latest`, release tags) are never touched. From c37fd7a8ad89328ec1428254470e7396914a3199 Mon Sep 17 00:00:00 2001 From: Kevin Spurrier Date: Mon, 28 Sep 2026 17:27:35 -0500 Subject: [PATCH 2/2] Podman / CI Reference: add two-tier Podman support for dependencies and bmi-forcing images Readme times based on spot checks from existing docker builds Dockerfile.dependencies: 47 minutes and 27 seconds Dockerfile.bmi-forcings: 2 minutes and 33 seconds --- .github/workflows/podman-smoke.yml | 93 +++++++++++++++++++++++++++--- PODMAN.md | 84 +++++++++++++++++++++++---- 2 files changed, 158 insertions(+), 19 deletions(-) diff --git a/.github/workflows/podman-smoke.yml b/.github/workflows/podman-smoke.yml index 48ccc440..400e190d 100644 --- a/.github/workflows/podman-smoke.yml +++ b/.github/workflows/podman-smoke.yml @@ -1,6 +1,6 @@ # .github/workflows/podman-smoke.yml # ADDITIVE WORKFLOW: Proves Podman build and smoke verification for ngen-forcing. -# Focuses on Dockerfile.bmi-forcings (primary CI/CD and downstream ngen dependency). +# Supports both Dockerfile.bmi-forcings (fast-path) and Dockerfile.dependencies (compiled stack). # Leaves existing .github/workflows/cicd.yml and release paths untouched. name: Podman Build & Smoke (Additive) @@ -13,12 +13,18 @@ on: required: true type: boolean default: false + rebuild_dependencies: + description: "Rebuild Dockerfile.dependencies with Podman (~45-60m)" + required: false + type: boolean + default: false pull_request: branches: - development paths: - '.github/workflows/podman-smoke.yml' - 'Dockerfile.bmi-forcings' + - 'Dockerfile.dependencies' - 'pyproject.toml' - 'NextGen_Forcings_Engine_BMI/**' - 'Forcing_Extraction_Scripts/**' @@ -30,9 +36,9 @@ permissions: jobs: podman-build-smoke: - name: Podman Build & Smoke (BMI Forcing) + name: Podman Build & Smoke (Forcing & Deps) runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 90 steps: - name: Checkout Repository @@ -45,22 +51,93 @@ jobs: podman version podman info --format 'OS: {{.Host.Distribution.Distribution}} {{.Host.Distribution.Version}} | Storage: {{.Store.GraphDriverName}}' - - name: Log in to GitHub Container Registry (for base image pull) + - name: Determine Dependencies Rebuild Strategy + id: strategy + run: | + set -euo pipefail + REBUILD_DEPS="false" + + if [ "${{ inputs.rebuild_dependencies }}" = "true" ]; then + echo "Manual workflow_dispatch requested dependencies rebuild." + REBUILD_DEPS="true" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + # Check if this PR touched Dockerfile.dependencies + if git diff --name-only "origin/${{ github.base_ref }}"...HEAD | grep -q "^Dockerfile.dependencies"; then + echo "Changes to Dockerfile.dependencies detected in PR diff; triggering dependencies rebuild." + REBUILD_DEPS="true" + else + echo "No changes to Dockerfile.dependencies in PR diff; using fast path with published base." + fi + fi + + echo "REBUILD_DEPS=${REBUILD_DEPS}" >> "${GITHUB_ENV}" + echo "rebuild_deps=${REBUILD_DEPS}" >> "${GITHUB_OUTPUT}" + + - name: Log in to GitHub Container Registry run: | echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io \ -u "${{ github.actor }}" --password-stdin + # ------------------------------------------------------------------------ + # Conditional Tier 1: Build Dockerfile.dependencies with Podman + # ------------------------------------------------------------------------ + - name: Build Dependencies Image with Podman (Docker Format) + if: env.REBUILD_DEPS == 'true' + id: build-deps + run: | + set -euo pipefail + ulimit -n 65535 || true + DEPS_IMAGE_TAG="localhost/ngen-dependencies-bookworm:podman-test" + + echo "Building ${DEPS_IMAGE_TAG} using Dockerfile.dependencies with --format docker..." + podman build \ + --ulimit nofile=65535:65535 \ + --format docker \ + -f Dockerfile.dependencies \ + -t "${DEPS_IMAGE_TAG}" \ + . + + echo "DEPS_IMAGE_LOCAL=${DEPS_IMAGE_TAG}" >> "${GITHUB_ENV}" + + - name: Smoke Probe Deps - WGRIB2, ESMF, GDAL & Boost Headers + if: env.REBUILD_DEPS == 'true' + run: | + set -euo pipefail + echo "Verifying wgrib2 utility..." + podman run --rm "${DEPS_IMAGE_LOCAL}" wgrib2 -v + echo "Verifying Python ESMF/ESMPy module..." + podman run --rm "${DEPS_IMAGE_LOCAL}" python -c "import ESMF; print('ESMF Version:', ESMF.__version__)" + echo "Verifying GDAL utility..." + podman run --rm "${DEPS_IMAGE_LOCAL}" gdalinfo --version + echo "Verifying Boost headers in /opt/boost..." + podman run --rm "${DEPS_IMAGE_LOCAL}" test -d /opt/boost + echo "Dependencies Probe Passed: wgrib2, ESMF, GDAL, and Boost verified." + + - name: Resolve Target Dependencies Base Image + run: | + if [ "${REBUILD_DEPS}" = "true" ]; then + TARGET_DEPS="${DEPS_IMAGE_LOCAL}" + echo "Using freshly built local dependencies image: ${TARGET_DEPS}" + else + TARGET_DEPS="ghcr.io/ngwpc/ngen-dependencies-bookworm:latest" + echo "Using published base dependencies image: ${TARGET_DEPS}" + fi + echo "DEPS_BASE_IMAGE=${TARGET_DEPS}" >> "${GITHUB_ENV}" + + # ------------------------------------------------------------------------ + # Tier 2: Build Dockerfile.bmi-forcings with Podman + # ------------------------------------------------------------------------ - name: Build BMI Forcing Image with Podman (Docker Format) - id: build + id: build-forcing run: | set -euo pipefail ulimit -n 65535 || true IMAGE_TAG="localhost/ngen-bmi-forcing:podman-test" - echo "Building ${IMAGE_TAG} using Dockerfile.bmi-forcings with --format docker..." + echo "Building ${IMAGE_TAG} using Dockerfile.bmi-forcings with base ${DEPS_BASE_IMAGE}..." podman build \ --ulimit nofile=65535:65535 \ - --build-arg DEPS_IMAGE="ghcr.io/ngwpc/ngen-dependencies-bookworm:latest" \ + --build-arg DEPS_IMAGE="${DEPS_BASE_IMAGE}" \ --build-arg CI_COMMIT_REF_NAME="${GITHUB_REF_NAME}" \ --format docker \ -f Dockerfile.bmi-forcings \ @@ -110,4 +187,4 @@ jobs: podman push "${TAG_SHA}" echo "Pushed ${TAG_TEST} and ${TAG_SHA}" - echo "NOTICE: Production aliases (:latest, branch tags) remain untouched." + echo "NOTICE: Production aliases (:latest, release tags) remain untouched." diff --git a/PODMAN.md b/PODMAN.md index bbbcdb4d..3fe239e0 100644 --- a/PODMAN.md +++ b/PODMAN.md @@ -2,7 +2,22 @@ This repository maintains Docker as its primary documented and production CI path. Podman is supported as an **additional** option for local development, rootless execution, and container build/smoke verification. -Existing Docker workflows (`.github/workflows/cicd.yml`) and production release tags remain untouched and active. +Existing Docker workflows (`.github/workflows/cicd.yml`, `.github/workflows/dependencies.yml`) and production release tags remain untouched and active. + +--- + +## Container Architecture: Two-Tier Build + +`ngen-forcing` maintains two distinct Dockerfiles to optimize developer feedback loops: + +1. **`Dockerfile.dependencies` (Base Image — ~45–60 min build):** + * Compiles large, slow-moving C, C++, and Fortran dependencies from source: **Boost 1.86.0**, **ESMF 8.8.0 + ESMPy**, **ecFlow 5.15.2**, and **WGRIB2**, along with system NetCDF, HDF5, and GDAL packages. + * This base image rarely changes and is published as `ghcr.io/ngwpc/ngen-dependencies-bookworm:latest`. + +2. **`Dockerfile.bmi-forcings` (Application Image — ~2 min build):** + * Inherits directly `FROM` the compiled dependencies image. + * Installs the `NextGen_Forcings_Engine_BMI` Python package and EWTS. + * This is what developers build and test during day-to-day work. --- @@ -21,7 +36,7 @@ For Ubuntu 24.04+ (Noble) or RHEL 8/9, Podman 4.9+ is recommended. ## Registry Authentication -`Dockerfile.bmi-forcings` builds from `ghcr.io/ngwpc/ngen-dependencies-bookworm:latest`. Ensure you are logged in to GHCR before building: +When using the published base image, `Dockerfile.bmi-forcings` pulls `ghcr.io/ngwpc/ngen-dependencies-bookworm:latest`. Ensure you are logged into GHCR: ```bash echo "" | podman login ghcr.io -u "" --password-stdin @@ -31,9 +46,9 @@ echo "" | podman login ghcr.io -u "" --pas ## Building with Podman -### NextGen BMI Forcing (`Dockerfile.bmi-forcings`) +### 1. Building the Application Image (`Dockerfile.bmi-forcings`) — *Standard Fast Path* -Build the image directly using `Dockerfile.bmi-forcings`. Following NOAA-OWP/WRES conventions, use `--format docker` to ensure standard OCI/Docker compatibility: +This is the standard command for daily development: ```bash podman build \ @@ -44,33 +59,80 @@ podman build \ . ``` -*Note: The Dockerfile uses BuildKit syntax (`--mount=type=cache`) for pip caching. Modern Podman (via Buildah $\ge$ 1.24) natively resolves cache mounts locally without requiring a Docker daemon. The `--ulimit nofile=65535:65535` flag ensures sufficient file descriptors are available during package installations.* +### 2. Building the Dependencies Base (`Dockerfile.dependencies`) — *Infrequent* + +When updating Boost, ESMF, or system libraries, build the base image locally: + +```bash +podman build \ + --ulimit nofile=65535:65535 \ + --format docker \ + -f Dockerfile.dependencies \ + -t local/ngen-dependencies-bookworm:podman-test \ + . +``` + +To build `Dockerfile.bmi-forcings` on top of your freshly built local base image: + +```bash +podman build \ + --ulimit nofile=65535:65535 \ + --build-arg DEPS_IMAGE=local/ngen-dependencies-bookworm:podman-test \ + --format docker \ + -f Dockerfile.bmi-forcings \ + -t local/ngen-bmi-forcing:podman-test \ + . +``` + +*Note: The Dockerfiles use BuildKit syntax (`--mount=type=cache`) for apt and pip caching. Modern Podman (via Buildah $\ge$ 1.24) natively resolves cache mounts locally without requiring a Docker daemon. The `--ulimit nofile=65535:65535` flag ensures sufficient file descriptors are available during heavy compilation.* --- ## Smoke Verification -### 1. Test Entrypoint & Bash Execution +### Application Image (`Dockerfile.bmi-forcings`) + +#### 1. Test Entrypoint & Bash Execution ```bash podman run --rm local/ngen-bmi-forcing:podman-test -c "echo 'Entrypoint functional' && /bin/bash --version" ``` -### 2. Verify Python Environment & Dependencies -Test that `NextGen_Forcings_Engine` and `ewts` are installed and importable: +#### 2. Verify Python Environment & Dependencies ```bash podman run --rm local/ngen-bmi-forcing:podman-test -c "python --version && python -c 'import NextGen_Forcings_Engine, ewts; print(\"Packages healthy:\", NextGen_Forcings_Engine.__version__)'" ``` -### 3. Verify Provenance Metadata +#### 3. Verify Provenance Metadata ```bash podman run --rm local/ngen-bmi-forcing:podman-test -c "cat /ngen-app/ngen-bmi-forcing_git_info.json && test -s /ngen-app/ngen-bmi-forcing_git_info.json" ``` +### Dependencies Image (`Dockerfile.dependencies`) + +```bash +# Verify wgrib2 binary +podman run --rm local/ngen-dependencies-bookworm:podman-test wgrib2 -v + +# Verify ESMF/ESMPy Python module +podman run --rm local/ngen-dependencies-bookworm:podman-test python -c "import ESMF; print('ESMF Version:', ESMF.__version__)" + +# Verify GDAL binary +podman run --rm local/ngen-dependencies-bookworm:podman-test gdalinfo --version + +# Verify Boost header installation +podman run --rm local/ngen-dependencies-bookworm:podman-test test -d /opt/boost +``` + --- ## CI / Automation * **Workflow:** `.github/workflows/podman-smoke.yml` -* **Triggers:** Manual (`workflow_dispatch`) and automated checks on pull requests modifying container/source files (`Dockerfile.bmi-forcings`, `pyproject.toml`, `NextGen_Forcings_Engine_BMI/**`, `Forcing_Extraction_Scripts/**`, `ESMF_Mesh_Domain_Configuration_Production/**`). +* **Triggers:** + * Automated checks on pull requests modifying container or source files (`Dockerfile.bmi-forcings`, `Dockerfile.dependencies`, `pyproject.toml`, `NextGen_Forcings_Engine_BMI/**`, `Forcing_Extraction_Scripts/**`, `ESMF_Mesh_Domain_Configuration_Production/**`). + * Manual triggers via `workflow_dispatch` (with optional `rebuild_dependencies: true`). +* **Smart Dependencies Strategy:** + * On normal PRs (only modifying Python files / `Dockerfile.bmi-forcings`), CI takes the **fast path (~2 mins)** using the prebuilt `ghcr.io/ngwpc/ngen-dependencies-bookworm:latest` base. + * If a PR modifies `Dockerfile.dependencies` (or `rebuild_dependencies: true` is dispatched), CI automatically executes the **dependencies build & smoke probe** first, and feeds that fresh local base directly into the BMI forcing build. * **Runner Environment:** Pinned to `ubuntu-24.04`. -* **Registry Policy:** By default, builds remain local to the runner. When `push_images=true` is dispatched, only `:podman-test` and `:-podman-test` tags are published to GHCR under `ghcr.io//ngen-bmi-forcing`. Production aliases (`:latest`, release tags) are never touched. +* **Registry Policy:** By default, builds remain local to the runner. When `push_images=true` is dispatched, only `:podman-test` and `:-podman-test` tags are published to GHCR. Production aliases (`:latest`, branch tags) are never touched.