diff --git a/.github/workflows/podman-smoke.yml b/.github/workflows/podman-smoke.yml new file mode 100644 index 00000000..400e190d --- /dev/null +++ b/.github/workflows/podman-smoke.yml @@ -0,0 +1,190 @@ +# .github/workflows/podman-smoke.yml +# ADDITIVE WORKFLOW: Proves Podman build and smoke verification for ngen-forcing. +# Supports both Dockerfile.bmi-forcings (fast-path) and Dockerfile.dependencies (compiled stack). +# Leaves existing .github/workflows/cicd.yml and release paths untouched. + +name: Podman Build & Smoke (Additive) + +on: + workflow_dispatch: + inputs: + push_images: + description: "Push test tags (:podman-test) to GHCR" + required: true + type: boolean + default: false + rebuild_dependencies: + description: "Rebuild Dockerfile.dependencies with Podman (~45-60m)" + required: false + type: boolean + default: false + pull_request: + branches: + - development + paths: + - '.github/workflows/podman-smoke.yml' + - 'Dockerfile.bmi-forcings' + - 'Dockerfile.dependencies' + - 'pyproject.toml' + - 'NextGen_Forcings_Engine_BMI/**' + - 'Forcing_Extraction_Scripts/**' + - 'ESMF_Mesh_Domain_Configuration_Production/**' + +permissions: + contents: read + packages: write + +jobs: + podman-build-smoke: + name: Podman Build & Smoke (Forcing & Deps) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Verify Preinstalled Podman Environment + run: | + podman version + podman info --format 'OS: {{.Host.Distribution.Distribution}} {{.Host.Distribution.Version}} | Storage: {{.Store.GraphDriverName}}' + + - name: Determine Dependencies Rebuild Strategy + id: strategy + run: | + set -euo pipefail + REBUILD_DEPS="false" + + if [ "${{ inputs.rebuild_dependencies }}" = "true" ]; then + echo "Manual workflow_dispatch requested dependencies rebuild." + REBUILD_DEPS="true" + elif [ "${{ github.event_name }}" = "pull_request" ]; then + # Check if this PR touched Dockerfile.dependencies + if git diff --name-only "origin/${{ github.base_ref }}"...HEAD | grep -q "^Dockerfile.dependencies"; then + echo "Changes to Dockerfile.dependencies detected in PR diff; triggering dependencies rebuild." + REBUILD_DEPS="true" + else + echo "No changes to Dockerfile.dependencies in PR diff; using fast path with published base." + fi + fi + + echo "REBUILD_DEPS=${REBUILD_DEPS}" >> "${GITHUB_ENV}" + echo "rebuild_deps=${REBUILD_DEPS}" >> "${GITHUB_OUTPUT}" + + - name: Log in to GitHub Container Registry + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io \ + -u "${{ github.actor }}" --password-stdin + + # ------------------------------------------------------------------------ + # Conditional Tier 1: Build Dockerfile.dependencies with Podman + # ------------------------------------------------------------------------ + - name: Build Dependencies Image with Podman (Docker Format) + if: env.REBUILD_DEPS == 'true' + id: build-deps + run: | + set -euo pipefail + ulimit -n 65535 || true + DEPS_IMAGE_TAG="localhost/ngen-dependencies-bookworm:podman-test" + + echo "Building ${DEPS_IMAGE_TAG} using Dockerfile.dependencies with --format docker..." + podman build \ + --ulimit nofile=65535:65535 \ + --format docker \ + -f Dockerfile.dependencies \ + -t "${DEPS_IMAGE_TAG}" \ + . + + echo "DEPS_IMAGE_LOCAL=${DEPS_IMAGE_TAG}" >> "${GITHUB_ENV}" + + - name: Smoke Probe Deps - WGRIB2, ESMF, GDAL & Boost Headers + if: env.REBUILD_DEPS == 'true' + run: | + set -euo pipefail + echo "Verifying wgrib2 utility..." + podman run --rm "${DEPS_IMAGE_LOCAL}" wgrib2 -v + echo "Verifying Python ESMF/ESMPy module..." + podman run --rm "${DEPS_IMAGE_LOCAL}" python -c "import ESMF; print('ESMF Version:', ESMF.__version__)" + echo "Verifying GDAL utility..." + podman run --rm "${DEPS_IMAGE_LOCAL}" gdalinfo --version + echo "Verifying Boost headers in /opt/boost..." + podman run --rm "${DEPS_IMAGE_LOCAL}" test -d /opt/boost + echo "Dependencies Probe Passed: wgrib2, ESMF, GDAL, and Boost verified." + + - name: Resolve Target Dependencies Base Image + run: | + if [ "${REBUILD_DEPS}" = "true" ]; then + TARGET_DEPS="${DEPS_IMAGE_LOCAL}" + echo "Using freshly built local dependencies image: ${TARGET_DEPS}" + else + TARGET_DEPS="ghcr.io/ngwpc/ngen-dependencies-bookworm:latest" + echo "Using published base dependencies image: ${TARGET_DEPS}" + fi + echo "DEPS_BASE_IMAGE=${TARGET_DEPS}" >> "${GITHUB_ENV}" + + # ------------------------------------------------------------------------ + # Tier 2: Build Dockerfile.bmi-forcings with Podman + # ------------------------------------------------------------------------ + - name: Build BMI Forcing Image with Podman (Docker Format) + id: build-forcing + run: | + set -euo pipefail + ulimit -n 65535 || true + IMAGE_TAG="localhost/ngen-bmi-forcing:podman-test" + + echo "Building ${IMAGE_TAG} using Dockerfile.bmi-forcings with base ${DEPS_BASE_IMAGE}..." + podman build \ + --ulimit nofile=65535:65535 \ + --build-arg DEPS_IMAGE="${DEPS_BASE_IMAGE}" \ + --build-arg CI_COMMIT_REF_NAME="${GITHUB_REF_NAME}" \ + --format docker \ + -f Dockerfile.bmi-forcings \ + -t "${IMAGE_TAG}" \ + . + + echo "IMAGE_LOCAL=${IMAGE_TAG}" >> "${GITHUB_ENV}" + + - name: Smoke Probe 1 - Entrypoint & Shell Check + run: | + set -euo pipefail + echo "Verifying entrypoint (/bin/bash) and shell execution..." + podman run --rm "${IMAGE_LOCAL}" -c "echo 'Probe 1 Passed: /bin/bash functional' && /bin/bash --version | head -n1" + + - name: Smoke Probe 2 - Python Environment & Package Import + run: | + set -euo pipefail + echo "Verifying Python packages (NextGen_Forcings_Engine, ewts)..." + podman run --rm "${IMAGE_LOCAL}" -c "python --version && python -c 'import NextGen_Forcings_Engine, ewts; print(\"Probe 2 Passed: NextGen_Forcings_Engine and ewts imported successfully. Version:\", NextGen_Forcings_Engine.__version__)'" + + - name: Smoke Probe 3 - Git Provenance Metadata Check + run: | + set -euo pipefail + echo "Verifying provenance metadata file existence and content..." + podman run --rm "${IMAGE_LOCAL}" -c "cat /ngen-app/ngen-bmi-forcing_git_info.json && test -s /ngen-app/ngen-bmi-forcing_git_info.json" + echo "Probe 3 Passed: Git provenance metadata file verified." + + - name: Inspect Built OCI Image + run: | + podman image inspect "${IMAGE_LOCAL}" \ + --format 'Image ID: {{.Id}} | Digest: {{.Digest}} | Entrypoint: {{.Config.Entrypoint}} | Cmd: {{.Config.Cmd}}' + + - name: Optional GHCR Push (:podman-test only) + if: ${{ inputs.push_images }} + run: | + set -euo pipefail + REPO_LOWER=$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]') + DEST_BASE="ghcr.io/${REPO_LOWER}/ngen-bmi-forcing" + TAG_TEST="${DEST_BASE}:podman-test" + TAG_SHA="${DEST_BASE}:${{ github.sha }}-podman-test" + + echo "Publishing test verification tags..." + podman tag "${IMAGE_LOCAL}" "${TAG_TEST}" + podman tag "${IMAGE_LOCAL}" "${TAG_SHA}" + + podman push "${TAG_TEST}" + podman push "${TAG_SHA}" + + echo "Pushed ${TAG_TEST} and ${TAG_SHA}" + echo "NOTICE: Production aliases (:latest, release tags) remain untouched." diff --git a/PODMAN.md b/PODMAN.md new file mode 100644 index 00000000..3fe239e0 --- /dev/null +++ b/PODMAN.md @@ -0,0 +1,138 @@ +# Podman Support (Additive Path) + +This repository maintains Docker as its primary documented and production CI path. Podman is supported as an **additional** option for local development, rootless execution, and container build/smoke verification. + +Existing Docker workflows (`.github/workflows/cicd.yml`, `.github/workflows/dependencies.yml`) and production release tags remain untouched and active. + +--- + +## Container Architecture: Two-Tier Build + +`ngen-forcing` maintains two distinct Dockerfiles to optimize developer feedback loops: + +1. **`Dockerfile.dependencies` (Base Image — ~45–60 min build):** + * Compiles large, slow-moving C, C++, and Fortran dependencies from source: **Boost 1.86.0**, **ESMF 8.8.0 + ESMPy**, **ecFlow 5.15.2**, and **WGRIB2**, along with system NetCDF, HDF5, and GDAL packages. + * This base image rarely changes and is published as `ghcr.io/ngwpc/ngen-dependencies-bookworm:latest`. + +2. **`Dockerfile.bmi-forcings` (Application Image — ~2 min build):** + * Inherits directly `FROM` the compiled dependencies image. + * Installs the `NextGen_Forcings_Engine_BMI` Python package and EWTS. + * This is what developers build and test during day-to-day work. + +--- + +## Prerequisites + +Verify that Podman is installed on your workstation: + +```bash +podman version +podman info +``` + +For Ubuntu 24.04+ (Noble) or RHEL 8/9, Podman 4.9+ is recommended. + +--- + +## Registry Authentication + +When using the published base image, `Dockerfile.bmi-forcings` pulls `ghcr.io/ngwpc/ngen-dependencies-bookworm:latest`. Ensure you are logged into GHCR: + +```bash +echo "" | podman login ghcr.io -u "" --password-stdin +``` + +--- + +## Building with Podman + +### 1. Building the Application Image (`Dockerfile.bmi-forcings`) — *Standard Fast Path* + +This is the standard command for daily development: + +```bash +podman build \ + --ulimit nofile=65535:65535 \ + --format docker \ + -f Dockerfile.bmi-forcings \ + -t local/ngen-bmi-forcing:podman-test \ + . +``` + +### 2. Building the Dependencies Base (`Dockerfile.dependencies`) — *Infrequent* + +When updating Boost, ESMF, or system libraries, build the base image locally: + +```bash +podman build \ + --ulimit nofile=65535:65535 \ + --format docker \ + -f Dockerfile.dependencies \ + -t local/ngen-dependencies-bookworm:podman-test \ + . +``` + +To build `Dockerfile.bmi-forcings` on top of your freshly built local base image: + +```bash +podman build \ + --ulimit nofile=65535:65535 \ + --build-arg DEPS_IMAGE=local/ngen-dependencies-bookworm:podman-test \ + --format docker \ + -f Dockerfile.bmi-forcings \ + -t local/ngen-bmi-forcing:podman-test \ + . +``` + +*Note: The Dockerfiles use BuildKit syntax (`--mount=type=cache`) for apt and pip caching. Modern Podman (via Buildah $\ge$ 1.24) natively resolves cache mounts locally without requiring a Docker daemon. The `--ulimit nofile=65535:65535` flag ensures sufficient file descriptors are available during heavy compilation.* + +--- + +## Smoke Verification + +### Application Image (`Dockerfile.bmi-forcings`) + +#### 1. Test Entrypoint & Bash Execution +```bash +podman run --rm local/ngen-bmi-forcing:podman-test -c "echo 'Entrypoint functional' && /bin/bash --version" +``` + +#### 2. Verify Python Environment & Dependencies +```bash +podman run --rm local/ngen-bmi-forcing:podman-test -c "python --version && python -c 'import NextGen_Forcings_Engine, ewts; print(\"Packages healthy:\", NextGen_Forcings_Engine.__version__)'" +``` + +#### 3. Verify Provenance Metadata +```bash +podman run --rm local/ngen-bmi-forcing:podman-test -c "cat /ngen-app/ngen-bmi-forcing_git_info.json && test -s /ngen-app/ngen-bmi-forcing_git_info.json" +``` + +### Dependencies Image (`Dockerfile.dependencies`) + +```bash +# Verify wgrib2 binary +podman run --rm local/ngen-dependencies-bookworm:podman-test wgrib2 -v + +# Verify ESMF/ESMPy Python module +podman run --rm local/ngen-dependencies-bookworm:podman-test python -c "import ESMF; print('ESMF Version:', ESMF.__version__)" + +# Verify GDAL binary +podman run --rm local/ngen-dependencies-bookworm:podman-test gdalinfo --version + +# Verify Boost header installation +podman run --rm local/ngen-dependencies-bookworm:podman-test test -d /opt/boost +``` + +--- + +## CI / Automation + +* **Workflow:** `.github/workflows/podman-smoke.yml` +* **Triggers:** + * Automated checks on pull requests modifying container or source files (`Dockerfile.bmi-forcings`, `Dockerfile.dependencies`, `pyproject.toml`, `NextGen_Forcings_Engine_BMI/**`, `Forcing_Extraction_Scripts/**`, `ESMF_Mesh_Domain_Configuration_Production/**`). + * Manual triggers via `workflow_dispatch` (with optional `rebuild_dependencies: true`). +* **Smart Dependencies Strategy:** + * On normal PRs (only modifying Python files / `Dockerfile.bmi-forcings`), CI takes the **fast path (~2 mins)** using the prebuilt `ghcr.io/ngwpc/ngen-dependencies-bookworm:latest` base. + * If a PR modifies `Dockerfile.dependencies` (or `rebuild_dependencies: true` is dispatched), CI automatically executes the **dependencies build & smoke probe** first, and feeds that fresh local base directly into the BMI forcing build. +* **Runner Environment:** Pinned to `ubuntu-24.04`. +* **Registry Policy:** By default, builds remain local to the runner. When `push_images=true` is dispatched, only `:podman-test` and `:-podman-test` tags are published to GHCR. Production aliases (`:latest`, branch tags) are never touched.