release.yml

name: Release

on:
  push:
    tags:
      - "rel-*"
  # Manual dispatch is always a rehearsal: it builds, verifies, and signs, but never
  # publishes to PyPI or a GitHub Release.  Both are gated on a `rel-` tag push and nothing
  # else, so there is no path to cutting a release from an arbitrary ref.  Choose the ref to
  # rehearse against with the branch/tag selector in the Actions UI.  The one thing a
  # dispatch can publish is the rehearsal build to test.pypi.org, and only when the
  # `testpypi` input is set (#76).
  workflow_dispatch:
    inputs:
      testpypi:
        description: "Also upload the rehearsal build to test.pypi.org (a sandbox; versions there cannot be re-uploaded either)"
        type: boolean
        default: false

# A publish must never be cancelled halfway through, so unlike CI this does not set
# cancel-in-progress.
concurrency:
  group: release-${{ github.ref }}

permissions:
  contents: read

jobs:
  build:
    name: Build and sign
    runs-on: ubuntu-latest
    permissions:
      contents: read
      # Required for keyless Sigstore signing: the OIDC token is what binds the signature
      # to this workflow's identity.
      id-token: write

    steps:
      - name: Checkout
        # Actions in this workflow are pinned to commit SHAs rather than floating major
        # tags: this pipeline holds an OIDC signing token and write access to releases, so
        # a compromised upstream tag here would be able to sign and publish artifacts.
        # Dependabot (.github/dependabot.yml) keeps the pins current.
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          # setuptools-scm derives the version from the tag, so full history and tags are
          # required.  Without this the release would be versioned 0.1.dev1.
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Derive and validate version
        id: version
        run: |
          set -euo pipefail
          TAG="${GITHUB_REF_NAME}"
          if [[ "${{ github.event_name }}" == "push" ]]; then
            if [[ "$TAG" != rel-* ]]; then
              echo "::error::Tag does not start with 'rel-': $TAG"
              exit 1
            fi
            VERSION="${TAG#rel-}"
            # Guard against a typo'd tag (rel-1.15, rel-v1.15.0) producing a bad release.
            # Anchored at both ends deliberately: the version check below is a plain string
            # compare against the wheel's version, and setuptools-scm emits PEP 440
            # normalized versions.  A tag like `rel-1.15.0-rc1` would normalize to
            # `1.15.0rc1` and fail that compare with a confusing message, so reject the
            # unsupported shape here where the error can say what is actually wrong.
            if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
              echo "::error::Tag '$TAG' must be rel-X.Y.Z with no suffix (got '$VERSION')"
              exit 1
            fi
            echo "tag=$TAG" >> "$GITHUB_OUTPUT"
            echo "version=$VERSION" >> "$GITHUB_OUTPUT"
            echo "Releasing version $VERSION from tag $TAG"
          else
            echo "Manual dispatch; version comes from setuptools-scm."
          fi

      - name: Verify release notes
        id: notes
        # Checked here, before the build, rather than left to action-gh-release: that action
        # fails on a missing body_path only after everything has been built, signed, and
        # uploaded, which reads as a late and unrelated failure.  A rehearsal (workflow_dispatch)
        # has no rel- tag and so no notes to look for, hence the push-only guard.
        # Every release from 1.16.0 on ships notes; see CLAUDE.md "Cutting a release".
        #
        # The notes file is the entire release body, so it must carry its own verification
        # section with this tag's signing identity.  CI's quality job runs the same checker
        # on every notes file at PR time; this is the backstop for a tag pushed anyway.
        if: github.event_name == 'push'
        run: |
          set -euo pipefail
          NOTES="doc/release-notes/${GITHUB_REF_NAME}.md"
          if [[ ! -f "$NOTES" ]]; then
            echo "::error::Release notes not found at $NOTES."
            echo "::error::Add the notes for ${{ steps.version.outputs.version }} and retag once they are on the tagged commit."
            exit 1
          fi
          python .github/scripts/check-release-notes.py "$NOTES"
          echo "notes=$NOTES" >> "$GITHUB_OUTPUT"

      - name: Build wheel and sdist
        run: |
          set -euo pipefail
          # A rehearsal builds an untagged commit, which setuptools-scm versions with a local
          # segment (1.16.1.dev61+g496f0e0).  PyPI and TestPyPI both reject local versions, so
          # a dispatch drops it (1.16.1.dev61) to stay uploadable to TestPyPI.  A tag build has
          # no local segment and is left alone.
          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
            export SETUPTOOLS_SCM_OVERRIDES_FOR_DP_PYTHON_LIB='{local_scheme="no-local-version"}'
          fi
          python -m pip install --upgrade pip build twine
          python -m build

      - name: Verify built version matches the tag
        # setuptools-scm derives the version independently from git.  If it disagrees with the
        # tag, something is wrong with the checkout (e.g. missing tags) and the artifacts would
        # be mislabelled -- fail loudly rather than publishing them.
        if: github.event_name == 'push'
        run: |
          set -euo pipefail
          EXPECTED="${{ steps.version.outputs.version }}"
          BUILT=$(ls dist/*.whl | head -1 | sed -E 's|.*/dp_python_lib-([^-]+)-py3.*|\1|')
          echo "tag version:   $EXPECTED"
          echo "built version: $BUILT"
          if [[ "$BUILT" != "$EXPECTED" ]]; then
            echo "::error::Built version '$BUILT' does not match tag version '$EXPECTED'."
            exit 1
          fi

      - name: Verify distribution metadata
        run: twine check --strict dist/*

      - name: Verify the wheel installs and imports cleanly
        run: |
          set -euo pipefail
          python -m venv /tmp/wheel-check
          /tmp/wheel-check/bin/pip install dist/*.whl
          /tmp/wheel-check/bin/python -c "
          from dp_python_lib.client import MldpClient, QueryParams
          from importlib.metadata import version
          print('installed version:', version('dp-python-lib'))
          "

      - name: Generate SHA256SUMS
        # One checksums file in the standard format, so consumers can verify every artifact
        # with a single `sha256sum -c SHA256SUMS`.
        working-directory: dist
        run: |
          set -euo pipefail
          sha256sum dp_python_lib-*.whl dp_python_lib-*.tar.gz > SHA256SUMS
          cat SHA256SUMS

      - name: Sign artifacts with Sigstore
        # Keyless signing: no key material to store or rotate.  A checksum published in the
        # same release as the artifact proves integrity but not authenticity -- anyone able to
        # rewrite the release can rewrite both files.  The Sigstore bundle is what ties these
        # artifacts to this workflow's identity.
        uses: sigstore/gh-action-sigstore-python@790bc6befb9d733738f18d8f895854b453640ec9 # v3.5.0
        with:
          inputs: ./dist/*.whl ./dist/*.tar.gz ./dist/SHA256SUMS

      - name: Stage the release notes
        # The notes file is the release body, verbatim -- nothing is added to it here.  That is
        # what makes `gh release edit <tag> --notes-file doc/release-notes/<tag>.md` a lossless
        # republish: anything a workflow appended would be exactly what such an edit drops, which
        # is how rel-1.16.0's page lost its verification instructions (#56).  The verification
        # section is therefore hand-written in each notes file and checked above.
        #
        # The publish job downloads artifacts but never checks out the repo, so the notes travel
        # with the dist/ upload.  Staged after signing, so they are not signed as an artifact.
        if: github.event_name == 'push'
        env:
          NOTES: ${{ steps.notes.outputs.notes }}
        run: cp "$NOTES" dist/RELEASE_NOTES.md

      - name: Upload build outputs
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: release-dist
          path: dist/
          # GitHub holds a deployment awaiting approval for up to 30 days, and publish-pypi
          # needs these exact files: a full re-run would rebuild and re-sign, and the new files
          # would not match the SHA256SUMS and bundles already on the GitHub Release.  So the
          # artifact outlives the approval window (#76, plan D2), with margin: its clock starts
          # here, the approval's only once publish-github-release finishes, and a late "Re-run
          # failed jobs" needs it too.  A repository or org retention maximum below this caps it
          # silently; this repository's maximum was 90 days when this was set.
          retention-days: 35

  publish-github-release:
    name: Publish GitHub Release
    needs: build
    runs-on: ubuntu-latest
    # Publishing happens only on a `rel-` tag push.  A workflow_dispatch run is therefore
    # always a rehearsal: it builds, verifies, and signs, but stops here.
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/rel-')
    permissions:
      contents: write

    steps:
      - name: Download build outputs
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: release-dist
          path: dist

      - name: List artifacts to be published
        run: ls -la dist/

      - name: Publish GitHub Release
        uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
        with:
          files: |
            dist/*.whl
            dist/*.tar.gz
            dist/SHA256SUMS
            dist/*.sigstore.json
          fail_on_unmatched_files: true
          # doc/release-notes/<tag>.md, verbatim.  RELEASE_NOTES.md is deliberately absent
          # from `files` above -- it is the release body, not an artifact to download.
          #
          # generate_release_notes is deliberately NOT set: GitHub's commit list would be the
          # one part of the body not in the file, and so the part a republish from the file
          # silently drops.  The notes carry a hand-written Full Changelog link instead.
          body_path: dist/RELEASE_NOTES.md

      - name: Verify the published body matches the notes
        # Closes the loop for the one path this workflow controls: if the action ever resolves
        # a different body (a changed default, a pre-existing release it declines to update),
        # the release is flagged here instead of being noticed by a reader.  Later hand edits of
        # the page are out of reach of any workflow; CLAUDE.md forbids them instead.
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
          TAG: ${{ github.ref_name }}
        run: |
          set -euo pipefail
          gh release view "$TAG" --json body --jq .body > published.md
          # GitHub does not preserve the file's trailing newline, so both sides go through
          # "$(cat ...)", which strips trailing newlines; CRLF line ends are normalized too.
          if ! diff -u --strip-trailing-cr <(printf '%s\n' "$(cat dist/RELEASE_NOTES.md)") \
                                           <(printf '%s\n' "$(cat published.md)"); then
            echo "::error::The published release body differs from doc/release-notes/${TAG}.md."
            exit 1
          fi
          echo "Published body matches doc/release-notes/${TAG}.md."

  # ---------------------------------------------------------------------------------------
  # PyPI publishing (#76; plan/tickets/76/plan.md).
  #
  # Trusted Publishing (OIDC): no API token is stored anywhere.  The publisher on pypi.org
  # names this repository, workflow release.yml, and environment `pypi`; the environment
  # requires an approval and admits only rel-* tags.  The environment must exist BEFORE any
  # job names it: GitHub creates a missing one on the fly, with no protection rules.
  #
  # publish-pypi runs last, after the GitHub Release is published and its body verified,
  # because a PyPI file can never be replaced.  It uploads the same files the build job
  # signed, then checks that the index serves them byte for byte (check-index-digests.py).
  #
  # PyPI rejects Sigstore bundles as uploads, hence the cleanup step -- the bundles still go
  # to the GitHub Release.  attestations: true emits PEP 740 attestations instead.
  #
  # publish-testpypi is the rehearsal of the same path, run by dispatch with testpypi=true,
  # through its own publisher and an unprotected `testpypi` environment.  The two jobs'
  # steps are duplicated on purpose; keep them in step.
  # ---------------------------------------------------------------------------------------
  publish-pypi:
    name: Publish to PyPI
    needs: [build, publish-github-release]
    runs-on: ubuntu-latest
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/rel-')
    environment:
      name: pypi
      url: https://pypi.org/p/dp-python-lib
    permissions:
      contents: read
      id-token: write

    steps:
      - name: Check out the digest checker
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          sparse-checkout: .github/scripts
          persist-credentials: false

      - name: Download build outputs
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: release-dist
          path: dist

      - name: Remove non-distribution files
        # Only sdists and wheels may be uploaded; the checksums file, the release notes, and
        # the Sigstore bundles would all be rejected.  SHA256SUMS is kept aside for the check.
        run: |
          mv dist/SHA256SUMS SHA256SUMS
          rm -f dist/RELEASE_NOTES.md dist/*.sigstore.json

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
        with:
          attestations: true
          # Files go up one at a time, so a wheel can land and the sdist fail; without this,
          # every re-run would then fail on the wheel.  A re-run uploads the same artifact, and
          # the digest check below compares skipped files too, so a real conflict still fails
          # (plan D3a).
          skip-existing: true

      - name: Verify PyPI serves the signed files
        run: python3 .github/scripts/check-index-digests.py SHA256SUMS --index-url https://pypi.org --project dp-python-lib

  publish-testpypi:
    name: Publish rehearsal to TestPyPI
    needs: build
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && inputs.testpypi
    environment:
      name: testpypi
      url: https://test.pypi.org/p/dp-python-lib
    permissions:
      contents: read
      id-token: write

    steps:
      - name: Check out the digest checker
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          sparse-checkout: .github/scripts
          persist-credentials: false

      - name: Download build outputs
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          name: release-dist
          path: dist

      - name: Remove non-distribution files
        # As in publish-pypi.  A dispatch stages no RELEASE_NOTES.md; rm -f covers that.
        run: |
          mv dist/SHA256SUMS SHA256SUMS
          rm -f dist/RELEASE_NOTES.md dist/*.sigstore.json

      - name: Publish to TestPyPI
        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
        with:
          repository-url: https://test.pypi.org/legacy/
          attestations: true
          # As in publish-pypi.  A .devN version counts commits since the last tag, so two
          # different commits can share one; the second rehearsal's upload is then skipped,
          # and the digest check below fails on the mismatch, as it should.
          skip-existing: true

      - name: Verify TestPyPI serves the signed files
        run: python3 .github/scripts/check-index-digests.py SHA256SUMS --index-url https://test.pypi.org --project dp-python-lib