diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..4b7bedb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,555 @@ +name: 'Publish' +# Trusted publisher. Builds the tag's commit in a secret-free job, smoke-tests +# the wheels, runs Gemini-generated tests, then uploads to PyPI through the +# `pypi` environment's trusted publisher. Dispatched by release.yml, or by a +# human with write on the target repo. Repeat-safe: validate re-checks, +# the upload skips files already on PyPI, and post-publish checks digests. + +on: + workflow_dispatch: + inputs: + repository: + description: 'Which repo to publish (must be enabled in release/repos/).' + required: true + type: choice + options: + - 'keras' + - 'keras-hub' + - 'keras-rs' + - 'kinetic' + tag: + description: 'Tag of a published (non-draft) GitHub release, e.g. v0.33.0.' + required: true + type: string + override_generated_tests: + description: 'Upload even if the generated tests fail. The smoke install/import test always blocks.' + required: false + type: boolean + default: false + requested_by: + description: 'Set by the release panel. Ignored unless the release App dispatched this run.' + required: false + type: string + default: '' + +# Parsed by the release panel's same-branch run lock. +run-name: 'publish ${{ inputs.repository }} ${{ inputs.tag }}' + +permissions: + contents: 'read' + +# Inputs reach scripts only through the environment. +env: + PUBLISH_REPOSITORY: '${{ inputs.repository }}' + PUBLISH_TAG: '${{ inputs.tag }}' + PUBLISH_OVERRIDE_GENERATED_TESTS: '${{ inputs.override_generated_tests }}' + PUBLISH_REQUESTED_BY: '${{ inputs.requested_by }}' + +jobs: + validate: + # Holds the App key (env release-read, main only); runs no target code. + runs-on: 'ubuntu-latest' + environment: 'release-read' + permissions: + contents: 'read' + outputs: + sha: '${{ steps.validate.outputs.sha }}' + version: '${{ steps.validate.outputs.version }}' + steps: + - name: 'Check out shared-workflows' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: 'Resolve target repo' + id: 'target' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { loadConfig, targetRepo } = require('./scripts/release/lib/config.js'); + const config = loadConfig(process.env.PUBLISH_REPOSITORY, process.env.GITHUB_WORKSPACE); + const [owner, name] = targetRepo(config, context.repo.owner).split('/'); + core.setOutput('owner', owner); + core.setOutput('name', name); + + - name: 'Mint read-only App token for the target repo' + id: 'app-token' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: '${{ secrets.RELEASE_APP_CLIENT_ID }}' + private-key: '${{ secrets.RELEASE_APP_PRIVATE_KEY }}' + owner: '${{ steps.target.outputs.owner }}' + repositories: '${{ steps.target.outputs.name }}' + permission-metadata: 'read' + permission-contents: 'read' + + - name: 'Validate actor, tag, release and version files' + id: 'validate' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PUBLISH_TRIGGERING_ACTOR: '${{ github.triggering_actor }}' + PUBLISH_TARGET_OWNER: '${{ steps.target.outputs.owner }}' + PUBLISH_TARGET_NAME: '${{ steps.target.outputs.name }}' + RELEASE_APP_SLUG: '${{ vars.RELEASE_APP_SLUG }}' + with: + github-token: '${{ steps.app-token.outputs.token }}' + script: | + const { main } = require('./scripts/release/validate_publish.js'); + await main({ github, context, core, env: process.env }); + + resolve: + needs: 'validate' + runs-on: 'ubuntu-latest' + permissions: + contents: 'read' + outputs: + target: '${{ steps.config.outputs.target }}' + tag_prefix: '${{ steps.config.outputs.tag_prefix }}' + build_setup_command: '${{ steps.config.outputs.build_setup_command }}' + build_command: '${{ steps.config.outputs.build_command }}' + smoke_snippet: '${{ steps.config.outputs.smoke_snippet }}' + dist_dirs: '${{ steps.config.outputs.dist_dirs }}' + packages: '${{ steps.config.outputs.packages }}' + import_names: '${{ steps.config.outputs.import_names }}' + backends: '${{ steps.config.outputs.backends }}' + steps: + - name: 'Check out shared-workflows' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: 'Load repo config' + id: 'config' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { loadConfig, targetRepo } = require('./scripts/release/lib/config.js'); + const config = loadConfig(process.env.PUBLISH_REPOSITORY, process.env.GITHUB_WORKSPACE); + core.setOutput('target', targetRepo(config, context.repo.owner)); + for (const key of ['tag_prefix', 'build_setup_command', 'build_command', 'smoke_snippet']) { + core.setOutput(key, config[key] || ''); + } + for (const key of ['dist_dirs', 'packages', 'import_names', 'backends']) { + core.setOutput(key, JSON.stringify(config[key])); + } + + build: + # Runs target repo code: no secrets, no environment, read-only token. + needs: + - 'validate' + - 'resolve' + runs-on: 'ubuntu-latest' + permissions: + contents: 'read' + outputs: + digests: '${{ steps.digests.outputs.digests }}' + steps: + - name: 'Check out target at the validated commit' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: '${{ needs.resolve.outputs.target }}' + ref: '${{ needs.validate.outputs.sha }}' + fetch-depth: 1 + persist-credentials: false + + - name: 'Set up Python' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: 'Build setup' + env: + BUILD_SETUP_COMMAND: '${{ needs.resolve.outputs.build_setup_command }}' + run: | + if [ -n "$BUILD_SETUP_COMMAND" ]; then + bash -euo pipefail -c "$BUILD_SETUP_COMMAND" + fi + + - name: 'Build' + env: + BUILD_COMMAND: '${{ needs.resolve.outputs.build_command }}' + run: bash -euo pipefail -c "$BUILD_COMMAND" + + - name: 'Record sha256 digests and stage the dist artifact' + id: 'digests' + env: + DIST_DIRS: '${{ needs.resolve.outputs.dist_dirs }}' + shell: python + run: | + import hashlib, json, os, shutil + staging = os.path.join(os.environ["RUNNER_TEMP"], "dist-artifact") + digests = {} + for dist_dir in json.loads(os.environ["DIST_DIRS"]): + rel = dist_dir.strip("/") + os.makedirs(os.path.join(staging, rel), exist_ok=True) + for name in sorted(os.listdir(rel)): + src = os.path.join(rel, name) + if not os.path.isfile(src) or os.path.islink(src): + raise SystemExit(f"{src} is not a regular file") + shutil.copyfile(src, os.path.join(staging, rel, name)) + with open(src, "rb") as f: + digests[f"{rel}/{name}"] = hashlib.sha256(f.read()).hexdigest() + if not digests: + raise SystemExit("The build produced no files in dist_dirs") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write("digests=" + json.dumps(digests, sort_keys=True) + "\n") + print(json.dumps(digests, indent=2, sort_keys=True)) + + - name: 'Upload dist' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: 'dist' + path: '${{ runner.temp }}/dist-artifact' + if-no-files-found: 'error' + + smoke: + # No checkout: installs the built wheels into a fresh venv only. + needs: + - 'resolve' + - 'build' + runs-on: 'ubuntu-latest' + permissions: + contents: 'read' + strategy: + fail-fast: false + matrix: + backend: '${{ fromJSON(needs.resolve.outputs.backends) }}' + steps: + - name: 'Download dist' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: 'dist' + path: 'dist-artifact' + + - name: 'Set up Python' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: 'Install all wheels in one pip command and import from site-packages' + env: + BACKEND: '${{ matrix.backend }}' + DIST_DIRS: '${{ needs.resolve.outputs.dist_dirs }}' + IMPORT_NAMES: '${{ needs.resolve.outputs.import_names }}' + SMOKE_SNIPPET: '${{ needs.resolve.outputs.smoke_snippet }}' + run: | + set -euo pipefail + python -m venv "$RUNNER_TEMP/venv" + venv_py="$RUNNER_TEMP/venv/bin/python" + case "$BACKEND" in + tensorflow) "$venv_py" -m pip install tensorflow-cpu ;; + jax) "$venv_py" -m pip install 'jax[cpu]' ;; + torch) "$venv_py" -m pip install torch --index-url https://download.pytorch.org/whl/cpu ;; + *) echo "::error::Unknown backend $BACKEND"; exit 1 ;; + esac + python -I -c ' + import glob, json, os + for d in json.loads(os.environ["DIST_DIRS"]): + found = sorted(glob.glob(os.path.join("dist-artifact", d, "*.whl"))) + if not found: + raise SystemExit(f"no wheel in {d}") + for p in found: + print(os.path.abspath(p)) + ' > "$RUNNER_TEMP/wheels.txt" + mapfile -t wheels < "$RUNNER_TEMP/wheels.txt" + "$venv_py" -m pip install "${wheels[@]}" + cd "$RUNNER_TEMP" + KERAS_BACKEND="$BACKEND" "$venv_py" -I -c ' + import importlib, json, os + exec(os.environ["SMOKE_SNIPPET"]) + for name in json.loads(os.environ["IMPORT_NAMES"]): + path = getattr(importlib.import_module(name), "__file__", None) or "" + if "site-packages" not in path: + raise SystemExit(f"{name} was imported from {path!r}, not site-packages") + print(f"{name} imported from {path}") + ' + + generate: + # Holds GEMINI_API (env gemini, main only) in one step; runs no target code. + needs: + - 'validate' + - 'resolve' + runs-on: 'ubuntu-latest' + environment: 'gemini' + permissions: + contents: 'read' + steps: + - name: 'Check out target with history' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: '${{ needs.resolve.outputs.target }}' + ref: '${{ needs.validate.outputs.sha }}' + path: 'target' + fetch-depth: 0 + fetch-tags: true + persist-credentials: false + + - name: 'Check out shared-workflows' + # No repository: override, so this is the running commit (github.sha). + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: 'sw' + persist-credentials: false + + - name: 'Set up Python' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: 'Find the previous release tag' + id: 'prev' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PUBLISH_VERSION: '${{ needs.validate.outputs.version }}' + TAG_PREFIX: '${{ needs.resolve.outputs.tag_prefix }}' + with: + script: | + const { execFileSync } = require('node:child_process'); + const { previousReleaseTag } = require('./sw/scripts/release/lib/version.js'); + const pep440 = require('./sw/scripts/release/lib/pep440.js'); + const prefix = process.env.TAG_PREFIX; + const out = execFileSync('git', ['-C', 'target', '-c', 'core.hooksPath=/dev/null', 'tag', '--list', `${prefix}*`], { encoding: 'utf8' }); + const versions = out.split('\n').filter(Boolean).map((t) => t.slice(prefix.length)).filter((v) => { + try { pep440.parse(v); return true; } catch { return false; } + }); + const previous = previousReleaseTag(process.env.PUBLISH_VERSION, versions, prefix); + core.info(`Previous release tag: ${previous || '(none; diffing the whole tree)'}`); + core.setOutput('tag', previous || ''); + + - name: 'Generate tests with Gemini' + env: + GEMINI_API_KEY: '${{ secrets.GEMINI_API }}' + TARGET_DIR: 'target' + BASE_REF: '${{ steps.prev.outputs.tag }}' + HEAD_REF: '${{ needs.validate.outputs.sha }}' + OUTPUT_DIR: 'generated' + PUBLISH_VERSION: '${{ needs.validate.outputs.version }}' + PACKAGES: '${{ needs.resolve.outputs.packages }}' + IMPORT_NAMES: '${{ needs.resolve.outputs.import_names }}' + run: python -I sw/scripts/release/py/generate_test.py + + - name: 'Upload generated tests' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: 'generated-tests' + path: 'generated' + if-no-files-found: 'error' + + generated-tests: + # Runs Gemini-written code: no secrets, no checkout. + needs: + - 'resolve' + - 'build' + - 'generate' + if: always() && needs.build.result == 'success' + runs-on: 'ubuntu-latest' + permissions: + contents: 'read' + strategy: + fail-fast: false + matrix: + backend: '${{ fromJSON(needs.resolve.outputs.backends) }}' + steps: + - name: 'Require a successful generate job' + # A Gemini outage becomes a generated-tests failure, which + # override_generated_tests can cover. + env: + GENERATE_RESULT: '${{ needs.generate.result }}' + run: | + if [ "$GENERATE_RESULT" != "success" ]; then + echo "::error::The generate job finished with '$GENERATE_RESULT', so there are no generated tests to run." + exit 1 + fi + + - name: 'Download dist' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: 'dist' + path: 'dist-artifact' + + - name: 'Download generated tests' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: 'generated-tests' + path: 'generated' + + - name: 'Set up Python' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: 'Install all wheels in one pip command and run the generated tests' + env: + BACKEND: '${{ matrix.backend }}' + DIST_DIRS: '${{ needs.resolve.outputs.dist_dirs }}' + run: | + set -euo pipefail + python -m venv "$RUNNER_TEMP/venv" + venv_py="$RUNNER_TEMP/venv/bin/python" + case "$BACKEND" in + tensorflow) "$venv_py" -m pip install tensorflow-cpu ;; + jax) "$venv_py" -m pip install 'jax[cpu]' ;; + torch) "$venv_py" -m pip install torch --index-url https://download.pytorch.org/whl/cpu ;; + *) echo "::error::Unknown backend $BACKEND"; exit 1 ;; + esac + python -I -c ' + import glob, json, os + for d in json.loads(os.environ["DIST_DIRS"]): + found = sorted(glob.glob(os.path.join("dist-artifact", d, "*.whl"))) + if not found: + raise SystemExit(f"no wheel in {d}") + for p in found: + print(os.path.abspath(p)) + ' > "$RUNNER_TEMP/wheels.txt" + mapfile -t wheels < "$RUNNER_TEMP/wheels.txt" + "$venv_py" -m pip install "${wheels[@]}" 'pytest==8.3.5' + cp -R generated "$RUNNER_TEMP/generated-run" + cd "$RUNNER_TEMP/generated-run" + KERAS_BACKEND="$BACKEND" "$venv_py" -m pytest -p no:cacheprovider -q . + + upload: + # The only job with id-token: write and the pypi environment. + needs: + - 'validate' + - 'resolve' + - 'build' + - 'smoke' + - 'generated-tests' + if: always() && github.repository == 'keras-team/shared-workflows' && github.ref == 'refs/heads/main' && needs.validate.result == 'success' && needs.build.result == 'success' && needs.smoke.result == 'success' && (needs.generated-tests.result == 'success' || (inputs.override_generated_tests == true && needs.generated-tests.result == 'failure')) + runs-on: 'ubuntu-latest' + environment: 'pypi' + permissions: + id-token: 'write' + contents: 'read' + outputs: + manifest: '${{ steps.split.outputs.manifest }}' + env: + DIST_DIRS: '${{ needs.resolve.outputs.dist_dirs }}' + PACKAGES: '${{ needs.resolve.outputs.packages }}' + PUBLISH_VERSION: '${{ needs.validate.outputs.version }}' + TAG_PREFIX: '${{ needs.resolve.outputs.tag_prefix }}' + steps: + - name: 'Check out shared-workflows' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: 'sw' + persist-credentials: false + + - name: 'Set up Python' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: 'Download dist' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: 'dist' + path: 'dist-artifact' + + - name: 'Recheck sha256 against the build job' + env: + DIGESTS: '${{ needs.build.outputs.digests }}' + run: python -I sw/scripts/release/py/validate_dist.py digests --root dist-artifact --dist-dirs "$DIST_DIRS" --digests "$DIGESTS" + + - name: 'Validate METADATA / PKG-INFO and filenames' + run: python -I sw/scripts/release/py/validate_dist.py metadata --root dist-artifact --dist-dirs "$DIST_DIRS" --packages "$PACKAGES" --tag "$PUBLISH_TAG" --tag-prefix "$TAG_PREFIX" + + - name: 'Set aside files already on PyPI' + id: 'split' + run: python -I sw/scripts/release/py/split_existing.py --root dist-artifact --existing-dir dist-existing --dist-dirs "$DIST_DIRS" --packages "$PACKAGES" --version "$PUBLISH_VERSION" + + - name: 'Upload dist_dirs[0] (keras-hub first)' + if: steps.split.outputs.upload_dir_0 != '' + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: '${{ steps.split.outputs.upload_dir_0 }}' + skip-existing: true + + - name: 'Upload dist_dirs[1]' + if: steps.split.outputs.upload_dir_1 != '' + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + packages-dir: '${{ steps.split.outputs.upload_dir_1 }}' + skip-existing: true + + post-publish: + needs: + - 'validate' + - 'resolve' + - 'build' + - 'upload' + runs-on: 'ubuntu-latest' + timeout-minutes: 30 + permissions: + contents: 'read' + steps: + - name: 'Check out shared-workflows' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: 'sw' + persist-credentials: false + + - name: 'Set up Python' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: 'Poll PyPI and check digests of the files this run uploaded' + env: + MANIFEST: '${{ needs.upload.outputs.manifest }}' + DIGESTS: '${{ needs.build.outputs.digests }}' + PACKAGES: '${{ needs.resolve.outputs.packages }}' + PUBLISH_VERSION: '${{ needs.validate.outputs.version }}' + run: python -I sw/scripts/release/py/post_publish.py --manifest "$MANIFEST" --digests "$DIGESTS" --packages "$PACKAGES" --version "$PUBLISH_VERSION" + + report: + # No secrets. + needs: + - 'validate' + - 'resolve' + - 'build' + - 'smoke' + - 'generate' + - 'generated-tests' + - 'upload' + - 'post-publish' + if: always() + runs-on: 'ubuntu-latest' + permissions: {} + steps: + - name: 'Write release report' + env: + VALIDATE_RESULT: '${{ needs.validate.result }}' + BUILD_RESULT: '${{ needs.build.result }}' + SMOKE_RESULT: '${{ needs.smoke.result }}' + GENERATE_RESULT: '${{ needs.generate.result }}' + GENERATED_TESTS_RESULT: '${{ needs.generated-tests.result }}' + UPLOAD_RESULT: '${{ needs.upload.result }}' + POST_PUBLISH_RESULT: '${{ needs.post-publish.result }}' + DIGESTS: '${{ needs.build.outputs.digests }}' + run: | + { + echo "## Publish report: \`${PUBLISH_REPOSITORY}\` \`${PUBLISH_TAG}\`" + echo "" + echo "| Job | Result |" + echo "| --- | --- |" + echo "| validate | ${VALIDATE_RESULT} |" + echo "| build | ${BUILD_RESULT} |" + echo "| smoke | ${SMOKE_RESULT} |" + echo "| generate | ${GENERATE_RESULT} |" + echo "| generated-tests | ${GENERATED_TESTS_RESULT} |" + echo "| upload | ${UPLOAD_RESULT} |" + echo "| post-publish | ${POST_PUBLISH_RESULT} |" + echo "" + if [ "$PUBLISH_OVERRIDE_GENERATED_TESTS" = "true" ] && [ "$GENERATED_TESTS_RESULT" = "failure" ] && [ "$UPLOAD_RESULT" = "success" ]; then + echo "⚠️ override_generated_tests was used: the upload went ahead although the generated tests failed." + echo "" + fi + echo "Build sha256 digests:" + echo "" + echo '```json' + echo "${DIGESTS:-(none: the build job did not finish)}" + echo '```' + echo "" + echo "PyPI digest comparison for uploaded files is in the post-publish job summary." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7135148 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,257 @@ +name: 'Release' +# Release panel. One button for every Keras repo: each run detects the next +# step for the chosen repo and release branch (prepare, continue, publish, +# fast-path, ...) and does it with the release App's token. The bot never +# merges a PR. When a non-draft release exists afterwards, `dispatch` starts +# publish.yml, which uploads to PyPI. See RELEASE_PROCESS.md. + +on: + workflow_dispatch: + inputs: + repository: + description: 'Which repo to release (must be enabled in release/repos/).' + required: true + type: choice + options: + - 'keras' + - 'keras-hub' + - 'keras-rs' + - 'kinetic' + channel: + description: 'dev = pre-release X.Y.Z.devN (GitHub pre-release); stable = final release.' + required: true + type: choice + options: + - 'dev' + - 'stable' + kind: + description: 'major = new X.Y line on a release branch (patch component 0); patch = X.Y.Z fix on an existing release branch.' + required: true + type: choice + options: + - 'major' + - 'patch' + release_branch: + description: 'Release branch, e.g. r0.33. Always required; used for the same-branch lock.' + required: true + type: string + commit_sha: + description: 'Major with a new branch only: the master commit to cut the branch from.' + required: false + type: string + default: '' + version: + description: 'Optional. Leave blank to use the computed version (the dry run shows it). Type one only to resume or re-publish a specific version.' + required: false + type: string + default: '' + cherry_picks: + description: 'Master PR numbers (recommended, e.g. #3101) or commit SHAs, any order; applied in master merge order. Allowed for patch, and for major when the release branch already exists (e.g. fixes before X.Y.0).' + required: false + type: string + default: '' + bump_master: + description: 'Also open a PR bumping master to the next dev version.' + required: false + type: boolean + default: false + release_mode: + description: 'auto: after you merge the version PR and run the panel again, tags, publishes the GitHub release and uploads to PyPI in one go. draft: creates a draft release with notes for you to review (no tag yet); run the panel again to publish, which creates the tag and uploads. The bot never merges the PR in either mode.' + required: true + type: choice + options: + - 'draft' + - 'auto' + override_generated_tests: + description: 'Upload even if the Gemini-generated tests fail (e.g. a wrong generated test or a Gemini outage). The smoke install/import test always blocks.' + required: false + type: boolean + default: false + dry_run: + description: 'Show what would happen and change nothing. Untick to act.' + required: false + type: boolean + default: true + +# Parsed field by field by the same-branch run lock (scripts/release/lib/lock.js). +run-name: "release ${{ inputs.repository }} ${{ inputs.release_branch }} ${{ inputs.kind }} ${{ inputs.version || 'auto' }}" + +permissions: + contents: 'read' + +# Inputs reach scripts only through the environment, never as ${{ }} inside +# run: or script:. +env: + RELEASE_REPOSITORY: '${{ inputs.repository }}' + RELEASE_CHANNEL: '${{ inputs.channel }}' + RELEASE_KIND: '${{ inputs.kind }}' + RELEASE_BRANCH: '${{ inputs.release_branch }}' + RELEASE_COMMIT_SHA: '${{ inputs.commit_sha }}' + RELEASE_VERSION: '${{ inputs.version }}' + RELEASE_CHERRY_PICKS: '${{ inputs.cherry_picks }}' + RELEASE_BUMP_MASTER: '${{ inputs.bump_master }}' + RELEASE_MODE: '${{ inputs.release_mode }}' + RELEASE_OVERRIDE_GENERATED_TESTS: '${{ inputs.override_generated_tests }}' + RELEASE_DRY_RUN: '${{ inputs.dry_run }}' + RELEASE_APP_SLUG: '${{ vars.RELEASE_APP_SLUG }}' + +jobs: + plan: + # No secrets. Validates inputs, refuses any ref but main, and checks the + # active-run signal of the same-branch lock. + runs-on: 'ubuntu-latest' + permissions: + actions: 'read' + contents: 'read' + outputs: + target_owner: '${{ steps.plan.outputs.target_owner }}' + target_name: '${{ steps.plan.outputs.target_name }}' + steps: + - name: 'Check out shared-workflows' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: 'Validate inputs and check the run lock' + id: 'plan' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { main } = require('./scripts/release/plan.js'); + await main({ github, context, core, env: process.env }); + + act: + # No job-level concurrency on purpose: a second click must be refused by + # the run lock, not queued or silently cancelled. + needs: 'plan' + runs-on: 'ubuntu-latest' + environment: 'release' + permissions: + contents: 'read' + outputs: + published: '${{ steps.act.outputs.published }}' + tag: '${{ steps.act.outputs.tag }}' + steps: + - name: 'Refuse re-runs' + # "Re-run failed jobs" restarts here and skips the plan job's run lock. + run: | + if [ "$GITHUB_RUN_ATTEMPT" != "1" ]; then + echo "::error::Re-runs are refused because they skip the run lock in the plan job. Start a new panel run; it resumes where this one stopped." + exit 1 + fi + + - name: 'Check out shared-workflows' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: 'Mint App token for the target repo' + id: 'app-token' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: '${{ secrets.RELEASE_APP_CLIENT_ID }}' + private-key: '${{ secrets.RELEASE_APP_PRIVATE_KEY }}' + owner: '${{ needs.plan.outputs.target_owner }}' + repositories: '${{ needs.plan.outputs.target_name }}' + # contents: write on dry runs too: GitHub lists draft releases only + # to callers with push access, and the draft lock needs them. + permission-contents: 'write' + permission-pull-requests: "${{ inputs.dry_run && 'read' || 'write' }}" + # `workflows` has no read level; dry runs leave it out entirely. + permission-workflows: "${{ !inputs.dry_run && 'write' || '' }}" + permission-actions: 'read' + permission-metadata: 'read' + + - name: 'Detect and run the release step' + id: 'act' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RELEASE_TARGET_OWNER: '${{ needs.plan.outputs.target_owner }}' + RELEASE_TARGET_NAME: '${{ needs.plan.outputs.target_name }}' + RELEASE_TRIGGERING_ACTOR: '${{ github.triggering_actor }}' + RELEASE_APP_TOKEN: '${{ steps.app-token.outputs.token }}' + with: + github-token: '${{ steps.app-token.outputs.token }}' + script: | + const { main } = require('./scripts/release/act.js'); + await main({ github, context, core, env: process.env }); + + dispatch: + needs: 'act' + if: inputs.dry_run == false && needs.act.outputs.published == 'true' + runs-on: 'ubuntu-latest' + environment: 'release' + permissions: + contents: 'read' + steps: + - name: 'Refuse re-runs' + run: | + if [ "$GITHUB_RUN_ATTEMPT" != "1" ]; then + echo "::error::Re-runs are refused because they skip the run lock in the plan job. Start a new panel run; it resumes where this one stopped." + exit 1 + fi + + - name: 'Check out shared-workflows' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: 'Mint App token for shared-workflows (actions: write only)' + id: 'app-token' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: '${{ secrets.RELEASE_APP_CLIENT_ID }}' + private-key: '${{ secrets.RELEASE_APP_PRIVATE_KEY }}' + owner: '${{ github.repository_owner }}' + repositories: 'shared-workflows' + permission-actions: 'write' + + - name: 'Dispatch publish.yml' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + RELEASE_TAG: '${{ needs.act.outputs.tag }}' + RELEASE_TRIGGERING_ACTOR: '${{ github.triggering_actor }}' + with: + github-token: '${{ steps.app-token.outputs.token }}' + script: | + const { main } = require('./scripts/release/dispatch.js'); + await main({ github, context, core, env: process.env }); + + report: + # No secrets. The act job's own summary lists the planned or taken actions. + needs: + - 'plan' + - 'act' + - 'dispatch' + if: always() + runs-on: 'ubuntu-latest' + permissions: {} + steps: + - name: 'Write run report' + env: + PLAN_RESULT: '${{ needs.plan.result }}' + ACT_RESULT: '${{ needs.act.result }}' + DISPATCH_RESULT: '${{ needs.dispatch.result }}' + PUBLISHED: '${{ needs.act.outputs.published }}' + TAG: '${{ needs.act.outputs.tag }}' + run: | + publish_runs="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/workflows/publish.yml" + { + echo "## Release run report" + echo "" + echo "| Job | Result |" + echo "| --- | --- |" + echo "| plan | ${PLAN_RESULT} |" + echo "| act | ${ACT_RESULT} |" + echo "| dispatch | ${DISPATCH_RESULT} |" + echo "" + echo "Repository: \`${RELEASE_REPOSITORY}\`. Dry run: \`${RELEASE_DRY_RUN}\`." + echo "The planned or taken actions are in the act job summary above." + if [ "$PUBLISHED" = "true" ] && [ "$DISPATCH_RESULT" = "success" ]; then + echo "" + echo "PyPI upload started: ${publish_runs} (run titled \`publish ${RELEASE_REPOSITORY} ${TAG}\`). Digests and override use are reported there." + elif [ "$PUBLISHED" = "true" ]; then + echo "" + echo "A release exists for \`${TAG}\` but publish.yml was not dispatched. Run the panel again with version set to this release's version to re-publish." + fi + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d171a6c..8b81224 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,3 +22,41 @@ jobs: - name: 'Run unit tests' run: node --test 'scripts/**/*.test.js' + + pytest: + runs-on: 'ubuntu-latest' + steps: + - name: 'Check out repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: 'Set up Python' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: 'Install pytest' + run: python -m pip install 'pytest==8.3.5' + + - name: 'Run release script tests' + run: python -m pytest -q scripts/release/py + + zizmor: + runs-on: 'ubuntu-latest' + steps: + - name: 'Check out repository' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: 'Set up Python' + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.11' + + - name: 'Install zizmor' + run: python -m pip install 'zizmor==1.30.1' + + - name: 'Audit the release workflows' + run: zizmor --offline .github/workflows/release.yml .github/workflows/publish.yml .github/workflows/test.yml diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md new file mode 100644 index 0000000..3bf0101 --- /dev/null +++ b/RELEASE_PROCESS.md @@ -0,0 +1,422 @@ +# Release Process + +This repo runs releases for Keras repos from one place. You press a button +here; a GitHub App does the git work in the target repo; a separate workflow +builds, tests and uploads to PyPI. + +Phase 1 covers **releases only** for **keras-hub**. Nightlies are out of scope +and still run from each repo's own `nightly.yml`. + +## 1. The Release panel + +The panel is the `release.yml` workflow in this repo: **Actions -> release -> +Run workflow**. + +| Workflow | What it does | Who starts it | +| :--- | :--- | :--- | +| `release.yml` (the panel) | Works out the next step, then cuts the branch, cherry-picks, opens PRs, creates the draft or the release and tag | You | +| `publish.yml` | Builds, smoke-tests, runs generated tests, uploads to PyPI | The panel, automatically. A human can also start it directly to retry | + +**Who can use it.** You need **write on the target repo** (e.g. keras-hub) +**and write on `shared-workflows`** (GitHub only shows Run workflow to +writers). Maintain counts as write. Triage does not. If you are a keras-hub +collaborator but not in a team with write here, ask an admin to add you. + +**Branch picker.** The "Use workflow from" picker must be **`main`**. Only +reviewed code on `main` can reach the bot key and PyPI. Any other branch is +refused, dry runs included. + +**The bot never merges.** It opens PRs. A human with write merges them, then +presses Release again. + +## 2. Form fields + +| Field | Type | Label shown in the form | Why it is there | +| :--- | :--- | :--- | :--- | +| `repository` | choice | Which repo to release (must be enabled in release/repos/). | Picks the target and its config file. Disabled repos are refused. | +| `channel` | choice | dev = pre-release X.Y.Z.devN (GitHub pre-release); stable = final release. | Decides whether the version gets a `.devN` suffix and the pre-release flag. | +| `kind` | choice | major = new X.Y line on a release branch (patch component 0); patch = X.Y.Z fix on an existing release branch. | Decides how the version is computed. | +| `release_branch` | string | Release branch, e.g. r0.33. Always required; used for the same-branch lock. | Every release happens on a release branch. The name also gives X.Y. | +| `commit_sha` | string | Major with a new branch only: the master commit to cut the branch from. | Where the new branch starts. Must be on master, and its version files must have the branch's X.Y. | +| `version` | string | Optional. Leave blank to use the computed version (the dry run shows it). Type one only to resume or re-publish a specific version. | Normally blank. Needed only to re-publish a version that is already released. | +| `cherry_picks` | string | Master PR numbers (recommended, e.g. #3101) or commit SHAs, any order; applied in master merge order. Allowed for patch, and for major when the release branch already exists (e.g. fixes before X.Y.0). | Fixes to carry from master onto the release branch. | +| `bump_master` | boolean | Also open a PR bumping master to the next dev version. | Saves a manual PR after cutting a branch. Skipped if master is already there. | +| `release_mode` | choice | auto: after you merge the version PR and run the panel again, tags, publishes the GitHub release and uploads to PyPI in one go. draft: creates a draft release with notes for you to review (no tag yet); run the panel again to publish, which creates the tag and uploads. The bot never merges the PR in either mode. | Lets you review release notes before anything is public. | +| `override_generated_tests` | boolean | Upload even if the Gemini-generated tests fail (e.g. a wrong generated test or a Gemini outage). The smoke install/import test always blocks. | See section 7. | +| `dry_run` | boolean, default on | Show what would happen and change nothing. Untick to act. | Always start with a dry run. | + +There is no "action" field. The panel detects the next step from the state of +the branch, PRs, drafts, tags and releases. + +`cherry_picks`, `commit_sha` and `bump_master` only apply when the detected +step is *prepare*. Giving them on any other step is refused, and the message +names the detected step. + +## 3. Walkthroughs + +### Always start with a dry run + +Leave `dry_run` ticked. The summary shows: + +- every release branch, its latest release and the suggested next version; +- for the chosen branch, the master PRs not yet on it (cherry-pick candidates); +- the detected step and the resolved version; +- stale bump PRs; +- a table of the exact form values to use for the real run. + +A dry run writes nothing. It does hold the same-branch lock while it runs. + +### What "Next:" means + +Every run's summary ends with a `Next:` line. It tells you exactly what to do +now: merge PR #N, run again, or fix something first. Every real run also prints +the resolved version and the detected step at the top. + +### How the panel decides what to do + +For version V on branch B, the first match wins: + +| # | State found | Step | What happens | +| :--- | :--- | :--- | :--- | +| 1 | A published (non-draft) release `vV` exists. Reached only with a typed `version`. | already-released | No repo writes. Re-runs `publish.yml` for that tag. | +| 2 | A draft with tag `vV` | publish | Re-checks the draft, publishes it (creates the tag), starts `publish.yml`. | +| 3 | Tag `vV` at the merge sha of the merged bump PR, no release | continue (resume) | Creates the release without re-tagging. | +| 4 | Merged `release-tool/bump-V` PR, no tag | continue | auto: tag + release + PyPI. draft: creates the draft. | +| 5 | Open `release-tool/bump-V` PR | no-op | "waiting for a human with write to merge PR #N". | +| 6 | Version files at B head already equal V, and no picks | fast-path | auto: tags B's head sha and publishes. draft: creates a draft at that sha. | +| 7 | Anything else | prepare | Cuts the branch if needed, applies picks, opens the bump PR. | + +### New minor release with dev releases (example: 0.33) + +Master's version must already be `0.33.0.devN` so the branch name and the +version agree. + +**Cut the branch and release 0.33.0.dev0.** + +1. Dry run: `repository=keras-hub`, `channel=dev`, `kind=major`, + `release_branch=r0.33`, `commit_sha=`. Optionally tick + `bump_master`. +2. Untick `dry_run` and run. The bot creates `r0.33` at `commit_sha`. + - If the new branch already has version `0.33.0.dev0`, it stops there: no + PR, no tag. `Next:` says run again. + - Otherwise it opens a bump PR into `r0.33`. Merge it. + - If you ticked `bump_master`, it also opens a PR bumping master to the next + dev version (branch `release-tool/master-bump-`). Merge it when ready. +3. Run again with the same repository and branch. The panel picks fast-path + (or continue, if there was a bump PR) and releases `0.33.0.dev0`. + +Cherry-picks are refused when the branch is being created. The branch is cut +from `commit_sha`, which already contains master's fixes. + +**Fixes before 0.33.0: release 0.33.0.dev1.** + +1. Merge the fix on master first. +2. Run with `channel=dev`, `kind=major`, `release_branch=r0.33`, + `cherry_picks=#3101`. Leave `commit_sha` blank (the branch exists). +3. The bot opens one PR into `r0.33` with the picks and a bump to + `0.33.0.dev1`. A human merges it. +4. Run again. The continue step tags the merge commit and releases. + +Each further dev release repeats this with the next `devN`. + +**Final release 0.33.0.** + +1. Run with `channel=stable`, `kind=major`, `release_branch=r0.33`. Add + `cherry_picks` if there are last fixes. +2. Merge the bump PR (version `0.33.0`). +3. Run again. Continue releases `0.33.0`. It gets the Latest badge if it is the + highest stable version in the repo. + +### Patch release (example: 0.33.1) + +1. Merge the fix on master. +2. Dry run with `channel=stable`, `kind=patch`, `release_branch=r0.33`, + `cherry_picks=#3120, #3125`. +3. Untick `dry_run`. The bot opens a bump PR into `r0.33` with the picks and + version `0.33.1`. A human merges it. +4. Run again. Continue releases `0.33.1`. + +The patch version is the highest stable `v0.33.W` plus one. A patch needs a +stable `v0.33.0` first; without it the panel says "use kind=major". A patch on +an older line (e.g. 0.31.2 after 0.33.0) does not take the Latest badge. + +### auto vs draft + +| | auto | draft | +| :--- | :--- | :--- | +| Run after merging the bump PR | Creates tag + release in one call, then starts `publish.yml` | Creates a draft release with notes. No tag yet. | +| Next run | — | Publish step: publishes the draft (this creates the tag), then starts `publish.yml` | +| Clicks after the merge | 1 | 2 | + +In draft mode you can edit the release notes. Do not change the tag or the +target commit: the publish step re-checks both and refuses an edited draft. + +> **Do not publish the bot's draft from the GitHub UI.** That skips the PyPI +> upload. Always publish by running the panel again. If it happens, see the +> failure table. + +The panel only publishes drafts created by the bot. A draft created by a human +is refused. + +## 4. Cherry-picks + +- Give master PR numbers (recommended) or commit SHAs, any order. +- Every PR must be merged into master. SHAs must be on master's first-parent + history (a merge commit or a squash commit). +- The bot applies them oldest first, in master merge order, with + `git cherry-pick -x`. + - Squash-merged PR: picks the squash commit. + - Merge-commit PR: picks the merge commit with `-m 1`. + - **Rebase-merged PR: refused.** It has several commits on master and no + single commit to pick. Give the individual SHAs instead. +- A pick already on the branch, or one that comes out empty, is skipped and + reported. It is not an error. + +**The pick record.** Every bump PR body carries a line like: + +``` +Cherry-picks: #3101 (abc1234), #3110 (def5678) +``` + +Later runs and the dry run read this line from PRs merged into the branch to +know what is already there. Any merge method is fine; squash-merging the bump +PR loses nothing. + +**Conflicts.** On a conflict the bot aborts the cherry-pick and pushes +nothing. The summary lists the conflicting commit, the conflicting files, and +earlier master commits touching those files that you did not select. Then +either: + +- run again with an adjusted list (often adding the missing earlier PR), or +- resolve by hand: + 1. Open a PR onto the release branch made with `git cherry-pick -x `. + 2. Add a `Cherry-picks: #N (sha)` line to its body, so later runs know it is + there. + 3. A human merges it. + 4. Run the panel. It detects prepare, applies any remaining picks, and opens + the bump PR. + 5. Merge the bump PR and run once more. + +## 5. Same-branch refusal + +Only one release can be in progress per repo and branch. The panel refuses +(it does not queue) when it sees any of these for the same branch: + +- another active panel run (including a dry run); +- an active `publish.yml` run for a tag on that branch's line; +- an open bump PR into the branch (except the no-op wait for that same PR); +- a draft release for that branch (except the publish step for that same + draft). + +Why: two runs on one branch could pick the same version, tag twice, or race +each other's picks. Refusing is safer than guessing which one you meant. + +- Two clicks at the same moment each see the other and both refuse. Click + again. +- A run title the panel cannot parse counts as a match (fails closed). +- An in-flight release for a different version than the one you typed is + refused and named. Two in-flight versions on one branch are refused and + listed. + +## 6. Failures and how to resume + +Every step is idempotent. Running the panel again resumes at the first +incomplete step. It never opens a second PR or creates a second tag. The bot +creates a tag and its release in one API call, so a failure leaves either +nothing or a complete release. + +| Situation | What you see | What to do | +| :--- | :--- | :--- | +| Prepare failed half-way | No PR, or a stale `release-tool/bump-V` branch | Run again. The bot rebuilds the bump branch from scratch and opens the PR. | +| Cherry-pick conflict | Conflict report; nothing pushed | See section 4. | +| `dispatch` job failed (release exists, `publish.yml` never started) | Panel run red at `dispatch` | Run the panel with `version=V`. The already-released step starts `publish.yml` again. | +| `publish.yml` failed (build, smoke, generated tests, upload) | `publish` run red | "Re-run failed jobs" on that run, or run the panel with `version=V`, or dispatch `publish.yml` directly with the tag. | +| Partial PyPI upload (e.g. keras-hub uploaded, keras-nlp not) | `publish` red after upload | Same as above. Files already on PyPI are skipped and listed as "published by an earlier run"; only missing files are uploaded and digest-checked. The run ends green once every expected file is on PyPI. | +| Released on GitHub but not on PyPI | First summary line: "vV is released on GitHub but not on PyPI; run with version=V to re-publish" | Run with `version=V`. Until fixed, prepare refuses to start the next version. | +| A human published the bot's draft from the GitHub UI | Release is public, nothing on PyPI (the tag ruleset may also block the UI publish for people outside the rollback team) | Run with `version=V` to upload. | +| Human-created draft on the branch | Publish refused; it also blocks the branch | Delete the draft. | +| Stale bump PR (merged, version already released or lower) | Listed as stale in the dry run | Nothing. It is ignored. | +| Closed, unmerged bump PR | — | Nothing. It is ignored; the next prepare opens a new PR. | +| Open bump PR you no longer want | No-op / branch refused | Close it. | +| Merged bump PR you want to abandon (never tagged; it blocks every other version) | In-flight release that you don't want | A human with write adds the label `release-tool:abandoned` to that PR, then runs again. A label added by a triage user is ignored. | +| Commits landed on the branch after the bump PR merged | Continue refuses: "these N commits would be left out of vV: ...; label PR #N `release-tool:abandoned` and run again to cut a version that includes them" | Add the label and run again. The next run usually takes fast-path at the new branch head, including those commits. | +| Tag `vV` created by hand at the bump PR's merge commit, no release | — | Run again. Continue creates the release without re-tagging. | +| You abandoned a bump PR, but tag `vV` already sits at its merge commit | Refused: "tag vV exists … not at the merge commit" | Ask the rollback team to delete tag `vV` (there is no release for it), then run again. | +| "Re-run failed jobs" on a panel (`release.yml`) run | `act`/`dispatch` refuse on attempt 2+ | Start a new panel run. A re-run skips the lock check in `plan`, so it is blocked. | +| "Re-run failed jobs" on `publish.yml` | Allowed | It re-uploads only the tag and digests that `validate` already approved. | +| Legacy publisher still enabled | Refused before any tag or publish | Disable keras-hub's `publish-to-pypi.yml` (section 8). | +| Gemini outage or a wrong generated test | `generate` or `generated-tests` red, upload skipped | Run again with `override_generated_tests` ticked (section 7). | +| Branch busy | Same-branch refusal | Wait for the other run, or finish/close the in-flight PR or draft. | + +## 7. `override_generated_tests` + +`publish.yml` runs two kinds of tests on the built wheels: + +- **smoke**: fresh install and import on each backend. **Always blocks.** +- **generated tests**: tests written by Gemini from the release diff. + +With the override ticked, the upload still runs when the generated tests +**failed**, including when Gemini itself failed (an outage counts as a +generated-tests failure). It does **not** cover: + +- a failed build, smoke test or `validate`; +- generated tests that were cancelled or skipped. + +The override only matters on the run that starts `publish.yml`: the continue +or fast-path run in auto mode, the publish run in draft mode, or an +already-released re-publish. The `publish` run report says when the override +was used. + +## 8. Coexistence and rollback + +- Phase 1 deletes nothing. keras-hub's legacy `publish-to-pypi.yml` stays in + the repo, **disabled** in the Actions UI. +- That workflow runs on every push and uploads when a tag is pushed. If it were + enabled, a tag created by the bot would upload a second time. So the panel + checks it before any tag or publish and refuses unless it is disabled. Any + error from that check also refuses. +- keras-hub's existing PyPI trusted publisher stays in place. The new one for + `shared-workflows` is added next to it. + +**To roll back:** + +1. Re-enable `publish-to-pypi.yml` in keras-hub. +2. Stop using the panel for that repo. +3. A member of the rollback team (on the bypass lists of rulesets (b1) and + (b2)) cuts the branch and creates the tag/release by hand, following the + "Rollback only" section of keras-hub's `RELEASE_PROCESS.md`. Changes to + release branches still go through PRs. + +**Later (phase 2):** once a real upload through `shared-workflows` is proven, +the legacy file is deleted in the target repo. Only after that deletion lands +does a follow-up PR here set `legacy_publish_workflow` to `null` in the repo +config. Never before: until then a missing file makes the check fail, which +refuses (fails closed). The config loader rejects a config with the field +absent. + +## 9. Admin setup checklist + +Do these in order. + +1. **[Admin] Environments.** On `shared-workflows`, Settings -> Environments, + create `pypi`, `release`, `release-read`, `gemini`. For each: Deployment + branches and tags -> "Selected branches and tags" -> add a rule of + **type Branch** (not Tag) named `main`. A Tag rule, or a name pattern + without the type, would let a writer push a *tag* called `main` and run a + modified workflow in the environment. No required reviewers on `release`, + `release-read`, `gemini`. Required reviewers on `pypi` are optional; they + pause auto mode until someone approves. +2. **[Admin] `main` ruleset.** Settings -> Rules -> Rulesets -> New branch + ruleset on `shared-workflows`, target `main`: + - require a pull request with >= 1 approval; + - require review from Code Owners; + - dismiss stale approvals on new pushes; + - require approval of the most recent push; + - require status checks by job name: `unit-tests` (`Tests` is the workflow + name, not a check), `pytest`, `zizmor`; + - block force pushes and deletion; + - **bypass list empty** (the App must never be on it). + + The existing CODEOWNERS (`* @keras-team/core @jeffcarp`) then covers + `release/**`, `scripts/release/**` (the code that runs with the App key) + and `.github/workflows/**`. +3. **[Admin] Secrets audit** (before step 5 adds writers). List the repo-level + secrets (Settings -> Secrets and variables -> Actions) and the org-level + secrets whose access includes `shared-workflows`. A workflow on any + writer's branch can read them, so remove or re-scope anything not needed + here. Leave `gemini-issue-triage.yml` and `jules-review.yml` alone: they + are `workflow_call` workflows whose secrets come from the calling repo. + Then Settings -> Actions -> General -> Workflow permissions = "Read + repository contents and packages permissions", and untick "Allow GitHub + Actions to create and approve pull requests". +4. **[Maintainer] Fork end-to-end run** is green; run links are in the PR + description; the `shared-workflows` PR is merged. Until the App exists, any + panel run fails at `plan` on the empty `RELEASE_APP_SLUG`. That is + expected. +5. **[Admin] Create the release App** (org Settings -> Developer settings -> + GitHub Apps -> New): + - Owned by `keras-team`; webhook inactive; no event subscriptions; "Only on + this account". + - Repository permissions, one set for every installed repo: Contents, Pull + requests, Workflows, Actions = Read and write; Metadata = Read. Nothing + else. Each job narrows its own token (e.g. `dispatch` only ever holds + `actions: write` on `shared-workflows`). + - Install on "Only select repositories": `shared-workflows` and `keras-hub`. + Add other targets when they are enabled. + - Generate a private key. Put `RELEASE_APP_CLIENT_ID` (the App's Client + ID, not its numeric App ID) and + `RELEASE_APP_PRIVATE_KEY` in envs `release` and `release-read` only, + never at repo or org level. + - Set repo variable `RELEASE_APP_SLUG` on `shared-workflows` to the App's + slug. + - Put `GEMINI_API` in the `gemini` env. + - Create three new rulesets in **keras-hub** (today `r*` branches have no + rules, so the App could push to them): + + | Ruleset | Type | Target | Rules | Bypass list | + | :--- | :--- | :--- | :--- | :--- | + | (a) Release branches, PR required | branch | `r[0-9]*` | Require a PR (>= 1 approval); dismiss stale approvals when new commits are pushed; require approval of the most recent reviewable push; block force pushes and deletion | **Empty** (never the App) | + | (b1) Release branch creation | branch | `r[0-9]*` | Restrict creations | Release App + rollback team | + | (b2) Release tags | tag | `v[0-9]*` | Restrict creations, updates and deletions | Release App + rollback team | + + (a) makes "the bot never pushes to or merges into a release branch" a + GitHub-enforced rule. Its two approval settings stop a writer from + pushing a commit onto an open bump PR and approving it themselves. Do not + enable "require signed commits": the bot's picks are unsigned. `[0-9]` + keeps names like `refactor-x` unaffected. The rollback team (e.g. + keras-hub release maintainers) lets a human still cut a branch and create + a tag by hand during rollback. + - **Write grants.** Grant each enabled target repo's writer team write on + `shared-workflows` (needed to press Run workflow). Only these people can + release; the maintainer sends the admin the list. Safe only after steps + 1, 2 and 3. +6. **[PyPI project Owner of `keras-hub` and `keras-nlp`, not a GitHub admin] + Last admin step.** On pypi.org, each project -> Manage -> Publishing -> add + a GitHub trusted publisher: + + | Field | Value | + | :--- | :--- | + | Owner | `keras-team` | + | Repository name | `shared-workflows` | + | Workflow name | `publish.yml` | + | Environment name | `pypi` | + + This is additional. Keep the existing keras-hub publisher for rollback. +7. **[Maintainer] Disable the legacy workflow.** Disable keras-hub's + `publish-to-pypi.yml` in the Actions UI (or + `PUT /repos/keras-team/keras-hub/actions/workflows/{id}/disable`) before + the first real run that can create a tag (continue, fast-path, or draft + publish). The panel refuses otherwise. +8. **[Maintainer] First release.** Run the first KerasHub release through the + panel. On a problem, re-enable the old workflow and roll back (section 8). + **Follow-ups:** repeat the trusted publisher mapping (step 6), disabling + the legacy workflow (step 7) and a live release for keras, keras-rs and + kinetic, one repo at a time. Add each to the App's installation when it is + enabled (step 5). + +## 10. Security model + +- **Only `main` holds keys.** Every environment with a secret (`release`, + `release-read`, `gemini`, `pypi`) is limited to the branch `main`, and + `main` needs a reviewed, Code-Owner-approved PR with no bypass. +- **Target code never sees a secret.** The target repo's build and tests run + in jobs with no App key, no Gemini key and no PyPI access. The jobs holding + the App key never run target repo code or git hooks, and never restore a + cache. +- **The bot cannot merge or push to release branches.** No code calls a merge + API, and keras-hub ruleset (a) blocks direct pushes with an empty bypass + list. Every release change reaches the branch through a PR a human merged. +- **Every release is re-checked before upload.** `validate` checks the + requester has write on the target, the tag's commit is on a release branch, + a published release exists, and the version files match the tag. The + panel only publishes its own drafts, anchored to the merged bump PR. +- **What is uploaded is what was built.** Files are checked by sha256 between + build and upload, and their metadata (name, version) is checked against the + config and the tag. PyPI's copy is digest-checked afterwards. +- **Least privilege tokens.** App tokens are minted per job, scoped to one + repo and the minimum permissions, and revoked at job end. PyPI uses OIDC + trusted publishing; there is no PyPI token. +- **No script injection.** Free-text inputs reach scripts only through + environment variables. zizmor runs on every PR. +- **Humans stay in the loop.** Pressing the panel needs write on both repos + (triage is refused), and a human merges every bump PR. diff --git a/release/repos/keras-hub.json b/release/repos/keras-hub.json new file mode 100644 index 0000000..33d738b --- /dev/null +++ b/release/repos/keras-hub.json @@ -0,0 +1,17 @@ +{ + "enabled": true, + "github_repo": "keras-team/keras-hub", + "default_branch": "master", + "release_branch_pattern": "^r(\\d+)\\.(\\d+)$", + "tag_prefix": "v", + "version_files": ["keras_hub/src/version.py"], + "packages": ["keras-hub", "keras-nlp"], + "import_names": ["keras_hub", "keras_nlp"], + "build_setup_command": "pip install -r requirements.txt", + "build_command": "python pip_build.py", + "dist_dirs": ["dist/", "keras_nlp/dist/"], + "dev_suffix_on_master": ".dev0", + "smoke_snippet": "import keras_hub, keras_nlp; print(keras_hub.__version__)", + "backends": ["tensorflow", "jax", "torch"], + "legacy_publish_workflow": "publish-to-pypi.yml" +} diff --git a/release/repos/keras-rs.json b/release/repos/keras-rs.json new file mode 100644 index 0000000..01edb38 --- /dev/null +++ b/release/repos/keras-rs.json @@ -0,0 +1,17 @@ +{ + "enabled": false, + "github_repo": "keras-team/keras-rs", + "default_branch": "main", + "release_branch_pattern": "^r(\\d+)\\.(\\d+)$", + "tag_prefix": "v", + "version_files": ["keras_rs/src/version.py"], + "packages": ["keras-rs"], + "import_names": ["keras_rs"], + "build_setup_command": "pip install -r requirements.txt build", + "build_command": "python pip_build.py", + "dist_dirs": ["dist/"], + "dev_suffix_on_master": ".dev0", + "smoke_snippet": "import keras_rs; print(keras_rs.__version__)", + "backends": ["tensorflow", "jax", "torch"], + "legacy_publish_workflow": "publish-to-pypi.yml" +} diff --git a/release/repos/keras.json b/release/repos/keras.json new file mode 100644 index 0000000..c0d7d61 --- /dev/null +++ b/release/repos/keras.json @@ -0,0 +1,17 @@ +{ + "enabled": false, + "github_repo": "keras-team/keras", + "default_branch": "master", + "release_branch_pattern": "^r(\\d+)\\.(\\d+)$", + "tag_prefix": "v", + "version_files": ["keras/src/version.py"], + "packages": ["keras"], + "import_names": ["keras"], + "build_setup_command": "pip install -r requirements.txt build", + "build_command": "python pip_build.py", + "dist_dirs": ["dist/"], + "dev_suffix_on_master": "", + "smoke_snippet": "import keras; print(keras.__version__)", + "backends": ["tensorflow", "jax", "torch"], + "legacy_publish_workflow": "publish-to-pypi.yml" +} diff --git a/release/repos/kinetic.json b/release/repos/kinetic.json new file mode 100644 index 0000000..16e31d7 --- /dev/null +++ b/release/repos/kinetic.json @@ -0,0 +1,17 @@ +{ + "enabled": false, + "github_repo": "keras-team/kinetic", + "default_branch": "main", + "release_branch_pattern": "^r(\\d+)\\.(\\d+)$", + "tag_prefix": "v", + "version_files": ["pyproject.toml"], + "packages": ["keras-kinetic"], + "import_names": ["kinetic"], + "build_setup_command": "pip install build", + "build_command": "python -m build", + "dist_dirs": ["dist/"], + "dev_suffix_on_master": "", + "smoke_snippet": "import kinetic", + "backends": ["jax"], + "legacy_publish_workflow": "publish-to-pypi.yml" +} diff --git a/scripts/release/act.js b/scripts/release/act.js new file mode 100644 index 0000000..e09cf79 --- /dev/null +++ b/scripts/release/act.js @@ -0,0 +1,342 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * release.yml `act` job (env `release`, App token on the target repo). + * + * Order: re-run guard -> inputs -> permission gate -> gatherState -> resolve + * V and detect the step -> PR/draft lock -> step guards -> legacy guard -> + * writes. A dry run takes the same path, performs every read and guard, and + * records "Would ..." instead of each write, plus a discovery report. + * + * Writes are limited to: `POST /git/refs` for a new release branch (major, + * branch absent), force-pushes of `release-tool/*` refs, opening PRs, and + * `POST /releases` / publishing the bot's draft. Nothing here merges. + */ + +const os = require("node:os"); +const path = require("node:path"); +const { RefusalError } = require("./lib/errors"); +const pep440 = require("./lib/pep440"); +const version = require("./lib/version"); +const { detect, SHA40, BUMP_PREFIX, ABANDONED_LABEL } = require("./lib/detect"); +const { prDraftConflicts } = require("./lib/lock"); +const { hasWrite, continueGuards, draftAnchorFailures } = require("./lib/guards"); +const gitLib = require("./lib/git"); +const { Gh, pypiVersions } = require("./lib/gh"); +const S = require("./lib/summary"); +const plan = require("./plan"); +const { prepare, zeroPicksAfterSkips } = require("./prepare"); + +const DRAFT_WARNING = "> **Do not publish this draft from the GitHub UI**: " + + "that skips the PyPI upload. Run the Release panel again instead.\n\n"; + +function refuseIf(failures, what, next) { + if (failures.length === 0) return; + throw new RefusalError(`${what}:\n` + + failures.map((f) => `- ${S.escapeMd(f)}`).join("\n"), next); +} + +function tagVersions(tags, prefix) { + return tags.map((t) => t.name).filter((n) => n.startsWith(prefix)) + .map((n) => n.slice(prefix.length)).filter(pep440.isValid); +} + +const tagSha = (ctx, name) => + (ctx.state.tags.find((t) => t.name === name) || {}).sha || null; + +async function permissionGate(gh, actor) { + const permission = await gh.permission(actor); + if (!hasWrite(permission)) { + throw new RefusalError(`${S.escapeMd(actor || "(unknown)")} has ` + + `\`${permission}\` permission on ${gh.fullName}; releasing needs write.`, + "ask someone with write on the target repo to run the release."); + } +} + +/** Refuses unless the legacy publisher is disabled; explicit null skips. */ +async function legacyGuard(ctx) { + const file = ctx.config.legacy_publish_workflow; + if (file === null) return; + const { status, state } = await ctx.gh.workflowState(file); + if (status !== 200 || typeof state !== "string" || + !state.startsWith("disabled")) { + throw new RefusalError(`Legacy publisher \`${file}\` on ${ctx.gh.fullName}` + + ` is not confirmed disabled (HTTP ${status}, state ` + + `\`${state || "unknown"}\`); both publishers could upload.`, + `disable ${file} in the target repo's Actions tab, then run again.`); + } +} + +/** One `POST /releases` creating tag + release (or a draft) at `sha`. */ +async function releaseAt(ctx, sha, draft) { + const { tag, report } = ctx; + if (!SHA40.test(sha || "")) throw new Error(`release target must be a sha: ${sha}`); + const flags = version.releaseFlags(ctx.version, ctx.allVersions); + const prev = version.previousReleaseTag(ctx.version, ctx.allVersions, ctx.prefix); + const what = draft ? `a draft release ${tag} (no tag yet)` : `release ${tag} with its tag`; + const detail = `at \`${sha}\` (prerelease ${flags.prerelease}, latest ` + + `${flags.make_latest}, notes since ${prev || "the first commit"})`; + if (ctx.dryRun) { + report.add(`Would create ${what} ${detail}.`); + return; + } + const notes = await ctx.gh.generateNotes(tag, sha, prev); + const rel = await ctx.gh.createRelease({ + tag_name: tag, target_commitish: sha, draft, + prerelease: flags.prerelease, make_latest: flags.make_latest, + name: tag, body: (draft ? DRAFT_WARNING : "") + notes.body, + }); + report.add(`Created ${what} ${detail}: ${rel.html_url}`); + if (!draft) ctx.published = true; +} + +function releasedNext(ctx) { + if (ctx.inputs.release_mode === "draft") { + return "review the draft's notes on GitHub (edit them if needed, but do " + + "not publish it there), then run the panel again with the same " + + "repository and branch to publish and upload."; + } + return `the dispatch job starts \`publish ${ctx.inputs.repository} ` + + `${ctx.tag}\`; watch that run for the PyPI upload.`; +} + +async function alreadyReleased(ctx) { + ctx.report.add(`Release ${ctx.tag} is already published; nothing is ` + + `written to ${ctx.gh.fullName}.`); + if (ctx.dryRun) { + ctx.report.add(`Would re-dispatch publish.yml for ${ctx.tag}.`); + return; + } + ctx.published = true; + ctx.report.setNext(`the dispatch job re-runs \`publish ` + + `${ctx.inputs.repository} ${ctx.tag}\` (repeat-safe); watch that run.`); +} + +async function wait(ctx) { + const n = ctx.detection.pr.number; + ctx.report.add(`Waiting for a human with write to merge PR #${n}.`); + ctx.report.setNext(`a human with write merges PR #${n} (the bot never ` + + "merges), then run the panel again with the same repository and branch."); +} + +async function continueStep(ctx) { + const { gh, config, detection, report } = ctx; + const pr = detection.pr; + const mergeSha = pr.merge_commit_sha; + const failures = continueGuards({ + pr, config, branch: ctx.branch, version: ctx.version, + botLogin: ctx.botLogin, targetRepo: gh.fullName, + mergedByPermission: await gh.permission(pr.merged_by_login), + versionAtMergeSha: await gh.versionAt(mergeSha, config.version_files), + tagShaForVersion: tagSha(ctx, ctx.tag), + highestTagOnLine: version.highest(ctx.lineVersions), + commitsAfterMerge: await gh.commitsAfter(ctx.branch, mergeSha), + }); + refuseIf(failures, `Continue guards failed for PR #${pr.number}`, + `fix the items above, or label PR #${pr.number} \`${ABANDONED_LABEL}\` ` + + "(someone with write) and run again to cut a version from the branch head."); + await legacyGuard(ctx); + if (tagSha(ctx, ctx.tag)) { + report.add(`Tag ${ctx.tag} already points at the merge sha; not re-tagging.`); + } + await releaseAt(ctx, mergeSha, ctx.inputs.release_mode === "draft"); + report.setNext(releasedNext(ctx)); +} + +/** Fast-path: release the B-head sha resolved at detection time. */ +async function fastPath(ctx, { picksSkipped = false } = {}) { + const { state } = ctx; + const sha = state.headSha; + const highest = version.highest(ctx.lineVersions); + const failures = []; + if (!sha || state.versionAtHead !== ctx.version) { + failures.push(`version files at ${ctx.branch} head are ` + + `${state.versionAtHead}, not ${ctx.version}`); + } + if (!picksSkipped && !zeroPicksAfterSkips(ctx)) failures.push("cherry-picks were requested"); + if (tagSha(ctx, ctx.tag)) failures.push(`tag ${ctx.tag} already exists`); + if (highest && pep440.compare(ctx.version, highest) <= 0) { + failures.push(`${ctx.version} is not above ${highest}, the highest tag on the line`); + } + refuseIf(failures, "Fast-path guards failed", + "fix the items above, then run again."); + await legacyGuard(ctx); + await releaseAt(ctx, sha, ctx.inputs.release_mode === "draft"); + ctx.report.setNext(releasedNext(ctx)); +} + +async function versionOrNull(ctx, sha) { + try { + return await ctx.gh.versionAt(sha, ctx.config.version_files); + } catch (err) { + if (err instanceof RefusalError) return null; + throw err; + } +} + +/** + * Independent anchor for the draft: the merge sha of the App-authored, + * human-merged bump-V PR into B; else (fast-path draft) the draft's own + * target if it is a full sha reachable from B. + */ +async function draftAnchor(ctx, draft) { + const pr = ctx.state.prs.find((p) => p.merged && !p.abandoned_by_writer && + p.base_ref === ctx.branch && p.head_ref === `${BUMP_PREFIX}${ctx.version}` && + p.head_repo_full_name === ctx.gh.fullName && p.author_login === ctx.botLogin && + p.merged_by_login && p.merged_by_login !== ctx.botLogin); + if (pr) return pr.merge_commit_sha; + const target = draft.target_commitish; + if (SHA40.test(target || "") && await ctx.gh.isAncestor(target, ctx.branch)) { + return target; + } + return null; +} + +async function publishStep(ctx) { + const draft = ctx.detection.draft; + const target = draft.target_commitish; + const failures = draftAnchorFailures({ + draft, version: ctx.version, botLogin: ctx.botLogin, prefix: ctx.prefix, + tagExists: Boolean(tagSha(ctx, ctx.tag)), + anchorSha: await draftAnchor(ctx, draft), + versionAtTarget: SHA40.test(target || "") ? await versionOrNull(ctx, target) : null, + }); + refuseIf(failures, `Draft ${ctx.tag} failed re-verification`, + "delete the draft (a human-made or edited draft is never published by " + + "the bot) and run the panel again to recreate it."); + await legacyGuard(ctx); + const flags = version.releaseFlags(ctx.version, ctx.allVersions); + if (ctx.dryRun) { + ctx.report.add(`Would publish draft ${ctx.tag} at \`${target}\` (creates ` + + `the tag; latest ${flags.make_latest}).`); + return; + } + const rel = await ctx.gh.publishDraft(draft.id, { make_latest: flags.make_latest }); + ctx.published = true; + ctx.report.add(`Published draft ${ctx.tag} at \`${target}\`: ${rel.html_url}`); + ctx.report.setNext(`the dispatch job starts \`publish ${ctx.inputs.repository} ` + + `${ctx.tag}\`; watch that run for the PyPI upload.`); +} + +function tryCompute(args) { + try { + return version.computeVersion(args); + } catch (err) { + return null; + } +} + +async function discovery(ctx) { + const { gh, config, state, inputs, detection, report, prefix } = ctx; + const rows = []; + for (const branch of await gh.releaseBranches(config.release_branch_pattern)) { + const xy = version.branchXY(branch, config.release_branch_pattern); + const lv = version.lineVersions(state.tags, xy, prefix); + const released = state.releases.filter((r) => !r.draft && + lv.includes(r.tag_name.slice(prefix.length))); + const kind = lv.includes(`${xy.x}.${xy.y}.0`) ? "patch" : "major"; + rows.push({ + branch, + latest: version.highest(released.map((r) => r.tag_name.slice(prefix.length))), + nextStable: tryCompute({ kind, channel: "stable", xy, lineVersions: lv }), + nextDev: tryCompute({ kind, channel: "dev", xy, lineVersions: lv }), + }); + } + report.add(S.branchesSection(rows)); + report.add(S.candidatesSection(ctx.branch, state.branchExists ? + await gh.pickCandidates({ branch: ctx.branch, + defaultBranch: config.default_branch, picked: state.pickedPrNumbers }) : null)); + report.add(S.listSection("Stale bump PRs (ignored)", + (detection.stale || []).map((n) => `#${n}`))); + const isPrepare = detection.step === "prepare"; + report.add(S.formSection({ + repository: inputs.repository, channel: inputs.channel, kind: inputs.kind, + release_branch: inputs.release_branch, + commit_sha: isPrepare && !state.branchExists ? inputs.commit_sha : "", + version: detection.step === "already-released" ? ctx.version : "", + cherry_picks: isPrepare ? inputs.cherry_picks : "", + bump_master: isPrepare ? inputs.bump_master : "false", + release_mode: inputs.release_mode, + override_generated_tests: inputs.override_generated_tests, + dry_run: "false", + })); + report.add("### Planned actions"); +} + +const STEPS = { + "already-released": alreadyReleased, + "publish": publishStep, + "resume-continue": continueStep, + "continue": continueStep, + "wait": wait, + "fast-path": (ctx) => fastPath(ctx), + "prepare": (ctx) => prepare(ctx, { fastPath }), +}; + +async function main({ github, context, core, env = process.env, + rootDir = plan.ROOT_DIR, git = gitLib.git, fetchPypi = null }) { + const report = new S.Report(); + const ctx = { report, published: false, tag: "" }; + try { + plan.assertFirstAttempt(env); + const { inputs, config, xy } = plan.loadInputsAndConfig(env, rootDir); + const slug = plan.requireAppSlug(env); + const [owner, repo] = plan.resolveTarget(config, context, + env.RELEASE_TARGET_OWNER, env.RELEASE_TARGET_NAME); + const gh = new Gh({ github, owner, repo }); + await permissionGate(gh, env.RELEASE_TRIGGERING_ACTOR); + const botLogin = `${slug}[bot]`; + const state = await gh.gatherState({ + config, branch: inputs.release_branch, botLogin, + fetchPypi: fetchPypi || ((pkg) => pypiVersions(pkg, fetch)), + }); + const detection = detect({ state, inputs, config, botLogin }); + report.setHeader(detection.version, detection.step); + for (const w of detection.warnings || []) report.warn(w); + const prefix = config.tag_prefix; + Object.assign(ctx, { + gh, git, env, inputs, config, xy, state, detection, botLogin, prefix, + branch: inputs.release_branch, version: detection.version, + tag: `${prefix}${detection.version}`, dryRun: inputs.dry_run === "true", + allVersions: tagVersions(state.tags, prefix), + lineVersions: version.lineVersions(state.tags, xy, prefix), + workdir: path.join(env.RUNNER_TEMP || os.tmpdir(), `release-${repo}-${process.pid}`), + }); + if (ctx.dryRun) await discovery(ctx); + const conflicts = prDraftConflicts({ + state, step: detection.step, version: detection.version, prefix, + pattern: config.release_branch_pattern, + }); + refuseIf(conflicts.map((c) => typeof c === "string" ? c : + c.reason || c.message || JSON.stringify(c)), + `Another release is in flight on ${S.escapeMd(inputs.release_branch)}`, + "finish or close it first, then run again."); + await STEPS[detection.step](ctx); + if (ctx.dryRun && detection.step !== "wait") { + report.setNext("untick dry_run and run again with the form values above."); + } + } catch (err) { + plan.failWith(core, report, err); + } + core.setOutput("published", ctx.published && !ctx.dryRun ? "true" : "false"); + core.setOutput("tag", ctx.tag); + await S.writeSummary(core, report); +} + +module.exports = { legacyGuard, main, permissionGate }; diff --git a/scripts/release/act.test.js b/scripts/release/act.test.js new file mode 100644 index 0000000..228e74b --- /dev/null +++ b/scripts/release/act.test.js @@ -0,0 +1,290 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); +const T = require("./test_support"); +const act = require("./act"); +const { escapeMd } = require("./lib/summary"); + +const { SHA } = T; +const ROOT = T.configRoot(); +const pypi = async () => new Set(["0.33.0"]); + +async function run({ fixture, env = {}, git = T.fakeGit(), fail, root = ROOT }) { + const gh = T.fakeGithub(fixture, fail); + const core = T.fakeCore(); + await act.main({ + github: gh.github, context: T.context(), core, env: T.panelEnv(env), + rootDir: root, git: git.fn, fetchPypi: pypi, + }); + return { gh, core, git }; +} + +/** r0.33 head = merge sha of bot PR #60 (bump-0.33.1), merged by alice. */ +function mergedBump({ tag = false, draft = false } = {}) { + const fx = T.baseFixture(); + fx.branches["r0.33"] = SHA.ms; + fx.files[`${SHA.ms}:${T.VERSION_FILE}`] = T.versionText("0.33.1"); + fx.pulls.push(T.bumpPr(60, "0.33.1", { + merged_at: "2026-09-02T00:00:00Z", merge_commit_sha: SHA.ms, + merged_by: { login: "alice" }, body: "Cherry-picks: #101 (4444444)", + })); + fx.compares[`${SHA.ms}...r0.33`] = { status: "identical", commits: [] }; + if (tag) fx.refs.push({ ref: "refs/tags/v0.33.1", object: { type: "commit", sha: SHA.ms } }); + if (draft) fx.releases.push(T.releaseObj(7, "v0.33.1", SHA.ms, { draft: true })); + return fx; +} + +/** r0.33 head b1 already reads 0.33.1; nothing in flight. */ +function fastPathFixture() { + const fx = T.baseFixture(); + fx.branches["r0.33"] = SHA.b1; + fx.files[`${SHA.b1}:${T.VERSION_FILE}`] = T.versionText("0.33.1"); + return fx; +} + +/** Fake git for a prepare run on top of b0 (0.33.0). */ +function prepareGit(overrides = {}) { + return T.fakeGit((args) => { + if (overrides[args[0]]) return overrides[args[0]](args); + if (args[0] === "show") return T.versionText("0.33.0"); + if (args[0] === "hash-object") return "abc123\n"; + if (args[0] === "ls-tree") return `100644 blob abc123\t${T.VERSION_FILE}\n`; + if (args[0] === "rev-list") return "2\n"; + return ""; + }); +} + +const lines = (core) => core.summaryText.split("\n"); + +describe("act gates", () => { + it("refuses a re-run attempt before any API call", async () => { + const { gh, core } = await run({ fixture: T.baseFixture(), env: { GITHUB_RUN_ATTEMPT: "2" } }); + assert.match(core.failed, /attempt 2/); + assert.equal(gh.calls.length, 0); + assert.equal(core.outputs.published, "false"); + }); + + for (const permission of ["read", "none"]) { + it(`refuses an actor whose .permission is ${permission} (triage reports read)`, async () => { + const fx = T.baseFixture(); + fx.permissions.alice = permission; + const { gh, core } = await run({ fixture: fx, env: { RELEASE_DRY_RUN: "true" } }); + assert.match(core.failed, /needs write/); + assert.equal(gh.called("repos.listReleases").length, 0); + }); + } + + it("allows maintain, which .permission reports as write", async () => { + const { core } = await run({ fixture: T.baseFixture(), env: { RELEASE_DRY_RUN: "true" } }); + assert.equal(core.failed, null); + }); + + it("refuses when the App token was minted for a different repo", async () => { + const { gh, core } = await run({ + fixture: T.baseFixture(), env: { RELEASE_TARGET_NAME: "keras" }, + }); + assert.match(core.failed, /scoped to/); + assert.equal(gh.calls.length, 0); + }); +}); + +describe("act dry run", () => { + it("prepare: zero writes, discovery report, escaped titles", async () => { + const git = T.fakeGit(); + const { gh, core } = await run({ + fixture: T.baseFixture(), git, + env: { RELEASE_DRY_RUN: "true", RELEASE_CHERRY_PICKS: "#101" }, + }); + assert.equal(core.failed, null); + assert.deepEqual(gh.writes(), []); + assert.equal(gh.called("repos.generateReleaseNotes").length, 0); + assert.equal(git.calls.length, 0); + assert.equal(core.outputs.published, "false"); + assert.match(lines(core)[0], /0\.33\.1/); + assert.match(lines(core)[0], /prepare/); + assert.match(lines(core).at(-1), /^Next: /); + for (const text of ["Release branches", "Would rebuild", "Form values"]) { + assert.ok(core.summaryText.includes(text), text); + } + assert.ok(core.summaryText.includes(escapeMd("Fix *thing* "))); + assert.ok(!core.summaryText.includes("Fix *thing* ")); + }); + + it("draft present: reports the publish step and writes nothing", async () => { + const { gh, core } = await run({ fixture: mergedBump({ draft: true }), env: { RELEASE_DRY_RUN: "true" } }); + assert.equal(core.failed, null); + assert.deepEqual(gh.writes(), []); + assert.match(lines(core)[0], /publish/); + assert.ok(core.summaryText.includes("Would publish draft v0.33.1")); + assert.equal(core.outputs.published, "false"); + }); +}); + +describe("act idempotent resume", () => { + it("partial prepare: rebuilds the bot ref from B head, opens one PR", async () => { + const fx = T.baseFixture(); + fx.pulls.push(T.bumpPr(50, "0.33.1")); + const git = prepareGit(); + const { gh, core } = await run({ fixture: fx, git, env: { RELEASE_CHERRY_PICKS: "#101" } }); + assert.equal(core.failed, null); + const created = gh.called("pulls.create"); + assert.equal(created.length, 1); + assert.equal(created[0].params.head, "release-tool/bump-0.33.1"); + assert.equal(created[0].params.base, "r0.33"); + assert.match(created[0].params.body, /^Cherry-picks: #101/m); + assert.equal(gh.called("git.createRef").length, 0); + const checkout = git.calls.find((c) => c.args[0] === "checkout"); + assert.deepEqual(checkout.args.slice(-3), ["-B", "release-tool/bump-0.33.1", SHA.b0]); + const pick = git.calls.find((c) => c.args[0] === "cherry-pick"); + assert.ok(pick.args.includes("-x") && pick.args.includes(SHA.m1)); + const pushes = git.pushes(); + assert.equal(pushes.length, 1); + assert.ok(pushes[0].args.includes("HEAD:refs/heads/release-tool/bump-0.33.1")); + assert.ok(pushes[0].args.includes("--force")); + const clone = git.calls.find((c) => c.args[0] === "clone"); + assert.ok(!clone.args.join(" ").includes("test-token")); + assert.equal(core.outputs.published, "false"); + }); + + it("tag without release: one POST /releases, no re-tag", async () => { + const { gh, core } = await run({ fixture: mergedBump({ tag: true }) }); + assert.equal(core.failed, null); + assert.match(lines(core)[0], /resume-continue/); + const rel = gh.called("repos.createRelease"); + assert.equal(rel.length, 1); + assert.equal(rel[0].params.tag_name, "v0.33.1"); + assert.equal(rel[0].params.target_commitish, SHA.ms); + assert.equal(gh.called("git.createRef").length, 0); + assert.equal(core.outputs.published, "true"); + assert.equal(core.outputs.tag, "v0.33.1"); + }); + + it("draft present (blank version): publishes the bot draft only", async () => { + const { gh, core } = await run({ fixture: mergedBump({ draft: true }) }); + assert.equal(core.failed, null); + const upd = gh.called("repos.updateRelease"); + assert.equal(upd.length, 1); + assert.equal(upd[0].params.release_id, 7); + assert.equal(upd[0].params.draft, false); + assert.equal(gh.called("repos.createRelease").length, 0); + assert.equal(core.outputs.published, "true"); + }); +}); + +describe("act cherry-pick conflict", () => { + it("aborts, pushes nothing, reports files and unselected commits", async () => { + const git = prepareGit({ + "cherry-pick": (args) => { + if (args[1] === "--abort") return ""; + throw Object.assign(new Error("conflict"), + { stdout: "", stderr: "CONFLICT (content): Merge conflict in keras_hub/a.py" }); + }, + "diff": () => "keras_hub/a.py\n", + "merge-base": () => `${SHA.p0}\n`, + "log": () => `${SHA.m0}\tRefactor a (#99)\n`, + }); + const { gh, core } = await run({ fixture: T.baseFixture(), git, env: { RELEASE_CHERRY_PICKS: "#101" } }); + assert.match(core.failed, /conflicts/); + assert.deepEqual(gh.writes(), []); + assert.equal(git.pushes().length, 0); + assert.ok(git.calls.some((c) => c.args.join(" ") === "cherry-pick --abort")); + assert.ok(core.summaryText.includes(escapeMd("keras_hub/a.py"))); + assert.ok(core.summaryText.includes(SHA.m0.slice(0, 7))); + assert.equal(core.outputs.published, "false"); + }); +}); + +describe("act atomic release", () => { + it("continue makes exactly one POST /releases with tag + sha", async () => { + const { gh, core } = await run({ fixture: mergedBump() }); + assert.equal(core.failed, null); + assert.match(lines(core)[0], /continue/); + const writes = gh.writes(); + assert.deepEqual(writes.map((w) => w.name), ["repos.createRelease"]); + assert.equal(writes[0].params.tag_name, "v0.33.1"); + assert.equal(writes[0].params.target_commitish, SHA.ms); + assert.equal(writes[0].params.draft, false); + const notes = gh.called("repos.generateReleaseNotes")[0].params; + assert.equal(notes.previous_tag_name, "v0.33.0"); + }); + + it("a failed release call leaves no tag behind", async () => { + const { gh, core } = await run({ fixture: mergedBump(), fail: { createRelease: true } }); + assert.ok(core.failed); + assert.deepEqual(gh.writes().map((w) => w.name), ["repos.createRelease"]); + assert.equal(core.outputs.published, "false"); + }); +}); + +describe("act fast-path", () => { + for (const mode of ["auto", "draft"]) { + it(`${mode}: targets the resolved sha, never the branch name`, async () => { + const { gh, core } = await run({ fixture: fastPathFixture(), env: { RELEASE_MODE: mode } }); + assert.equal(core.failed, null); + assert.match(lines(core)[0], /fast-path/); + const rel = gh.called("repos.createRelease"); + assert.equal(rel.length, 1); + assert.equal(rel[0].params.target_commitish, SHA.b1); + assert.equal(rel[0].params.draft, mode === "draft"); + assert.equal(core.outputs.published, mode === "auto" ? "true" : "false"); + }); + } +}); + +describe("act legacy publisher guard", () => { + const cases = { + "HTTP 403": { status: 403 }, + "HTTP 404": undefined, + "active": { status: 200, state: "active" }, + }; + for (const [name, workflow] of Object.entries(cases)) { + it(`refuses when the workflow lookup is ${name}`, async () => { + const fx = fastPathFixture(); + fx.workflows = workflow ? { "publish-to-pypi.yml": workflow } : {}; + const { gh, core } = await run({ fixture: fx }); + assert.match(core.failed, /publish-to-pypi\.yml/); + assert.deepEqual(gh.writes(), []); + }); + } + + it("is skipped only for an explicit null", async () => { + const root = T.configRoot({ legacy_publish_workflow: null }); + const { gh, core } = await run({ fixture: fastPathFixture(), root }); + assert.equal(core.failed, null); + assert.equal(gh.called("actions.getWorkflow").length, 0); + assert.equal(gh.called("repos.createRelease").length, 1); + }); +}); + +describe("act already-released", () => { + it("typed version with a published release: no writes, published=true", async () => { + const git = T.fakeGit(); + const { gh, core } = await run({ + fixture: T.baseFixture(), git, + env: { RELEASE_VERSION: "0.33.0", RELEASE_KIND: "major" }, + }); + assert.equal(core.failed, null); + assert.match(lines(core)[0], /already-released/); + assert.deepEqual(gh.writes(), []); + assert.equal(git.calls.length, 0); + assert.equal(core.outputs.published, "true"); + assert.equal(core.outputs.tag, "v0.33.0"); + }); +}); diff --git a/scripts/release/dispatch.js b/scripts/release/dispatch.js new file mode 100644 index 0000000..748f68e --- /dev/null +++ b/scripts/release/dispatch.js @@ -0,0 +1,76 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * release.yml `dispatch` job: starts publish.yml on shared-workflows main. + * + * Runs with an App token scoped to shared-workflows and `actions: write` + * only. Refuses a re-run attempt. `requested_by` is always the triggering + * actor of this panel run (from env), never a free-text input. + */ + +const { RefusalError } = require("./lib/errors"); +const pep440 = require("./lib/pep440"); +const { Gh } = require("./lib/gh"); +const S = require("./lib/summary"); +const plan = require("./plan"); + +const TAG_RE = /^v(\d+\.\d+\.\d+(?:\.dev\d+)?)$/; + +async function main({ github, context, core, env = process.env }) { + const report = new S.Report("Publish dispatch"); + report.setHeader(null, "dispatch"); + try { + plan.assertFirstAttempt(env); + const repository = String(env.RELEASE_REPOSITORY || ""); + const tag = String(env.RELEASE_TAG || ""); + const override = String(env.RELEASE_OVERRIDE_GENERATED_TESTS || ""); + const requestedBy = String(env.RELEASE_TRIGGERING_ACTOR || ""); + plan.assertKnownRepository(repository, "start a new panel run."); + const m = tag.match(TAG_RE); + if (!m) { + throw new RefusalError(`act produced no valid tag ("${S.escapeMd(tag)}").`, + "check the act job's summary."); + } + pep440.parse(m[1]); + report.setHeader(m[1], "dispatch"); + if (override !== "true" && override !== "false") { + throw new RefusalError(`Invalid override_generated_tests "${S.escapeMd(override)}".`, + "start a new panel run."); + } + if (!requestedBy) { + throw new RefusalError("The triggering actor is unknown.", "start a new panel run."); + } + const { owner, repo } = context.repo; + const gh = new Gh({ github, owner, repo }); + await gh.dispatch("publish.yml", "main", { + repository, tag, override_generated_tests: override, + requested_by: requestedBy, + }); + const runs = `${context.serverUrl}/${owner}/${repo}/actions/workflows/publish.yml`; + report.add(`Dispatched \`publish ${repository} ${tag}\` on main ` + + `(requested by ${S.escapeMd(requestedBy)}, override_generated_tests ` + + `${override}).`); + report.setNext(`watch the run titled \`publish ${repository} ${tag}\` at ${runs}.`); + } catch (err) { + plan.failWith(core, report, err); + } + await S.writeSummary(core, report); +} + +module.exports = { main }; diff --git a/scripts/release/dispatch.test.js b/scripts/release/dispatch.test.js new file mode 100644 index 0000000..761cce3 --- /dev/null +++ b/scripts/release/dispatch.test.js @@ -0,0 +1,77 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); +const T = require("./test_support"); +const dispatch = require("./dispatch"); + +async function run(env = {}) { + const gh = T.fakeGithub({}); + const core = T.fakeCore(); + await dispatch.main({ + github: gh.github, context: T.context(), core, + env: { + GITHUB_RUN_ATTEMPT: "1", RELEASE_REPOSITORY: "keras-hub", + RELEASE_TAG: "v0.33.1", RELEASE_OVERRIDE_GENERATED_TESTS: "false", + RELEASE_TRIGGERING_ACTOR: "alice", GITHUB_TRIGGERING_ACTOR: "mallory", + ...env, + }, + }); + return { gh, core }; +} + +describe("dispatch", () => { + it("refuses a re-run attempt and dispatches nothing", async () => { + const { gh, core } = await run({ GITHUB_RUN_ATTEMPT: "2" }); + assert.match(core.failed, /attempt 2/); + assert.equal(gh.calls.length, 0); + }); + + it("dispatches publish.yml on main with the triggering actor", async () => { + const { gh, core } = await run({ RELEASE_OVERRIDE_GENERATED_TESTS: "true" }); + assert.equal(core.failed, null); + const calls = gh.called("actions.createWorkflowDispatch"); + assert.equal(calls.length, 1); + const p = calls[0].params; + assert.equal(p.workflow_id, "publish.yml"); + assert.equal(p.ref, "main"); + assert.equal(p.owner, "keras-team"); + assert.equal(p.repo, "shared-workflows"); + assert.deepEqual(p.inputs, { + repository: "keras-hub", tag: "v0.33.1", + override_generated_tests: "true", requested_by: "alice", + }); + assert.match(core.summaryText.split("\n")[0], /0\.33\.1 — step: dispatch/); + assert.match(core.summaryText.split("\n").at(-1), /^Next: /); + }); + + for (const [name, env] of Object.entries({ + "an empty tag": { RELEASE_TAG: "" }, + "a branch-name tag": { RELEASE_TAG: "r0.33" }, + "an unknown repository": { RELEASE_REPOSITORY: "other" }, + "a non-boolean override": { RELEASE_OVERRIDE_GENERATED_TESTS: "1" }, + "an empty triggering actor": { RELEASE_TRIGGERING_ACTOR: "" }, + })) { + it(`refuses ${name}`, async () => { + const { gh, core } = await run(env); + assert.ok(core.failed); + assert.equal(gh.called("actions.createWorkflowDispatch").length, 0); + }); + } +}); diff --git a/scripts/release/expr.js b/scripts/release/expr.js new file mode 100644 index 0000000..c76786a --- /dev/null +++ b/scripts/release/expr.js @@ -0,0 +1,269 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Evaluator for the subset of GitHub Actions expressions used in the release + * workflows. Test helper only: lets tests evaluate the literal `if:` strings + * read from the YAML instead of a hand-maintained JS mirror. + * + * Supported: `==`, `!=`, `&&`, `||`, `!`, parentheses, string literals + * ('...' with '' escape), `true`, `false`, `null`, integer literals, property + * paths (`needs.generated-tests.result`, `inputs.dry_run`, `github.ref`, ...) + * and the status functions `always()`, `success()`, `failure()`, + * `cancelled()`. Semantics follow the GitHub docs: `&&` / `||` return an + * operand (not a boolean), `==` is loose (mismatched types compare as + * numbers), and string equality ignores case. A path whose root is not in the + * supplied context throws, so a typo such as `input.dry_run` fails the test + * instead of evaluating to null. + */ + +class ExprError extends Error {} + +function tokenize(src) { + const tokens = []; + let i = 0; + while (i < src.length) { + const c = src[i]; + if (/\s/.test(c)) { + i += 1; + continue; + } + const two = src.slice(i, i + 2); + if (["==", "!=", "&&", "||"].includes(two)) { + tokens.push({ t: "op", v: two }); + i += 2; + continue; + } + if (c === "!" || c === "(" || c === ")" || c === ",") { + tokens.push({ t: c === "!" ? "op" : c, v: c }); + i += 1; + continue; + } + if (c === "'") { + let out = ""; + let j = i + 1; + for (;;) { + if (j >= src.length) throw new ExprError("unterminated string literal"); + if (src[j] === "'") { + if (src[j + 1] === "'") { + out += "'"; + j += 2; + continue; + } + break; + } + out += src[j]; + j += 1; + } + tokens.push({ t: "lit", v: out }); + i = j + 1; + continue; + } + const num = /^-?\d+/.exec(src.slice(i)); + if (num) { + tokens.push({ t: "lit", v: Number(num[0]) }); + i += num[0].length; + continue; + } + const ident = /^[A-Za-z_][A-Za-z0-9_-]*(\.[A-Za-z_][A-Za-z0-9_-]*)*/.exec(src.slice(i)); + if (ident) { + const word = ident[0]; + if (word === "true") tokens.push({ t: "lit", v: true }); + else if (word === "false") tokens.push({ t: "lit", v: false }); + else if (word === "null") tokens.push({ t: "lit", v: null }); + else tokens.push({ t: "path", v: word }); + i += word.length; + continue; + } + throw new ExprError(`unsupported character ${JSON.stringify(c)} at ${i}`); + } + return tokens; +} + +// AST: {k:"lit",v} {k:"path",v} {k:"call",name} {k:"not",e} {k:"bin",op,l,r} +function parseTokens(tokens) { + let pos = 0; + const peek = () => tokens[pos]; + const take = () => tokens[pos++]; + + function primary() { + const tok = take(); + if (!tok) throw new ExprError("unexpected end of expression"); + if (tok.t === "lit") return { k: "lit", v: tok.v }; + if (tok.t === "(") { + const e = orExpr(); + const close = take(); + if (!close || close.t !== ")") throw new ExprError("missing )"); + return e; + } + if (tok.t === "path") { + if (peek() && peek().t === "(") { + take(); + const close = take(); + if (!close || close.t !== ")") throw new ExprError(`only zero-argument calls are supported: ${tok.v}`); + if (!["always", "success", "failure", "cancelled"].includes(tok.v)) { + throw new ExprError(`unsupported function ${tok.v}()`); + } + return { k: "call", name: tok.v }; + } + return { k: "path", v: tok.v }; + } + throw new ExprError(`unexpected token ${JSON.stringify(tok.v)}`); + } + + function unary() { + if (peek() && peek().t === "op" && peek().v === "!") { + take(); + return { k: "not", e: unary() }; + } + return primary(); + } + + function eqExpr() { + let l = unary(); + while (peek() && peek().t === "op" && (peek().v === "==" || peek().v === "!=")) { + const op = take().v; + l = { k: "bin", op, l, r: unary() }; + } + return l; + } + + function andExpr() { + let l = eqExpr(); + while (peek() && peek().t === "op" && peek().v === "&&") { + take(); + l = { k: "bin", op: "&&", l, r: eqExpr() }; + } + return l; + } + + function orExpr() { + let l = andExpr(); + while (peek() && peek().t === "op" && peek().v === "||") { + take(); + l = { k: "bin", op: "||", l, r: andExpr() }; + } + return l; + } + + const ast = orExpr(); + if (pos !== tokens.length) throw new ExprError(`unexpected token ${JSON.stringify(tokens[pos].v)}`); + return ast; +} + +function unwrap(src) { + const s = src.trim(); + const m = /^\$\{\{([\s\S]*)\}\}$/.exec(s); + return m ? m[1] : s; +} + +function parse(src) { + return parseTokens(tokenize(unwrap(src))); +} + +function truthy(v) { + return !(v === false || v === null || v === undefined || v === "" || v === 0 || Number.isNaN(v)); +} + +function toNumber(v) { + if (v === null || v === undefined) return 0; + if (typeof v === "boolean") return v ? 1 : 0; + if (typeof v === "number") return v; + if (typeof v === "string") return v.trim() === "" ? 0 : Number(v); + return NaN; +} + +function looseEquals(a, b) { + const na = a === undefined ? null : a; + const nb = b === undefined ? null : b; + if (typeof na === "string" && typeof nb === "string") return na.toLowerCase() === nb.toLowerCase(); + if (typeof na === typeof nb && na !== null && nb !== null) return na === nb; + if (na === null && nb === null) return true; + return toNumber(na) === toNumber(nb); +} + +function lookup(path, ctx) { + const parts = path.split("."); + if (!Object.prototype.hasOwnProperty.call(ctx, parts[0])) { + throw new ExprError(`unknown context "${parts[0]}" in ${path}`); + } + let cur = ctx[parts[0]]; + for (const p of parts.slice(1)) { + if (cur === null || cur === undefined || typeof cur !== "object") return null; + cur = Object.prototype.hasOwnProperty.call(cur, p) ? cur[p] : null; + } + return cur === undefined ? null : cur; +} + +function needsResults(ctx) { + return Object.values(ctx.needs || {}).map((n) => n && n.result); +} + +function evalAst(ast, ctx) { + switch (ast.k) { + case "lit": + return ast.v; + case "path": + return lookup(ast.v, ctx); + case "call": + if (ast.name === "always") return true; + if (ast.name === "cancelled") return Boolean(ctx.cancelled); + if (ast.name === "failure") return needsResults(ctx).some((r) => r === "failure"); + return !ctx.cancelled && needsResults(ctx).every((r) => r === "success"); + case "not": + return !truthy(evalAst(ast.e, ctx)); + case "bin": { + if (ast.op === "&&") { + const l = evalAst(ast.l, ctx); + return truthy(l) ? evalAst(ast.r, ctx) : l; + } + if (ast.op === "||") { + const l = evalAst(ast.l, ctx); + return truthy(l) ? l : evalAst(ast.r, ctx); + } + const eq = looseEquals(evalAst(ast.l, ctx), evalAst(ast.r, ctx)); + return ast.op === "==" ? eq : !eq; + } + default: + throw new ExprError(`bad node ${ast.k}`); + } +} + +function hasStatusCall(ast) { + if (ast.k === "call") return true; + if (ast.k === "not") return hasStatusCall(ast.e); + if (ast.k === "bin") return hasStatusCall(ast.l) || hasStatusCall(ast.r); + return false; +} + +/** Evaluates an expression and returns its value (not coerced). */ +function evaluate(src, ctx) { + return evalAst(parse(src), ctx); +} + +/** + * Evaluates a job-level `if:` the way the runner does: when the expression + * contains no status function, GitHub implicitly prepends `success() &&`. + */ +function evaluateJobIf(src, ctx) { + const ast = parse(src); + if (!hasStatusCall(ast) && !evalAst({ k: "call", name: "success" }, ctx)) return false; + return truthy(evalAst(ast, ctx)); +} + +module.exports = { evaluate, evaluateJobIf, truthy, ExprError }; diff --git a/scripts/release/lib/config.js b/scripts/release/lib/config.js new file mode 100644 index 0000000..b77fb7c --- /dev/null +++ b/scripts/release/lib/config.js @@ -0,0 +1,212 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Loads and validates release/repos/.json, and maps a config to the + * repo a run targets (the fork override of plan section 3). This is the one + * module both release.yml and publish.yml use for that mapping. + */ + +const fs = require("node:fs"); +const path = require("node:path"); +const { RefusalError } = require("./errors"); + +const DEFAULT_ROOT = path.resolve(__dirname, "..", "..", ".."); +const NAME_RE = /^[a-z0-9][a-z0-9-]*$/; +const OWNER_RE = /^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$/; +const REPO_RE = /^[A-Za-z0-9-]+\/[A-Za-z0-9._-]+$/; +const WORKFLOW_RE = /^[A-Za-z0-9._-]+\.ya?ml$/; +const UPSTREAM_OWNER = "keras-team"; + +const FIX_NEXT = (name) => + `fix release/repos/${name}.json in keras-team/shared-workflows (via a ` + + "reviewed PR), then run the panel again."; + +function isNonEmptyString(v) { + return typeof v === "string" && v.trim() !== ""; +} + +function isRelativePath(v) { + return ( + isNonEmptyString(v) && + !v.startsWith("/") && + !v.includes("\\") && + !v.split("/").includes("..") + ); +} + +function nonEmptyArrayOf(check) { + return (v) => Array.isArray(v) && v.length > 0 && v.every(check); +} + +/** Field name -> [validator, human description]. Every field is required. */ +const SCHEMA = { + enabled: [(v) => typeof v === "boolean", "a boolean"], + github_repo: [(v) => typeof v === "string" && REPO_RE.test(v), '"owner/name"'], + default_branch: [isNonEmptyString, "a non-empty string"], + release_branch_pattern: [isNonEmptyString, "a regex string"], + tag_prefix: [(v) => v === "v", 'exactly "v"'], + version_files: [nonEmptyArrayOf(isRelativePath), "a non-empty array of relative paths"], + packages: [nonEmptyArrayOf(isNonEmptyString), "a non-empty array of strings"], + import_names: [nonEmptyArrayOf(isNonEmptyString), "a non-empty array of strings"], + build_setup_command: [isNonEmptyString, "a non-empty string"], + build_command: [isNonEmptyString, "a non-empty string"], + dist_dirs: [nonEmptyArrayOf(isRelativePath), "a non-empty array of relative paths"], + dev_suffix_on_master: [(v) => typeof v === "string", "a string (may be empty)"], + smoke_snippet: [isNonEmptyString, "a non-empty string"], + backends: [nonEmptyArrayOf(isNonEmptyString), "a non-empty array of strings"], + legacy_publish_workflow: [ + (v) => v === null || (typeof v === "string" && WORKFLOW_RE.test(v)), + "a workflow file name or explicit null", + ], +}; + +function countCaptureGroups(re) { + // Appending an always-matching alternative makes exec("") return one slot + // per capture group. + return new RegExp(`${re.source}|`).exec("").length - 1; +} + +/** + * Validates a parsed config object. Does not look at `enabled` beyond its + * type, so disabled configs can be checked too. Returns the config. + */ +function validateConfig(config, name) { + const next = FIX_NEXT(name); + if (config === null || typeof config !== "object" || Array.isArray(config)) { + throw new RefusalError(`Config ${name}.json is not a JSON object.`, next); + } + const unknown = Object.keys(config).filter((k) => !(k in SCHEMA)); + if (unknown.length) { + throw new RefusalError( + `Config ${name}.json has unknown field(s): ${unknown.join(", ")}.`, + next, + ); + } + for (const [field, [check, desc]] of Object.entries(SCHEMA)) { + if (!Object.prototype.hasOwnProperty.call(config, field)) { + const hint = + field === "legacy_publish_workflow" + ? " (set it to explicit null only after the legacy file is deleted)" + : ""; + throw new RefusalError( + `Config ${name}.json is missing required field "${field}"${hint}.`, + next, + ); + } + if (!check(config[field])) { + throw new RefusalError( + `Config ${name}.json field "${field}" must be ${desc}.`, + next, + ); + } + } + const pattern = config.release_branch_pattern; + let re; + try { + re = new RegExp(pattern); + } catch (e) { + throw new RefusalError( + `Config ${name}.json release_branch_pattern is not a valid regex: ${e.message}`, + next, + ); + } + if (!pattern.startsWith("^") || !pattern.endsWith("$")) { + throw new RefusalError( + `Config ${name}.json release_branch_pattern must be anchored with ^ and $.`, + next, + ); + } + if (countCaptureGroups(re) !== 2) { + throw new RefusalError( + `Config ${name}.json release_branch_pattern must have exactly two ` + + "capture groups (X and Y).", + next, + ); + } + if (re.test(config.default_branch)) { + throw new RefusalError( + `Config ${name}.json default_branch matches release_branch_pattern.`, + next, + ); + } + return config; +} + +/** + * Reads, validates and returns release/repos/.json. Refuses repos with + * `enabled: false`. + */ +function loadConfig(name, rootDir = DEFAULT_ROOT) { + if (typeof name !== "string" || !NAME_RE.test(name)) { + throw new RefusalError( + `"${name}" is not a valid repository name.`, + "pick one of the repositories offered by the panel.", + ); + } + const file = path.join(rootDir, "release", "repos", `${name}.json`); + let text; + try { + text = fs.readFileSync(file, "utf8"); + } catch (e) { + throw new RefusalError( + `No release config for "${name}" (release/repos/${name}.json).`, + "pick one of the repositories offered by the panel.", + ); + } + let config; + try { + config = JSON.parse(text); + } catch (e) { + throw new RefusalError( + `Config ${name}.json is not valid JSON: ${e.message}`, + FIX_NEXT(name), + ); + } + validateConfig(config, name); + if (config.enabled !== true) { + throw new RefusalError( + `Releases for "${name}" are not enabled (release/repos/${name}.json has ` + + '"enabled": false).', + "ask a shared-workflows maintainer to enable the repo once its rollout " + + "steps in RELEASE_PROCESS.md are done.", + ); + } + return config; +} + +/** + * Plan section 3: upstream runs target config.github_repo; a run in a fork of + * shared-workflows targets the same-named repo under the fork owner. Owners + * are compared case-insensitively, as GitHub does, so a fork owner can never + * be mistaken for keras-team or vice versa. + */ +function targetRepo(config, repositoryOwner) { + if (typeof repositoryOwner !== "string" || !OWNER_RE.test(repositoryOwner)) { + throw new Error(`Invalid repository owner "${repositoryOwner}".`); + } + if (!config || typeof config.github_repo !== "string" || !REPO_RE.test(config.github_repo)) { + throw new Error("Config has no valid github_repo."); + } + if (repositoryOwner.toLowerCase() === UPSTREAM_OWNER) { + return config.github_repo; + } + return `${repositoryOwner}/${config.github_repo.split("/")[1]}`; +} + +module.exports = { loadConfig, validateConfig, targetRepo, SCHEMA, DEFAULT_ROOT, LOGIN_RE: OWNER_RE }; diff --git a/scripts/release/lib/config.test.js b/scripts/release/lib/config.test.js new file mode 100644 index 0000000..fcd6d22 --- /dev/null +++ b/scripts/release/lib/config.test.js @@ -0,0 +1,144 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { loadConfig, validateConfig, targetRepo, SCHEMA, DEFAULT_ROOT } = require("./config"); +const { RefusalError } = require("./errors"); + +const REPOS = ["keras-hub", "keras", "keras-rs", "kinetic"]; + +function readReal(name) { + return JSON.parse(fs.readFileSync(path.join(DEFAULT_ROOT, "release", "repos", `${name}.json`), "utf8")); +} + +/** Writes `config` as release/repos/.json under a fresh temp root. */ +function withConfig(config, fn, name = "demo") { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "sw-config-")); + try { + fs.mkdirSync(path.join(root, "release", "repos"), { recursive: true }); + const text = typeof config === "string" ? config : JSON.stringify(config); + fs.writeFileSync(path.join(root, "release", "repos", `${name}.json`), text); + return fn(root, name); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +} + +function refusesWith(config, re) { + withConfig(config, (root, name) => { + assert.throws(() => loadConfig(name, root), (e) => e instanceof RefusalError && re.test(e.message)); + }); +} + +test("every shipped config is valid; only keras-hub is enabled", () => { + for (const name of REPOS) { + const c = readReal(name); + validateConfig(c, name); + assert.equal(c.enabled, name === "keras-hub", name); + assert.equal(c.tag_prefix, "v"); + assert.deepEqual(Object.keys(c).sort(), Object.keys(SCHEMA).sort(), name); + } + assert.equal(readReal("keras").default_branch, "master"); + assert.equal(readReal("keras-rs").default_branch, "main"); + assert.equal(readReal("kinetic").default_branch, "main"); +}); + +test("keras-hub config matches the target repo facts", () => { + const c = loadConfig("keras-hub"); + assert.equal(c.github_repo, "keras-team/keras-hub"); + assert.equal(c.default_branch, "master"); + assert.deepEqual(c.version_files, ["keras_hub/src/version.py"]); + assert.deepEqual(c.packages, ["keras-hub", "keras-nlp"]); + assert.deepEqual(c.dist_dirs, ["dist/", "keras_nlp/dist/"]); + assert.equal(c.build_setup_command, "pip install -r requirements.txt"); + assert.equal(c.build_command, "python pip_build.py"); + assert.equal(c.legacy_publish_workflow, "publish-to-pypi.yml"); +}); + +test("disabled repos are refused by loadConfig", () => { + for (const name of ["keras", "keras-rs", "kinetic"]) { + assert.throws(() => loadConfig(name), (e) => e instanceof RefusalError && /not enabled/.test(e.message)); + } +}); + +test("unknown and missing repos are refused, names are validated", () => { + assert.throws(() => loadConfig("nope"), /No release config/); + assert.throws(() => loadConfig("../keras-hub"), /not a valid repository name/); + assert.throws(() => loadConfig(""), /not a valid repository name/); +}); + +test("loader rejections", () => { + const base = { ...readReal("keras-hub") }; + refusesWith("{not json", /not valid JSON/); + refusesWith("[]", /not a JSON object/); + refusesWith({ ...base, extra: 1 }, /unknown field\(s\): extra/); + for (const field of Object.keys(SCHEMA)) { + const c = { ...base }; + delete c[field]; + refusesWith(c, new RegExp(`missing required field "${field}"`)); + } + refusesWith({ ...base, enabled: "true" }, /"enabled" must be/); + refusesWith({ ...base, tag_prefix: "" }, /"tag_prefix" must be exactly "v"/); + refusesWith({ ...base, tag_prefix: "keras-v" }, /"tag_prefix"/); + refusesWith({ ...base, version_files: [] }, /"version_files"/); + refusesWith({ ...base, version_files: "keras_hub/src/version.py" }, /"version_files"/); + refusesWith({ ...base, version_files: ["../x.py"] }, /"version_files"/); + refusesWith({ ...base, version_files: ["/etc/passwd"] }, /"version_files"/); + refusesWith({ ...base, packages: [] }, /"packages"/); + refusesWith({ ...base, dist_dirs: ["../dist"] }, /"dist_dirs"/); + refusesWith({ ...base, github_repo: "keras-hub" }, /"github_repo"/); + refusesWith({ ...base, legacy_publish_workflow: "" }, /"legacy_publish_workflow"/); + refusesWith({ ...base, legacy_publish_workflow: "../x.yml" }, /"legacy_publish_workflow"/); + refusesWith({ ...base, release_branch_pattern: "^r(\\d+$" }, /not a valid regex/); + refusesWith({ ...base, release_branch_pattern: "r(\\d+)\\.(\\d+)" }, /anchored/); + refusesWith({ ...base, release_branch_pattern: "^r(\\d+)$" }, /exactly two capture groups/); + refusesWith({ ...base, release_branch_pattern: "^r(\\d+)\\.(\\d+)(x)?$" }, /exactly two/); + refusesWith({ ...base, release_branch_pattern: "^(.*)(.*)$" }, /default_branch matches/); +}); + +test("legacy_publish_workflow may be explicit null, never absent", () => { + const base = { ...readReal("keras-hub"), legacy_publish_workflow: null }; + withConfig(base, (root, name) => { + assert.equal(loadConfig(name, root).legacy_publish_workflow, null); + }); + const absent = { ...base }; + delete absent.legacy_publish_workflow; + refusesWith(absent, /missing required field "legacy_publish_workflow"/); +}); + +test("non-capturing groups do not count as X/Y groups", () => { + const c = { ...readReal("keras-hub"), release_branch_pattern: "^(?:rel-)?r(\\d+)\\.(\\d+)$" }; + withConfig(c, (root, name) => assert.ok(loadConfig(name, root))); +}); + +test("targetRepo maps forks and never resolves to keras-team for a fork", () => { + const c = readReal("keras-hub"); + assert.equal(targetRepo(c, "keras-team"), "keras-team/keras-hub"); + assert.equal(targetRepo(c, "Keras-Team"), "keras-team/keras-hub"); + assert.equal(targetRepo(c, "laxmareddyp"), "laxmareddyp/keras-hub"); + for (const owner of ["laxmareddyp", "keras-team-fork", "evil-keras-team", "a"]) { + assert.ok(!targetRepo(c, owner).toLowerCase().startsWith("keras-team/"), owner); + } + assert.throws(() => targetRepo(c, ""), /Invalid repository owner/); + assert.throws(() => targetRepo(c, "a/b"), /Invalid repository owner/); + assert.throws(() => targetRepo({}, "keras-team"), /github_repo/); +}); diff --git a/scripts/release/lib/detect.js b/scripts/release/lib/detect.js new file mode 100644 index 0000000..a4def6e --- /dev/null +++ b/scripts/release/lib/detect.js @@ -0,0 +1,300 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * In-flight scan, version resolution and step detection (plan section 4). + * PURE: everything comes in through `state` (built by gh.gatherState). + */ + +const pep440 = require("./pep440"); +const V = require("./version"); +const { RefusalError } = require("./errors"); +const { parsePicks } = require("./picks"); + +const BUMP_PREFIX = "release-tool/bump-"; +const ABANDONED_LABEL = "release-tool:abandoned"; +const SHA40 = /^[0-9a-f]{40}$/i; + +/** Version named by a release bump head ref, or null. */ +function bumpVersion(headRef) { + if (typeof headRef !== "string" || !headRef.startsWith(BUMP_PREFIX)) return null; + const v = headRef.slice(BUMP_PREFIX.length); + return pep440.isValid(v) ? v : null; +} + +/** + * X.Y of a `X.Y...` tag, read loosely so that tags outside the + * supported version subset still map to a line (fail closed); else null. + */ +function tagLineXY(tag, prefix) { + if (typeof tag !== "string" || !tag.startsWith(prefix)) return null; + const m = /^(\d+)\.(\d+)(?:\D|$)/.exec(tag.slice(prefix.length)); + return m ? { x: Number(m[1]), y: Number(m[2]) } : null; +} + +function draftMapsToLine(tag, xy, prefix) { + const t = tagLineXY(tag, prefix); + return t !== null && t.x === xy.x && t.y === xy.y; +} + +function isAbandoned(pr) { + return (pr.labels || []).includes(ABANDONED_LABEL) && pr.abandoned_by_writer === true; +} + +/** Bump PRs into the branch, excluding closed-unmerged ones. */ +function bumpPrs(state) { + return (state.prs || []) + .filter((pr) => pr.base_ref === state.branch && bumpVersion(pr.head_ref) !== null) + .filter((pr) => pr.state === "open" || pr.merged === true) + .map((pr) => ({ pr, version: bumpVersion(pr.head_ref) })); +} + +function indexState(state, prefix) { + const tagSha = new Map(); + for (const t of state.tags || []) { + if (t.name.startsWith(prefix)) tagSha.set(t.name.slice(prefix.length), t.sha); + } + const published = new Map(); + const drafts = []; + for (const r of state.releases || []) { + if (r.draft) drafts.push(r); + else if (r.tag_name.startsWith(prefix)) published.set(r.tag_name.slice(prefix.length), r); + } + return { tagSha, published, drafts }; +} + +/** Plan section 4 "In-flight release on B". Returns { signals, stale, warnings }. */ +function scanInFlight({ state, prefix, xy, botLogin, idx, lineTop }) { + const signals = []; + const stale = []; + const warnings = []; + const hasAnyRelease = (v) => + idx.published.has(v) || idx.drafts.some((d) => d.tag_name === prefix + v); + for (const { pr, version } of bumpPrs(state)) { + if (pr.state === "open") { + signals.push({ kind: "open-pr", version, pr }); + continue; + } + if ((pr.labels || []).includes(ABANDONED_LABEL)) { + if (isAbandoned(pr)) continue; + warnings.push( + `PR #${pr.number} has the ${ABANDONED_LABEL} label, but it was not added ` + + "by someone with write access, so it is ignored.", + ); + } + const sha = idx.tagSha.get(version); + if (sha !== undefined) { + if (sha === pr.merge_commit_sha && pr.author_login === botLogin && !hasAnyRelease(version)) { + signals.push({ kind: "tagged-pr", version, pr }); + } else if (!hasAnyRelease(version)) { + stale.push({ pr, version }); + } + continue; + } + if (lineTop === null || pep440.compare(version, lineTop) > 0) { + signals.push({ kind: "merged-pr", version, pr }); + } else { + stale.push({ pr, version }); + } + } + for (const d of idx.drafts) { + if (!draftMapsToLine(d.tag_name, xy, prefix)) continue; + const version = d.tag_name.slice(prefix.length); + if (!pep440.isValid(version)) { + throw new RefusalError( + `Draft release "${d.tag_name}" maps to ${state.branch} but is not a version ` + + "the release tool understands.", + "delete or publish that draft by hand, then run the panel again.", + ); + } + signals.push({ kind: "draft", version, draft: d }); + } + return { signals, stale, warnings }; +} + +/** Plan section 4 "Released on GitHub but not on PyPI". */ +function pypiCheck({ state, xy, idx }) { + const released = V.highest([...idx.published.keys()].filter((v) => { + const p = pep440.tryParse(v); + return p && p.x === xy.x && p.y === xy.y; + })); + if (released === null) return null; + if (state.pypiVersions === null || state.pypiVersions === undefined) { + return `Could not check PyPI for ${released}; v${released} may be released on ` + + "GitHub but not on PyPI."; + } + if (!state.pypiVersions.has(released)) { + return `v${released} is released on GitHub but not on PyPI; run with ` + + `version=${released} to re-publish.`; + } + return null; +} + +function refuseIfMultiple(list, what, version) { + if (list.length > 1) { + throw new RefusalError( + `Found ${list.length} ${what} for ${version}; the release tool cannot tell ` + + "which one to use.", + "clean up the duplicates by hand, then run the panel again.", + ); + } + return list[0]; +} + +function detect({ state, inputs, config, botLogin }) { + const prefix = config.tag_prefix; + const { kind, channel } = inputs; + V.checkKindChannel(kind, channel); + const xy = V.branchXY(state.branch, config.release_branch_pattern); + const idx = indexState(state, prefix); + const lineTags = V.lineVersions(state.tags, xy, prefix); + const lineTop = V.highest(lineTags); + const warnings = []; + const pypiWarning = pypiCheck({ state, xy, idx }); + if (pypiWarning) warnings.push(pypiWarning); + + const scan = scanInFlight({ state, prefix, xy, botLogin, idx, lineTop }); + warnings.push(...scan.warnings); + const inFlightVersions = [...new Set(scan.signals.map((s) => s.version))]; + if (inFlightVersions.length > 1) { + throw new RefusalError( + `More than one release is in flight on ${state.branch}: ` + + `${inFlightVersions.join(", ")}.`, + `finish or clean up all but one (label a merged bump PR ${ABANDONED_LABEL}, ` + + "close an open one, or delete a draft), then run the panel again.", + ); + } + const inFlight = inFlightVersions[0] || null; + if (inFlight !== null) { + try { + V.checkConsistent(inFlight, { kind, channel, xy }); + } catch (e) { + throw new RefusalError( + `Release ${inFlight} is in flight on ${state.branch}, but it does not match ` + + `the inputs: ${e.message}`, + `rerun with the kind and channel of ${inFlight}, or clean it up first.`, + ); + } + } + + const typed = (inputs.version || "").trim().replace(new RegExp(`^${prefix}`), ""); + const version = V.resolveVersion({ + typed, inFlight, kind, channel, xy, lineVersions: lineTags, + released: new Set(idx.published.keys()), branch: state.branch, prefix, + }); + const released = idx.published.get(version) || null; + + const stale = scan.stale.filter((s) => s.version !== version).map((s) => s.pr.number); + const tag = prefix + version; + const result = (step, extra = {}) => ({ step, version, tag, stale, warnings, ...extra }); + const bumps = bumpPrs(state).filter((b) => b.version === version).map((b) => b.pr); + const merged = bumps.filter((pr) => pr.merged === true && !isAbandoned(pr)); + const tagSha = idx.tagSha.get(version); + + let detection; + if (released) { + detection = result("already-released", { release: released }); + } else if (idx.drafts.some((d) => d.tag_name === tag)) { + const draft = refuseIfMultiple(idx.drafts.filter((d) => d.tag_name === tag), "drafts", tag); + detection = result("publish", { draft }); + } else if (tagSha !== undefined) { + const pr = refuseIfMultiple(merged.filter((p) => p.merge_commit_sha === tagSha), "merged bump PRs", version); + if (!pr) { + throw new RefusalError( + `Tag ${tag} exists at ${tagSha.slice(0, 7)} without a release and not at the ` + + "merge commit of a release-tool bump PR.", + `delete the tag ${tag} if it is wrong, or release it by hand.`, + ); + } + detection = result("resume-continue", { pr }); + } else if (merged.length) { + detection = result("continue", { pr: refuseIfMultiple(merged, "merged bump PRs", version) }); + } else if (bumps.some((pr) => pr.state === "open")) { + detection = result("wait", { pr: bumps.find((pr) => pr.state === "open") }); + } else if (state.branchExists && state.versionAtHead === version && parsePicks(inputs.cherry_picks).length === 0) { + detection = result("fast-path"); + } else { + detection = result("prepare"); + } + checkPrepareOnlyInputs(detection.step, inputs); + if (detection.step === "prepare") checkPrepare({ state, inputs, pypiWarning }); + return detection; +} + +function checkPrepareOnlyInputs(step, inputs) { + const given = []; + if (parsePicks(inputs.cherry_picks).length) given.push("cherry_picks"); + if ((inputs.commit_sha || "").trim()) given.push("commit_sha"); + if (inputs.bump_master === "true") given.push("bump_master"); + if (step !== "prepare" && given.length) { + throw new RefusalError( + `The detected step is ${step}, which does not use ${given.join(", ")} ` + + "(they only apply when preparing a new version).", + `run again without ${given.join(", ")}.`, + ); + } +} + +/** Prepare-step rules that need no I/O. */ +function checkPrepare({ state, inputs, pypiWarning }) { + if (pypiWarning) { + throw new RefusalError( + `${pypiWarning} Refusing to prepare a newer version until that is fixed.`, + "re-publish the missing version first (see the line above).", + ); + } + const commitSha = (inputs.commit_sha || "").trim(); + if (!state.branchExists) { + if (inputs.kind !== "major") { + throw new RefusalError( + `Release branch ${state.branch} does not exist; a patch needs an existing branch.`, + "use kind=major with commit_sha to cut the branch, or fix release_branch.", + ); + } + if (!SHA40.test(commitSha)) { + throw new RefusalError( + `Release branch ${state.branch} does not exist; commit_sha must be the full ` + + "40-character master commit to cut it from.", + "set commit_sha to a full commit sha on the default branch.", + ); + } + if (parsePicks(inputs.cherry_picks).length) { + throw new RefusalError( + "Cherry-picks are not allowed when cutting a new branch: it is cut from " + + "commit_sha, which already contains master's fixes.", + "run again without cherry_picks.", + ); + } + } else if (commitSha) { + throw new RefusalError( + `Release branch ${state.branch} already exists; commit_sha is only for cutting a new branch.`, + "run again without commit_sha.", + ); + } +} + +module.exports = { + detect, + checkCommitShaVersion: V.checkCommitShaVersion, + bumpVersion, + draftMapsToLine, + tagLineXY, + BUMP_PREFIX, + ABANDONED_LABEL, + SHA40, +}; diff --git a/scripts/release/lib/detect.test.js b/scripts/release/lib/detect.test.js new file mode 100644 index 0000000..af999a5 --- /dev/null +++ b/scripts/release/lib/detect.test.js @@ -0,0 +1,275 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { detect, checkCommitShaVersion, bumpVersion, draftMapsToLine } = require("./detect"); +const { loadConfig } = require("./config"); +const { refusalMatching: refusal } = require("./errors"); + +const CONFIG = loadConfig("keras-hub"); +const { BOT } = require("../test_support"); +const sha = (c) => c.repeat(40); + +const tag = (v, s = sha("0")) => ({ name: `v${v}`, sha: s }); +const rel = (v, over = {}) => ({ + id: 1, tag_name: `v${v}`, draft: false, prerelease: v.includes("dev"), + target_commitish: sha("a"), author_login: BOT, ...over, +}); +const draft = (v, over = {}) => rel(v, { draft: true, ...over }); +const pr = (v, over = {}) => ({ + number: 10, state: "closed", merged: true, merge_commit_sha: sha("a"), + head_ref: `release-tool/bump-${v}`, head_repo_full_name: "keras-team/keras-hub", + base_ref: "r0.33", author_login: BOT, merged_by_login: "alice", body: "", + labels: [], abandoned_by_writer: false, ...over, +}); +const openPr = (v, over = {}) => pr(v, { state: "open", merged: false, merge_commit_sha: null, ...over }); + +function run(state = {}, inputs = {}) { + return detect({ + state: { + branch: "r0.33", branchExists: true, headSha: sha("f"), versionAtHead: "0.33.0.dev0", + defaultBranchVersion: "0.34.0.dev0", tags: [tag("0.33.0.dev0")], releases: [rel("0.33.0.dev0")], + prs: [], pickedPrNumbers: new Set(), pypiVersions: new Set(["0.33.0.dev0"]), ...state, + }, + inputs: { + repository: "keras-hub", channel: "dev", kind: "major", release_branch: "r0.33", + commit_sha: "", version: "", cherry_picks: "", bump_master: "false", release_mode: "auto", + override_generated_tests: "false", dry_run: "true", ...inputs, + }, + config: CONFIG, + botLogin: BOT, + }); +} + +test("step 1: typed V with a non-draft release -> already-released", () => { + const d = run({}, { version: "0.33.0.dev0" }); + assert.equal(d.step, "already-released"); + assert.equal(d.release.tag_name, "v0.33.0.dev0"); + assert.equal(d.tag, "v0.33.0.dev0"); +}); + +test("blank V never resolves to already-released", () => { + assert.equal(run().step, "prepare"); + assert.equal(run().version, "0.33.0.dev1"); + // Inconsistent data (a release whose tag is missing) still never auto-resolves. + assert.throws(() => run({ releases: [rel("0.33.0.dev0"), rel("0.33.0.dev1")] }), refusal(/already released/)); +}); + +test("step 2: merged bump-V PR + draft vV -> one candidate -> publish", () => { + const d = run({ prs: [pr("0.33.0.dev1")], releases: [rel("0.33.0.dev0"), draft("0.33.0.dev1")] }); + assert.equal(d.step, "publish"); + assert.equal(d.version, "0.33.0.dev1"); + assert.equal(d.draft.tag_name, "v0.33.0.dev1"); +}); + +test("step 3: tag at merge sha without release -> resume-continue (not stale)", () => { + const d = run({ prs: [pr("0.33.0.dev1")], tags: [tag("0.33.0.dev0"), tag("0.33.0.dev1", sha("a"))] }); + assert.equal(d.step, "resume-continue"); + assert.equal(d.version, "0.33.0.dev1"); + assert.equal(d.pr.number, 10); + assert.deepEqual(d.stale, []); +}); + +test("tag at merge sha of a human-authored PR is not in flight", () => { + const state = { prs: [pr("0.33.0.dev1", { author_login: "mallory" })], tags: [tag("0.33.0.dev0"), tag("0.33.0.dev1", sha("a"))] }; + const d = run(state, { version: "", channel: "dev" }); + assert.equal(d.version, "0.33.0.dev2"); + assert.deepEqual(d.stale, [10]); +}); + +test("step 4: merged bump-V PR not yet tagged -> continue", () => { + const d = run({ prs: [pr("0.33.0.dev1")] }); + assert.equal(d.step, "continue"); + assert.equal(d.pr.number, 10); +}); + +test("step 5: open bump-V PR -> wait; in-flight version reused over computed", () => { + const d = run({ prs: [openPr("0.33.0.dev3", { number: 42 })] }); + assert.equal(d.step, "wait"); + assert.equal(d.version, "0.33.0.dev3"); + assert.equal(d.pr.number, 42); +}); + +test("step 6: version files at head == V and zero picks -> fast-path", () => { + assert.equal(run({ versionAtHead: "0.33.0.dev1" }).step, "fast-path"); + assert.equal(run({ versionAtHead: "0.33.0.dev1" }, { cherry_picks: "#5" }).step, "prepare"); +}); + +test("step 7: otherwise -> prepare (F1: picks before X.Y.0)", () => { + const d = run({}, { cherry_picks: "#3101, #3110" }); + assert.equal(d.step, "prepare"); + assert.equal(d.version, "0.33.0.dev1"); +}); + +test("prepare-only inputs given in a non-prepare step -> refused", () => { + const state = { prs: [openPr("0.33.0.dev1")] }; + assert.throws(() => run(state, { cherry_picks: "#5" }), refusal(/detected step is wait.*cherry_picks/)); + assert.throws(() => run(state, { commit_sha: sha("c") }), refusal(/commit_sha/)); + assert.throws(() => run(state, { bump_master: "true" }), refusal(/bump_master/)); + assert.throws(() => run({ versionAtHead: "0.33.0.dev1" }, { bump_master: "true" }), refusal(/fast-path/)); + assert.throws(() => run({ prs: [pr("0.33.0.dev1")] }, { bump_master: "true" }), refusal(/continue/)); +}); + +test("in-flight for a different version than typed -> refused", () => { + const state = { prs: [openPr("0.33.0.dev1")] }; + assert.throws(() => run(state, { version: "0.33.0.dev2" }), refusal(/0.33.0.dev1 is in flight/)); + assert.throws(() => run(state, { version: "0.33.0.dev0" }), refusal(/0.33.0.dev1 is in flight/)); + assert.equal(run(state, { version: "v0.33.0.dev1" }).step, "wait"); +}); + +test("two distinct in-flight versions -> refused and listed", () => { + const state = { prs: [openPr("0.33.0.dev1")], releases: [rel("0.33.0.dev0"), draft("0.33.0.dev2")] }; + assert.throws(() => run(state), refusal(/0.33.0.dev1, 0.33.0.dev2/)); +}); + +test("old release-less tag on the line is not in flight (r0.32 v0.32.0.dev0)", () => { + const state = { branch: "r0.32", versionAtHead: "0.32.0.dev0", tags: [tag("0.32.0.dev0")], releases: [] }; + assert.equal(run(state).version, "0.32.0.dev1"); + assert.equal(run(state, { channel: "stable" }).version, "0.32.0"); +}); + +test("stale merged bump PRs are ignored and listed; completed ones are not", () => { + const state = { + tags: [tag("0.33.0.dev0"), tag("0.33.0.dev1", sha("b")), tag("0.33.0.dev2", sha("c"))], + releases: [rel("0.33.0.dev0"), rel("0.33.0.dev2")], + prs: [ + pr("0.32.9", { number: 1, base_ref: "r0.32" }), + pr("0.33.0.dev0", { number: 2, merge_commit_sha: sha("0") }), + pr("0.33.0.dev1", { number: 3, merge_commit_sha: sha("a") }), + pr("0.33.0.dev2", { number: 4, merge_commit_sha: sha("c") }), + ], + pypiVersions: new Set(["0.33.0.dev2"]), + }; + const d = run(state); + assert.equal(d.step, "prepare"); + assert.equal(d.version, "0.33.0.dev3"); + assert.deepEqual(d.stale, [3]); +}); + +test("untagged merged PR with V <= highest tag is stale", () => { + const d = run({ tags: [tag("0.33.0.dev0"), tag("0.33.0.dev2")], prs: [pr("0.33.0.dev1")] }); + assert.equal(d.version, "0.33.0.dev3"); + assert.deepEqual(d.stale, [10]); +}); + +test("abandoned label: writer -> not in flight; triage -> ignored with warning", () => { + const labelled = { labels: ["release-tool:abandoned"] }; + const byWriter = run({ versionAtHead: "0.33.0.dev1", prs: [pr("0.33.0.dev1", { ...labelled, abandoned_by_writer: true })] }); + assert.equal(byWriter.step, "fast-path"); + const byTriage = run({ versionAtHead: "0.33.0.dev1", prs: [pr("0.33.0.dev1", { ...labelled, abandoned_by_writer: false })] }); + assert.equal(byTriage.step, "continue"); + assert.match(byTriage.warnings.join("\n"), /not added by someone with write/); +}); + +test("closed unmerged bump PR is ignored", () => { + const d = run({ prs: [pr("0.33.0.dev1", { merged: false, merge_commit_sha: null })] }); + assert.equal(d.step, "prepare"); + assert.equal(d.pr, undefined); +}); + +test("bump PRs whose base is not the branch are ignored", () => { + assert.equal(run({ prs: [openPr("0.33.0.dev1", { base_ref: "master" })] }).step, "prepare"); + assert.equal(run({ prs: [pr("0.33.0.dev1", { base_ref: "r0.32" })] }).step, "prepare"); + assert.equal(run({ prs: [openPr("0.33.0.dev1", { head_ref: "release-tool/master-bump-0.33.0.dev1" })] }).step, "prepare"); +}); + +test("in-flight V contradicting inputs -> refused", () => { + assert.throws(() => run({ prs: [openPr("0.33.0.dev1")] }, { channel: "stable" }), refusal(/in flight.*channel=stable/)); + const state = { tags: [tag("0.33.0")], releases: [rel("0.33.0")], prs: [openPr("0.33.1")], pypiVersions: new Set(["0.33.0"]) }; + assert.throws(() => run(state, { channel: "stable", kind: "major" }), refusal(/in flight.*kind=patch/)); + assert.equal(run(state, { channel: "stable", kind: "patch" }).step, "wait"); +}); + +test("typed version rules", () => { + const state = { tags: [tag("0.31.1"), tag("0.33.0")], releases: [rel("0.31.1"), rel("0.33.0")], pypiVersions: new Set(["0.33.0"]) }; + const patch = { kind: "patch", channel: "stable" }; + assert.throws(() => run(state, { ...patch, version: "0.31.1" }), refusal(/not on the 0.33 line/)); + assert.throws(() => run(state, { ...patch, version: "0.33.3" }), refusal(/cannot be skipped/)); + assert.equal(run(state, { ...patch, version: "0.33.1" }).step, "prepare"); + assert.throws(() => run({}, { version: "0.33.0.dev0", channel: "stable" }), refusal(/channel=stable/)); + const legacy = { tags: [tag("0.33.0.dev0"), tag("0.33.0.dev1")] }; + assert.throws(() => run(legacy, { version: "0.33.0.dev1" }), refusal(/not newer/)); +}); + +test("downgrade at resolution: major dev on r0.32 with v0.32.0 present", () => { + const state = { branch: "r0.32", tags: [tag("0.32.0")], releases: [rel("0.32.0")], pypiVersions: new Set(["0.32.0"]) }; + assert.throws(() => run(state), refusal(/not newer than v0.32.0/)); +}); + +test("PyPI missing: warning first, prepare of V+1 refused", () => { + const missing = { pypiVersions: new Set(["0.32.0"]) }; + assert.throws(() => run(missing), refusal(/v0.33.0.dev0 is released on GitHub but not on PyPI.*Refusing/)); + const d = run(missing, { version: "0.33.0.dev0" }); + assert.equal(d.step, "already-released"); + assert.match(d.warnings[0], /^v0.33.0.dev0 is released on GitHub but not on PyPI; run with version=0.33.0.dev0/); + assert.throws(() => run({ pypiVersions: null }), refusal(/Could not check PyPI/)); + assert.deepEqual(run().warnings, []); +}); + +test("prepare with branch absent", () => { + const absent = { branchExists: false, headSha: null, versionAtHead: null, tags: [], releases: [], pypiVersions: new Set() }; + const d = run(absent, { commit_sha: sha("c") }); + assert.equal(d.step, "prepare"); + assert.equal(d.version, "0.33.0.dev0"); + assert.throws(() => run(absent), refusal(/40-character/)); + assert.throws(() => run(absent, { commit_sha: "abc1234" }), refusal(/40-character/)); + assert.throws(() => run(absent, { commit_sha: sha("c"), cherry_picks: "#1" }), refusal(/not allowed when cutting/)); + const patchAbsent = { ...absent, tags: [tag("0.33.0")], releases: [rel("0.33.0")], pypiVersions: new Set(["0.33.0"]) }; + assert.throws(() => run(patchAbsent, { kind: "patch", channel: "stable" }), refusal(/patch needs an existing branch/)); + // F10: after the branch is cut at 0.33.0.dev0, the next run is fast-path. + assert.equal(run({ ...absent, branchExists: true, versionAtHead: "0.33.0.dev0" }).step, "fast-path"); +}); + +test("commit_sha with an existing branch -> refused", () => { + const state = { prs: [] }; + assert.throws(() => run(state, { commit_sha: sha("c") }), refusal(/already exists; commit_sha/)); +}); + +test("checkCommitShaVersion", () => { + checkCommitShaVersion("0.33.0.dev0", { x: 0, y: 33 }); + assert.throws(() => checkCommitShaVersion("0.34.0.dev0", { x: 0, y: 33 }), refusal(/not on the\s+0.33 line/)); + assert.throws(() => checkCommitShaVersion("garbage", { x: 0, y: 33 }), refusal(/garbage/)); +}); + +test("tag vV elsewhere without release while V is in flight -> refused", () => { + const state = { prs: [openPr("0.33.0.dev1")], tags: [tag("0.33.0.dev0"), tag("0.33.0.dev1", sha("b"))] }; + assert.throws(() => run(state), refusal(/not at the merge commit/)); +}); + +test("duplicate drafts and unreadable drafts on the line -> refused", () => { + const dup = { releases: [rel("0.33.0.dev0"), draft("0.33.0.dev1", { id: 2 }), draft("0.33.0.dev1", { id: 3 })] }; + assert.throws(() => run(dup), refusal(/2 drafts/)); + assert.throws(() => run({ releases: [draft("0.33.0rc1")] }), refusal(/not a version/)); + assert.equal(run({ releases: [rel("0.33.0.dev0"), draft("0.32.0.dev1")] }).step, "prepare"); +}); + +test("unknown kind or channel -> refused", () => { + assert.throws(() => run({}, { kind: "minor" }), refusal(/Unknown kind/)); + assert.throws(() => run({}, { channel: "rc" }), refusal(/Unknown channel/)); +}); + +test("helpers", () => { + assert.equal(bumpVersion("release-tool/bump-0.33.1"), "0.33.1"); + assert.equal(bumpVersion("release-tool/bump-x"), null); + assert.equal(bumpVersion("release-tool/master-bump-0.34.0.dev0"), null); + assert.equal(draftMapsToLine("v0.33.1", { x: 0, y: 33 }, "v"), true); + assert.equal(draftMapsToLine("v0.33rc1", { x: 0, y: 33 }, "v"), true); + assert.equal(draftMapsToLine("v0.331.0", { x: 0, y: 33 }, "v"), false); + assert.equal(draftMapsToLine("0.33.1", { x: 0, y: 33 }, "v"), false); +}); diff --git a/scripts/release/lib/errors.js b/scripts/release/lib/errors.js new file mode 100644 index 0000000..c641105 --- /dev/null +++ b/scripts/release/lib/errors.js @@ -0,0 +1,60 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * A refusal is an expected, user-facing stop: the release tool decided not to + * act. Its message is shown verbatim in the run summary, followed by a + * "Next:" line telling the user what to do. + */ + +const DEFAULT_NEXT = "fix the problem above and run the panel again."; + +class RefusalError extends Error { + /** + * @param {string} message What was refused and why. + * @param {string} [next] What the user should do next. + */ + constructor(message, next) { + super(message); + this.name = "RefusalError"; + this.next = next && String(next).trim() ? String(next).trim() : DEFAULT_NEXT; + } +} + +/** + * Renders any thrown value for the run summary. Refusals end with their + * "Next:" line; anything else is an unexpected failure. + */ +function formatRefusal(err) { + if (err instanceof RefusalError) { + return `${err.message}\n\nNext: ${err.next}`; + } + const message = err && err.message ? err.message : String(err); + return ( + `Unexpected error: ${message}\n\n` + + "Next: this is a bug in the release tool; open an issue on " + + "keras-team/shared-workflows with a link to this run." + ); +} + +/** Predicate (e.g. for assert.throws): a RefusalError whose message matches `re`. */ +function refusalMatching(re) { + return (e) => e instanceof RefusalError && re.test(e.message); +} + +module.exports = { RefusalError, formatRefusal, refusalMatching, DEFAULT_NEXT }; diff --git a/scripts/release/lib/errors.test.js b/scripts/release/lib/errors.test.js new file mode 100644 index 0000000..d32102f --- /dev/null +++ b/scripts/release/lib/errors.test.js @@ -0,0 +1,53 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { RefusalError, formatRefusal, refusalMatching, DEFAULT_NEXT } = require("./errors"); + +test("refusalMatching matches only RefusalErrors with a matching message", () => { + assert.equal(refusalMatching(/No/)(new RefusalError("No.")), true); + assert.equal(refusalMatching(/Yes/)(new RefusalError("No.")), false); + assert.equal(refusalMatching(/No/)(new Error("No.")), false); +}); + +test("RefusalError carries message and next", () => { + const e = new RefusalError("No.", "do this."); + assert.ok(e instanceof Error); + assert.equal(e.name, "RefusalError"); + assert.equal(e.message, "No."); + assert.equal(e.next, "do this."); +}); + +test("RefusalError defaults next", () => { + assert.equal(new RefusalError("No.").next, DEFAULT_NEXT); + assert.equal(new RefusalError("No.", " ").next, DEFAULT_NEXT); +}); + +test("formatRefusal ends with the Next: line", () => { + const text = formatRefusal(new RefusalError("No.", "do this.")); + assert.equal(text, "No.\n\nNext: do this."); + assert.match(text.split("\n").pop(), /^Next: /); +}); + +test("formatRefusal marks other errors as unexpected, still with Next:", () => { + const text = formatRefusal(new TypeError("boom")); + assert.match(text, /^Unexpected error: boom/); + assert.match(text.split("\n").pop(), /^Next: /); + assert.match(formatRefusal("raw"), /^Unexpected error: raw/); +}); diff --git a/scripts/release/lib/gh.js b/scripts/release/lib/gh.js new file mode 100644 index 0000000..303586d --- /dev/null +++ b/scripts/release/lib/gh.js @@ -0,0 +1,438 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * The only module that talks to GitHub for target-repo data. + * + * Wraps the octokit that actions/github-script provides (`github.rest.*`, + * `github.paginate`). There is deliberately no merge call anywhere in here: + * the bot opens PRs, humans merge them. + */ + +const { RefusalError } = require("./errors"); +const { readVersionFromFile } = require("./version"); +const { parseCherryPicksLines } = require("./picks"); +const { hasWrite } = require("./guards"); +const { ACTIVE_STATUSES } = require("./lock"); +const { ABANDONED_LABEL } = require("./detect"); + +const CANDIDATE_PR_LOOKUPS = 30; + +const isNotFound = (err) => Boolean(err) && err.status === 404; +const firstLine = (text) => String(text || "").split("\n")[0]; + +/** Versions of `pkg` on PyPI: Set, empty Set on 404, null if lookup failed. */ +async function pypiVersions(pkg, fetchImpl) { + try { + const url = `https://pypi.org/pypi/${encodeURIComponent(pkg)}/json`; + const res = await fetchImpl(url); + if (res.status === 404) return new Set(); + if (!res.ok) return null; + const data = await res.json(); + return new Set(Object.keys(data.releases || {})); + } catch (err) { + return null; + } +} + +class Gh { + constructor({ github, owner, repo }) { + this.github = github; + this.rest = github.rest; + this.base = { owner, repo }; + this.fullName = `${owner}/${repo}`; + } + + async permission(user) { + if (!user) return "none"; + try { + const { data } = await this.rest.repos.getCollaboratorPermissionLevel({ + ...this.base, username: user, + }); + return data.permission || "none"; + } catch (err) { + if (isNotFound(err)) return "none"; + throw err; + } + } + + async fileAt(ref, path) { + try { + const { data } = await this.rest.repos.getContent({ + ...this.base, path, ref, + }); + if (Array.isArray(data) || data.type !== "file") return null; + return Buffer.from(data.content, data.encoding || "base64") + .toString("utf8"); + } catch (err) { + if (isNotFound(err)) return null; + throw err; + } + } + + /** The version all `files` carry at `ref`; refuses if missing or split. */ + async versionAt(ref, files) { + const found = []; + for (const file of files) { + const text = await this.fileAt(ref, file); + if (text === null) { + throw new RefusalError(`Version file ${file} is missing at ${ref}.`, + "check `version_files` in the repo config."); + } + found.push(readVersionFromFile(text)); + } + if (new Set(found).size !== 1) { + const pairs = files.map((f, i) => `${f}=${found[i]}`).join(", "); + throw new RefusalError(`Version files disagree at ${ref}: ${pairs}.`, + "make every version file carry the same version, then run again."); + } + return found[0]; + } + + async branchHead(name) { + try { + const { data } = await this.rest.repos.getBranch({ + ...this.base, branch: name, + }); + return data.commit.sha; + } catch (err) { + if (isNotFound(err)) return null; + throw err; + } + } + + /** `{ sha, parents, title }` for a commit-ish, or null if unknown. */ + async commit(ref) { + try { + const { data } = await this.rest.repos.getCommit({ ...this.base, ref }); + return { + sha: data.sha, + parents: data.parents.map((p) => p.sha), + title: firstLine(data.commit && data.commit.message), + }; + } catch (err) { + if (isNotFound(err) || err.status === 422) return null; + throw err; + } + } + + async compare(base, head) { + const { data } = await this.rest.repos.compareCommitsWithBasehead({ + ...this.base, basehead: `${base}...${head}`, + }); + return data; + } + + /** True when `sha` is an ancestor of (or equal to) `ref`. */ + async isAncestor(sha, ref) { + try { + const data = await this.compare(sha, ref); + return data.status === "ahead" || data.status === "identical"; + } catch (err) { + if (isNotFound(err)) return false; + throw err; + } + } + + /** Commits on `branch` after `sha`, oldest first: `[{ sha, message }]`. */ + async commitsAfter(branch, sha) { + const data = await this.compare(sha, branch); + if (data.status !== "ahead" && data.status !== "identical") { + throw new RefusalError(`Commit ${sha} is not on ${branch}.`, + `check the history of ${branch}.`); + } + return data.commits.map((c) => ({ sha: c.sha, message: c.commit.message })); + } + + /** First-parent chain of `branch`, newest first, at most `limit` shas. */ + async firstParentShas(branch, limit = 3000) { + let cur = await this.branchHead(branch); + const parentsOf = new Map(); + const out = []; + const pages = this.github.paginate.iterator(this.rest.repos.listCommits, { + ...this.base, sha: branch, per_page: 100, + }); + for await (const { data } of pages) { + for (const c of data) parentsOf.set(c.sha, c.parents.map((p) => p.sha)); + while (cur && parentsOf.has(cur) && out.length < limit) { + out.push(cur); + cur = parentsOf.get(cur)[0]; + } + if (!cur || out.length >= limit) break; + } + return out; + } + + async pull(number) { + try { + const { data } = await this.rest.pulls.get({ + ...this.base, pull_number: number, + }); + return data; + } catch (err) { + if (isNotFound(err)) return null; + throw err; + } + } + + async commitPulls(sha) { + const { data } = await this.rest.repos + .listPullRequestsAssociatedWithCommit({ ...this.base, commit_sha: sha }); + return data.map((p) => p.number); + } + + /** True only if the latest `label` event was made by a user with write. */ + async labelAddedByWriter(prNumber, label) { + const events = await this.github.paginate(this.rest.issues.listEvents, { + ...this.base, issue_number: prNumber, per_page: 100, + }); + const added = events.filter((e) => + e.event === "labeled" && e.label && e.label.name === label); + if (added.length === 0) return false; + const actor = added[added.length - 1].actor; + return hasWrite(await this.permission(actor && actor.login)); + } + + /** Follows annotated tag objects down to the commit sha. */ + async derefTagObject(object) { + let obj = object; + for (let i = 0; i < 5 && obj.type === "tag"; i++) { + const { data } = await this.rest.git.getTag({ + ...this.base, tag_sha: obj.sha, + }); + obj = data.object; + } + return obj.type === "commit" ? obj.sha : null; + } + + async resolveTag(name) { + try { + const { data } = await this.rest.git.getRef({ + ...this.base, ref: `tags/${name}`, + }); + return await this.derefTagObject(data.object); + } catch (err) { + if (isNotFound(err)) return null; + throw err; + } + } + + async listTags(prefix) { + const refs = await this.github.paginate(this.rest.git.listMatchingRefs, { + ...this.base, ref: `tags/${prefix}`, per_page: 100, + }); + const tags = []; + for (const ref of refs) { + const sha = await this.derefTagObject(ref.object); + if (sha) tags.push({ name: ref.ref.replace(/^refs\/tags\//, ""), sha }); + } + return tags; + } + + async listReleases() { + const all = await this.github.paginate(this.rest.repos.listReleases, { + ...this.base, per_page: 100, + }); + return all.map((r) => ({ + id: r.id, tag_name: r.tag_name, draft: r.draft, + prerelease: r.prerelease, target_commitish: r.target_commitish, + author_login: r.author ? r.author.login : null, + })); + } + + async releaseBranches(pattern) { + const re = new RegExp(pattern); + const all = await this.github.paginate(this.rest.repos.listBranches, { + ...this.base, per_page: 100, + }); + return all.map((b) => b.name).filter((n) => re.test(n)); + } + + listPulls(baseRef, state) { + return this.github.paginate(this.rest.pulls.list, { + ...this.base, base: baseRef, state, per_page: 100, + }); + } + + async toolPr(p) { + const merged = Boolean(p.merged_at); + const full = merged ? await this.pull(p.number) : p; + const labels = (p.labels || []).map((l) => typeof l === "string" ? l : l.name); + return { + number: p.number, state: p.state, merged, + merge_commit_sha: full.merge_commit_sha || null, + head_ref: p.head.ref, + head_repo_full_name: p.head.repo ? p.head.repo.full_name : null, + base_ref: p.base.ref, + base_repo_full_name: p.base.repo ? p.base.repo.full_name : null, + author_login: p.user ? p.user.login : null, + merged_by_login: full.merged_by ? full.merged_by.login : null, + body: p.body || "", + labels, + abandoned_by_writer: labels.includes(ABANDONED_LABEL) && + await this.labelAddedByWriter(p.number, ABANDONED_LABEL), + }; + } + + /** State consumed by the pure modules; see impl_spec "State". */ + async gatherState({ config, branch, fetchPypi }) { + const files = config.version_files; + const headSha = await this.branchHead(branch); + const defaultHead = await this.branchHead(config.default_branch); + if (!defaultHead) { + throw new RefusalError(`Default branch ${config.default_branch} not found.`, + "check `default_branch` in the repo config."); + } + const branchPulls = await this.listPulls(branch, "all"); + const defaultOpen = await this.listPulls(config.default_branch, "open"); + const seen = new Set(); + const prs = []; + for (const p of [...branchPulls, ...defaultOpen]) { + if (!p.head || !p.head.ref.startsWith("release-tool/")) continue; + // Only the bot's own branches in the target repo count; anyone can open + // a fork PR whose head is named release-tool/*. + if (!p.head.repo || p.head.repo.full_name !== this.fullName) continue; + if (seen.has(p.number)) continue; + seen.add(p.number); + prs.push(await this.toolPr(p)); + } + const mergedBodies = branchPulls.filter((p) => p.merged_at) + .map((p) => p.body || ""); + return { + branch, + branchExists: headSha !== null, + headSha, + versionAtHead: headSha ? await this.versionAt(headSha, files) : null, + defaultBranchVersion: await this.versionAt(defaultHead, files), + defaultHeadSha: defaultHead, + tags: await this.listTags(config.tag_prefix), + releases: await this.listReleases(), + prs, + pickedPrNumbers: parseCherryPicksLines(mergedBodies), + pypiVersions: fetchPypi ? await fetchPypi(config.packages[0]) : null, + }; + } + + /** + * Master PRs not yet on `branch`, oldest first. `picked` marks PRs listed + * in a merged `Cherry-picks:` line; `hint` marks commits whose sha appears + * in a `(cherry picked from commit ...)` trailer on the branch. + */ + async pickCandidates({ branch, defaultBranch, picked }) { + const missing = await this.compare(branch, defaultBranch); + const onBranch = await this.compare(defaultBranch, branch); + const trailers = new Set(); + for (const c of onBranch.commits) { + const re = /\(cherry picked from commit ([0-9a-f]{40})\)/g; + for (const m of c.commit.message.matchAll(re)) trailers.add(m[1]); + } + const byPr = new Map(); + let lookups = 0; + for (const c of missing.commits) { + const title = firstLine(c.commit.message); + const m = title.match(/\(#(\d+)\)\s*$/) || + title.match(/^Merge pull request #(\d+)/); + let number = m ? Number(m[1]) : null; + if (number === null && lookups < CANDIDATE_PR_LOOKUPS) { + lookups += 1; + number = (await this.commitPulls(c.sha))[0] || null; + } + const key = number === null ? c.sha : number; + if (byPr.has(key) && !m) continue; + byPr.set(key, { + number, sha: c.sha, title, + picked: number !== null && picked.has(number), + hint: trailers.has(c.sha), + }); + } + return { + candidates: [...byPr.values()], + truncated: missing.total_commits > missing.commits.length, + }; + } + + async createBranch(name, sha) { + await this.rest.git.createRef({ + ...this.base, ref: `refs/heads/${name}`, sha, + }); + } + + async openPr({ head, base, title, body }) { + const { data } = await this.rest.pulls.create({ + ...this.base, head, base, title, body, + }); + return { number: data.number, html_url: data.html_url }; + } + + async createRelease(params) { + const { data } = await this.rest.repos.createRelease({ + ...this.base, ...params, + }); + return data; + } + + async publishDraft(id, { make_latest }) { + const { data } = await this.rest.repos.updateRelease({ + ...this.base, release_id: id, draft: false, make_latest, + }); + return data; + } + + async generateNotes(tag, target, previousTag) { + const params = { ...this.base, tag_name: tag, target_commitish: target }; + if (previousTag) params.previous_tag_name = previousTag; + const { data } = await this.rest.repos.generateReleaseNotes(params); + return { name: data.name, body: data.body }; + } + + /** `{ status, state }`; any failure is reported, never thrown. */ + async workflowState(file) { + try { + const res = await this.rest.actions.getWorkflow({ + ...this.base, workflow_id: file, + }); + return { status: res.status, state: res.data ? res.data.state : null }; + } catch (err) { + return { status: err.status || 0, state: null }; + } + } + + async listActiveRuns(workflowFile) { + const byId = new Map(); + for (const status of ACTIVE_STATUSES) { + const runs = await this.github.paginate(this.rest.actions.listWorkflowRuns, { + ...this.base, workflow_id: workflowFile, status, per_page: 100, + }); + for (const r of runs) { + byId.set(r.id, { + id: r.id, workflow: workflowFile, + display_title: r.display_title, status: r.status, + }); + } + } + return [...byId.values()]; + } + + async dispatch(workflowFile, ref, inputs) { + await this.rest.actions.createWorkflowDispatch({ + ...this.base, workflow_id: workflowFile, ref, inputs, + }); + } +} + +module.exports = { ABANDONED_LABEL, Gh, pypiVersions }; diff --git a/scripts/release/lib/gh.test.js b/scripts/release/lib/gh.test.js new file mode 100644 index 0000000..d281ec8 --- /dev/null +++ b/scripts/release/lib/gh.test.js @@ -0,0 +1,177 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); +const T = require("../test_support"); +const { Gh, pypiVersions } = require("./gh"); +const { refusalMatching } = require("./errors"); + +const { SHA } = T; + +function gh(fixture, fail) { + const fake = T.fakeGithub(fixture, fail); + return { fake, gh: new Gh({ github: fake.github, owner: "keras-team", repo: "keras-hub" }) }; +} + +function stateFixture() { + const fx = T.baseFixture(); + fx.refs.push({ ref: "refs/tags/v0.33.1", object: { type: "tag", sha: SHA.tagobj } }); + fx.tagObjects = { [SHA.tagobj]: { object: { type: "commit", sha: SHA.ms } } }; + fx.releases.push(T.releaseObj(7, "v0.33.2", SHA.b1, { draft: true })); + fx.pulls.push( + T.bumpPr(60, "0.33.1", { + merged_at: "t", merge_commit_sha: SHA.ms, merged_by: { login: "alice" }, + labels: [{ name: "release-tool:abandoned" }], + }), + { ...T.bumpPr(61, "x"), head: { ref: "feature", repo: null }, merged_at: "t", + body: "Picks\nCherry-picks: #101 (4444444), #102 (5555555)" }, + { ...T.bumpPr(70, "y"), state: "open", + head: { ref: "release-tool/master-bump-0.34.0.dev0", + repo: { full_name: "keras-team/keras-hub" } }, base: { ref: "master" } }, + { ...T.bumpPr(71, "0.32.1"), state: "open", base: { ref: "r0.32" } }, + { ...T.bumpPr(72, "z"), head: { ref: "release-tool/master-bump-0.33.0.dev0", repo: null }, + base: { ref: "master" } }, + { ...T.bumpPr(73, "0.33.9"), state: "open", user: { login: "mallory" }, + head: { ref: "release-tool/bump-0.33.9", repo: { full_name: "mallory/keras-hub" } } }, + ); + return fx; +} + +const labelEvent = (login) => + ({ event: "labeled", label: { name: "release-tool:abandoned" }, actor: { login } }); + +describe("Gh.gatherState", () => { + it("dereferences annotated tags and keeps drafts", async () => { + const { gh: g } = gh(stateFixture()); + const state = await g.gatherState({ config: T.CONFIG, branch: "r0.33" }); + assert.deepEqual(state.tags.find((t) => t.name === "v0.33.1"), + { name: "v0.33.1", sha: SHA.ms }); + assert.ok(state.releases.some((r) => r.tag_name === "v0.33.2" && r.draft)); + assert.equal(state.releases[0].author_login, T.BOT); + assert.equal(state.versionAtHead, "0.33.0"); + assert.equal(state.defaultBranchVersion, "0.34.0.dev0"); + assert.equal(state.pypiVersions, null); + }); + + it("keeps same-repo release-tool PRs into the branch and open ones into default, never fork PRs", async () => { + const { gh: g } = gh(stateFixture()); + const state = await g.gatherState({ config: T.CONFIG, branch: "r0.33" }); + assert.deepEqual(state.prs.map((p) => p.number).sort(), [60, 70]); + const pr = state.prs.find((p) => p.number === 60); + assert.equal(pr.merged, true); + assert.equal(pr.merged_by_login, "alice"); + assert.equal(pr.merge_commit_sha, SHA.ms); + assert.deepEqual([...state.pickedPrNumbers].sort(), [101, 102]); + }); + + it("honours the abandoned label only when a writer added it last", async () => { + for (const [events, expected] of [ + [[labelEvent("carol")], false], + [[labelEvent("alice")], true], + [[labelEvent("alice"), labelEvent("carol")], false], + [[], false], + ]) { + const fx = stateFixture(); + fx.permissions.carol = "read"; + fx.events = { 60: events }; + const { gh: g } = gh(fx); + const state = await g.gatherState({ config: T.CONFIG, branch: "r0.33" }); + assert.equal(state.prs.find((p) => p.number === 60).abandoned_by_writer, expected); + } + }); + + it("reports a missing branch and reads PyPI for packages[0]", async () => { + const fx = T.baseFixture(); + delete fx.branches["r0.33"]; + const asked = []; + const { gh: g } = gh(fx); + const state = await g.gatherState({ + config: T.CONFIG, branch: "r0.33", + fetchPypi: async (pkg) => { asked.push(pkg); return new Set(["0.33.0"]); }, + }); + assert.equal(state.branchExists, false); + assert.equal(state.headSha, null); + assert.equal(state.versionAtHead, null); + assert.deepEqual(asked, ["keras-hub"]); + }); + + it("refuses when version files disagree", async () => { + const config = { ...T.CONFIG, version_files: [T.VERSION_FILE, "setup.py"] }; + const fx = T.baseFixture(); + fx.files[`${SHA.b0}:setup.py`] = 'version = "0.32.0"\n'; + const { gh: g } = gh(fx); + await assert.rejects(g.versionAt(SHA.b0, config.version_files), + refusalMatching(/disagree/)); + }); +}); + +describe("Gh helpers", () => { + it("walks the first-parent chain only", async () => { + const fx = T.baseFixture(); + fx.history = [ + { sha: SHA.master, parents: [SHA.m1, SHA.other] }, + { sha: SHA.other, parents: [SHA.p0] }, + { sha: SHA.m1, parents: [SHA.p0] }, + { sha: SHA.p0, parents: [] }, + ]; + const { gh: g } = gh(fx); + assert.deepEqual(await g.firstParentShas("master"), [SHA.master, SHA.m1, SHA.p0]); + }); + + it("returns workflow status instead of throwing", async () => { + const fx = T.baseFixture(); + fx.workflows = { "a.yml": { status: 403 } }; + const { gh: g } = gh(fx); + assert.deepEqual(await g.workflowState("a.yml"), { status: 403, state: null }); + assert.deepEqual(await g.workflowState("missing.yml"), { status: 404, state: null }); + fx.workflows["a.yml"] = { status: 200, state: "disabled_manually" }; + assert.deepEqual(await g.workflowState("a.yml"), { status: 200, state: "disabled_manually" }); + }); + + it("reads .permission and maps 404 to none", async () => { + const { gh: g } = gh(T.baseFixture()); + assert.equal(await g.permission("alice"), "write"); + assert.equal(await g.permission("stranger"), "none"); + }); + + it("lists active runs across every active status, deduplicated", async () => { + const fx = T.baseFixture(); + fx.runs = { "release.yml": [ + { id: 1, status: "queued", display_title: "release keras-hub r0.33 patch auto" }, + { id: 2, status: "completed", display_title: "release keras-hub r0.33 patch auto" }, + { id: 3, status: "waiting", display_title: "release keras r0.1 patch auto" }, + ] }; + const { gh: g } = gh(fx); + const runs = await g.listActiveRuns("release.yml"); + assert.deepEqual(runs.map((r) => r.id).sort(), [1, 3]); + assert.equal(runs[0].workflow, "release.yml"); + }); +}); + +describe("pypiVersions", () => { + const res = (status, body) => async () => + ({ status, ok: status === 200, json: async () => body }); + it("maps 404 to empty, errors to null, 200 to release keys", async () => { + assert.deepEqual([...await pypiVersions("x", res(404))], []); + assert.equal(await pypiVersions("x", res(500)), null); + assert.equal(await pypiVersions("x", async () => { throw new Error("net"); }), null); + const ok = await pypiVersions("x", res(200, { releases: { "0.33.0": [] } })); + assert.deepEqual([...ok], ["0.33.0"]); + }); +}); diff --git a/scripts/release/lib/git.js b/scripts/release/lib/git.js new file mode 100644 index 0000000..33b1602 --- /dev/null +++ b/scripts/release/lib/git.js @@ -0,0 +1,214 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Git wrapper for the release bot. + * + * - Runs `git` through `execFileSync`, never a shell, with + * `core.hooksPath=/dev/null` so no target-repo hook can ever execute. + * - Never logs the authenticated remote URL: every error message, stdout and + * stderr is redacted before it is rethrown. + * - `pushBotRef` is the only push helper and refuses any destination outside + * `refs/heads/release-tool/`. + */ + +const { execFileSync } = require("node:child_process"); + +const BOT_REF_PREFIX = "release-tool/"; + +function redact(text) { + return String(text == null ? "" : text).replace( + /x-access-token:[^@\s]*@/g, + "x-access-token:***@", + ); +} + +class GitError extends Error { + constructor(args, err) { + const stderr = redact(err.stderr); + super(`git ${redact(args.join(" "))} failed: ${stderr.trim()}`); + this.name = "GitError"; + this.status = err.status; + this.stdout = redact(err.stdout); + this.stderr = stderr; + } +} + +/** + * Runs git with hooks disabled. Returns stdout as a string. + * `input` is written to stdin (used for `hash-object --stdin`). + */ +function git(args, { cwd, env, input } = {}) { + const fullArgs = [ + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + "-c", + "protocol.file.allow=never", + ...args, + ]; + try { + return execFileSync("git", fullArgs, { + cwd, + env: { ...process.env, GIT_TERMINAL_PROMPT: "0", ...(env || {}) }, + input, + encoding: "utf8", + stdio: ["pipe", "pipe", "pipe"], + maxBuffer: 256 * 1024 * 1024, + }); + } catch (err) { + throw new GitError(args, err); + } +} + +/** Builds `https://x-access-token:@github.com//.git`. */ +function withAuthRemote(url, token) { + const parsed = new URL(url); + parsed.username = "x-access-token"; + parsed.password = token; + return parsed.toString(); +} + +/** + * Force-pushes HEAD to `refs/heads/`; `ref` must start with + * `release-tool/`. Release branches and tags are never pushed with git. + */ +function pushBotRef(gitFn, { cwd, remoteUrl, ref }) { + if (typeof ref !== "string" || !ref.startsWith(BOT_REF_PREFIX) || + ref.includes("..") || ref.includes(":")) { + throw new Error(`refusing to push to non-bot ref: ${ref}`); + } + // The prefix is written literally so static.test.js can verify the + // destination; `suffix` is the part of `ref` after `release-tool/`. + const suffix = ref.slice(BOT_REF_PREFIX.length); + return gitFn( + ["push", "--force", remoteUrl, `HEAD:refs/heads/release-tool/${suffix}`], + { cwd }); +} + +/** + * A local clone of the (public) target repo. Every command goes through the + * injected `gitFn`, so tests replace git entirely. Commits are authored by + * `identity` via GIT_* env vars; nothing is written to git config. + */ +class Workspace { + constructor({ gitFn, cwd, identity, pushUrl }) { + this.gitFn = gitFn; + this.cwd = cwd; + this.pushUrl = pushUrl; + const email = `${identity}@users.noreply.github.com`; + this.env = { + GIT_AUTHOR_NAME: identity, GIT_AUTHOR_EMAIL: email, + GIT_COMMITTER_NAME: identity, GIT_COMMITTER_EMAIL: email, + }; + } + + run(args, input) { + return String(this.gitFn(args, { cwd: this.cwd, env: this.env, input })); + } + + /** Blobless clone over anonymous https; the push URL is used only to push. */ + clone(url) { + this.gitFn(["clone", "--quiet", "--no-tags", "--filter=blob:none", + "--no-checkout", url, this.cwd], { env: this.env }); + } + + resetBranch(name, sha) { + this.run(["checkout", "--quiet", "--force", "-B", name, sha]); + } + + /** + * Applies one pick; `args` is the full argv from picks.cherryPickArgs. + * Returns `{ result: "applied" | "empty" }` or `{ result: "conflict", + * files }` after `cherry-pick --abort`. + */ + cherryPick(args) { + if (args[0] !== "cherry-pick") throw new Error(`not a cherry-pick: ${args.join(" ")}`); + try { + this.run(args); + return { result: "applied" }; + } catch (err) { + const files = this.run(["diff", "--name-only", "--diff-filter=U"]) + .split("\n").filter(Boolean); + const output = `${err.stdout || ""}\n${err.stderr || ""}`; + if (files.length === 0 && /is now empty|nothing to commit/i.test(output)) { + this.run(["cherry-pick", "--skip"]); + return { result: "empty" }; + } + try { + this.run(["cherry-pick", "--abort"]); + } catch (abortErr) { + // No cherry-pick in progress: nothing to abort. + } + if (files.length === 0) throw err; + return { result: "conflict", files }; + } + } + + /** First-parent commits in `range` touching `files`: `[{ sha, subject }]`. */ + commitsTouching(range, files) { + return this.run(["log", "--first-parent", "--format=%H%x09%s", range, + "--", ...files]).split("\n").filter(Boolean).map((line) => { + const tab = line.indexOf("\t"); + return { sha: line.slice(0, tab), subject: line.slice(tab + 1) }; + }); + } + + mergeBase(a, b) { + return this.run(["merge-base", a, b]).trim(); + } + + /** + * Rewrites every file whose version differs from `version` in the index + * and commits. Returns false (no commit) when all files already match. + */ + commitVersion(files, version, { read, rewrite, message }) { + let changed = false; + for (const file of files) { + const text = this.run(["show", `HEAD:${file}`]); + if (read(text) === version) continue; + const blob = this.run(["hash-object", "-w", "--stdin"], + rewrite(text, version)).trim(); + const mode = this.run(["ls-tree", "HEAD", "--", file]).split(/\s+/)[0]; + this.run(["update-index", "--cacheinfo", `${mode || "100644"},${blob},${file}`]); + changed = true; + } + if (changed) this.run(["commit", "--quiet", "-m", message]); + return changed; + } + + commitCount(base) { + return Number(this.run(["rev-list", "--count", `${base}..HEAD`]).trim()); + } + + pushBotRef(ref) { + return pushBotRef((args, opts) => this.gitFn(args, { ...opts, env: this.env }), + { cwd: this.cwd, remoteUrl: this.pushUrl, ref }); + } +} + +module.exports = { + BOT_REF_PREFIX, + GitError, + Workspace, + git, + pushBotRef, + redact, + withAuthRemote, +}; diff --git a/scripts/release/lib/guards.js b/scripts/release/lib/guards.js new file mode 100644 index 0000000..d041b60 --- /dev/null +++ b/scripts/release/lib/guards.js @@ -0,0 +1,173 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Continue guards (plan section 6), draft re-verification (plan section 2), + * and publish.yml actor resolution (plan section 5). PURE. + */ + +const pep440 = require("./pep440"); +const { RefusalError } = require("./errors"); +const { LOGIN_RE } = require("./config"); +const { BUMP_PREFIX, ABANDONED_LABEL, SHA40 } = require("./detect"); + +/** `.permission` of GET /collaborators/{user}/permission: maintain reads as write. */ +function hasWrite(permission) { + return permission === "admin" || permission === "write"; +} + +function sameLogin(a, b) { + return typeof a === "string" && typeof b === "string" && a !== "" && + a.toLowerCase() === b.toLowerCase(); +} + +function versionsAgree(found, version) { + const list = Array.isArray(found) ? found : [found]; + return list.length > 0 && list.every((v) => v === version); +} + +function shortCommit(c) { + if (typeof c === "string") return c.slice(0, 7); + const subject = (c.message || "").split("\n")[0]; + return `${String(c.sha).slice(0, 7)} ${subject}`.trim(); +} + +/** + * Every continue guard of plan section 6, plus the post-merge-commits + * refusal. Returns a list of failure messages; empty means pass. + * `targetRepo` is the base repo full name, used when the PR object has no + * `base_repo_full_name`. + */ +function continueGuards({ + pr, config, branch, version, botLogin, mergedByPermission, versionAtMergeSha, + tagShaForVersion, highestTagOnLine, commitsAfterMerge, targetRepo, +}) { + const f = []; + if (!config || config.enabled !== true) f.push("the repo is not configured and enabled"); + if (!botLogin) f.push("the App bot login is empty (RELEASE_APP_SLUG unset)"); + if (!pr) return [...f, "there is no bump PR"]; + if (pr.merged !== true) f.push(`PR #${pr.number} is not merged`); + const pattern = config && config.release_branch_pattern; + if (!pattern || !new RegExp(pattern).test(pr.base_ref || "") || pr.base_ref !== branch) { + f.push(`PR #${pr.number} base "${pr.base_ref}" is not the release branch ${branch}`); + } + const baseRepo = pr.base_repo_full_name || targetRepo; + if (!baseRepo || !sameLogin(pr.head_repo_full_name, baseRepo)) { + f.push(`PR #${pr.number} head repo "${pr.head_repo_full_name}" is not the base repo`); + } + if (pr.head_ref !== `${BUMP_PREFIX}${version}`) { + f.push(`PR #${pr.number} head ref "${pr.head_ref}" is not ${BUMP_PREFIX}${version}`); + } + if (!botLogin || !sameLogin(pr.author_login, botLogin)) { + f.push(`PR #${pr.number} was opened by ${pr.author_login}, not ${botLogin}`); + } + if (!pr.merged_by_login || (botLogin && sameLogin(pr.merged_by_login, botLogin))) { + f.push(`PR #${pr.number} was not merged by a human (merged_by: ${pr.merged_by_login})`); + } + if (!hasWrite(mergedByPermission)) { + f.push( + `PR #${pr.number} was merged by ${pr.merged_by_login}, whose permission is ` + + `"${mergedByPermission}" (needs write)`, + ); + } + if (!versionsAgree(versionAtMergeSha, version)) { + f.push(`the version files at the merge commit say ${versionAtMergeSha}, not ${version}`); + } + const tagAtMerge = tagShaForVersion && tagShaForVersion === pr.merge_commit_sha; + if (tagShaForVersion && !tagAtMerge) { + f.push(`tag v${version} already exists at ${tagShaForVersion.slice(0, 7)}, not at the merge commit`); + } + if (highestTagOnLine && pep440.isValid(version)) { + const c = pep440.compare(version, highestTagOnLine); + if (c < 0 || (c === 0 && !tagAtMerge)) { + f.push(`${version} is not newer than the highest tag on the line (${highestTagOnLine})`); + } + } + if (commitsAfterMerge && commitsAfterMerge.length) { + const n = commitsAfterMerge.length; + f.push( + `these ${n} commits would be left out of v${version}: ` + + `${commitsAfterMerge.map(shortCommit).join("; ")}; label PR #${pr.number} ` + + `\`${ABANDONED_LABEL}\` and run again to cut a version that includes them`, + ); + } + return f; +} + +/** + * Draft publish re-verification (plan section 2). `anchorSha` is recomputed + * by the caller: the bump PR's merge sha (continue path) or, for a fast-path + * draft, a sha reachable from B whose version files equal V (null if none). + */ +function draftAnchorFailures({ + draft, version, botLogin, tagExists, anchorSha, versionAtTarget, prefix = "v", +}) { + const f = []; + if (!draft) return ["there is no draft"]; + if (draft.draft !== true) f.push(`release ${draft.tag_name} is not a draft`); + if (!botLogin || !sameLogin(draft.author_login, botLogin)) { + f.push(`draft ${draft.tag_name} was created by ${draft.author_login}, not ${botLogin}`); + } + if (draft.tag_name !== `${prefix}${version}`) { + f.push(`draft tag "${draft.tag_name}" is not ${prefix}${version}`); + } + if (tagExists) f.push(`tag ${prefix}${version} already exists`); + const target = draft.target_commitish; + if (typeof target !== "string" || !SHA40.test(target)) { + f.push(`draft target "${target}" is not a full commit sha`); + } else if (!anchorSha || target !== anchorSha) { + f.push(`draft target ${target.slice(0, 7)} is not the verified commit ${anchorSha ? anchorSha.slice(0, 7) : "(none found)"}`); + } + if (!versionsAgree(versionAtTarget, version)) { + f.push(`the version files at the draft target say ${versionAtTarget}, not ${version}`); + } + return f; +} + +/** + * publish.yml actor (plan section 5): the App bot may act for `requestedBy`; + * a human acts for themselves (requestedBy ignored); any other bot is refused. + */ +function resolvePublishActor({ triggeringActor, requestedBy, appSlug }) { + const slug = (appSlug || "").trim(); + if (!slug) { + throw new RefusalError("vars.RELEASE_APP_SLUG is empty.", "ask an admin to set the RELEASE_APP_SLUG repo variable."); + } + const actor = (triggeringActor || "").trim(); + if (!actor) throw new RefusalError("The triggering actor is empty."); + if (sameLogin(actor, `${slug}[bot]`)) { + const who = (requestedBy || "").trim(); + if (!LOGIN_RE.test(who)) { + throw new RefusalError( + `Dispatched by ${actor} with an invalid requested_by "${requestedBy}".`, + "start the release from the panel (release.yml).", + ); + } + return who; + } + if (actor.toLowerCase().endsWith("[bot]")) { + throw new RefusalError( + `${actor} may not run publish.yml; only ${slug}[bot] and humans can.`, + "start the release from the panel (release.yml).", + ); + } + if (!LOGIN_RE.test(actor)) throw new RefusalError(`Invalid triggering actor "${actor}".`); + return actor; +} + +module.exports = { hasWrite, continueGuards, draftAnchorFailures, resolvePublishActor }; diff --git a/scripts/release/lib/guards.test.js b/scripts/release/lib/guards.test.js new file mode 100644 index 0000000..4532e02 --- /dev/null +++ b/scripts/release/lib/guards.test.js @@ -0,0 +1,164 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const { hasWrite, continueGuards, draftAnchorFailures, resolvePublishActor } = require("./guards"); +const { loadConfig } = require("./config"); +const { refusalMatching: refusal } = require("./errors"); + +const CONFIG = loadConfig("keras-hub"); +const { BOT } = require("../test_support"); +const MERGE = "a".repeat(40); +const OTHER = "b".repeat(40); + +test("hasWrite reads the .permission field", () => { + assert.equal(hasWrite("admin"), true); + assert.equal(hasWrite("write"), true); + for (const p of ["read", "triage", "maintain", "none", "", undefined]) assert.equal(hasWrite(p), false, String(p)); +}); + +function guardArgs() { + return { + pr: { + number: 12, state: "closed", merged: true, merge_commit_sha: MERGE, + head_ref: "release-tool/bump-0.33.1", head_repo_full_name: "keras-team/keras-hub", + base_repo_full_name: "keras-team/keras-hub", base_ref: "r0.33", author_login: BOT, + merged_by_login: "alice", labels: [], + }, + config: CONFIG, branch: "r0.33", version: "0.33.1", botLogin: BOT, + mergedByPermission: "write", versionAtMergeSha: "0.33.1", tagShaForVersion: null, + highestTagOnLine: "0.33.0", commitsAfterMerge: [], + }; +} + +test("continue guards pass on a clean merged bot PR", () => { + assert.deepEqual(continueGuards(guardArgs()), []); + assert.deepEqual(continueGuards({ ...guardArgs(), tagShaForVersion: MERGE, highestTagOnLine: "0.33.1" }), []); + assert.deepEqual(continueGuards({ ...guardArgs(), versionAtMergeSha: ["0.33.1", "0.33.1"] }), []); + assert.deepEqual(continueGuards({ ...guardArgs(), highestTagOnLine: null }), []); +}); + +// Each entry flips exactly one guard and must produce exactly one failure, so +// deleting any single guard turns its case red. +const FLIPS = { + "disabled config": (a) => { a.config = { ...CONFIG, enabled: false }; }, + "not merged": (a) => { a.pr.merged = false; }, + "base not a release branch": (a) => { a.pr.base_ref = "master"; a.branch = "master"; }, + "base is another release branch": (a) => { a.pr.base_ref = "r0.32"; }, + "head repo is a fork": (a) => { a.pr.head_repo_full_name = "mallory/keras-hub"; }, + "head ref wrong": (a) => { a.pr.head_ref = "release-tool/bump-0.33.2"; }, + "author is a human": (a) => { a.pr.author_login = "alice"; }, + "merged by the bot": (a) => { a.pr.merged_by_login = BOT; }, + "merged by nobody": (a) => { a.pr.merged_by_login = null; }, + "merger has only read": (a) => { a.mergedByPermission = "read"; }, + "merger is triage": (a) => { a.mergedByPermission = "triage"; }, + "version file mismatch": (a) => { a.versionAtMergeSha = "0.33.0"; }, + "one of two version files mismatches": (a) => { a.versionAtMergeSha = ["0.33.1", "0.33.0"]; }, + "tag elsewhere": (a) => { a.tagShaForVersion = OTHER; }, + "downgrade": (a) => { a.highestTagOnLine = "0.33.2"; }, + "equal without tag at merge sha": (a) => { a.highestTagOnLine = "0.33.1"; }, + "commits after merge": (a) => { a.commitsAfterMerge = [{ sha: "c".repeat(40), message: "Fix x\n\nbody" }]; }, +}; + +test("continue guards: every guard flipped individually is rejected", () => { + for (const [name, flip] of Object.entries(FLIPS)) { + const a = guardArgs(); + flip(a); + assert.equal(continueGuards(a).length, 1, name); + } + assert.ok(continueGuards({ ...guardArgs(), config: null }).length >= 1); + assert.ok(continueGuards({ ...guardArgs(), botLogin: "" }).some((m) => /bot login is empty/.test(m))); +}); + +test("head repo check falls back to targetRepo when the PR lacks base_repo_full_name", () => { + const a = guardArgs(); + delete a.pr.base_repo_full_name; + assert.equal(continueGuards(a).length, 1); + assert.deepEqual(continueGuards({ ...a, targetRepo: "keras-team/keras-hub" }), []); + assert.equal(continueGuards({ ...a, targetRepo: "laxmareddyp/keras-hub" }).length, 1); +}); + +test("post-merge commits are listed with the abandon instruction", () => { + const a = guardArgs(); + a.commitsAfterMerge = [{ sha: "c".repeat(40), message: "Fix x\n\nbody" }, "d".repeat(40)]; + const [msg] = continueGuards(a); + assert.match(msg, /these 2 commits would be left out of v0.33.1: ccccccc Fix x; ddddddd; label PR #12 `release-tool:abandoned`/); +}); + +test("no PR at all fails", () => { + assert.deepEqual(continueGuards({ ...guardArgs(), pr: null }), ["there is no bump PR"]); +}); + +function anchorArgs() { + return { + draft: { tag_name: "v0.33.1", draft: true, target_commitish: MERGE, author_login: BOT, body: "notes" }, + version: "0.33.1", botLogin: BOT, tagExists: false, anchorSha: MERGE, versionAtTarget: "0.33.1", + }; +} + +test("draft anchor: bot draft at the anchor sha with edited notes -> passes", () => { + const a = anchorArgs(); + a.draft.body = "Edited by a human."; + a.draft.name = "Renamed"; + assert.deepEqual(draftAnchorFailures(a), []); +}); + +test("draft anchor: each tampering is refused", () => { + const cases = { + "human-authored draft": (a) => { a.draft.author_login = "alice"; }, + "retargeted to another sha": (a) => { a.draft.target_commitish = OTHER; }, + "retargeted to a branch name": (a) => { a.draft.target_commitish = "r0.33"; }, + "short sha target": (a) => { a.draft.target_commitish = MERGE.slice(0, 7); }, + "tag edited": (a) => { a.draft.tag_name = "v0.33.2"; }, + "tag already exists": (a) => { a.tagExists = true; }, + "no anchor found": (a) => { a.anchorSha = null; }, + "not a draft": (a) => { a.draft.draft = false; }, + "version at target differs": (a) => { a.versionAtTarget = "0.33.0"; }, + "empty bot login": (a) => { a.botLogin = ""; }, + }; + for (const [name, tamper] of Object.entries(cases)) { + const a = anchorArgs(); + tamper(a); + assert.equal(draftAnchorFailures(a).length, 1, name); + } + assert.deepEqual(draftAnchorFailures({ ...anchorArgs(), draft: null }), ["there is no draft"]); +}); + +test("actor: human with a fake requested_by is evaluated as the human", () => { + assert.equal(resolvePublishActor({ triggeringActor: "alice", requestedBy: "bob", appSlug: "keras-release" }), "alice"); + assert.equal(resolvePublishActor({ triggeringActor: "alice", requestedBy: "", appSlug: "keras-release" }), "alice"); +}); + +test("actor: the App bot is evaluated as requested_by", () => { + assert.equal(resolvePublishActor({ triggeringActor: "keras-release[bot]", requestedBy: "bob", appSlug: "keras-release" }), "bob"); + assert.throws(() => resolvePublishActor({ triggeringActor: "keras-release[bot]", requestedBy: "", appSlug: "keras-release" }), refusal(/invalid requested_by/)); + assert.throws(() => resolvePublishActor({ triggeringActor: "keras-release[bot]", requestedBy: "other[bot]", appSlug: "keras-release" }), refusal(/invalid requested_by/)); +}); + +test("actor: github-actions[bot] and other bots are rejected", () => { + for (const actor of ["github-actions[bot]", "dependabot[bot]", "other-app[BOT]"]) { + assert.throws(() => resolvePublishActor({ triggeringActor: actor, requestedBy: "bob", appSlug: "keras-release" }), refusal(/may not run publish.yml/), actor); + } +}); + +test("actor: empty slug or actor is rejected", () => { + assert.throws(() => resolvePublishActor({ triggeringActor: "alice", requestedBy: "", appSlug: "" }), refusal(/RELEASE_APP_SLUG is empty/)); + assert.throws(() => resolvePublishActor({ triggeringActor: "[bot]", requestedBy: "bob", appSlug: "" }), refusal(/RELEASE_APP_SLUG is empty/)); + assert.throws(() => resolvePublishActor({ triggeringActor: "", requestedBy: "bob", appSlug: "keras-release" }), refusal(/actor is empty/)); +}); diff --git a/scripts/release/lib/lock.js b/scripts/release/lib/lock.js new file mode 100644 index 0000000..ce1ec62 --- /dev/null +++ b/scripts/release/lib/lock.js @@ -0,0 +1,131 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Same-branch lock (plan section 4). PURE. + * - Run signal (`plan` job): other active release.yml / publish.yml runs. + * - PR and draft signals (`act` job): open bump PRs into B, drafts on B's + * line, with exactly the plan's per-step exemptions. + * Anything that cannot be parsed counts as a conflict (fail closed). + */ + +const V = require("./version"); +const { BUMP_PREFIX, draftMapsToLine, tagLineXY } = require("./detect"); + +const ACTIVE_STATUSES = new Set(["requested", "queued", "pending", "waiting", "in_progress"]); +const REPO_TOKEN = /^[a-z0-9][a-z0-9-]*$/; +const KINDS = new Set(["major", "patch"]); + +function tokens(title) { + if (typeof title !== "string") return null; + const parts = title.split(" "); + return parts.every((p) => p !== "") ? parts : null; +} + +/** `release ` (release.yml run-name). */ +function parseReleaseTitle(t) { + const p = tokens(t); + if (!p || p.length !== 5 || p[0] !== "release") return null; + const [, repository, branch, kind, version] = p; + if (!REPO_TOKEN.test(repository) || !KINDS.has(kind)) return null; + return { repository, branch, kind, version }; +} + +/** `publish ` (publish.yml run-name). */ +function parsePublishTitle(t) { + const p = tokens(t); + if (!p || p.length !== 3 || p[0] !== "publish") return null; + const [, repository, tag] = p; + if (!REPO_TOKEN.test(repository)) return null; + return { repository, tag }; +} + +/** + * Active runs that hold the lock for repository+branch. `runs` is + * [{ id, workflow: "release.yml"|"publish.yml", display_title, status }]. + * Returns [{ run, reason }]. + */ +function runConflicts({ runs, selfRunId, repository, branch, pattern, prefix }) { + const xy = V.branchXY(branch, pattern); + const out = []; + for (const run of runs || []) { + if (!ACTIVE_STATUSES.has(run.status)) continue; + if (String(run.id) === String(selfRunId)) continue; + const title = run.display_title; + if (run.workflow === "release.yml") { + const parsed = parseReleaseTitle(title); + if (!parsed) { + out.push({ run, reason: `release run "${title}" has an unparseable title` }); + } else if (parsed.repository === repository && parsed.branch === branch) { + out.push({ run, reason: `release run "${title}" is active on ${repository} ${branch}` }); + } + } else if (run.workflow === "publish.yml") { + const parsed = parsePublishTitle(title); + if (!parsed) { + out.push({ run, reason: `publish run "${title}" has an unparseable title` }); + } else if (parsed.repository === repository && draftMapsToLine(parsed.tag, xy, prefix)) { + out.push({ run, reason: `publish run "${title}" is active on ${branch}'s line` }); + } else if (parsed.repository === repository && tagLineXY(parsed.tag, prefix) === null) { + out.push({ run, reason: `publish run "${title}" has an unparseable tag` }); + } + } else { + out.push({ run, reason: `run of unexpected workflow "${run.workflow}"` }); + } + } + return out; +} + +/** + * PR and draft signals for `state.branch`, after step detection. Exemptions: + * - wait step: the open `release-tool/bump-` PR into B only; + * - publish step: the draft whose tag_name is `` only; + * - every other step: none. + * Returns [{ type: "pr"|"draft", number?, tag?, reason }]. + */ +function prDraftConflicts({ state, step, version, prefix, pattern }) { + const xy = V.branchXY(state.branch, pattern); + const out = []; + for (const pr of state.prs || []) { + if (pr.state !== "open" || pr.base_ref !== state.branch) continue; + if (typeof pr.head_ref !== "string" || !pr.head_ref.startsWith(BUMP_PREFIX)) continue; + if (step === "wait" && pr.head_ref === `${BUMP_PREFIX}${version}`) continue; + out.push({ + type: "pr", + number: pr.number, + reason: `open bump PR #${pr.number} (${pr.head_ref}) into ${state.branch}`, + }); + } + for (const r of state.releases || []) { + if (!r.draft || !draftMapsToLine(r.tag_name, xy, prefix)) continue; + if (step === "publish" && r.tag_name === `${prefix}${version}`) continue; + out.push({ + type: "draft", + tag: r.tag_name, + reason: `draft release ${r.tag_name} on ${state.branch}'s line`, + }); + } + return out; +} + +module.exports = { + ACTIVE_STATUSES, + parseReleaseTitle, + parsePublishTitle, + runConflicts, + prDraftConflicts, +}; diff --git a/scripts/release/lib/lock.test.js b/scripts/release/lib/lock.test.js new file mode 100644 index 0000000..70225e4 --- /dev/null +++ b/scripts/release/lib/lock.test.js @@ -0,0 +1,136 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const lock = require("./lock"); + +const PATTERN = "^r(\\d+)\\.(\\d+)$"; +const LOCK = { selfRunId: 100, repository: "keras-hub", branch: "r0.33", pattern: PATTERN, prefix: "v" }; +const relRun = (display_title, over = {}) => ({ id: 1, workflow: "release.yml", display_title, status: "in_progress", ...over }); +const pubRun = (display_title, over = {}) => ({ id: 2, workflow: "publish.yml", display_title, status: "queued", ...over }); +const conflicts = (runs, over = {}) => lock.runConflicts({ runs, ...LOCK, ...over }); + +test("ACTIVE_STATUSES is exactly the plan's list", () => { + assert.deepEqual([...lock.ACTIVE_STATUSES].sort(), ["in_progress", "pending", "queued", "requested", "waiting"]); +}); + +test("parseReleaseTitle", () => { + assert.deepEqual(lock.parseReleaseTitle("release keras-hub r0.33 patch auto"), + { repository: "keras-hub", branch: "r0.33", kind: "patch", version: "auto" }); + assert.deepEqual(lock.parseReleaseTitle("release keras r0.33 major 0.33.1"), + { repository: "keras", branch: "r0.33", kind: "major", version: "0.33.1" }); + for (const bad of [ + "release keras-hub r0.33 patch", "release keras-hub r0.33 patch auto x", "publish keras-hub r0.33 patch auto", + "release keras-hub r0.33 minor auto", "release keras-hub r0.33 patch auto", "release Keras r0.33 patch auto", "", null, + ]) { + assert.equal(lock.parseReleaseTitle(bad), null, String(bad)); + } +}); + +test("parsePublishTitle", () => { + assert.deepEqual(lock.parsePublishTitle("publish keras-hub v0.33.1"), { repository: "keras-hub", tag: "v0.33.1" }); + for (const bad of ["publish keras-hub", "publish keras-hub v0.33.1 x", "release keras-hub v0.33.1", "Publish keras v1.0.0"]) { + assert.equal(lock.parsePublishTitle(bad), null, bad); + } +}); + +test("release runs: same repo+branch conflicts; 'auto' and '0.33.1' lock each other", () => { + assert.equal(conflicts([relRun("release keras-hub r0.33 patch auto")]).length, 1); + assert.equal(conflicts([relRun("release keras-hub r0.33 patch 0.33.1")]).length, 1); + assert.equal(conflicts([relRun("release keras-hub r0.33 major auto")]).length, 1); + assert.equal(conflicts([relRun("release keras-hub r0.32 patch auto")]).length, 0); +}); + +test("lock title parsing: keras does not match a keras-hub run and vice versa", () => { + assert.equal(conflicts([relRun("release keras-hub r0.33 patch auto")], { repository: "keras" }).length, 0); + assert.equal(conflicts([relRun("release keras r0.33 patch auto")]).length, 0); + assert.equal(conflicts([pubRun("publish keras-hub v0.33.1")], { repository: "keras" }).length, 0); + assert.equal(conflicts([pubRun("publish keras v0.33.1")]).length, 0); +}); + +test("publish run on the same line locks; on another line does not", () => { + assert.equal(conflicts([pubRun("publish keras-hub v0.33.1")]).length, 1); + assert.equal(conflicts([pubRun("publish keras-hub v0.33.0.dev2")]).length, 1); + assert.equal(conflicts([pubRun("publish keras-hub v0.33.1")], { branch: "r0.32" }).length, 0); + assert.equal(conflicts([pubRun("publish keras-hub v0.330.1")]).length, 0); +}); + +test("unparseable titles of active runs are conflicts (fail closed)", () => { + assert.equal(conflicts([relRun("release keras-hub r0 33 patch auto")]).length, 1); + assert.equal(conflicts([relRun("Release")]).length, 1); + assert.equal(conflicts([pubRun("publish")]).length, 1); + assert.equal(conflicts([pubRun("publish keras-hub latest")]).length, 1); + assert.equal(conflicts([pubRun("publish keras-hub vnext")]).length, 1); + assert.equal(conflicts([{ id: 3, workflow: "other.yml", display_title: "x", status: "queued" }]).length, 1); +}); + +test("inactive runs and our own run are ignored", () => { + for (const status of ["completed", "cancelled", "action_required", undefined]) { + assert.equal(conflicts([relRun("release keras-hub r0.33 patch auto", { status })]).length, 0, String(status)); + } + assert.equal(conflicts([relRun("release keras-hub r0.33 patch auto", { id: 100 })]).length, 0); + assert.equal(conflicts([relRun("release keras-hub r0.33 patch auto", { id: "100" })]).length, 0); + for (const status of ["requested", "queued", "pending", "waiting", "in_progress"]) { + assert.equal(conflicts([relRun("release keras-hub r0.33 patch auto", { status })]).length, 1, status); + } +}); + +// PR and draft signals. +const state = (prs = [], releases = []) => ({ branch: "r0.33", prs, releases }); +const open = (v, over = {}) => ({ number: 7, state: "open", merged: false, head_ref: `release-tool/bump-${v}`, base_ref: "r0.33", ...over }); +const draft = (tag) => ({ tag_name: tag, draft: true }); +const prDraft = (s, step, version) => lock.prDraftConflicts({ state: s, step, version, prefix: "v", pattern: PATTERN }); + +test("PR signal: open bump PR into the branch refuses every step except wait for bump-V", () => { + const s = state([open("0.33.1")]); + assert.deepEqual(prDraft(s, "wait", "0.33.1"), []); + for (const step of ["prepare", "continue", "resume-continue", "fast-path", "publish", "already-released"]) { + assert.equal(prDraft(s, step, "0.33.1").length, 1, step); + } + assert.equal(prDraft(s, "wait", "0.33.2").length, 1); +}); + +test("PR signal ignores closed PRs, other bases and non-bump heads", () => { + assert.deepEqual(prDraft(state([open("0.33.1", { state: "closed" })]), "prepare", "0.33.2"), []); + assert.deepEqual(prDraft(state([open("0.33.1", { base_ref: "master" })]), "prepare", "0.33.2"), []); + assert.deepEqual(prDraft(state([open("0.34.0.dev0", { head_ref: "release-tool/master-bump-0.34.0.dev0", base_ref: "master" })]), "prepare", "0.33.2"), []); + assert.deepEqual(prDraft(state([open("x", { head_ref: "feature/foo" })]), "prepare", "0.33.2"), []); + assert.equal(prDraft(state([open("x", { head_ref: "release-tool/bump-garbage" })]), "prepare", "0.33.2").length, 1); +}); + +test("draft signal: drafts on the line refuse every step except publish for vV", () => { + const s = state([], [draft("v0.33.1")]); + assert.deepEqual(prDraft(s, "publish", "0.33.1"), []); + for (const step of ["prepare", "continue", "resume-continue", "fast-path", "wait", "already-released"]) { + assert.equal(prDraft(s, step, "0.33.1").length, 1, step); + } + assert.equal(prDraft(s, "publish", "0.33.2").length, 1); +}); + +test("draft signal ignores published releases and other lines; odd drafts fail closed", () => { + assert.deepEqual(prDraft(state([], [{ tag_name: "v0.33.1", draft: false }]), "prepare", "0.33.2"), []); + assert.deepEqual(prDraft(state([], [draft("v0.32.1")]), "prepare", "0.33.2"), []); + assert.equal(prDraft(state([], [draft("v0.33.1rc1")]), "prepare", "0.33.2").length, 1); +}); + +test("exemptions are exact: wait does not exempt a draft, publish does not exempt a PR", () => { + const s = state([open("0.33.1")], [draft("v0.33.1")]); + assert.deepEqual(prDraft(s, "wait", "0.33.1").map((c) => c.type), ["draft"]); + assert.deepEqual(prDraft(s, "publish", "0.33.1").map((c) => c.type), ["pr"]); +}); diff --git a/scripts/release/lib/pep440.js b/scripts/release/lib/pep440.js new file mode 100644 index 0000000..8c77254 --- /dev/null +++ b/scripts/release/lib/pep440.js @@ -0,0 +1,81 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * The PEP 440 subset the release tool accepts: X.Y.Z and X.Y.Z.devN, in + * normalized form only. Leading zeros are rejected so that every version has + * exactly one spelling, and therefore exactly one tag name. + */ + +const NUM = "(0|[1-9]\\d*)"; +const VERSION_RE = new RegExp(`^${NUM}\\.${NUM}\\.${NUM}(?:\\.dev${NUM})?$`); + +function tryParse(v) { + if (typeof v !== "string") return null; + const m = VERSION_RE.exec(v); + if (!m) return null; + return { + x: Number(m[1]), + y: Number(m[2]), + z: Number(m[3]), + dev: m[4] === undefined ? null : Number(m[4]), + }; +} + +function parse(v) { + const parsed = tryParse(v); + if (!parsed) { + throw new Error( + `"${v}" is not a supported version (expected X.Y.Z or X.Y.Z.devN).`, + ); + } + return parsed; +} + +function isValid(v) { + return tryParse(v) !== null; +} + +function cmpNum(a, b) { + return a < b ? -1 : a > b ? 1 : 0; +} + +/** -1 | 0 | 1. A dev release sorts before its final release. */ +function compare(a, b) { + const pa = typeof a === "string" ? parse(a) : a; + const pb = typeof b === "string" ? parse(b) : b; + for (const key of ["x", "y", "z"]) { + const c = cmpNum(pa[key], pb[key]); + if (c !== 0) return c; + } + if (pa.dev === pb.dev) return 0; + if (pa.dev === null) return 1; + if (pb.dev === null) return -1; + return cmpNum(pa.dev, pb.dev); +} + +function isDev(v) { + return parse(v).dev !== null; +} + +function format({ x, y, z, dev }) { + const base = `${x}.${y}.${z}`; + return dev === null || dev === undefined ? base : `${base}.dev${dev}`; +} + +module.exports = { parse, tryParse, isValid, compare, isDev, format }; diff --git a/scripts/release/lib/pep440.test.js b/scripts/release/lib/pep440.test.js new file mode 100644 index 0000000..4af0a21 --- /dev/null +++ b/scripts/release/lib/pep440.test.js @@ -0,0 +1,59 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const pep440 = require("./pep440"); + +test("parse accepts X.Y.Z and X.Y.Z.devN", () => { + assert.deepEqual(pep440.parse("0.33.1"), { x: 0, y: 33, z: 1, dev: null }); + assert.deepEqual(pep440.parse("0.33.0.dev2"), { x: 0, y: 33, z: 0, dev: 2 }); + assert.deepEqual(pep440.parse("10.0.0.dev0"), { x: 10, y: 0, z: 0, dev: 0 }); +}); + +test("parse rejects everything else", () => { + for (const bad of [ + "", "0.33", "v0.33.0", "0.33.0rc1", "0.33.0.post1", "0.33.0-dev1", + "0.33.0.dev", "01.2.3", "0.033.0", "0.33.0.dev01", " 0.33.0", "0.33.0\n", "1.2.3.4", + ]) { + assert.throws(() => pep440.parse(bad), /not a supported version/, bad); + assert.equal(pep440.tryParse(bad), null, bad); + assert.equal(pep440.isValid(bad), false, bad); + } + assert.equal(pep440.tryParse(undefined), null); +}); + +test("compare orders dev before final and numerically", () => { + const sorted = [ + "0.9.0", "0.10.0.dev0", "0.10.0.dev1", "0.10.0.dev10", "0.10.0", "0.10.1", "1.0.0", + ]; + const shuffled = [...sorted].reverse(); + assert.deepEqual(shuffled.sort(pep440.compare), sorted); + assert.equal(pep440.compare("0.33.0", "0.33.0"), 0); + assert.equal(pep440.compare("0.33.0.dev1", "0.33.0"), -1); + assert.equal(pep440.compare("0.33.0", "0.33.0.dev1"), 1); + assert.equal(pep440.compare("0.33.0.dev2", "0.33.0.dev10"), -1); +}); + +test("isDev and format round trip", () => { + assert.equal(pep440.isDev("0.33.0.dev0"), true); + assert.equal(pep440.isDev("0.33.0"), false); + for (const v of ["0.33.0", "0.33.0.dev0", "3.16.12"]) { + assert.equal(pep440.format(pep440.parse(v)), v); + } +}); diff --git a/scripts/release/lib/picks.js b/scripts/release/lib/picks.js new file mode 100644 index 0000000..5aaa2d7 --- /dev/null +++ b/scripts/release/lib/picks.js @@ -0,0 +1,172 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Cherry-pick selection (plan section 6): parsing, PR validation, merge + * classification, first-parent ordering, git arguments and the + * `Cherry-picks:` record line. PURE: callers fetch PR and commit data. + */ + +const { RefusalError } = require("./errors"); + +const PR_TOKEN = /^#([1-9]\d*)$/; +const SHA_TOKEN = /^[0-9a-f]{7,40}$/i; +const LINE_RE = /^[ \t]*Cherry-picks:(.*)$/gim; + +/** `#3101, #3110 abc1234` -> [{type:"pr",number}, {type:"sha",sha}]; deduplicated. */ +function parsePicks(text) { + const out = []; + const seen = new Set(); + for (const token of String(text || "").split(/[\s,]+/).filter(Boolean)) { + let pick; + const pr = PR_TOKEN.exec(token); + if (pr) { + pick = { type: "pr", number: Number(pr[1]) }; + } else if (SHA_TOKEN.test(token)) { + pick = { type: "sha", sha: token.toLowerCase() }; + } else { + throw new RefusalError( + `"${token}" in cherry_picks is neither a PR number (#3101) nor a commit ` + + "sha (at least 7 hex characters).", + "fix cherry_picks and run again.", + ); + } + const key = pick.type === "pr" ? `#${pick.number}` : pick.sha; + if (!seen.has(key)) { + seen.add(key); + out.push(pick); + } + } + return out; +} + +/** A `#PR` pick must be a PR merged into the default branch. */ +function validatePrForPick(pr, defaultBranch) { + if (!pr || pr.merged !== true || !pr.merge_commit_sha) { + throw new RefusalError( + `PR #${pr && pr.number} is not merged, so it cannot be cherry-picked.`, + "remove it from cherry_picks, or merge it into the default branch first.", + ); + } + if (pr.base_ref !== defaultBranch) { + throw new RefusalError( + `PR #${pr.number} was merged into ${pr.base_ref}, not ${defaultBranch}.`, + `only PRs merged into ${defaultBranch} can be cherry-picked.`, + ); + } +} + +/** + * How a PR landed, from its merge_commit_sha: + * 2 parents -> "merge"; 1 parent and several commits whose + * merge_commit_sha~1 also maps to the same PR -> "rebase"; else "squash". + */ +function classifyMerge({ parents, commits, parentMapsToSamePr }) { + const n = Array.isArray(parents) ? parents.length : parents; + if (n === 2) return "merge"; + if (n !== 1) throw new RefusalError(`Unsupported commit with ${n} parents.`); + return commits > 1 && parentMapsToSamePr === true ? "rebase" : "squash"; +} + +/** + * Resolves each pick's sha against the default branch's first-parent chain + * (`git rev-list --first-parent`, newest first) and returns the picks with + * full shas, oldest first. PR picks must carry `sha` (their merge commit). + */ +function orderPicks(picks, firstParentShas) { + const chain = (firstParentShas || []).map((s) => s.toLowerCase()); + const resolved = picks.map((pick) => { + if (!pick.sha) throw new Error(`Pick ${JSON.stringify(pick)} has no sha.`); + const name = pick.type === "pr" ? `PR #${pick.number} (${pick.sha.slice(0, 7)})` : pick.sha; + const prefix = pick.sha.toLowerCase(); + const hits = chain.map((s, i) => [s, i]).filter(([s]) => s.startsWith(prefix)); + if (hits.length > 1) { + throw new RefusalError( + `Commit ${name} is ambiguous on the default branch.`, + "use a longer sha prefix.", + ); + } + if (!hits.length) { + throw new RefusalError( + `Commit ${name} is not on the default branch's first-parent history.`, + "give the PR number, or the merge commit sha that landed it on the default branch.", + ); + } + return { pick: { ...pick, sha: hits[0][0] }, index: hits[0][1] }; + }); + const seen = new Map(); + for (const r of resolved) { + if (seen.has(r.index)) { + throw new RefusalError( + `Picks ${seen.get(r.index)} and ${label(r.pick)} are the same commit.`, + "list each change once.", + ); + } + seen.set(r.index, label(r.pick)); + } + return resolved.sort((a, b) => b.index - a.index).map((r) => r.pick); +} + +function label(pick) { + return pick.type === "pr" ? `#${pick.number}` : pick.sha.slice(0, 7); +} + +/** git arguments for one pick; `method` comes from classifyMerge. */ +function cherryPickArgs(pick) { + if (!pick.sha) throw new Error("Pick has no sha."); + if (pick.method === "rebase") { + throw new RefusalError( + `PR #${pick.number} was rebase-merged, so it has no single commit to pick.`, + "list the PR's individual commit shas in cherry_picks instead of the PR number.", + ); + } + if (pick.method === "merge") return ["cherry-pick", "-x", "-m", "1", pick.sha]; + if (pick.method === "squash") return ["cherry-pick", "-x", pick.sha]; + throw new Error(`Pick ${label(pick)} has unknown method "${pick.method}".`); +} + +/** `Cherry-picks: #3101 (abc1234), def5678` or "" for no picks. */ +function cherryPicksLine(picks) { + if (!picks || !picks.length) return ""; + const parts = picks.map((p) => + p.type === "pr" ? `#${p.number} (${p.sha.slice(0, 7)})` : p.sha.slice(0, 7), + ); + return `Cherry-picks: ${parts.join(", ")}`; +} + +/** PR numbers named in `Cherry-picks:` lines of the given PR bodies. */ +function parseCherryPicksLines(bodies) { + const out = new Set(); + for (const body of bodies || []) { + if (typeof body !== "string") continue; + for (const m of body.matchAll(LINE_RE)) { + for (const n of m[1].matchAll(/#([1-9]\d*)\b/g)) out.add(Number(n[1])); + } + } + return out; +} + +module.exports = { + parsePicks, + validatePrForPick, + classifyMerge, + orderPicks, + cherryPickArgs, + cherryPicksLine, + parseCherryPicksLines, +}; diff --git a/scripts/release/lib/picks.test.js b/scripts/release/lib/picks.test.js new file mode 100644 index 0000000..ea240be --- /dev/null +++ b/scripts/release/lib/picks.test.js @@ -0,0 +1,118 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const P = require("./picks"); +const { refusalMatching: refusal } = require("./errors"); + +// Default-branch first-parent chain, newest first (git rev-list order). +const CHAIN = ["e".repeat(40), "d".repeat(40), "c1" + "0".repeat(38), "c2" + "0".repeat(38), "b".repeat(40), "a".repeat(40)]; + +test("parsePicks: PR numbers and sha prefixes, commas or whitespace", () => { + assert.deepEqual(P.parsePicks("#3101, #3110 abc1234\n\tDEF5678ab"), [ + { type: "pr", number: 3101 }, { type: "pr", number: 3110 }, + { type: "sha", sha: "abc1234" }, { type: "sha", sha: "def5678ab" }, + ]); + assert.deepEqual(P.parsePicks("#1,#1, abc1234 ABC1234"), [{ type: "pr", number: 1 }, { type: "sha", sha: "abc1234" }]); +}); + +test("parsePicks: empty means zero picks", () => { + for (const empty of ["", " ", " , ,", undefined, null]) assert.deepEqual(P.parsePicks(empty), []); +}); + +test("parsePicks: unknown tokens are refused", () => { + for (const bad of ["3101", "#", "#0", "#12a", "abc123", "xyz1234", "https://github.com/x/pull/1", "#-1", "a".repeat(41)]) { + assert.throws(() => P.parsePicks(bad), refusal(/neither a PR number/), bad); + } +}); + +test("validatePrForPick: not merged or not on the default branch -> refused", () => { + P.validatePrForPick({ number: 1, merged: true, merge_commit_sha: CHAIN[0], base_ref: "master" }, "master"); + assert.throws(() => P.validatePrForPick({ number: 1, merged: false, merge_commit_sha: null, base_ref: "master" }, "master"), refusal(/not merged/)); + assert.throws(() => P.validatePrForPick({ number: 1, merged: true, merge_commit_sha: CHAIN[0], base_ref: "r0.33" }, "master"), refusal(/merged into r0.33, not master/)); + assert.throws(() => P.validatePrForPick(null, "master"), refusal(/not merged/)); +}); + +test("classifyMerge: merge, squash, rebase", () => { + assert.equal(P.classifyMerge({ parents: 2, commits: 5, parentMapsToSamePr: false }), "merge"); + assert.equal(P.classifyMerge({ parents: ["x", "y"], commits: 1, parentMapsToSamePr: false }), "merge"); + assert.equal(P.classifyMerge({ parents: 1, commits: 1, parentMapsToSamePr: false }), "squash"); + assert.equal(P.classifyMerge({ parents: 1, commits: 1, parentMapsToSamePr: true }), "squash"); + assert.equal(P.classifyMerge({ parents: 1, commits: 4, parentMapsToSamePr: false }), "squash"); + assert.equal(P.classifyMerge({ parents: 1, commits: 4, parentMapsToSamePr: true }), "rebase"); + assert.throws(() => P.classifyMerge({ parents: 3, commits: 1 }), refusal(/3 parents/)); +}); + +test("orderPicks: sorts oldest first and expands prefixes", () => { + const picks = [ + { type: "sha", sha: "eeeeeee" }, + { type: "pr", number: 7, sha: "a".repeat(40) }, + { type: "sha", sha: "DDDDDDD" }, + ]; + assert.deepEqual(P.orderPicks(picks, CHAIN), [ + { type: "pr", number: 7, sha: "a".repeat(40) }, + { type: "sha", sha: "d".repeat(40) }, + { type: "sha", sha: "e".repeat(40) }, + ]); +}); + +test("orderPicks: off-first-parent, ambiguous and duplicate commits are refused", () => { + assert.throws(() => P.orderPicks([{ type: "sha", sha: "f".repeat(7) }], CHAIN), refusal(/not on the default branch's first-parent/)); + assert.throws(() => P.orderPicks([{ type: "pr", number: 9, sha: "9".repeat(40) }], CHAIN), refusal(/PR #9.*first-parent/)); + assert.throws(() => P.orderPicks([{ type: "sha", sha: "c" }], CHAIN), refusal(/ambiguous/)); + assert.throws( + () => P.orderPicks([{ type: "pr", number: 7, sha: "a".repeat(40) }, { type: "sha", sha: "aaaaaaa" }], CHAIN), + refusal(/#7 and aaaaaaa are the same commit/), + ); + assert.throws(() => P.orderPicks([{ type: "pr", number: 7 }], CHAIN), /has no sha/); +}); + +test("cherryPickArgs: always -x; -m 1 for merge commits; rebase rejected", () => { + assert.deepEqual(P.cherryPickArgs({ type: "pr", number: 1, sha: CHAIN[0], method: "squash" }), ["cherry-pick", "-x", CHAIN[0]]); + assert.deepEqual(P.cherryPickArgs({ type: "pr", number: 1, sha: CHAIN[0], method: "merge" }), ["cherry-pick", "-x", "-m", "1", CHAIN[0]]); + assert.throws(() => P.cherryPickArgs({ type: "pr", number: 1, sha: CHAIN[0], method: "rebase" }), refusal(/rebase-merged/)); + assert.throws(() => P.cherryPickArgs({ type: "sha", sha: CHAIN[0] }), /unknown method/); + assert.throws(() => P.cherryPickArgs({ type: "sha" }), /no sha/); +}); + +test("Cherry-picks line round trip", () => { + const picks = [ + { type: "pr", number: 3101, sha: "abc1234" + "0".repeat(33) }, + { type: "pr", number: 3110, sha: "def5678" + "0".repeat(33) }, + { type: "sha", sha: "1234567" + "0".repeat(33) }, + ]; + const line = P.cherryPicksLine(picks); + assert.equal(line, "Cherry-picks: #3101 (abc1234), #3110 (def5678), 1234567"); + const body = `Bump to 0.33.1.\n\n${line}\n\nOther text #999 is not a pick.`; + assert.deepEqual([...P.parseCherryPicksLines([body])].sort(), [3101, 3110]); + assert.equal(P.cherryPicksLine([]), ""); +}); + +test("parseCherryPicksLines: several bodies, CRLF, human-written lines, junk", () => { + const bodies = [ + "Cherry-picks: #1 (aaaaaaa)\r\nmore", + " cherry-picks: #2, #3", + "Squash-merged hand pick.\n\nCherry-picks: #4", + "No line here #5", + null, + "Cherry-picks:", + ]; + assert.deepEqual([...P.parseCherryPicksLines(bodies)].sort((a, b) => a - b), [1, 2, 3, 4]); + assert.deepEqual([...P.parseCherryPicksLines([])], []); +}); diff --git a/scripts/release/lib/summary.js b/scripts/release/lib/summary.js new file mode 100644 index 0000000..aba4530 --- /dev/null +++ b/scripts/release/lib/summary.js @@ -0,0 +1,155 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Markdown run-summary rendering. + * + * Every summary's first line names the version and the step (plus any + * warning, e.g. "released on GitHub but not on PyPI"), and its last line + * starts with "Next:". Anything that came from the target repo (PR titles, + * commit subjects, file names) goes through `escapeMd`. + */ + +const { RefusalError } = require("./errors"); + +const MD_SPECIAL = /[\\`*_{}[\]()<>#+\-.!|~@]/g; + +function escapeMd(text) { + return String(text == null ? "" : text) + .replace(/[\r\n]+/g, " ") + .replace(MD_SPECIAL, "\\$&"); +} + +function table(headers, rows) { + const out = [`| ${headers.join(" | ")} |`, `|${headers.map(() => " --- |").join("")}`]; + for (const row of rows) out.push(`| ${row.join(" | ")} |`); + return out.join("\n"); +} + +class Report { + constructor(title = "Release") { + this.title = title; + this.version = null; + this.step = null; + this.warnings = []; + this.body = []; + this.next = null; + this.failed = false; + } + + setHeader(version, step) { + if (version) this.version = version; + if (step) this.step = step; + } + + warn(text) { + this.warnings.push(text); + } + + /** Raw markdown; callers escape untrusted parts themselves. */ + add(markdown) { + this.body.push(markdown); + } + + setNext(text) { + this.next = text; + } + + refuse(err) { + this.failed = true; + const isRefusal = err instanceof RefusalError; + this.add(`**Refused:** ${err.message}`); + this.setNext(isRefusal && err.next ? err.next : + "fix the error above, then click Run workflow again (a new run, " + + "not Re-run)."); + } + + /** Version is PEP 440-validated and step is a fixed enum: not escaped. */ + firstLine() { + const head = `**${this.title} ${this.version || "(version not resolved)"}` + + ` — step: ${this.step || "none"}**`; + const warnings = this.warnings.map((w) => ` · ⚠️ **${escapeMd(w)}**`); + return head + warnings.join(""); + } + + toMarkdown() { + const next = this.next || "nothing to do."; + return [this.firstLine(), "", ...this.body, "", `Next: ${next}`].join("\n"); + } +} + +async function writeSummary(core, report) { + const md = report.toMarkdown(); + core.info(md); + await core.summary.addRaw(md, true).write(); + return md; +} + +function branchesSection(rows) { + if (rows.length === 0) return "### Release branches\n\nNone found."; + return "### Release branches\n\n" + table( + ["Branch", "Latest release", "Next stable", "Next dev"], + rows.map((r) => [r.branch, r.latest || "—", r.nextStable || "—", + r.nextDev || "—"].map(escapeMd)), + ); +} + +function candidatesSection(branch, result) { + const head = `### Cherry-pick candidates for ${escapeMd(branch)}`; + if (!result) return `${head}\n\nBranch does not exist yet; picks are refused ` + + "for a new branch (it is cut from `commit_sha`)."; + const open = result.candidates.filter((c) => !c.picked); + const rows = open.map((c) => [ + c.number === null ? "—" : `#${c.number}`, + escapeMd(c.sha.slice(0, 7)), + escapeMd(c.title), + c.hint ? "maybe (cherry-pick trailer found)" : "", + ]); + const picked = result.candidates.filter((c) => c.picked) + .map((c) => `#${c.number}`); + const parts = [head, "", + rows.length ? table(["PR", "Commit", "Title", "Already on branch?"], rows) : + "No master PRs missing from this branch."]; + if (picked.length) parts.push("", `Already picked (Cherry-picks: lines): ${picked.join(", ")}`); + if (result.truncated) parts.push("", "List truncated to the first 250 commits."); + return parts.join("\n"); +} + +function formSection(values) { + return "### Form values for the real run\n\n" + table( + ["Field", "Value"], + Object.entries(values).map(([k, v]) => [escapeMd(k), + v === "" ? "(blank)" : `\`${String(v).replaceAll("`", "")}\``]), + ); +} + +function listSection(title, items) { + if (items.length === 0) return `### ${title}\n\nNone.`; + return `### ${title}\n\n` + items.map((i) => `- ${i}`).join("\n"); +} + +module.exports = { + Report, + candidatesSection, + branchesSection, + escapeMd, + formSection, + listSection, + table, + writeSummary, +}; diff --git a/scripts/release/lib/summary.test.js b/scripts/release/lib/summary.test.js new file mode 100644 index 0000000..28f9bb3 --- /dev/null +++ b/scripts/release/lib/summary.test.js @@ -0,0 +1,101 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); +const S = require("./summary"); +const { RefusalError } = require("./errors"); + +describe("escapeMd", () => { + it("neutralises markdown, links, html, mentions and newlines", () => { + const out = S.escapeMd("[x](http://e) *b* _i_ `c` @team #1\nline|2"); + for (const raw of ["[x]", "", "*b*", "_i_", "`c`", "\n"]) { + assert.ok(!out.includes(raw), raw); + } + assert.ok(out.includes("\\(http") && out.includes("\\@team")); + assert.equal(S.escapeMd("a*b"), "a\\*b"); + assert.equal(S.escapeMd(null), ""); + }); +}); + +describe("Report", () => { + it("first line names version and step; last line starts with Next:", () => { + const r = new S.Report(); + r.setHeader("0.33.1", "fast-path"); + r.add("body"); + r.setNext("watch the publish run."); + const lines = r.toMarkdown().split("\n"); + assert.equal(lines[0], "**Release 0.33.1 — step: fast-path**"); + assert.equal(lines.at(-1), "Next: watch the publish run."); + }); + + it("puts warnings on the first line", () => { + const r = new S.Report(); + r.setHeader("0.33.2", "prepare"); + r.warn("v0.33.1 is released on GitHub but not on PyPI; run with version=0.33.1 to re-publish"); + const first = r.toMarkdown().split("\n")[0]; + assert.match(first, /0\.33\.2 — step: prepare/); + assert.match(first, /not on PyPI/); + }); + + it("renders a refusal with its Next text exactly once", () => { + const r = new S.Report(); + r.refuse(new RefusalError("Nope.", "do the thing.")); + const md = r.toMarkdown(); + assert.ok(md.includes("**Refused:** Nope.")); + assert.equal(md.split("\n").at(-1), "Next: do the thing."); + assert.equal(md.match(/Next:/g).length, 1); + assert.equal(r.failed, true); + }); + + it("gives unexpected errors a generic Next", () => { + const r = new S.Report(); + r.refuse(new Error("boom")); + assert.match(r.toMarkdown().split("\n").at(-1), /^Next: .*new run/); + }); +}); + +describe("discovery sections", () => { + it("escapes candidate titles and lists already-picked PRs separately", () => { + const md = S.candidatesSection("r0.33", { + candidates: [ + { number: 5, sha: "a".repeat(40), title: "Fix [link](x)", picked: false, hint: false }, + { number: 6, sha: "b".repeat(40), title: "Old", picked: true, hint: false }, + ], + truncated: false, + }); + assert.ok(md.includes(S.escapeMd("Fix [link](x)"))); + assert.ok(!md.includes("[link](x)")); + assert.match(md, /Already picked.*#6/); + assert.ok(!md.includes("| #6 |")); + }); + + it("explains a missing branch and renders form values", () => { + assert.match(S.candidatesSection("r0.99", null), /does not exist/); + const form = S.formSection({ version: "", dry_run: "false" }); + assert.match(form, /\| version \| \(blank\) \|/); + assert.match(form, /\| dry\\_run \| `false` \|/); + }); + + it("renders release branches with next versions", () => { + const md = S.branchesSection([ + { branch: "r0.33", latest: "0.33.0", nextStable: "0.33.1", nextDev: "0.33.1.dev0" }, + ]); + assert.match(md, /r0\\\.33 \| 0\\\.33\\\.0 \| 0\\\.33\\\.1 \| 0\\\.33\\\.1\\\.dev0/); + }); +}); diff --git a/scripts/release/lib/version.js b/scripts/release/lib/version.js new file mode 100644 index 0000000..3e897b4 --- /dev/null +++ b/scripts/release/lib/version.js @@ -0,0 +1,263 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Version math: branch X.Y, computed next version, typed-version consistency, + * previous-release tag, release flags, and version-file read/rewrite. + */ + +const pep440 = require("./pep440"); +const { RefusalError } = require("./errors"); + +function branchXY(branch, pattern) { + const m = typeof branch === "string" ? new RegExp(pattern).exec(branch) : null; + if (!m || !/^\d+$/.test(m[1] || "") || !/^\d+$/.test(m[2] || "")) { + throw new RefusalError( + `"${branch}" is not a release branch (must match ${pattern}).`, + "set release_branch to a release branch name such as r0.33.", + ); + } + return { x: Number(m[1]), y: Number(m[2]) }; +} + +function tagName(t) { + return typeof t === "string" ? t : t && t.name; +} + +/** Versions (prefix stripped) of tags on the X.Y line, ascending. */ +function lineVersions(tags, xy, prefix) { + const out = []; + for (const t of tags || []) { + const name = tagName(t); + if (typeof name !== "string" || !name.startsWith(prefix)) continue; + const v = name.slice(prefix.length); + const p = pep440.tryParse(v); + if (p && p.x === xy.x && p.y === xy.y) out.push(v); + } + return out.sort(pep440.compare); +} + +function highest(versions) { + const valid = (versions || []).filter(pep440.isValid); + if (!valid.length) return null; + return valid.reduce((a, b) => (pep440.compare(a, b) >= 0 ? a : b)); +} + +function highestStable(versions) { + return highest((versions || []).filter((v) => pep440.isValid(v) && !pep440.isDev(v))); +} + +/** Throws on a kind or channel outside the panel's choices. */ +function checkKindChannel(kind, channel) { + if (kind !== "major" && kind !== "patch") { + throw new RefusalError(`Unknown kind "${kind}".`, "choose kind major or patch."); + } + if (channel !== "dev" && channel !== "stable") { + throw new RefusalError(`Unknown channel "${channel}".`, "choose channel dev or stable."); + } +} + +/** Plan section 4 "Version resolution", for a blank `version`. */ +function computeVersion({ kind, channel, xy, lineVersions: versions }) { + checkKindChannel(kind, channel); + const parsed = (versions || []).map(pep440.tryParse).filter(Boolean); + let z = 0; + if (kind === "patch") { + const stable = parsed.filter((p) => p.dev === null && p.x === xy.x && p.y === xy.y); + if (!stable.some((p) => p.z === 0)) { + throw new RefusalError( + `There is no stable ${xy.x}.${xy.y}.0 tag yet, so there is nothing to patch.`, + `use kind=major to release ${xy.x}.${xy.y}.0 (or a ${xy.x}.${xy.y}.0.devN).`, + ); + } + z = Math.max(...stable.map((p) => p.z)) + 1; + } + const target = { x: xy.x, y: xy.y, z, dev: null }; + if (channel === "stable") return pep440.format(target); + const devs = parsed + .filter((p) => p.x === target.x && p.y === target.y && p.z === target.z && p.dev !== null) + .map((p) => p.dev); + target.dev = devs.length ? Math.max(...devs) + 1 : 0; + return pep440.format(target); +} + +/** Throws when V does not belong to the branch line, channel or kind. */ +function checkConsistent(v, { kind, channel, xy }) { + const p = pep440.tryParse(v); + if (!p) { + throw new RefusalError( + `"${v}" is not a supported version (X.Y.Z or X.Y.Z.devN).`, + "leave version blank to use the computed version, or fix it.", + ); + } + if (p.x !== xy.x || p.y !== xy.y) { + throw new RefusalError( + `Version ${v} is not on the ${xy.x}.${xy.y} line of the release branch.`, + `use a ${xy.x}.${xy.y}.* version, or the release branch for ${p.x}.${p.y}.`, + ); + } + if ((p.dev !== null) !== (channel === "dev")) { + throw new RefusalError( + `Version ${v} does not match channel=${channel} ` + + "(dev releases end in .devN, stable ones do not).", + "fix channel or version so they agree.", + ); + } + if (kind === "patch" && p.z === 0) { + throw new RefusalError( + `Version ${v} has patch component 0, which is a kind=major release.`, + "use kind=major, or a version with a non-zero patch component.", + ); + } + if (kind === "major" && p.z !== 0) { + throw new RefusalError( + `Version ${v} has a non-zero patch component, which is a kind=patch release.`, + "use kind=patch, or an X.Y.0 version.", + ); + } +} + +/** + * Plan section 2: major (Z == 0) -> highest stable A.B.0 with (A,B) < (X,Y); + * patch -> highest stable X.Y.W with W < Z. Dev versions are never candidates. + */ +function previousReleaseTag(v, allVersions, prefix) { + const p = pep440.parse(v); + const candidates = (allVersions || []) + .map((s) => [s, pep440.tryParse(s)]) + .filter(([, q]) => q && q.dev === null) + .filter(([, q]) => + p.z === 0 + ? q.z === 0 && (q.x < p.x || (q.x === p.x && q.y < p.y)) + : q.x === p.x && q.y === p.y && q.z < p.z, + ) + .map(([s]) => s); + const best = highest(candidates); + return best === null ? null : `${prefix}${best}`; +} + +/** Plan section 4 "Release flags". */ +function releaseFlags(v, allVersions) { + const prerelease = pep440.isDev(v); + if (prerelease) return { prerelease: true, make_latest: "false" }; + const top = highestStable([...(allVersions || []), v]); + return { prerelease: false, make_latest: pep440.compare(v, top) === 0 ? "true" : "false" }; +} + +/** + * Plan section 4 "Version resolution": the in-flight version, a typed one, + * or the computed one; then the downgrade check and the no-skipping rule for + * a V that is neither in flight nor already released. `released` is the Set + * of versions with a non-draft release. + */ +function resolveVersion({ typed, inFlight, kind, channel, xy, lineVersions: line, released, branch, prefix }) { + let version; + if (typed) { + checkConsistent(typed, { kind, channel, xy }); + if (inFlight !== null && typed !== inFlight) { + throw new RefusalError( + `Release ${inFlight} is in flight on ${branch}; refusing to work on ${typed}.`, + `leave version blank (or type ${inFlight}) to continue ${inFlight}, or clean it up first.`, + ); + } + version = typed; + } else { + version = inFlight !== null ? inFlight : computeVersion({ kind, channel, xy, lineVersions: line }); + } + if (!typed && released.has(version)) { + throw new RefusalError( + `The computed version ${version} is already released.`, + `type version=${version} to re-publish it, or pick another kind/channel.`, + ); + } + if (version === inFlight || released.has(version)) return version; + const top = highest(line); + if (top !== null && pep440.compare(version, top) <= 0) { + throw new RefusalError( + `${version} is not newer than ${prefix}${top}, the highest tag on ${branch} (no downgrades).`, + "pick a kind/channel whose next version is newer.", + ); + } + const computed = typed ? computeVersion({ kind, channel, xy, lineVersions: line }) : version; + if (computed !== version) { + throw new RefusalError( + `Typed version ${typed} is not the next version (${computed}); versions cannot be skipped.`, + "leave version blank to use the computed version.", + ); + } + return version; +} + +/** The version at commit_sha must be on the branch's X.Y line. */ +function checkCommitShaVersion(versionAtCommit, xy) { + const p = pep440.tryParse(versionAtCommit); + if (!p || p.x !== xy.x || p.y !== xy.y) { + throw new RefusalError( + `The version files at commit_sha say ${versionAtCommit}, which is not on the ` + + `${xy.x}.${xy.y} line of the release branch.`, + `pick a master commit whose version is ${xy.x}.${xy.y}.*, or fix the branch name.`, + ); + } +} + +// One assignment per line: `__version__ = "..."` or `version = "..."` at +// column 0, optional trailing comment, LF or CRLF endings. +const VERSION_LINE_RE = + /^(__version__|version)([ \t]*=[ \t]*)(["'])([^"'\r\n]+)\3([ \t]*(?:#[^\r\n]*)?)(\r?)$/gm; + +function versionMatches(text) { + if (typeof text !== "string") throw new Error("Version file content must be a string."); + return [...text.matchAll(VERSION_LINE_RE)]; +} + +function readVersionFromFile(text) { + const matches = versionMatches(text); + if (matches.length !== 1) { + throw new RefusalError( + `Expected exactly one \`__version__ = "..."\` or \`version = "..."\` line ` + + `in the version file, found ${matches.length}.`, + "fix the version file in the target repo, or version_files in the config.", + ); + } + return matches[0][4]; +} + +function rewriteVersionInFile(text, v) { + pep440.parse(v); + readVersionFromFile(text); + return text.replace( + VERSION_LINE_RE, + (_all, name, eq, quote, _old, trailer, cr) => `${name}${eq}${quote}${v}${quote}${trailer}${cr}`, + ); +} + +module.exports = { + branchXY, + lineVersions, + highest, + highestStable, + checkKindChannel, + computeVersion, + checkConsistent, + previousReleaseTag, + releaseFlags, + resolveVersion, + checkCommitShaVersion, + readVersionFromFile, + rewriteVersionInFile, +}; diff --git a/scripts/release/lib/version.test.js b/scripts/release/lib/version.test.js new file mode 100644 index 0000000..5b964f6 --- /dev/null +++ b/scripts/release/lib/version.test.js @@ -0,0 +1,149 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const V = require("./version"); +const { RefusalError, refusalMatching: refusal } = require("./errors"); +const { loadConfig } = require("./config"); + +const PATTERN = loadConfig("keras-hub").release_branch_pattern; + +test("branchXY reads X and Y from the pattern", () => { + assert.deepEqual(V.branchXY("r0.33", PATTERN), { x: 0, y: 33 }); + for (const bad of ["master", "r0.33.1", "xr0.33", "r0.x", "", undefined]) { + assert.throws(() => V.branchXY(bad, PATTERN), refusal(/not a release branch/), String(bad)); + } +}); + +test("lineVersions keeps only the X.Y line, prefix stripped, sorted", () => { + const tags = [ + { name: "v0.33.0" }, { name: "v0.32.0" }, "v0.33.0.dev1", { name: "v0.33.1" }, + { name: "0.33.2" }, { name: "v0.33.0rc1" }, { name: "keras-v0.33.3" }, + ]; + assert.deepEqual(V.lineVersions(tags, { x: 0, y: 33 }, "v"), ["0.33.0.dev1", "0.33.0", "0.33.1"]); +}); + +test("highest and highestStable", () => { + assert.equal(V.highest(["0.33.0.dev1", "0.33.0", "0.32.9"]), "0.33.0"); + assert.equal(V.highest(["0.34.0.dev0", "0.33.0"]), "0.34.0.dev0"); + assert.equal(V.highestStable(["0.34.0.dev0", "0.33.0"]), "0.33.0"); + assert.equal(V.highest([]), null); + assert.equal(V.highestStable(["0.34.0.dev0"]), null); +}); + +// keras-hub history fixtures (plan section 7, version resolution). +test("r0.32 {v0.32.0.dev0}: major dev -> 0.32.0.dev1, major stable -> 0.32.0", () => { + const args = { xy: { x: 0, y: 32 }, lineVersions: ["0.32.0.dev0"] }; + assert.equal(V.computeVersion({ ...args, kind: "major", channel: "dev" }), "0.32.0.dev1"); + assert.equal(V.computeVersion({ ...args, kind: "major", channel: "stable" }), "0.32.0"); +}); + +test("r0.31 {v0.31.0, v0.31.1}: patch stable -> 0.31.2, patch dev -> 0.31.2.dev0", () => { + const args = { xy: { x: 0, y: 31 }, lineVersions: ["0.31.0", "0.31.1"] }; + assert.equal(V.computeVersion({ ...args, kind: "patch", channel: "stable" }), "0.31.2"); + assert.equal(V.computeVersion({ ...args, kind: "patch", channel: "dev" }), "0.31.2.dev0"); + assert.equal( + V.computeVersion({ ...args, lineVersions: [...args.lineVersions, "0.31.2.dev0"], kind: "patch", channel: "dev" }), + "0.31.2.dev1", + ); +}); + +test("major with no tags -> X.Y.0 / X.Y.0.dev0", () => { + const args = { xy: { x: 0, y: 34 }, lineVersions: [] }; + assert.equal(V.computeVersion({ ...args, kind: "major", channel: "stable" }), "0.34.0"); + assert.equal(V.computeVersion({ ...args, kind: "major", channel: "dev" }), "0.34.0.dev0"); +}); + +test("patch with no stable X.Y.0 -> refused, use kind=major", () => { + for (const lineVersions of [[], ["0.33.0.dev0", "0.33.0.dev1"]]) { + assert.throws( + () => V.computeVersion({ kind: "patch", channel: "stable", xy: { x: 0, y: 33 }, lineVersions }), + (e) => e instanceof RefusalError && /kind=major/.test(e.next), + ); + } +}); + +test("computeVersion rejects unknown kind/channel", () => { + const xy = { x: 0, y: 33 }; + assert.throws(() => V.computeVersion({ kind: "minor", channel: "dev", xy, lineVersions: [] }), refusal(/Unknown kind/)); + assert.throws(() => V.computeVersion({ kind: "major", channel: "rc", xy, lineVersions: [] }), refusal(/Unknown channel/)); +}); + +test("checkConsistent: X.Y, channel and kind", () => { + const ok = { kind: "patch", channel: "stable", xy: { x: 0, y: 33 } }; + V.checkConsistent("0.33.1", ok); + V.checkConsistent("0.33.0.dev2", { kind: "major", channel: "dev", xy: { x: 0, y: 33 } }); + assert.throws(() => V.checkConsistent("0.31.1", ok), refusal(/not on the 0.33 line/)); + assert.throws(() => V.checkConsistent("0.33.1.dev0", ok), refusal(/channel=stable/)); + assert.throws(() => V.checkConsistent("0.33.1", { ...ok, channel: "dev" }), refusal(/channel=dev/)); + assert.throws(() => V.checkConsistent("0.33.0", ok), refusal(/kind=major release/)); + assert.throws(() => V.checkConsistent("0.33.1", { ...ok, kind: "major" }), refusal(/kind=patch release/)); + assert.throws(() => V.checkConsistent("v0.33.1", ok), refusal(/not a supported version/)); +}); + +test("previousReleaseTag", () => { + const all = ["0.31.0", "0.31.0.dev0", "0.31.1", "0.32.0", "0.32.1", "0.33.0.dev1", "0.17.0", "0.17.1"]; + assert.equal(V.previousReleaseTag("0.33.0", all, "v"), "v0.32.0"); + assert.equal(V.previousReleaseTag("0.33.0.dev2", all, "v"), "v0.32.0"); + assert.equal(V.previousReleaseTag("0.31.1", ["0.31.0", "0.31.1.dev0", "0.31.0.dev0"], "v"), "v0.31.0"); + assert.equal(V.previousReleaseTag("0.31.2", all, "v"), "v0.31.1"); + assert.equal(V.previousReleaseTag("1.0.0", ["0.17.0", "0.17.1", "0.9.0", "1.0.0.dev0"], "v"), "v0.17.0"); + assert.equal(V.previousReleaseTag("0.1.0", ["0.1.0.dev0", "0.0.1"], "v"), null); + assert.equal(V.previousReleaseTag("0.31.1", ["0.31.1.dev0", "0.30.0"], "v"), null); +}); + +test("releaseFlags", () => { + const all = ["0.31.1", "0.32.0", "0.33.0", "0.34.0.dev0"]; + assert.deepEqual(V.releaseFlags("0.34.0.dev1", all), { prerelease: true, make_latest: "false" }); + assert.deepEqual(V.releaseFlags("0.31.2", all), { prerelease: false, make_latest: "false" }); + assert.deepEqual(V.releaseFlags("0.33.1", all), { prerelease: false, make_latest: "true" }); + assert.deepEqual(V.releaseFlags("0.33.0", all), { prerelease: false, make_latest: "true" }); + assert.deepEqual(V.releaseFlags("0.34.0", all), { prerelease: false, make_latest: "true" }); + assert.deepEqual(V.releaseFlags("0.1.0", []), { prerelease: false, make_latest: "true" }); +}); + +test("readVersionFromFile: __version__ and version =", () => { + const hub = 'from x import y\n\n# Unique source of truth.\n__version__ = "0.33.0.dev0"\n\n\ndef version():\n return __version__\n'; + assert.equal(V.readVersionFromFile(hub), "0.33.0.dev0"); + const toml = '[project]\nname = "keras-kinetic"\nversion = "0.0.3"\ntarget-version = "py311"\n'; + assert.equal(V.readVersionFromFile(toml), "0.0.3"); + assert.equal(V.readVersionFromFile("__version__ = '1.2.3' # pinned\r\n"), "1.2.3"); +}); + +test("readVersionFromFile refuses zero or several version lines", () => { + assert.throws(() => V.readVersionFromFile("x = 1\n"), refusal(/found 0/)); + assert.throws(() => V.readVersionFromFile('__version__ = "1"\nversion = "2"\n'), refusal(/found 2/)); + assert.throws(() => V.readVersionFromFile(' version = "1.0.0"\n'), refusal(/found 0/)); +}); + +test("rewriteVersionInFile touches only the version line", () => { + const hub = 'a = 1\n__version__ = "0.33.0.dev0"\nb = "version = 2"\n'; + assert.equal(V.rewriteVersionInFile(hub, "0.33.0.dev1"), 'a = 1\n__version__ = "0.33.0.dev1"\nb = "version = 2"\n'); + const toml = '[project]\nversion = "0.0.3"\ndependencies = []\n'; + assert.equal(V.rewriteVersionInFile(toml, "0.0.4"), '[project]\nversion = "0.0.4"\ndependencies = []\n'); + const crlf = "x\r\n__version__ = '1.2.3' # keep\r\ny\r\n"; + assert.equal(V.rewriteVersionInFile(crlf, "1.2.4"), "x\r\n__version__ = '1.2.4' # keep\r\ny\r\n"); + const noNewline = '__version__ = "1.2.3"'; + assert.equal(V.rewriteVersionInFile(noNewline, "1.2.4"), '__version__ = "1.2.4"'); +}); + +test("rewriteVersionInFile refuses bad versions and ambiguous files", () => { + assert.throws(() => V.rewriteVersionInFile('__version__ = "1.2.3"\n', "1.2"), /not a supported version/); + assert.throws(() => V.rewriteVersionInFile('__version__ = "1"\n__version__ = "2"\n', "1.2.3"), refusal(/found 2/)); +}); diff --git a/scripts/release/plan.js b/scripts/release/plan.js new file mode 100644 index 0000000..e454de2 --- /dev/null +++ b/scripts/release/plan.js @@ -0,0 +1,220 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * release.yml `plan` job (no secrets; GITHUB_TOKEN with actions: read). + * + * - Refuses unless running from refs/heads/main and RELEASE_APP_SLUG is set. + * - Validates every panel input. Inputs are read from `process.env` only + * (the workflow maps each `inputs.*` into `env:`), never interpolated. + * - Run-signal half of the same-branch lock: refuses when another active + * release.yml run targets the same repo+branch, or an active publish.yml + * run targets a tag on the branch's X.Y line. + * + * Also exports the input reader and the re-run guard used by act/dispatch. + */ + +const path = require("node:path"); +const { RefusalError } = require("./lib/errors"); +const { loadConfig, targetRepo } = require("./lib/config"); +const { branchXY, checkConsistent } = require("./lib/version"); +const pep440 = require("./lib/pep440"); +const { parsePicks } = require("./lib/picks"); +const { runConflicts } = require("./lib/lock"); +const { Gh } = require("./lib/gh"); +const { Report, escapeMd, writeSummary } = require("./lib/summary"); + +const ROOT_DIR = path.resolve(__dirname, "..", ".."); +const REPOSITORIES = ["keras", "keras-hub", "keras-rs", "kinetic"]; +const BOOL = ["true", "false"]; +const CHOICES = { + channel: ["dev", "stable"], + kind: ["major", "patch"], + release_mode: ["draft", "auto"], + bump_master: BOOL, + override_generated_tests: BOOL, + dry_run: BOOL, +}; +const ENV_NAMES = { + repository: "RELEASE_REPOSITORY", + channel: "RELEASE_CHANNEL", + kind: "RELEASE_KIND", + release_branch: "RELEASE_BRANCH", + commit_sha: "RELEASE_COMMIT_SHA", + version: "RELEASE_VERSION", + cherry_picks: "RELEASE_CHERRY_PICKS", + bump_master: "RELEASE_BUMP_MASTER", + release_mode: "RELEASE_MODE", + override_generated_tests: "RELEASE_OVERRIDE_GENERATED_TESTS", + dry_run: "RELEASE_DRY_RUN", +}; + +function readInputs(env) { + const inputs = {}; + for (const [key, name] of Object.entries(ENV_NAMES)) { + inputs[key] = String(env[name] == null ? "" : env[name]).trim(); + } + return inputs; +} + +/** Refuses a "Re-run failed jobs" attempt: it would skip plan's lock. */ +function assertFirstAttempt(env) { + const attempt = String(env.GITHUB_RUN_ATTEMPT || ""); + if (attempt !== "1") { + throw new RefusalError( + `This is run attempt ${escapeMd(attempt || "(unknown)")}. Re-running ` + + "jobs is not supported: it restarts after the plan job and skips the " + + "same-branch lock.", + "start a new run with Run workflow; it resumes where the last one stopped."); + } +} + +function refuseInput(what, value, next) { + throw new RefusalError(`Invalid ${what}: "${escapeMd(value)}".`, next); +} + +/** Returns the trimmed `vars.RELEASE_APP_SLUG`, or refuses when it is empty. */ +function requireAppSlug(env) { + const slug = String(env.RELEASE_APP_SLUG || "").trim(); + if (!slug) { + throw new RefusalError("`vars.RELEASE_APP_SLUG` is empty.", + "ask an admin to set the RELEASE_APP_SLUG repository variable."); + } + return slug; +} + +function assertKnownRepository(repository, + next = `pick one of ${REPOSITORIES.join(", ")}.`) { + if (!REPOSITORIES.includes(repository)) refuseInput("repository", repository, next); +} + +/** Reads, checks and returns `{ inputs, config, xy }`. */ +function loadInputsAndConfig(env, rootDir) { + const inputs = readInputs(env); + assertKnownRepository(inputs.repository); + const config = loadConfig(inputs.repository, rootDir); + for (const [key, allowed] of Object.entries(CHOICES)) { + if (!allowed.includes(inputs[key])) { + refuseInput(key, inputs[key], `use one of ${allowed.join(", ")}.`); + } + } + const xy = branchXY(inputs.release_branch, config.release_branch_pattern); + if (inputs.commit_sha) { + if (!/^[0-9a-f]{7,40}$/.test(inputs.commit_sha)) { + refuseInput("commit_sha", inputs.commit_sha, + "give a hex commit sha from the default branch."); + } + if (inputs.kind !== "major") { + throw new RefusalError("`commit_sha` applies only to kind=major with a " + + "new release branch.", "clear commit_sha and run again."); + } + } + if (inputs.version) { + try { + pep440.parse(inputs.version); + } catch (err) { + refuseInput("version", inputs.version, + "use X.Y.Z or X.Y.Z.devN without a prefix, or leave it blank."); + } + checkConsistent(inputs.version, { + kind: inputs.kind, channel: inputs.channel, xy, + }); + } + parsePicks(inputs.cherry_picks); + return { inputs, config, xy }; +} + +function failWith(core, report, err) { + report.refuse(err); + core.setFailed(err.next ? `${err.message} Next: ${err.next}` : err.message); +} + +/** + * Returns [owner, repo] from the config and refuses if the owner/name the + * App token was minted for (plan's outputs, passed back in as env) differ. + */ +function resolveTarget(config, context, tokenOwner, tokenName) { + const target = targetRepo(config, context.repo.owner); + const scoped = `${String(tokenOwner || "")}/${String(tokenName || "")}`; + if (scoped !== target) { + throw new RefusalError(`The App token is scoped to \`${escapeMd(scoped)}\`, ` + + `but the config targets \`${target}\`.`, + "start a new run with Run workflow; do not re-run single jobs."); + } + return target.split("/"); +} + +async function main({ github, context, core, env = process.env, + rootDir = ROOT_DIR }) { + const report = new Report("Release plan"); + report.setHeader(null, "plan"); + try { + if (context.ref !== "refs/heads/main") { + throw new RefusalError(`The panel ran from ${escapeMd(context.ref)}. ` + + "Only reviewed code on main can reach the bot key and PyPI.", + "choose `main` in the branch picker and run again."); + } + requireAppSlug(env); + const { inputs, config } = loadInputsAndConfig(env, rootDir); + report.setHeader(inputs.version || "auto", "plan"); + const target = targetRepo(config, context.repo.owner); + const gh = new Gh({ + github, owner: context.repo.owner, repo: context.repo.repo, + }); + const runs = [ + ...await gh.listActiveRuns("release.yml"), + ...await gh.listActiveRuns("publish.yml"), + ]; + const conflicts = runConflicts({ + runs, selfRunId: context.runId, repository: inputs.repository, + branch: inputs.release_branch, pattern: config.release_branch_pattern, + prefix: config.tag_prefix, + }); + if (conflicts.length > 0) { + const list = conflicts.map(({ run, reason }) => + `- ${run.workflow} run ${run.id} (${run.status}): ${escapeMd(reason)}`); + throw new RefusalError( + `Another active run holds the lock for ${escapeMd(inputs.repository)} ` + + `${escapeMd(inputs.release_branch)}:\n${list.join("\n")}`, + "wait for that run to finish, then click Run workflow again."); + } + const [owner, name] = target.split("/"); + core.setOutput("target_owner", owner); + core.setOutput("target_name", name); + report.add(`Inputs valid. Target: \`${target}\`. No other active run ` + + `holds ${escapeMd(inputs.release_branch)}.`); + report.setNext("the act job runs next and reports the detected step."); + } catch (err) { + failWith(core, report, err); + } + await writeSummary(core, report); +} + +module.exports = { + ENV_NAMES, + REPOSITORIES, + ROOT_DIR, + assertFirstAttempt, + assertKnownRepository, + failWith, + loadInputsAndConfig, + main, + readInputs, + requireAppSlug, + resolveTarget, +}; diff --git a/scripts/release/plan.test.js b/scripts/release/plan.test.js new file mode 100644 index 0000000..76c654a --- /dev/null +++ b/scripts/release/plan.test.js @@ -0,0 +1,110 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); +const T = require("./test_support"); +const plan = require("./plan"); + +const ROOT = T.configRoot(); + +async function run({ runs = {}, env = {}, context = {} } = {}) { + const gh = T.fakeGithub({ runs }); + const core = T.fakeCore(); + await plan.main({ + github: gh.github, context: T.context(context), core, + env: T.panelEnv(env), rootDir: ROOT, + }); + return { gh, core }; +} + +const run_ = (id, status, title) => ({ id, status, display_title: title }); + +describe("plan preconditions", () => { + it("refuses any ref other than main, before any API call", async () => { + const { gh, core } = await run({ context: { ref: "refs/heads/feature" } }); + assert.match(core.failed, /refs\/heads\/feature/); + assert.equal(gh.calls.length, 0); + }); + + it("refuses an empty RELEASE_APP_SLUG", async () => { + const { core } = await run({ env: { RELEASE_APP_SLUG: " " } }); + assert.match(core.failed, /RELEASE_APP_SLUG/); + }); + + const bad = { + "an unknown repository": { RELEASE_REPOSITORY: "../../etc" }, + "an invalid channel": { RELEASE_CHANNEL: "nightly" }, + "a non-boolean dry_run": { RELEASE_DRY_RUN: "yes" }, + "commit_sha with kind=patch": { RELEASE_COMMIT_SHA: "a".repeat(40) }, + "a malformed commit_sha": { RELEASE_KIND: "major", RELEASE_COMMIT_SHA: "not-a-sha" }, + "a malformed version": { RELEASE_VERSION: "v0.33.1" }, + }; + for (const [name, env] of Object.entries(bad)) { + it(`refuses ${name}`, async () => { + const { core } = await run({ env }); + assert.ok(core.failed, name); + assert.equal(core.outputs.target_owner, undefined); + }); + } +}); + +describe("plan run lock", () => { + it("lists active release.yml and publish.yml runs", async () => { + const { gh, core } = await run(); + assert.equal(core.failed, null); + const workflows = new Set(gh.called("actions.listWorkflowRuns") + .map((c) => c.params.workflow_id)); + assert.deepEqual([...workflows].sort(), ["publish.yml", "release.yml"]); + assert.equal(core.outputs.target_owner, "keras-team"); + assert.equal(core.outputs.target_name, "keras-hub"); + }); + + it("an active publish run on the branch's line refuses; another line does not", async () => { + const runs = { "publish.yml": [run_(9, "in_progress", "publish keras-hub v0.33.1")] }; + const same = await run({ runs }); + assert.match(same.core.failed, /publish\.yml run 9/); + const other = await run({ runs, env: { RELEASE_BRANCH: "r0.32" } }); + assert.equal(other.core.failed, null); + }); + + it("ignores its own run and refuses another on the same branch", async () => { + const own = { "release.yml": [run_(1, "in_progress", "release keras-hub r0.33 patch auto")] }; + assert.equal((await run({ runs: own })).core.failed, null); + const twin = { "release.yml": [run_(2, "queued", "release keras-hub r0.33 patch 0.33.1")] }; + assert.match((await run({ runs: twin })).core.failed, /Another active run/); + }); + + it("maps a fork owner to the fork's target repo", async () => { + const { core } = await run({ + context: { repo: { owner: "laxmareddyp", repo: "shared-workflows" } }, + }); + assert.equal(core.outputs.target_owner, "laxmareddyp"); + assert.equal(core.outputs.target_name, "keras-hub"); + }); +}); + +describe("assertFirstAttempt", () => { + it("accepts attempt 1 only", () => { + assert.doesNotThrow(() => plan.assertFirstAttempt({ GITHUB_RUN_ATTEMPT: "1" })); + for (const attempt of ["2", "", undefined]) { + assert.throws(() => plan.assertFirstAttempt({ GITHUB_RUN_ATTEMPT: attempt }), + /Re-running/); + } + }); +}); diff --git a/scripts/release/prepare.js b/scripts/release/prepare.js new file mode 100644 index 0000000..c960cb0 --- /dev/null +++ b/scripts/release/prepare.js @@ -0,0 +1,270 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * The prepare step of `act` (plan section 4 "State machine" and section 6). + * + * - Branch absent (major only): validates `commit_sha` (on the default + * branch, X.Y matches the branch name), creates B with `POST /git/refs`, + * and stops if B already carries V (F10: the next run is the fast-path). + * - Branch present: rebuilds `release-tool/bump-V` from scratch as B head + + * the picks in master first-parent order (`-x`, `-m 1` for merge commits) + * + a version commit (skipped when already at V), force-pushes that bot + * ref and opens the PR with a `Cherry-picks:` line. B is never pushed. + * - Conflict: `cherry-pick --abort`, nothing pushed, refusal lists the + * conflicting files and the unselected earlier master commits touching them. + * - `bump_master`: separate `release-tool/master-bump-` PR. + */ + +const { RefusalError } = require("./lib/errors"); +const pep440 = require("./lib/pep440"); +const version = require("./lib/version"); +const picksLib = require("./lib/picks"); +const { BUMP_PREFIX } = require("./lib/detect"); +const { Workspace, withAuthRemote } = require("./lib/git"); +const S = require("./lib/summary"); + +const short = (sha) => sha.slice(0, 7); +const pickLabel = (p) => + p.number ? `#${p.number} (${short(p.sha)})` : short(p.sha); + +/** True when every requested pick is a PR already listed on B. */ +function zeroPicksAfterSkips(ctx) { + return picksLib.parsePicks(ctx.inputs.cherry_picks).every((p) => + p.type === "pr" && ctx.state.pickedPrNumbers.has(p.number)); +} + +/** Resolves picks to commits, classifies them and sorts oldest first. */ +async function resolvePicks(ctx, picks) { + const { gh } = ctx; + const defaultBranch = ctx.config.default_branch; + const bySha = new Map(); + for (const pick of picks) { + let resolved; + if (pick.type === "pr") { + const pr = await gh.pull(pick.number); + picksLib.validatePrForPick(pr && { + number: pick.number, merged: pr.merged === true, + merge_commit_sha: pr.merge_commit_sha, base_ref: pr.base.ref, + }, defaultBranch); + const c = await gh.commit(pr.merge_commit_sha); + const parentMapsToSamePr = c.parents.length === 1 && pr.commits > 1 && + (await gh.commitPulls(c.parents[0])).includes(pick.number); + const method = picksLib.classifyMerge({ + parents: c.parents, commits: pr.commits, parentMapsToSamePr, + }); + resolved = { type: "pr", number: pick.number, sha: c.sha, method }; + } else { + const c = await gh.commit(pick.sha); + if (!c) { + throw new RefusalError(`Commit ${S.escapeMd(pick.sha)} not found.`, + "check the SHA and run again."); + } + const method = picksLib.classifyMerge({ + parents: c.parents, commits: 1, parentMapsToSamePr: false, + }); + resolved = { type: "sha", number: null, sha: c.sha, method }; + } + picksLib.cherryPickArgs(resolved); // refuses a rebase-merged PR up front + bySha.set(resolved.sha, resolved); + } + if (bySha.size === 0) return []; + return picksLib.orderPicks([...bySha.values()], + await gh.firstParentShas(defaultBranch)); +} + +function workspace(ctx) { + if (ctx.ws) return ctx.ws; + const token = String(ctx.env.RELEASE_APP_TOKEN || ""); + if (!token) throw new Error("RELEASE_APP_TOKEN is not set in the act step's env."); + const url = `https://github.com/${ctx.gh.fullName}.git`; + ctx.ws = new Workspace({ + gitFn: ctx.git, cwd: ctx.workdir, identity: ctx.botLogin, + pushUrl: withAuthRemote(url, token), + }); + ctx.ws.clone(url); + return ctx.ws; +} + +function conflictRefusal(ctx, ws, pick, files, selected) { + const defaultBranch = ctx.config.default_branch; + const base = ws.mergeBase(ctx.baseSha, `origin/${defaultBranch}`); + const earlier = ws.commitsTouching(`${base}..${pick.sha}~1`, files) + .filter((c) => { + if (selected.has(c.sha)) return false; + const m = c.subject.match(/\(#(\d+)\)\s*$/); + return !(m && ctx.state.pickedPrNumbers.has(Number(m[1]))); + }); + const lines = [ + `Cherry-pick of ${pickLabel(pick)} onto ${S.escapeMd(ctx.branch)} ` + + "conflicts. It was aborted and nothing was pushed.", + "", "Conflicting files:", ...files.map((f) => `- ${S.escapeMd(f)}`), "", + `Earlier ${S.escapeMd(defaultBranch)} commits touching those files that ` + + "were not selected:", + ...(earlier.length ? earlier.map((c) => + `- ${short(c.sha)} ${S.escapeMd(c.subject)}`) : ["- none found"]), + ]; + return new RefusalError(lines.join("\n"), + "run again with an adjusted cherry_picks list (e.g. add the prerequisites " + + "above), or resolve it by hand: a PR onto the branch made with `git " + + "cherry-pick -x` whose body has a `Cherry-picks:` line, merged by a human; " + + "then run the panel again."); +} + +async function bumpMaster(ctx) { + const { inputs, config, state, report } = ctx; + if (inputs.bump_master !== "true") return; + const v = pep440.parse(ctx.version); + const nextDev = `${v.x}.${v.y + 1}.0${config.dev_suffix_on_master}`; + pep440.parse(nextDev); + const master = config.default_branch; + if (pep440.compare(state.defaultBranchVersion, nextDev) >= 0) { + report.add(`Master bump skipped: ${master} is at ` + + `${state.defaultBranchVersion} (>= ${nextDev}).`); + return; + } + const head = `release-tool/master-bump-${nextDev}`; + const open = state.prs.find((p) => p.state === "open" && + p.head_ref === head && p.base_ref === master); + if (open) { + report.add(`Master bump PR #${open.number} is already open.`); + return; + } + if (ctx.dryRun) { + report.add(`Would open a PR from \`${head}\` bumping ${master} to ${nextDev}.`); + return; + } + const ws = workspace(ctx); + ws.resetBranch(head, state.defaultHeadSha); + ws.commitVersion(config.version_files, nextDev, { + read: version.readVersionFromFile, rewrite: version.rewriteVersionInFile, + message: `Bump ${master} version to ${nextDev}`, + }); + ws.pushBotRef(head); + const pr = await ctx.gh.openPr({ + head, base: master, title: `Bump ${master} version to ${nextDev}`, + body: `Bumps ${master} to ${nextDev} after ${ctx.tag}. A human with ` + + "write merges this PR; the bot never merges.", + }); + report.add(`Opened master bump PR #${pr.number}: ${pr.html_url}`); +} + +/** + * Cuts B at commit_sha. detect.checkPrepare already enforced kind=major, a + * 40-hex commit_sha and no picks; this adds the checks that need I/O. + */ +async function createBranchAt(ctx) { + const { gh, inputs, config, report } = ctx; + const c = await gh.commit(inputs.commit_sha); + if (!c) { + throw new RefusalError(`commit_sha ${short(inputs.commit_sha)} was not found.`, + `give the ${config.default_branch} commit to cut ${ctx.branch} from.`); + } + if (!await gh.isAncestor(c.sha, config.default_branch)) { + throw new RefusalError(`commit_sha ${short(c.sha)} is not on ` + + `${config.default_branch}.`, `pick a commit from ${config.default_branch}.`); + } + const atSha = await gh.versionAt(c.sha, config.version_files); + version.checkCommitShaVersion(atSha, ctx.xy); + if (ctx.dryRun) { + report.add(`Would create branch ${S.escapeMd(ctx.branch)} at \`${c.sha}\`.`); + } else { + await gh.createBranch(ctx.branch, c.sha); + report.add(`Created branch ${S.escapeMd(ctx.branch)} at \`${c.sha}\`.`); + } + return { sha: c.sha, versionAtSha: atSha }; +} + +async function prepare(ctx, { fastPath }) { + const { gh, inputs, config, state, report } = ctx; + ctx.baseSha = state.headSha; + if (!state.branchExists) { + const cut = await createBranchAt(ctx); + ctx.baseSha = cut.sha; + if (cut.versionAtSha === ctx.version) { + report.add(`${S.escapeMd(ctx.branch)} already carries ${ctx.version}; ` + + "no bump PR (it would be empty) and no tag in this run."); + await bumpMaster(ctx); + report.setNext(`run the panel again with commit_sha blank; the ` + + `fast-path step will release ${ctx.version}.`); + return; + } + } + const skipped = []; + const wanted = picksLib.parsePicks(inputs.cherry_picks).filter((p) => { + const done = p.type === "pr" && state.pickedPrNumbers.has(p.number); + if (done) skipped.push(`#${p.number}: already in a Cherry-picks: line on the branch`); + return !done; + }); + const picks = await resolvePicks(ctx, wanted); + const head = `${BUMP_PREFIX}${ctx.version}`; + if (ctx.dryRun) { + report.add(`Would rebuild \`${head}\` from ${S.escapeMd(ctx.branch)} at ` + + `\`${ctx.baseSha}\` with picks in order: ` + + `${picks.map((p) => `${pickLabel(p)} [${p.method}]`).join(", ") || "none"}` + + (state.versionAtHead === ctx.version ? "; no version commit (already " + + `${ctx.version}).` : `; then bump the version to ${ctx.version}.`)); + report.add(S.listSection("Skipped picks", skipped)); + await bumpMaster(ctx); + return; + } + const ws = workspace(ctx); + ws.resetBranch(head, ctx.baseSha); + const selected = new Set(picks.map((p) => p.sha)); + const applied = []; + for (const pick of picks) { + const out = ws.cherryPick(picksLib.cherryPickArgs(pick)); + if (out.result === "conflict") { + throw conflictRefusal(ctx, ws, pick, out.files, selected); + } + if (out.result === "empty") skipped.push(`${pickLabel(pick)}: empty after cherry-pick`); + else applied.push(pick); + } + const bumped = ws.commitVersion(config.version_files, ctx.version, { + read: version.readVersionFromFile, rewrite: version.rewriteVersionInFile, + message: `Bump version to ${ctx.version}`, + }); + report.add(S.listSection("Skipped picks", skipped)); + if (ws.commitCount(ctx.baseSha) === 0) { + report.add("Every pick was skipped and the version is already set: no " + + "PR; continuing as the fast-path step."); + report.setHeader(null, "fast-path"); + await fastPath(ctx, { picksSkipped: true }); + await bumpMaster(ctx); + return; + } + ws.pushBotRef(head); + const body = [ + `Prepares ${ctx.tag} on \`${ctx.branch}\`: ${applied.length} cherry-pick(s)` + + (bumped ? ` and the version bump to ${ctx.version}.` : + `; the version files already read ${ctx.version}.`), "", + ...(applied.length ? [picksLib.cherryPicksLine(applied), ""] : []), + ...(skipped.length ? ["Skipped:", ...skipped.map((s) => `- ${s}`), ""] : []), + "A human with write merges this PR (the bot never merges). Then run the " + + `Release panel again with the same repository and branch to release ${ctx.version}.`, + ].join("\n"); + const pr = await gh.openPr({ + head, base: ctx.branch, title: `Release ${ctx.version}`, body, + }); + report.add(`Opened PR #${pr.number}: ${pr.html_url}`); + await bumpMaster(ctx); + report.setNext(`a human with write reviews and merges PR #${pr.number}, ` + + "then run the panel again with the same repository and branch."); +} + +module.exports = { prepare, resolvePicks, zeroPicksAfterSkips }; diff --git a/scripts/release/py/fakes.py b/scripts/release/py/fakes.py new file mode 100644 index 0000000..589d027 --- /dev/null +++ b/scripts/release/py/fakes.py @@ -0,0 +1,31 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Test doubles shared by the release script tests.""" + +import io + + +class FakeResponse(io.BytesIO): + """Stands in for the context manager urllib's `urlopen` returns.""" + + def __init__(self, payload, status=200): + super().__init__(payload) + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False diff --git a/scripts/release/py/generate_test.py b/scripts/release/py/generate_test.py new file mode 100644 index 0000000..4dda4c4 --- /dev/null +++ b/scripts/release/py/generate_test.py @@ -0,0 +1,235 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Generate release tests with Gemini (publish.yml `generate`). + +Stdlib only; run as `python -I sw/scripts/release/py/generate_test.py` from +outside the target checkout. Diffs the target repo between the previous +release tag and the release commit (git with `--no-ext-diff --no-textconv` +and hooks off, so the target's .gitattributes cannot run anything), truncates +the diff at file boundaries, and asks Gemini for a pytest file. + +The API key is sent only in the `x-goog-api-key` header: never in the prompt, +the URL, a log line, or the git subprocess environment. A non-200 response +exits 1 after 3 attempts with backoff, printing only the status code. + +Environment: + GEMINI_API_KEY required + TARGET_DIR target checkout (full history and tags) + BASE_REF previous release tag; empty = diff the whole tree + HEAD_REF release commit sha + OUTPUT_DIR where test_generated_release.py is written + PUBLISH_VERSION version being released + PACKAGES JSON list of PyPI names + IMPORT_NAMES JSON list of import names + GEMINI_MODEL optional, default below + MAX_DIFF_BYTES optional, default below +""" + +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +API_URL = ( + "https://generativelanguage.googleapis.com/v1beta/models/" + "{model}:generateContent" +) +DEFAULT_MODEL = "gemini-2.5-pro" +DEFAULT_MAX_DIFF_BYTES = 400_000 +ATTEMPTS = 3 +EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" +OUTPUT_FILE = "test_generated_release.py" + + +class GenerationError(Exception): + pass + + +def git_env(env): + """Environment for git: the API key never reaches a subprocess.""" + clean = {k: v for k, v in env.items() if k != "GEMINI_API_KEY"} + clean["GIT_CONFIG_NOSYSTEM"] = "1" + clean["GIT_TERMINAL_PROMPT"] = "0" + return clean + + +def git_diff_args(target_dir, base_ref, head_ref): + return [ + "git", "-C", target_dir, "-c", "core.hooksPath=/dev/null", + "diff", "--no-ext-diff", "--no-textconv", "--no-color", + base_ref or EMPTY_TREE, head_ref, "--", + ] + + +def run_git_diff(target_dir, base_ref, head_ref, env, run=subprocess.run): + result = run( + git_diff_args(target_dir, base_ref, head_ref), + env=git_env(env), capture_output=True, check=False, + ) + if result.returncode != 0: + raise GenerationError(f"git diff failed with exit {result.returncode}") + return result.stdout.decode("utf-8", errors="replace") + + +def split_files(diff): + """Splits a unified git diff into one chunk per file.""" + chunks, current = [], [] + for line in diff.splitlines(keepends=True): + if line.startswith("diff --git ") and current: + chunks.append("".join(current)) + current = [] + current.append(line) + if current: + chunks.append("".join(current)) + return chunks + + +def chunk_path(chunk): + m = re.match(r"diff --git a/(\S+) b/", chunk) + return m.group(1) if m else chunk.split("\n", 1)[0] + + +def truncate_diff(diff, max_bytes): + """Keeps whole files only. Returns (kept_text, omitted_paths).""" + kept, omitted, size = [], [], 0 + for chunk in split_files(diff): + n = len(chunk.encode("utf-8")) + if size + n <= max_bytes: + kept.append(chunk) + size += n + else: + omitted.append(chunk_path(chunk)) + return "".join(kept), omitted + + +def build_prompt(diff, omitted, packages, import_names, version, base_ref): + omitted_note = "" + if omitted: + omitted_note = ( + "\nThese changed files were left out of the diff for size: " + + ", ".join(omitted) + "\n" + ) + base = base_ref or "the empty tree (first release)" + return ( + "You write pytest tests for a Python library release.\n" + f"Packages being released (PyPI): {', '.join(packages)}; version " + f"{version}.\n" + f"Import names: {', '.join(import_names)}.\n" + f"Below is the git diff from {base} to the release commit." + f"{omitted_note}\n" + "Write ONE self-contained pytest file that checks behaviour changed or " + "added in this diff through the public API of the installed packages. " + "Rules: import only the packages above, pytest, numpy and the Python " + "standard library; no network access; write files only under pytest's " + "tmp_path; keep every test small and fast on CPU; the tests must pass " + "with KERAS_BACKEND set to tensorflow, jax or torch; do not download " + "presets or weights. Reply with only the Python source, no prose.\n\n" + "```diff\n" + diff + "```\n" + ) + + +def extract_code(text): + m = re.search(r"```(?:python)?\s*\n(.*?)```", text, flags=re.S) + return (m.group(1) if m else text).strip() + "\n" + + +def call_gemini(prompt, api_key, model, opener=urllib.request.urlopen, + sleep=time.sleep): + url = API_URL.format(model=urllib.parse.quote(model, safe="")) + body = json.dumps( + {"contents": [{"role": "user", "parts": [{"text": prompt}]}]} + ).encode("utf-8") + last = None + for attempt in range(ATTEMPTS): + if attempt: + sleep(2 ** attempt) + request = urllib.request.Request(url, data=body, method="POST") + request.add_header("Content-Type", "application/json") + request.add_header("x-goog-api-key", api_key) + try: + with opener(request, timeout=300) as resp: + status = resp.status + payload = resp.read() + if status == 200: + return json.loads(payload.decode("utf-8")) + last = f"HTTP {status}" + except urllib.error.HTTPError as e: + # Only the status code: never headers or the request. + last = f"HTTP {e.code}" + except (urllib.error.URLError, OSError) as e: + last = type(e).__name__ + print(f"Gemini attempt {attempt + 1}/{ATTEMPTS} failed: {last}") + raise GenerationError(f"Gemini request failed after {ATTEMPTS} attempts: {last}") + + +def response_text(data): + try: + parts = data["candidates"][0]["content"]["parts"] + except (KeyError, IndexError, TypeError): + raise GenerationError("Gemini returned no candidates") + text = "".join(p.get("text", "") for p in parts if isinstance(p, dict)) + if not text.strip(): + raise GenerationError("Gemini returned an empty answer") + return text + + +def main(env=None, opener=urllib.request.urlopen, sleep=time.sleep, + run=subprocess.run): + env = dict(os.environ if env is None else env) + try: + api_key = env.get("GEMINI_API_KEY", "") + if not api_key: + raise GenerationError("GEMINI_API_KEY is not set") + for name in ("TARGET_DIR", "HEAD_REF", "OUTPUT_DIR", "PUBLISH_VERSION", + "PACKAGES", "IMPORT_NAMES"): + if not env.get(name): + raise GenerationError(f"{name} is not set") + max_bytes = int(env.get("MAX_DIFF_BYTES") or DEFAULT_MAX_DIFF_BYTES) + base_ref = env.get("BASE_REF", "") + diff = run_git_diff(env["TARGET_DIR"], base_ref, env["HEAD_REF"], env, + run=run) + kept, omitted = truncate_diff(diff, max_bytes) + print(f"Diff: {len(kept.encode('utf-8'))} bytes kept, " + f"{len(omitted)} files omitted for size") + prompt = build_prompt( + kept, omitted, json.loads(env["PACKAGES"]), + json.loads(env["IMPORT_NAMES"]), env["PUBLISH_VERSION"], base_ref, + ) + data = call_gemini(prompt, api_key, + env.get("GEMINI_MODEL") or DEFAULT_MODEL, + opener=opener, sleep=sleep) + code = extract_code(response_text(data)) + try: + compile(code, OUTPUT_FILE, "exec") + except SyntaxError as e: + raise GenerationError(f"generated code does not compile: line {e.lineno}") + os.makedirs(env["OUTPUT_DIR"], exist_ok=True) + path = os.path.join(env["OUTPUT_DIR"], OUTPUT_FILE) + with open(path, "w", encoding="utf-8") as f: + f.write(code) + print(f"Wrote {path} ({len(code.splitlines())} lines)") + return 0 + except GenerationError as e: + print(f"::error::{e}") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/py/post_publish.py b/scripts/release/py/post_publish.py new file mode 100644 index 0000000..6e9c2eb --- /dev/null +++ b/scripts/release/py/post_publish.py @@ -0,0 +1,186 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Post-publish check (publish.yml `post-publish`). + +Stdlib only and safe under `python -I`. Polls PyPI (up to 20 minutes) until +an exact-pin install of every package works and every expected file is listed +in the PyPI JSON for the version. Then, for each file THIS run uploaded, the +PyPI sha256 must equal the build job's sha256. Files set aside as already on +PyPI are listed as "published by an earlier run" with their PyPI digest and +are not compared (builds are not byte-reproducible). Writes a table to +$GITHUB_STEP_SUMMARY; a mismatch renders ❌ and fails the job. +""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time + +# `python -I` does not put this script's directory on sys.path; add it +# explicitly so the sibling module from the same trusted checkout is found. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from split_existing import PypiError # noqa: E402 +from split_existing import fetch_release_files # noqa: E402 + +TIMEOUT_SECONDS = 20 * 60 +INTERVAL_SECONDS = 30 + + +def compare(manifest, digests, pypi_files): + """Returns (rows, ok). `pypi_files` maps filename -> sha256.""" + rows, ok = [], True + for key in manifest["uploaded"]: + name = key.rsplit("/", 1)[-1] + built = digests.get(key) + on_pypi = pypi_files.get(name) + if built is None: + rows.append(("❌", key, "missing from the build digests", "", "")) + ok = False + elif on_pypi is None: + rows.append(("❌", key, "not on PyPI", built, "")) + ok = False + elif on_pypi != built: + rows.append(("❌", key, "sha256 mismatch", built, on_pypi)) + ok = False + else: + rows.append(("✅", key, "uploaded by this run", built, on_pypi)) + for item in manifest["existing"]: + name = item["file"].rsplit("/", 1)[-1] + on_pypi = pypi_files.get(name) + if on_pypi is None: + rows.append(("❌", item["file"], "no longer on PyPI", "", "")) + ok = False + else: + rows.append( + ("✅", item["file"], "published by an earlier run", "", on_pypi) + ) + return rows, ok + + +def expected_names(manifest): + names = [k.rsplit("/", 1)[-1] for k in manifest["uploaded"]] + names += [i["file"].rsplit("/", 1)[-1] for i in manifest["existing"]] + return names + + +def pypi_files_for(packages, version, fetch): + files = {} + for package in packages: + files.update(fetch(package, version)) + return files + + +def pip_install_ok(packages, version, run=subprocess.run): + """Exact-pin install into a throwaway venv; True when pip succeeds.""" + with tempfile.TemporaryDirectory() as tmp: + venv = os.path.join(tmp, "venv") + subprocess.run([sys.executable, "-m", "venv", venv], check=True) + pins = [f"{p}=={version}" for p in packages] + result = run( + [ + os.path.join(venv, "bin", "python"), "-m", "pip", "install", + "--no-deps", "--no-cache-dir", + "--index-url", "https://pypi.org/simple", *pins, + ], + check=False, + ) + return result.returncode == 0 + + +def poll(ready, timeout, interval, clock=time.monotonic, sleep=time.sleep): + """Calls `ready()` until it returns True or `timeout` seconds pass.""" + deadline = clock() + timeout + while True: + if ready(): + return True + if clock() + interval > deadline: + return False + sleep(interval) + + +def render(rows, version): + lines = [ + f"## PyPI check for {version}", + "", + "| | File | Status | Build sha256 | PyPI sha256 |", + "| --- | --- | --- | --- | --- |", + ] + for mark, key, status, built, on_pypi in rows: + lines.append(f"| {mark} | `{key}` | {status} | `{built}` | `{on_pypi}` |") + return "\n".join(lines) + "\n" + + +def run_check( + manifest, digests, packages, version, fetch=fetch_release_files, + install=pip_install_ok, timeout=TIMEOUT_SECONDS, + interval=INTERVAL_SECONDS, clock=time.monotonic, sleep=time.sleep, +): + """Returns (rows, ok, message).""" + wanted = expected_names(manifest) + state = {} + + def ready(): + try: + files = pypi_files_for(packages, version, fetch) + except PypiError as e: + print(f"PyPI JSON not ready: {e}") + return False + state["files"] = files + missing = [n for n in wanted if n not in files] + if missing: + print(f"Waiting for PyPI to list: {', '.join(missing)}") + return False + if not install(packages, version): + print("Waiting for an exact-pin install to succeed") + return False + return True + + if not poll(ready, timeout, interval, clock, sleep): + rows, _ = compare(manifest, digests, state.get("files", {})) + return rows, False, "PyPI did not serve every file within the timeout" + rows, ok = compare(manifest, digests, state["files"]) + return rows, ok, "" if ok else "PyPI digests do not match the build" + + +def main(argv=None, **kwargs): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", required=True, help="JSON from upload") + parser.add_argument("--digests", required=True, help="JSON from build") + parser.add_argument("--packages", required=True, help="JSON list") + parser.add_argument("--version", required=True) + args = parser.parse_args(argv) + manifest = json.loads(args.manifest) + rows, ok, message = run_check( + manifest, json.loads(args.digests), json.loads(args.packages), + args.version, **kwargs, + ) + table = render(rows, args.version) + print(table) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as f: + f.write(table) + if not ok: + print(f"::error::{message}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/py/split_existing.py b/scripts/release/py/split_existing.py new file mode 100644 index 0000000..4c6f835 --- /dev/null +++ b/scripts/release/py/split_existing.py @@ -0,0 +1,142 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Move files already on PyPI out of the upload dirs (publish.yml `upload`). + +Stdlib only and safe under `python -I`. Runs before the PyPA upload steps. +For each `packages[i]` it reads the PyPI JSON for the version; files of +`dist_dirs[i]` already listed there are moved to `--existing-dir` and +recorded as "existing" with PyPI's sha256; the rest are "uploaded". Only the +"uploaded" files are digest-checked by post_publish.py: builds are not +byte-reproducible, so a resume's rebuilt copy of an earlier upload never +matches. `skip-existing` stays on in the PyPA step as a race backstop. + +Writes to $GITHUB_OUTPUT: + manifest {"uploaded": [key, ...], "existing": [{"file", "pypi_sha256"}]} + upload_dir_0 artifact path of dist_dirs[0] if it still has files, else "" + upload_dir_1 same for dist_dirs[1] +""" + +import argparse +import json +import os +import shutil +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +PYPI_JSON = "https://pypi.org/pypi/{package}/{version}/json" +MAX_DIST_DIRS = 2 # publish.yml has one PyPA upload step per dist dir. +RETRIES = 3 + + +class PypiError(Exception): + pass + + +def fetch_release_files( + package, version, opener=urllib.request.urlopen, sleep=time.sleep +): + """Returns {filename: sha256} for `package==version` on PyPI. + + 404 means the version is not on PyPI yet and returns {}. Other failures + are retried with backoff and then raise PypiError: never guess. + """ + url = PYPI_JSON.format( + package=urllib.parse.quote(package, safe=""), + version=urllib.parse.quote(version, safe=""), + ) + last = None + for attempt in range(RETRIES): + if attempt: + sleep(2 ** attempt) + try: + with opener(url, timeout=30) as resp: + data = json.loads(resp.read().decode("utf-8")) + return { + f["filename"]: f["digests"]["sha256"] + for f in data.get("urls", []) + } + except urllib.error.HTTPError as e: + if e.code == 404: + return {} + last = f"HTTP {e.code}" + except (urllib.error.URLError, OSError, ValueError, KeyError) as e: + last = type(e).__name__ + raise PypiError(f"PyPI JSON for {package} {version} failed: {last}") + + +def split(root, existing_dir, dist_dirs, packages, version, fetch): + if len(dist_dirs) > MAX_DIST_DIRS: + raise PypiError( + f"{len(dist_dirs)} dist_dirs configured; publish.yml uploads at " + f"most {MAX_DIST_DIRS}" + ) + if len(dist_dirs) != len(packages): + raise PypiError("dist_dirs[i] must hold packages[i]") + uploaded, existing, upload_dirs = [], [], [] + for dist_dir, package in zip(dist_dirs, packages): + rel = dist_dir.strip("/") + on_pypi = fetch(package, version) + src_dir = os.path.join(root, rel) + remaining = 0 + for name in sorted(os.listdir(src_dir)): + key = f"{rel}/{name}" + if name in on_pypi: + dest = os.path.join(existing_dir, rel) + os.makedirs(dest, exist_ok=True) + shutil.move(os.path.join(src_dir, name), os.path.join(dest, name)) + existing.append({"file": key, "pypi_sha256": on_pypi[name]}) + else: + uploaded.append(key) + remaining += 1 + upload_dirs.append(src_dir + "/" if remaining else "") + while len(upload_dirs) < MAX_DIST_DIRS: + upload_dirs.append("") + return {"uploaded": uploaded, "existing": existing}, upload_dirs + + +def main(argv=None, fetch=fetch_release_files): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", required=True) + parser.add_argument("--existing-dir", required=True) + parser.add_argument("--dist-dirs", required=True, help="JSON list") + parser.add_argument("--packages", required=True, help="JSON list") + parser.add_argument("--version", required=True) + args = parser.parse_args(argv) + try: + manifest, upload_dirs = split( + args.root, args.existing_dir, json.loads(args.dist_dirs), + json.loads(args.packages), args.version, fetch, + ) + except PypiError as e: + print(f"::error::{e}") + return 1 + for item in manifest["existing"]: + print(f"Already on PyPI (published by an earlier run): {item['file']}") + for key in manifest["uploaded"]: + print(f"To upload: {key}") + output = os.environ.get("GITHUB_OUTPUT") + if output: + with open(output, "a", encoding="utf-8") as f: + f.write("manifest=" + json.dumps(manifest, sort_keys=True) + "\n") + for i, d in enumerate(upload_dirs): + f.write(f"upload_dir_{i}={d}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/py/test_generate_test.py b/scripts/release/py/test_generate_test.py new file mode 100644 index 0000000..0334f99 --- /dev/null +++ b/scripts/release/py/test_generate_test.py @@ -0,0 +1,215 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import email.message +import io +import json +import subprocess +import urllib.error + +import generate_test +from fakes import FakeResponse + +KEY = "AIzaSECRET-test-key-0123456789" +HEADER_MARKER = "x-leaky-header-value-42" + + +def file_chunk(path, body_lines): + lines = [ + f"diff --git a/{path} b/{path}\n", + f"--- a/{path}\n", + f"+++ b/{path}\n", + "@@ -1 +1 @@\n", + ] + [f"+{line}\n" for line in body_lines] + return "".join(lines) + + +A = file_chunk("keras_hub/a.py", ["x = 1"] * 5) +B = file_chunk("keras_hub/b.py", ["y = 2"] * 50) +C = file_chunk("keras_hub/c.py", ["z = 3"] * 5) +DIFF = A + B + C + + +def ok_payload(text="```python\ndef test_ok():\n assert True\n```"): + return json.dumps( + {"candidates": [{"content": {"parts": [{"text": text}]}}]} + ).encode() + + +def http_error(code): + hdrs = email.message.Message() + hdrs["X-Debug"] = HEADER_MARKER + return urllib.error.HTTPError( + "https://generativelanguage.googleapis.com/x", code, "err", hdrs, + io.BytesIO(b'{"error": "' + HEADER_MARKER.encode() + b'"}'), + ) + + +class Recorder: + def __init__(self, responses): + self.responses = list(responses) + self.requests = [] + + def __call__(self, request, timeout): + self.requests.append(request) + item = self.responses.pop(0) + if isinstance(item, Exception): + raise item + return item + + +def fake_git(diff=DIFF, calls=None): + def run(args, env, capture_output, check): + if calls is not None: + calls.append((args, env)) + return subprocess.CompletedProcess(args, 0, stdout=diff.encode(), stderr=b"") + + return run + + +def base_env(tmp_path, **extra): + env = { + "GEMINI_API_KEY": KEY, + "TARGET_DIR": "target", + "BASE_REF": "v0.32.0", + "HEAD_REF": "a" * 40, + "OUTPUT_DIR": str(tmp_path / "generated"), + "PUBLISH_VERSION": "0.33.0", + "PACKAGES": '["keras-hub", "keras-nlp"]', + "IMPORT_NAMES": '["keras_hub", "keras_nlp"]', + } + env.update(extra) + return env + + +def test_truncation_keeps_whole_files_only(): + limit = len((A + C).encode()) + kept, omitted = generate_test.truncate_diff(DIFF, limit) + assert kept == A + C + assert omitted == ["keras_hub/b.py"] + # Never cuts inside a file: every kept chunk is intact. + assert generate_test.split_files(kept) == [A, C] + + +def test_truncation_just_below_a_boundary_drops_the_file(): + kept, omitted = generate_test.truncate_diff(A + B, len((A + B).encode()) - 1) + assert kept == A + assert omitted == ["keras_hub/b.py"] + + +def test_no_truncation_when_under_limit(): + assert generate_test.truncate_diff(DIFF, 10**9) == (DIFF, []) + + +def test_success_writes_file_key_only_in_header(tmp_path, capsys): + opener = Recorder([FakeResponse(ok_payload())]) + git_calls = [] + rc = generate_test.main( + env=base_env(tmp_path), opener=opener, sleep=lambda s: None, + run=fake_git(calls=git_calls), + ) + assert rc == 0 + out = (tmp_path / "generated" / "test_generated_release.py").read_text() + assert "def test_ok" in out + (request,) = opener.requests + assert request.get_header("X-goog-api-key") == KEY + assert KEY not in request.full_url + body = request.data.decode() + assert KEY not in body + assert "keras_hub/a.py" in json.loads(body)["contents"][0]["parts"][0]["text"] + captured = capsys.readouterr() + assert KEY not in captured.out + captured.err + # The key never reaches the git subprocess environment. + (args, env), = git_calls + assert "GEMINI_API_KEY" not in env + assert KEY not in json.dumps(env) + assert "--no-ext-diff" in args and "--no-textconv" in args + assert "core.hooksPath=/dev/null" in args + + +def test_prompt_never_contains_key(tmp_path): + prompt = generate_test.build_prompt( + DIFF, ["x.py"], ["keras-hub"], ["keras_hub"], "0.33.0", "v0.32.0" + ) + assert KEY not in prompt + assert "x.py" in prompt + + +def test_non_200_exits_1_without_printing_headers(tmp_path, capsys): + opener = Recorder([http_error(403)] * 3) + rc = generate_test.main( + env=base_env(tmp_path), opener=opener, sleep=lambda s: None, + run=fake_git(), + ) + assert rc == 1 + captured = capsys.readouterr() + text = captured.out + captured.err + assert "HTTP 403" in text + assert HEADER_MARKER not in text + assert KEY not in text + assert not (tmp_path / "generated").exists() + + +def test_non_200_status_without_exception_also_fails(tmp_path, capsys): + opener = Recorder([FakeResponse(b"{}", status=204)] * 3) + rc = generate_test.main( + env=base_env(tmp_path), opener=opener, sleep=lambda s: None, + run=fake_git(), + ) + assert rc == 1 + assert "HTTP 204" in capsys.readouterr().out + + +def test_three_attempts_with_backoff(tmp_path): + sleeps = [] + opener = Recorder([http_error(503)] * 3) + rc = generate_test.main( + env=base_env(tmp_path), opener=opener, sleep=sleeps.append, + run=fake_git(), + ) + assert rc == 1 + assert len(opener.requests) == 3 + assert sleeps == [2, 4] + + +def test_retry_recovers(tmp_path): + opener = Recorder([http_error(503), FakeResponse(ok_payload())]) + rc = generate_test.main( + env=base_env(tmp_path), opener=opener, sleep=lambda s: None, + run=fake_git(), + ) + assert rc == 0 + assert len(opener.requests) == 2 + + +def test_missing_key_exits_1(tmp_path): + env = base_env(tmp_path) + del env["GEMINI_API_KEY"] + opener = Recorder([]) + assert generate_test.main(env=env, opener=opener, sleep=lambda s: None, run=fake_git()) == 1 + assert opener.requests == [] + + +def test_uncompilable_answer_exits_1(tmp_path): + opener = Recorder([FakeResponse(ok_payload("def broken(:\n"))]) + rc = generate_test.main( + env=base_env(tmp_path), opener=opener, sleep=lambda s: None, + run=fake_git(), + ) + assert rc == 1 + + +def test_blank_base_ref_diffs_from_empty_tree(): + args = generate_test.git_diff_args("target", "", "abc") + assert generate_test.EMPTY_TREE in args diff --git a/scripts/release/py/test_post_publish.py b/scripts/release/py/test_post_publish.py new file mode 100644 index 0000000..2a8aa31 --- /dev/null +++ b/scripts/release/py/test_post_publish.py @@ -0,0 +1,155 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import json + +import post_publish +import split_existing + +HUB = "dist/keras_hub-0.33.1-py3-none-any.whl" +NLP = "keras_nlp/dist/keras_nlp-0.33.1-py3-none-any.whl" +HUB_NAME = HUB.rsplit("/", 1)[1] +NLP_NAME = NLP.rsplit("/", 1)[1] + +# Resume after a partial upload: keras-hub was uploaded by an earlier run +# (its rebuilt copy differs), keras-nlp is uploaded now. +RESUME_MANIFEST = { + "uploaded": [NLP], + "existing": [{"file": HUB, "pypi_sha256": "hub-from-earlier-run"}], +} +DIGESTS = {HUB: "hub-rebuilt-differs", NLP: "nlp-built"} + + +class FakeClock: + def __init__(self): + self.now = 0.0 + + def __call__(self): + return self.now + + def sleep(self, s): + self.now += s + + +def fetch_from(files_by_package): + return lambda p, v: dict(files_by_package.get(p, {})) + + +def run(manifest, digests, pypi, install=lambda p, v: True, timeout=60): + clock = FakeClock() + return post_publish.run_check( + manifest, digests, ["keras-hub", "keras-nlp"], "0.33.1", + fetch=fetch_from(pypi), install=install, timeout=timeout, interval=10, + clock=clock, sleep=clock.sleep, + ) + + +def test_resume_digest_scope_only_compares_this_runs_uploads(): + pypi = { + "keras-hub": {HUB_NAME: "hub-from-earlier-run"}, + "keras-nlp": {NLP_NAME: "nlp-built"}, + } + rows, ok, _ = run(RESUME_MANIFEST, DIGESTS, pypi) + assert ok + assert ("✅", NLP, "uploaded by this run", "nlp-built", "nlp-built") in rows + assert ("✅", HUB, "published by an earlier run", "", "hub-from-earlier-run") in rows + + +def test_mismatch_on_uploaded_file_fails_and_renders_cross(): + pypi = { + "keras-hub": {HUB_NAME: "hub-from-earlier-run"}, + "keras-nlp": {NLP_NAME: "tampered"}, + } + rows, ok, message = run(RESUME_MANIFEST, DIGESTS, pypi) + assert not ok + assert "do not match" in message + assert ("❌", NLP, "sha256 mismatch", "nlp-built", "tampered") in rows + assert "| ❌ |" in post_publish.render(rows, "0.33.1") + + +def test_times_out_when_a_file_never_appears(): + pypi = {"keras-hub": {HUB_NAME: "hub-from-earlier-run"}, "keras-nlp": {}} + rows, ok, message = run(RESUME_MANIFEST, DIGESTS, pypi) + assert not ok + assert "timeout" in message + assert ("❌", NLP, "not on PyPI", "nlp-built", "") in rows + + +def test_waits_for_install_before_comparing(): + pypi = { + "keras-hub": {HUB_NAME: "hub-from-earlier-run"}, + "keras-nlp": {NLP_NAME: "nlp-built"}, + } + attempts = [] + + def install(packages, version): + attempts.append(list(packages)) + return len(attempts) >= 3 + + _, ok, _ = run(RESUME_MANIFEST, DIGESTS, pypi, install=install) + assert ok + assert attempts == [["keras-hub", "keras-nlp"]] * 3 + + +def test_install_never_succeeds_fails(): + pypi = { + "keras-hub": {HUB_NAME: "hub-from-earlier-run"}, + "keras-nlp": {NLP_NAME: "nlp-built"}, + } + _, ok, message = run(RESUME_MANIFEST, DIGESTS, pypi, install=lambda p, v: False) + assert not ok and "timeout" in message + + +def test_pypi_errors_are_retried_by_the_poll(): + calls = [] + + def fetch(p, v): + calls.append(p) + if len(calls) <= 2: + raise split_existing.PypiError("HTTP 503") + return {HUB_NAME: "h", NLP_NAME: "n"} + + clock = FakeClock() + _, ok, _ = post_publish.run_check( + {"uploaded": [HUB, NLP], "existing": []}, {HUB: "h", NLP: "n"}, + ["keras-hub", "keras-nlp"], "0.33.1", fetch=fetch, + install=lambda p, v: True, timeout=60, interval=10, clock=clock, + sleep=clock.sleep, + ) + assert ok + + +def test_poll_stops_at_deadline(): + clock = FakeClock() + calls = [] + assert not post_publish.poll(lambda: calls.append(1) or False, 1200, 30, clock, clock.sleep) + assert clock.now <= 1200 + assert len(calls) == 41 + + +def test_main_writes_summary_and_exit_code(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + clock = FakeClock() + rc = post_publish.main( + [ + "--manifest", json.dumps({"uploaded": [HUB], "existing": []}), + "--digests", json.dumps({HUB: "h"}), + "--packages", '["keras-hub"]', "--version", "0.33.1", + ], + fetch=lambda p, v: {HUB_NAME: "other"}, install=lambda p, v: True, + timeout=10, interval=5, clock=clock, sleep=clock.sleep, + ) + assert rc == 1 + assert "| ❌ |" in summary.read_text() diff --git a/scripts/release/py/test_split_existing.py b/scripts/release/py/test_split_existing.py new file mode 100644 index 0000000..958614c --- /dev/null +++ b/scripts/release/py/test_split_existing.py @@ -0,0 +1,172 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import json +import os +import urllib.error + +import pytest + +import split_existing +from fakes import FakeResponse + +HUB_WHL = "keras_hub-0.33.1-py3-none-any.whl" +NLP_WHL = "keras_nlp-0.33.1-py3-none-any.whl" + + +def pypi_payload(files): + return json.dumps( + {"urls": [{"filename": f, "digests": {"sha256": d}} for f, d in files.items()]} + ).encode() + + +def http_error(code): + return urllib.error.HTTPError("https://pypi.org/x", code, "err", {}, None) + + +def make_opener(responses, calls): + """`responses` is a list consumed per call: bytes or an exception.""" + + def opener(url, timeout): + calls.append(url) + item = responses.pop(0) + if isinstance(item, Exception): + raise item + return FakeResponse(item) + + return opener + + +def test_fetch_parses_files(): + calls = [] + opener = make_opener([pypi_payload({HUB_WHL: "aa"})], calls) + files = split_existing.fetch_release_files("keras-hub", "0.33.1", opener, sleep=lambda s: None) + assert files == {HUB_WHL: "aa"} + assert calls == ["https://pypi.org/pypi/keras-hub/0.33.1/json"] + + +def test_fetch_404_means_not_on_pypi(): + opener = make_opener([http_error(404)], []) + assert split_existing.fetch_release_files("keras-hub", "0.33.1", opener, sleep=lambda s: None) == {} + + +def test_fetch_retries_then_succeeds(): + sleeps = [] + opener = make_opener([http_error(503), http_error(502), pypi_payload({})], []) + assert split_existing.fetch_release_files("p", "1.0.0", opener, sleep=sleeps.append) == {} + assert sleeps == [2, 4] + + +def test_fetch_fails_closed_after_retries(): + opener = make_opener([http_error(500)] * 3, []) + with pytest.raises(split_existing.PypiError, match="HTTP 500"): + split_existing.fetch_release_files("p", "1.0.0", opener, sleep=lambda s: None) + + +def hub_tree(root): + for rel, name in (("dist", HUB_WHL), ("keras_nlp/dist", NLP_WHL)): + os.makedirs(os.path.join(root, rel), exist_ok=True) + with open(os.path.join(root, rel, name), "wb") as f: + f.write(name.encode()) + + +def test_partial_resume_moves_only_files_already_on_pypi(tmp_path): + # keras-hub wheel already on PyPI, keras-nlp missing. + root, existing = tmp_path / "a", tmp_path / "existing" + hub_tree(root) + pypi = {"keras-hub": {HUB_WHL: "pypi-hub"}, "keras-nlp": {}} + manifest, dirs = split_existing.split( + str(root), str(existing), ["dist/", "keras_nlp/dist/"], + ["keras-hub", "keras-nlp"], "0.33.1", lambda p, v: pypi[p], + ) + assert manifest == { + "uploaded": [f"keras_nlp/dist/{NLP_WHL}"], + "existing": [{"file": f"dist/{HUB_WHL}", "pypi_sha256": "pypi-hub"}], + } + assert dirs == ["", f"{root}/keras_nlp/dist/"] + assert not (root / "dist" / HUB_WHL).exists() + assert (existing / "dist" / HUB_WHL).exists() + assert (root / "keras_nlp" / "dist" / NLP_WHL).exists() + + +def test_nothing_on_pypi_keeps_order_keras_hub_first(tmp_path): + hub_tree(tmp_path) + manifest, dirs = split_existing.split( + str(tmp_path), str(tmp_path / "e"), ["dist/", "keras_nlp/dist/"], + ["keras-hub", "keras-nlp"], "0.33.1", lambda p, v: {}, + ) + assert manifest["existing"] == [] + assert dirs == [f"{tmp_path}/dist/", f"{tmp_path}/keras_nlp/dist/"] + + +def test_single_dist_dir_pads_second_upload_dir(tmp_path): + os.makedirs(tmp_path / "dist") + (tmp_path / "dist" / "keras-3.0.0.tar.gz").write_bytes(b"x") + _, dirs = split_existing.split( + str(tmp_path), str(tmp_path / "e"), ["dist/"], ["keras"], "3.0.0", + lambda p, v: {}, + ) + assert dirs == [f"{tmp_path}/dist/", ""] + + +def test_more_than_two_dist_dirs_refused(tmp_path): + with pytest.raises(split_existing.PypiError, match="at most 2"): + split_existing.split(str(tmp_path), "e", ["a/", "b/", "c/"], ["a", "b", "c"], "1", None) + + +def test_dist_dirs_and_packages_must_pair(tmp_path): + hub_tree(tmp_path) + with pytest.raises(split_existing.PypiError, match=r"dist_dirs\[i\] must hold packages\[i\]"): + split_existing.split( + str(tmp_path), str(tmp_path / "e"), ["dist/", "keras_nlp/dist/"], + ["keras-hub"], "0.33.1", lambda p, v: {}, + ) + + +def test_main_writes_outputs(tmp_path, monkeypatch): + hub_tree(tmp_path / "r") + out = tmp_path / "out" + monkeypatch.setenv("GITHUB_OUTPUT", str(out)) + rc = split_existing.main( + [ + "--root", str(tmp_path / "r"), "--existing-dir", str(tmp_path / "e"), + "--dist-dirs", '["dist/", "keras_nlp/dist/"]', + "--packages", '["keras-hub", "keras-nlp"]', "--version", "0.33.1", + ], + fetch=lambda p, v: {HUB_WHL: "h"} if p == "keras-hub" else {}, + ) + assert rc == 0 + lines = out.read_text().splitlines() + manifest = json.loads(lines[0].split("=", 1)[1]) + assert manifest["uploaded"] == [f"keras_nlp/dist/{NLP_WHL}"] + assert lines[1] == "upload_dir_0=" + assert lines[2] == f"upload_dir_1={tmp_path / 'r'}/keras_nlp/dist/" + + +def test_main_fails_when_pypi_unreachable(tmp_path, capsys): + hub_tree(tmp_path) + + def fetch(p, v): + raise split_existing.PypiError("PyPI JSON for keras-hub 0.33.1 failed: HTTP 503") + + rc = split_existing.main( + [ + "--root", str(tmp_path), "--existing-dir", str(tmp_path / "e"), + "--dist-dirs", '["dist/", "keras_nlp/dist/"]', + "--packages", '["keras-hub", "keras-nlp"]', "--version", "0.33.1", + ], + fetch=fetch, + ) + assert rc == 1 + assert "::error::PyPI JSON" in capsys.readouterr().out diff --git a/scripts/release/py/test_validate_dist.py b/scripts/release/py/test_validate_dist.py new file mode 100644 index 0000000..50e52a7 --- /dev/null +++ b/scripts/release/py/test_validate_dist.py @@ -0,0 +1,227 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import hashlib +import io +import json +import os +import tarfile +import zipfile + +import pytest + +import validate_dist + +HUB_DIRS = ["dist/", "keras_nlp/dist/"] +HUB_PACKAGES = ["keras-hub", "keras-nlp"] + + +def metadata(name, version): + return f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n\n".encode() + + +def write_wheel(path, name, version, meta_name=None, meta_version=None): + os.makedirs(os.path.dirname(path), exist_ok=True) + dist = name.replace("-", "_") + with zipfile.ZipFile(path, "w") as zf: + zf.writestr( + f"{dist}-{version}.dist-info/METADATA", + metadata(meta_name or name, meta_version or version), + ) + zf.writestr(f"{dist}/__init__.py", "") + + +def write_sdist(path, top, name, version): + os.makedirs(os.path.dirname(path), exist_ok=True) + with tarfile.open(path, "w:gz") as tf: + data = metadata(name, version) + info = tarfile.TarInfo(f"{top}/PKG-INFO") + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + egg = metadata(name, version) + info = tarfile.TarInfo(f"{top}/{name}.egg-info/PKG-INFO") + info.size = len(egg) + tf.addfile(info, io.BytesIO(egg)) + + +def hub_tree(root, version="0.33.0"): + write_wheel( + os.path.join(root, "dist", f"keras_hub-{version}-py3-none-any.whl"), + "keras-hub", version, + ) + # PEP 625 normalized sdist name. + write_sdist( + os.path.join(root, "dist", f"keras_hub-{version}.tar.gz"), + f"keras_hub-{version}", "keras-hub", version, + ) + write_wheel( + os.path.join( + root, "keras_nlp", "dist", f"keras_nlp-{version}-py3-none-any.whl" + ), + "keras-nlp", version, + ) + + +def check(root, packages=HUB_PACKAGES, dirs=HUB_DIRS, tag="v0.33.0"): + return validate_dist.check_metadata(str(root), dirs, packages, tag, "v") + + +def test_valid_keras_hub_tree_with_pep625_sdist_passes(tmp_path): + hub_tree(tmp_path) + assert check(tmp_path) == [] + + +def test_legacy_hyphenated_sdist_name_accepted(tmp_path): + hub_tree(tmp_path) + os.remove(tmp_path / "dist" / "keras_hub-0.33.0.tar.gz") + write_sdist( + str(tmp_path / "dist" / "keras-hub-0.33.0.tar.gz"), + "keras-hub-0.33.0", "keras-hub", "0.33.0", + ) + assert check(tmp_path) == [] + + +def test_keras_rs_filename_with_keras_metadata_rejected(tmp_path): + write_wheel( + str(tmp_path / "dist" / "keras_rs-0.3.0-py3-none-any.whl"), + "keras-rs", "0.3.0", meta_name="keras", + ) + errors = check(tmp_path, packages=["keras-rs"], dirs=["dist/"], tag="v0.3.0") + assert any("metadata Name 'keras'" in e for e in errors), errors + + +def test_stray_file_rejected(tmp_path): + hub_tree(tmp_path) + (tmp_path / "dist" / "notes.txt").write_text("hi") + errors = check(tmp_path) + assert errors == ["dist/notes.txt: only .whl and .tar.gz files may be uploaded"] + + +def test_zip_sdist_rejected(tmp_path): + hub_tree(tmp_path) + (tmp_path / "dist" / "keras_hub-0.33.0.zip").write_bytes(b"PK") + assert any(".zip" in e for e in check(tmp_path)) + + +def test_version_must_equal_tag(tmp_path): + hub_tree(tmp_path, version="0.33.1") + errors = check(tmp_path, tag="v0.33.0") + assert errors and all("!= tag version '0.33.0'" in e for e in errors), errors + + +def test_filename_version_must_match_metadata(tmp_path): + hub_tree(tmp_path) + write_wheel( + str(tmp_path / "dist" / "keras_hub-0.33.0-py3-none-any.whl"), + "keras-hub", "0.33.0", meta_version="0.33.0.dev1", + ) + errors = check(tmp_path) + assert any("filename version '0.33.0' != metadata" in e for e in errors) + + +def test_package_in_wrong_dist_dir_rejected(tmp_path): + hub_tree(tmp_path) + write_wheel( + str(tmp_path / "dist" / "keras_nlp-0.33.0-py3-none-any.whl"), + "keras-nlp", "0.33.0", + ) + errors = check(tmp_path) + assert any("expected 'keras-hub'" in e for e in errors), errors + + +def test_empty_or_missing_dist_dir_rejected(tmp_path): + hub_tree(tmp_path) + for f in os.listdir(tmp_path / "keras_nlp" / "dist"): + os.remove(tmp_path / "keras_nlp" / "dist" / f) + assert check(tmp_path) == ["keras_nlp/dist/: no files to upload"] + os.rmdir(tmp_path / "keras_nlp" / "dist") + assert "missing" in check(tmp_path)[0] + + +def test_dist_dirs_and_packages_must_pair(tmp_path): + hub_tree(tmp_path) + assert "dist_dirs[i] must hold packages[i]" in check( + tmp_path, packages=["keras-hub"] + )[0] + + +def test_tag_without_prefix_rejected(tmp_path): + hub_tree(tmp_path) + assert "prefix" in check(tmp_path, tag="0.33.0")[0] + + +def test_wheel_without_metadata_rejected(tmp_path): + path = tmp_path / "dist" / "keras_hub-0.33.0-py3-none-any.whl" + os.makedirs(path.parent) + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("keras_hub/__init__.py", "") + errors = check(tmp_path, packages=["keras-hub"], dirs=["dist/"]) + assert any("METADATA" in e for e in errors) + + +def digests_of(root): + out = {} + for d in HUB_DIRS: + rel = d.strip("/") + for name in os.listdir(os.path.join(root, rel)): + with open(os.path.join(root, rel, name), "rb") as f: + out[f"{rel}/{name}"] = hashlib.sha256(f.read()).hexdigest() + return out + + +def test_digests_match(tmp_path): + hub_tree(tmp_path) + assert validate_dist.check_digests(str(tmp_path), HUB_DIRS, digests_of(tmp_path)) == [] + + +def test_digest_mismatch_extra_and_missing_files_rejected(tmp_path): + hub_tree(tmp_path) + expected = digests_of(tmp_path) + key = "dist/keras_hub-0.33.0-py3-none-any.whl" + expected[key] = "0" * 64 + expected["dist/gone.whl"] = "1" * 64 + (tmp_path / "dist" / "extra.whl").write_bytes(b"x") + errors = validate_dist.check_digests(str(tmp_path), HUB_DIRS, expected) + assert any(e.startswith(f"{key}: sha256") for e in errors) + assert "dist/extra.whl: not produced by the build job" in errors + assert "dist/gone.whl: recorded by the build job but missing now" in errors + + +def test_main_exit_codes(tmp_path, capsys): + hub_tree(tmp_path) + args = [ + "metadata", "--root", str(tmp_path), "--dist-dirs", json.dumps(HUB_DIRS), + "--packages", json.dumps(HUB_PACKAGES), "--tag", "v0.33.0", + "--tag-prefix", "v", + ] + assert validate_dist.main(args) == 0 + (tmp_path / "dist" / "stray.json").write_text("{}") + assert validate_dist.main(args) == 1 + assert "::error::dist/stray.json" in capsys.readouterr().out + + +@pytest.mark.parametrize( + "name,expected", + [ + ("keras_hub-0.33.0-py3-none-any.whl", ("keras_hub", "0.33.0")), + ("keras_hub-0.33.0-1-py3-none-any.whl", ("keras_hub", "0.33.0")), + ], +) +def test_parse_wheel_filename(name, expected): + assert validate_dist.parse_wheel_filename(name) == expected + + +def test_parse_wheel_filename_rejects_garbage(): + with pytest.raises(ValueError): + validate_dist.parse_wheel_filename("keras_hub.whl") diff --git a/scripts/release/py/validate_dist.py b/scripts/release/py/validate_dist.py new file mode 100644 index 0000000..1c99b83 --- /dev/null +++ b/scripts/release/py/validate_dist.py @@ -0,0 +1,260 @@ +# Copyright 2026 The Keras Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Validate the dist artifact before the PyPI upload (publish.yml `upload`). + +Stdlib only and safe under `python -I`. Reads wheel METADATA and sdist +PKG-INFO with zipfile / tarfile; never executes or extracts target code. + +Two modes: + + digests Every file in `dist_dirs` must appear in the build job's + sha256 map with the same digest, and every mapped file must exist. + metadata Every file must be a `.whl` or `.tar.gz`; `dist_dirs[i]` holds + only `packages[i]` (so the upload order follows `packages`, i.e. + keras-hub first); filename name/version and METADATA/PKG-INFO + Name/Version must agree with each other, with the configured + package, and Version must equal the tag's version. +""" + +import argparse +import email.parser +import hashlib +import json +import os +import re +import sys +import tarfile +import zipfile + +MAX_METADATA_BYTES = 1024 * 1024 + + +def normalize(name): + """PEP 503 normalized project name.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def rel_key(dist_dir, name): + """Key used by the build job's digest map, e.g. `keras_nlp/dist/x.whl`.""" + return f"{dist_dir.strip('/')}/{name}" + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def list_dir(root, dist_dir): + path = os.path.join(root, dist_dir.strip("/")) + if not os.path.isdir(path): + return None + return sorted(os.listdir(path)) + + +def check_digests(root, dist_dirs, expected): + errors = [] + seen = set() + for dist_dir in dist_dirs: + names = list_dir(root, dist_dir) + if names is None: + errors.append(f"{dist_dir}: directory missing from the artifact") + continue + for name in names: + key = rel_key(dist_dir, name) + seen.add(key) + path = os.path.join(root, key) + if key not in expected: + errors.append(f"{key}: not produced by the build job") + continue + if not os.path.isfile(path): + errors.append(f"{key}: not a regular file") + continue + actual = sha256_file(path) + if actual != expected[key]: + errors.append( + f"{key}: sha256 {actual} != build job's {expected[key]}" + ) + for key in sorted(set(expected) - seen): + errors.append(f"{key}: recorded by the build job but missing now") + return errors + + +def parse_wheel_filename(filename): + """`{name}-{ver}(-{build})?-{py}-{abi}-{plat}.whl` -> (name, version).""" + stem = filename[: -len(".whl")] + parts = stem.split("-") + if len(parts) not in (5, 6): + raise ValueError(f"{filename}: not a valid wheel filename") + return parts[0], parts[1] + + +def parse_sdist_filename(filename): + """`{name}-{version}.tar.gz` -> (name, version). + + Accepts the PEP 625 normalized form (`keras_hub-0.33.0.tar.gz`) and the + legacy hyphenated one; names are compared after normalization. + """ + stem = filename[: -len(".tar.gz")] + if "-" not in stem: + raise ValueError(f"{filename}: not a valid sdist filename") + name, version = stem.rsplit("-", 1) + if not name or not version: + raise ValueError(f"{filename}: not a valid sdist filename") + return name, version + + +def _parse_headers(raw): + msg = email.parser.BytesHeaderParser().parsebytes(raw) + return msg.get("Name"), msg.get("Version") + + +def read_wheel_metadata(path): + with zipfile.ZipFile(path) as zf: + members = [ + n for n in zf.namelist() + if re.fullmatch(r"[^/]+\.dist-info/METADATA", n) + ] + if len(members) != 1: + raise ValueError( + f"expected exactly one *.dist-info/METADATA, found " + f"{len(members)}" + ) + info = zf.getinfo(members[0]) + if info.file_size > MAX_METADATA_BYTES: + raise ValueError("METADATA is too large") + return _parse_headers(zf.read(info)) + + +def read_sdist_metadata(path): + with tarfile.open(path, mode="r:gz") as tf: + members = [ + m for m in tf.getmembers() + if re.fullmatch(r"[^/]+/PKG-INFO", m.name) + ] + if len(members) != 1: + raise ValueError( + f"expected exactly one top-level PKG-INFO, found " + f"{len(members)}" + ) + member = members[0] + if not member.isfile(): + raise ValueError("PKG-INFO is not a regular file") + if member.size > MAX_METADATA_BYTES: + raise ValueError("PKG-INFO is too large") + f = tf.extractfile(member) + return _parse_headers(f.read()) + + +def check_file(path, filename, package, version): + """Returns a list of error strings for one dist file.""" + if filename.endswith(".whl"): + parse_name, read_meta = parse_wheel_filename, read_wheel_metadata + elif filename.endswith(".tar.gz"): + parse_name, read_meta = parse_sdist_filename, read_sdist_metadata + else: + return [f"{filename}: only .whl and .tar.gz files may be uploaded"] + if not os.path.isfile(path) or os.path.islink(path): + return [f"{filename}: not a regular file"] + try: + file_name, file_version = parse_name(filename) + meta_name, meta_version = read_meta(path) + except (ValueError, zipfile.BadZipFile, tarfile.TarError, OSError) as e: + return [f"{filename}: {e}"] + errors = [] + if not meta_name or not meta_version: + return [f"{filename}: metadata lacks Name or Version"] + if normalize(file_name) != normalize(package): + errors.append( + f"{filename}: filename names {file_name!r}, expected {package!r}" + ) + if normalize(meta_name) != normalize(package): + errors.append( + f"{filename}: metadata Name {meta_name!r}, expected {package!r}" + ) + if file_version != meta_version: + errors.append( + f"{filename}: filename version {file_version!r} != metadata " + f"Version {meta_version!r}" + ) + if meta_version != version: + errors.append( + f"{filename}: metadata Version {meta_version!r} != tag version " + f"{version!r}" + ) + return errors + + +def check_metadata(root, dist_dirs, packages, tag, tag_prefix): + if not tag.startswith(tag_prefix) or len(tag) == len(tag_prefix): + return [f"tag {tag!r} does not start with prefix {tag_prefix!r}"] + version = tag[len(tag_prefix):] + if len(dist_dirs) != len(packages): + return [ + f"config has {len(dist_dirs)} dist_dirs but {len(packages)} " + "packages; dist_dirs[i] must hold packages[i]" + ] + errors = [] + for dist_dir, package in zip(dist_dirs, packages): + names = list_dir(root, dist_dir) + if names is None: + errors.append(f"{dist_dir}: directory missing from the artifact") + continue + if not names: + errors.append(f"{dist_dir}: no files to upload") + continue + for name in names: + path = os.path.join(root, dist_dir.strip("/"), name) + errors.extend( + f"{dist_dir.strip('/')}/{e}" + for e in check_file(path, name, package, version) + ) + return errors + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["digests", "metadata"]) + parser.add_argument("--root", required=True) + parser.add_argument("--dist-dirs", required=True, help="JSON list") + parser.add_argument("--digests", help="JSON map from the build job") + parser.add_argument("--packages", help="JSON list") + parser.add_argument("--tag") + parser.add_argument("--tag-prefix") + args = parser.parse_args(argv) + dist_dirs = json.loads(args.dist_dirs) + if args.mode == "digests": + if not args.digests: + parser.error("--digests is required in digests mode") + errors = check_digests(args.root, dist_dirs, json.loads(args.digests)) + else: + if not args.packages or not args.tag or args.tag_prefix is None: + parser.error("--packages, --tag and --tag-prefix are required") + errors = check_metadata( + args.root, dist_dirs, json.loads(args.packages), args.tag, + args.tag_prefix, + ) + for e in errors: + print(f"::error::{e}") + if errors: + return 1 + print(f"{args.mode}: OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/static.test.js b/scripts/release/static.test.js new file mode 100644 index 0000000..d19ed7e --- /dev/null +++ b/scripts/release/static.test.js @@ -0,0 +1,486 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Static no-merge / no-direct-push test (plan section 7). + * + * Scans every non-test `.js` file under scripts/release/, plus the run: and + * script: bodies of release.yml / publish.yml / test.yml, and fails on: + * - any merge API: `pulls.merge`, `repos.merge`, `POST /merges`, + * `PUT .../pulls/{n}/merge` (also concatenated), GraphQL + * `mergePullRequest` / `enablePullRequestAutoMerge` / `mergeBranch` / + * `enqueuePullRequest`, `mergeUpstream`, a bare `merge` identifier, + * `gh pr merge`; + * - any git push whose refspec is not an inline literal under + * `release-tool/` (or `refs/heads/release-tool/`), `--tags` / `--all` / + * `--mirror` / `--delete` pushes, and shell-string `git push`; + * - creating a ref other than an inline `refs/heads/...` literal, and + * updating or deleting a ref outside `release-tool/` (branch creation via + * `createRef` stays allowed: it is how the bot cuts B at `commit_sha`). + * + * The scanner tokenizes JS (strings, templates, comments, regex literals), so + * comments that mention a merge API do not trip it and a merge call split + * across tokens still does. + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); + +const REGEX_PRECEDERS = new Set("(,=:[!&|?{};+-*%<>~^".split("")); +const REGEX_KEYWORDS = new Set([ + "return", "typeof", "case", "in", "of", "delete", "void", "throw", "new", + "else", "do", "yield", "await", +]); + +function readQuoted(src, i) { + const q = src[i]; + let j = i + 1; + let value = ""; + while (j < src.length && src[j] !== q) { + if (src[j] === "\\") { + value += src[j + 1]; + j += 2; + continue; + } + if (src[j] === "\n") throw new Error(`unterminated string at ${i}`); + value += src[j]; + j += 1; + } + if (j >= src.length) throw new Error(`unterminated string at ${i}`); + return { value, end: j + 1 }; +} + +// Skips a JS expression inside `${ ... }` starting at i; returns index after `}`. +function skipTemplateExpr(src, i) { + let depth = 1; + let j = i; + while (j < src.length) { + const c = src[j]; + if (c === "'" || c === '"') { + j = readQuoted(src, j).end; + continue; + } + if (c === "`") { + j = readTemplate(src, j).end; + continue; + } + if (c === "{") depth += 1; + if (c === "}") { + depth -= 1; + if (depth === 0) return j + 1; + } + j += 1; + } + throw new Error(`unterminated template expression at ${i}`); +} + +function readTemplate(src, i) { + let j = i + 1; + while (j < src.length) { + const c = src[j]; + if (c === "\\") { + j += 2; + continue; + } + if (c === "`") return { value: src.slice(i + 1, j), end: j + 1 }; + if (c === "$" && src[j + 1] === "{") { + j = skipTemplateExpr(src, j + 2); + continue; + } + j += 1; + } + throw new Error(`unterminated template at ${i}`); +} + +function readRegex(src, i) { + let j = i + 1; + let inClass = false; + while (j < src.length) { + const c = src[j]; + if (c === "\\") { + j += 2; + continue; + } + if (c === "\n") throw new Error(`unterminated regex at ${i}`); + if (c === "[") inClass = true; + else if (c === "]") inClass = false; + else if (c === "/" && !inClass) { + j += 1; + while (/[a-z]/i.test(src[j] || "")) j += 1; + return j; + } + j += 1; + } + throw new Error(`unterminated regex at ${i}`); +} + +/** + * Tokens: { type: "str"|"tpl"|"ident"|"num"|"punct"|"regex", value, + * parent (index of the enclosing opener token or -1), + * close (for openers: index of the matching closer) }. + */ +function tokenize(src) { + const tokens = []; + const stack = []; + let i = 0; + const push = (tok) => { + tok.parent = stack.length ? stack[stack.length - 1] : -1; + tokens.push(tok); + return tokens.length - 1; + }; + const last = () => tokens[tokens.length - 1]; + while (i < src.length) { + const c = src[i]; + if (/\s/.test(c)) { + i += 1; + continue; + } + if (c === "/" && src[i + 1] === "/") { + const nl = src.indexOf("\n", i); + i = nl === -1 ? src.length : nl + 1; + continue; + } + if (c === "/" && src[i + 1] === "*") { + const end = src.indexOf("*/", i + 2); + if (end === -1) throw new Error("unterminated comment"); + i = end + 2; + continue; + } + if (c === "'" || c === '"') { + const { value, end } = readQuoted(src, i); + push({ type: "str", value }); + i = end; + continue; + } + if (c === "`") { + const { value, end } = readTemplate(src, i); + push({ type: "tpl", value }); + i = end; + continue; + } + if (c === "/") { + const prev = last(); + const regexOk = !prev + || (prev.type === "punct" && REGEX_PRECEDERS.has(prev.value)) + || (prev.type === "ident" && REGEX_KEYWORDS.has(prev.value)); + if (regexOk) { + const end = readRegex(src, i); + push({ type: "regex", value: src.slice(i, end) }); + i = end; + continue; + } + } + const ident = /^[A-Za-z_$][A-Za-z0-9_$]*/.exec(src.slice(i, i + 200)); + if (ident) { + push({ type: "ident", value: ident[0] }); + i += ident[0].length; + continue; + } + const num = /^[0-9][0-9a-fA-FxX._n]*/.exec(src.slice(i, i + 50)); + if (num) { + push({ type: "num", value: num[0] }); + i += num[0].length; + continue; + } + if (src.startsWith("...", i)) { + push({ type: "punct", value: "..." }); + i += 3; + continue; + } + if ("([{".includes(c)) { + stack.push(push({ type: "punct", value: c })); + i += 1; + continue; + } + if (")]}".includes(c)) { + const open = stack.pop(); + if (open === undefined) throw new Error(`unbalanced ${c} at ${i}`); + const idx = push({ type: "punct", value: c }); + tokens[open].close = idx; + i += 1; + continue; + } + push({ type: "punct", value: c }); + i += 1; + } + if (stack.length) throw new Error("unbalanced brackets at end of file"); + return tokens; +} + +const MERGE_PATTERNS = [ + [/\bpulls\s*\.\s*merge\b/, "pulls.merge"], + [/\brepos\s*\.\s*merge\b/, "repos.merge"], + [/\b(pulls|repos)\s*\[\s*"merge"\s*\]/, "merge via bracket access"], + [/\/merges\b/, "POST /merges"], + [/\/pulls\/\S*\/merge\b/, "PUT /pulls/{n}/merge"], + [/mergePullRequest/, "GraphQL mergePullRequest"], + [/enablePullRequestAutoMerge/, "GraphQL enablePullRequestAutoMerge"], + [/\bgh\s+pr\s+merge\b/, "gh pr merge"], + [/"pr"\s*,\s*"merge"/, "gh pr merge (argument array)"], + [/\bmergeUpstream\b|merge-upstream/, "merge upstream (sync fork)"], + [/\bmergeBranch\b/, "GraphQL mergeBranch"], + [/\benqueuePullRequest\b/, "GraphQL merge-queue enqueue"], +]; + +const FORBIDDEN_PUSH_FLAGS = /^(--tags|--all|--mirror|--delete|-d|--prune|--follow-tags)(=|$)/; +const RELEASE_TOOL_DEST = /^(refs\/heads\/)?release-tool\//; + +function isLiteral(tok) { + return tok.type === "str" || tok.type === "tpl"; +} + +function elementsOf(tokens, openIdx, startIdx) { + const elements = []; + let cur = []; + for (let k = startIdx; k < tokens[openIdx].close; k += 1) { + const tok = tokens[k]; + if (tok.type === "punct" && tok.value === "," && tok.parent === openIdx) { + if (cur.length) elements.push(cur); + cur = []; + continue; + } + cur.push(tok); + } + if (cur.length) elements.push(cur); + return elements; +} + +function checkPushArray(tokens, pushIdx, violations) { + const openIdx = tokens[pushIdx].parent; + const opener = openIdx >= 0 ? tokens[openIdx] : null; + if (!opener || opener.value !== "[") { + violations.push('string "push" outside a git argument array (cannot verify the refspec)'); + return; + } + const elements = elementsOf(tokens, openIdx, pushIdx + 1); + const positional = []; + for (const el of elements) { + if (el.length === 1 && isLiteral(el[0]) && el[0].value.startsWith("-")) { + if (FORBIDDEN_PUSH_FLAGS.test(el[0].value)) { + violations.push(`git push with forbidden flag ${el[0].value}`); + } + continue; + } + positional.push(el); + } + // Tokens before "push" (e.g. ["-c", "x=y", "push", ...]) are git options + // and are not inspected. + if (positional.length < 2) { + violations.push("git push without an explicit release-tool/ refspec"); + return; + } + for (const el of positional.slice(1)) { + if (el.length !== 1 || !isLiteral(el[0])) { + violations.push("git push refspec is not an inline string literal (cannot verify it targets release-tool/)"); + continue; + } + const spec = el[0].value; + const dest = spec.slice(spec.lastIndexOf(":") + 1).replace(/^\+/, ""); + if (!RELEASE_TOOL_DEST.test(dest)) { + violations.push(`git push to ${JSON.stringify(spec)} (only release-tool/* may be pushed)`); + } + } +} + +function callArgs(tokens, calleeIdx) { + const open = tokens[calleeIdx + 1]; + if (!open || open.value !== "(") return null; + return tokens.slice(calleeIdx + 2, open.close); +} + +function checkRefCalls(tokens, violations) { + tokens.forEach((tok, idx) => { + if (tok.type !== "ident" && !isLiteral(tok)) return; + let kind = null; + let args = null; + if (tok.type === "ident" && ["createRef", "updateRef", "deleteRef"].includes(tok.value)) { + kind = tok.value === "createRef" ? "create" : "change"; + args = callArgs(tokens, idx); + } else if (isLiteral(tok) && /\/git\/refs\b/.test(tok.value)) { + const method = tok.value.trim().split(/\s+/)[0].toUpperCase(); + if (method === "POST") kind = "create"; + else if (method === "PATCH" || method === "DELETE") kind = "change"; + const parent = tok.parent >= 0 ? tokens[tok.parent] : null; + if (parent && parent.value === "(") args = tokens.slice(tok.parent + 1, parent.close); + } + if (!kind || !args) return; + const literals = args.filter(isLiteral).map((t) => t.value); + if (kind === "create" && literals.some((v) => /(^|\/)tags\//.test(v) && !/\/git\/refs\b/.test(v))) { + violations.push("tag ref created through the git refs API (tags come only from POST /releases)"); + } + if (kind === "create") { + const at = args.findIndex((t, k) => t.type === "ident" && t.value === "ref" + && args[k + 1] && args[k + 1].value === ":"); + const value = at >= 0 ? args[at + 2] : null; + if (!value || !isLiteral(value) || !value.value.startsWith("refs/heads/")) { + violations.push("ref created without an inline `ref: \"refs/heads/...\"` literal"); + } + } + if (kind === "change" && !literals.some((v) => v.includes("release-tool/"))) { + violations.push("ref updated or deleted outside release-tool/"); + } + }); +} + +function scanSource(src) { + const violations = []; + const tokens = tokenize(src); + const flat = tokens + .filter((t) => t.type !== "regex") + .map((t) => (isLiteral(t) ? JSON.stringify(t.value) : t.value)) + .join(" "); + for (const [re, label] of MERGE_PATTERNS) { + if (re.test(flat)) violations.push(`merge API: ${label}`); + } + tokens.forEach((tok, idx) => { + if (tok.type === "ident" && tok.value === "merge") { + violations.push("identifier `merge` (e.g. destructured from pulls/repos)"); + } + if (!isLiteral(tok)) return; + if (/\/merges?\/?$/.test(tok.value)) violations.push("literal ending in /merge or /merges"); + if (/\bgit\s+push\b/.test(tok.value)) violations.push("shell-string git push"); + if (tok.value === "push") checkPushArray(tokens, idx, violations); + }); + checkRefCalls(tokens, violations); + return violations; +} + +function releaseSources(dir) { + const out = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) out.push(...releaseSources(full)); + else if (entry.name.endsWith(".js") && !entry.name.endsWith(".test.js") + && entry.name !== "test_support.js") out.push(full); + } + return out.sort(); +} + +test("scanner flags every forbidden merge and push form", () => { + const bad = { + "pulls.merge": "await github.rest.pulls.merge({ owner, repo, pull_number: 1 });", + "pulls.merge split": "await github.rest.pulls\n .merge({});", + "repos.merge": "await github.rest.repos.merge({ owner, repo, base, head });", + "bracket": 'await github.rest.pulls["merge"]({});', + "POST /merges": 'await github.request("POST /repos/{owner}/{repo}/merges", {});', + "PUT pull merge": 'await github.request("PUT /repos/{owner}/{repo}/pulls/{pull_number}/merge");', + "graphql merge": 'await github.graphql("mutation { mergePullRequest(input: {}) { clientMutationId } }");', + "graphql auto-merge": "await github.graphql(`mutation { enablePullRequestAutoMerge(input: {}) { x } }`);", + "gh shell": 'execSync("gh pr merge 12 --squash");', + "gh args": 'execFileSync("gh", ["pr", "merge", "12"]);', + "push to release branch": 'git(["push", "origin", "HEAD:refs/heads/r0.33"]);', + "push short branch": 'git(["push", "origin", "r0.33"]);', + "push tags": 'git(["push", "--tags", "origin", "release-tool/x"]);', + "push variable refspec": "git([\"push\", remote, refspec]);", + "push spread": 'git(["push", remote, ...specs]);', + "push no refspec": 'git(["push", "origin"]);', + "push outside array": 'const verb = "push"; git([verb, "origin", "release-tool/x"]);', + "shell push": 'execSync("git push origin main");', + "tag createRef": "await github.rest.git.createRef({ owner, repo, ref: `refs/tags/v${v}`, sha });", + "tag via request": 'await github.request("POST /repos/{owner}/{repo}/git/refs", { ref: "refs/tags/v1.0.0", sha });', + "updateRef release branch": 'await github.rest.git.updateRef({ owner, repo, ref: "heads/r0.33", sha, force: true });', + "deleteRef": "await github.rest.git.deleteRef({ owner, repo, ref: `heads/${b}` });", + "mergeUpstream": "await github.rest.repos.mergeUpstream({ owner, repo, branch });", + "merge-upstream URL": 'await github.request("POST /repos/{owner}/{repo}/merge-upstream", {});', + "graphql mergeBranch": 'await github.graphql("mutation { mergeBranch(input: {}) { x } }");', + "graphql merge queue": 'await github.graphql("mutation { enqueuePullRequest(input: {}) { x } }");', + "concatenated merge URL": 'await github.request("PUT /repos/o/r/pulls/" + n + "/merge");', + "destructured merge": "const { merge } = github.rest.pulls; await merge({});", + "createRef variable ref": "await github.rest.git.createRef({ owner, repo, ref: someVar, sha });", + }; + for (const [name, src] of Object.entries(bad)) { + assert.ok(scanSource(src).length > 0, `expected a violation for: ${name}`); + } +}); + +test("scanner allows the bot's legitimate operations", () => { + const good = [ + "git([\"push\", \"--force\", remote, `${sha}:refs/heads/release-tool/bump-${v}`]);", + 'git(["-c", "core.hooksPath=/dev/null", "push", "origin", "+HEAD:release-tool/master-bump-1.0.0"]);', + "await github.rest.git.createRef({ owner, repo, ref: `refs/heads/${branch}`, sha });", + "await github.rest.git.updateRef({ owner, repo, ref: `heads/release-tool/bump-${v}`, sha, force: true });", + 'const args = []; args.push("x"); picks.push({ sha });', + "// The bot never calls pulls.merge or gh pr merge.\nconst x = 1;", + "/* git push origin main */ const y = 2;", + 'const re = /[\\]]"push"/; git(["push", "origin", "HEAD:refs/heads/release-tool/x"]);', + 'await github.rest.pulls.create({ title: "Merge fixes into r0.33", head, base });', + 'await github.rest.pulls.list({ state: "closed" }); const merged = pr.merged;', + "await github.rest.repos.createRelease({ tag_name: `v${v}`, target_commitish: sha, draft: false });", + ]; + for (const src of good) { + assert.deepEqual(scanSource(src), [], `unexpected violation for: ${src}`); + } +}); + +test("release scripts never merge and push only to release-tool/*", () => { + const dir = __dirname; + const files = releaseSources(dir); + const rel = files.map((f) => path.relative(dir, f)); + // Guard against a vacuous pass: the modules that talk to GitHub and git + // must be among the scanned files. + for (const required of ["lib/gh.js", "lib/git.js", "act.js"]) { + assert.ok(rel.includes(required), `${required} was not scanned; found ${rel.join(", ")}`); + } + const report = []; + for (const file of files) { + for (const v of scanSource(fs.readFileSync(file, "utf8"))) { + report.push(`${path.relative(dir, file)}: ${v}`); + } + } + assert.deepEqual(report, []); +}); + +test("only test files load test_support.js, so excluding it from the scan hides nothing", () => { + const offenders = releaseSources(__dirname) + .filter((f) => /require\(\s*["'][./]*test_support(\.js)?["']\s*\)/.test(fs.readFileSync(f, "utf8"))) + .map((f) => path.relative(__dirname, f)); + assert.deepEqual(offenders, []); +}); + +test("workflow run: and script: bodies never merge or push", () => { + const yaml = require("./yaml_lite.js"); + const root = path.join(__dirname, "..", ".."); + const report = []; + let scanned = 0; + for (const file of ["release.yml", "publish.yml", "test.yml"]) { + const wf = yaml.parseFile(path.join(root, ".github", "workflows", file)); + for (const [name, job] of Object.entries(wf.jobs)) { + (job.steps || []).forEach((step, i) => { + const where = `${file}/${name}/steps[${i}]`; + if (typeof step.run === "string") { + scanned += 1; + for (const [re, label] of MERGE_PATTERNS) { + if (re.test(step.run)) report.push(`${where}: merge API: ${label}`); + } + if (/\bgit\s+push\b/.test(step.run)) report.push(`${where}: git push in run:`); + } + const script = (step.with || {}).script; + if (typeof script === "string") { + scanned += 1; + for (const v of scanSource(script)) report.push(`${where}: ${v}`); + } + }); + } + } + assert.ok(scanned >= 10, `only ${scanned} run:/script: bodies were scanned`); + assert.deepEqual(report, []); +}); + +module.exports = { scanSource, tokenize }; diff --git a/scripts/release/test_support.js b/scripts/release/test_support.js new file mode 100644 index 0000000..c60bea4 --- /dev/null +++ b/scripts/release/test_support.js @@ -0,0 +1,282 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Test doubles shared by the release *.test.js files (not itself a test): + * a fixture-backed fake octokit that records every call, a fake git with a + * call spy, a fake `core`, a temp config root, and a keras-hub-like fixture. + */ + +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +const BOT = "keras-release[bot]"; +const SLUG = "keras-release"; +const VERSION_FILE = "keras_hub/src/version.py"; +const SHA = { + master: "1".repeat(40), b0: "2".repeat(40), ms: "3".repeat(40), + m1: "4".repeat(40), p0: "5".repeat(40), tagobj: "6".repeat(40), + b1: "7".repeat(40), other: "8".repeat(40), m0: "9".repeat(40), +}; + +/** + * Every method name that changes remote state. The fake has no merge method + * at all, so a merge call fails the test with a TypeError. + */ +const WRITE_METHODS = new Set([ + "git.createRef", "git.updateRef", "git.deleteRef", "git.createTag", + "pulls.create", "pulls.update", + "repos.createRelease", "repos.updateRelease", "repos.deleteRelease", + "repos.createOrUpdateFileContents", "actions.createWorkflowDispatch", + "issues.addLabels", "issues.createComment", +]); + +const versionText = (v) => `__version__ = "${v}"\n`; + +function httpError(status) { + return Object.assign(new Error(`HTTP ${status}`), { status }); +} + +function need(value, status = 404) { + if (value === undefined || value === null) throw httpError(status); + return value; +} + +function handlers(f, fail) { + return { + repos: { + getCollaboratorPermissionLevel: ({ username }) => + ({ permission: need(f.permissions[username]) }), + getContent: ({ path: p, ref }) => ({ + type: "file", encoding: "base64", + content: Buffer.from(need(f.files[`${ref}:${p}`])).toString("base64"), + }), + getBranch: ({ branch }) => ({ commit: { sha: need(f.branches[branch]) } }), + getCommit: ({ ref }) => { + const sha = f.branches[ref] || ref; + const c = need(f.commits[sha]); + return { sha, parents: c.parents.map((s) => ({ sha: s })), + commit: { message: c.message || "" } }; + }, + compareCommitsWithBasehead: ({ basehead }) => { + const c = need(f.compares[basehead]); + const commits = (c.commits || []).map((x) => + ({ sha: x.sha, commit: { message: x.message } })); + return { status: c.status, commits, total_commits: commits.length }; + }, + listCommits: () => f.history.map((h) => + ({ sha: h.sha, parents: h.parents.map((s) => ({ sha: s })) })), + listPullRequestsAssociatedWithCommit: ({ commit_sha }) => + (f.commitPulls[commit_sha] || []).map((n) => ({ number: n })), + listReleases: () => f.releases, + listBranches: () => Object.keys(f.branches).map((name) => ({ name })), + createRelease: (p) => { + if (fail.createRelease) throw httpError(500); + return { id: 4242, html_url: "https://example.test/release", ...p }; + }, + updateRelease: (p) => ({ html_url: "https://example.test/release", ...p }), + generateReleaseNotes: (p) => ({ name: p.tag_name, body: "Generated notes" }), + }, + git: { + listMatchingRefs: ({ ref }) => f.refs.filter((r) => r.ref.startsWith(`refs/${ref}`)), + getRef: ({ ref }) => need(f.refs.find((r) => r.ref === `refs/${ref}`)), + getTag: ({ tag_sha }) => need(f.tagObjects[tag_sha]), + createRef: (p) => ({ ref: p.ref }), + }, + pulls: { + list: ({ base, state }) => f.pulls.filter((p) => + p.base.ref === base && (state === "all" || p.state === state)), + get: ({ pull_number }) => need(f.pulls.find((p) => p.number === pull_number)), + create: () => ({ number: 9000, html_url: "https://example.test/pull/9000" }), + }, + issues: { + listEvents: ({ issue_number }) => f.events[issue_number] || [], + }, + actions: { + getWorkflow: ({ workflow_id }) => { + const w = f.workflows[workflow_id]; + if (!w || w.status !== 200) throw httpError(w ? w.status : 404); + return { state: w.state }; + }, + listWorkflowRuns: ({ workflow_id, status }) => ({ + workflow_runs: (f.runs[workflow_id] || []).filter((r) => r.status === status), + }), + createWorkflowDispatch: () => ({}), + }, + }; +} + +/** Fake octokit. `calls` records `{ name, params }` for every REST call. */ +function fakeGithub(fixture, fail = {}) { + const f = { + branches: {}, files: {}, refs: [], tagObjects: {}, releases: [], + pulls: [], events: {}, permissions: {}, workflows: {}, runs: {}, + commits: {}, commitPulls: {}, compares: {}, history: [], ...fixture, + }; + const calls = []; + const rest = {}; + for (const [ns, methods] of Object.entries(handlers(f, fail))) { + rest[ns] = {}; + for (const [name, fn] of Object.entries(methods)) { + rest[ns][name] = async (params) => { + calls.push({ name: `${ns}.${name}`, params }); + return { status: 200, data: fn(params) }; + }; + } + } + const items = (data) => Array.isArray(data) ? data : data.workflow_runs; + const paginate = async (fn, params) => items((await fn(params)).data); + paginate.iterator = async function* iterator(fn, params) { + const all = items((await fn(params)).data); + for (let i = 0; i < all.length; i += 100) yield { data: all.slice(i, i + 100) }; + }; + const github = { rest, paginate }; + return { + github, calls, + called: (name) => calls.filter((c) => c.name === name), + writes: () => calls.filter((c) => WRITE_METHODS.has(c.name)), + }; +} + +/** Fake git; `respond(args)` may return stdout or throw. */ +function fakeGit(respond = () => "") { + const calls = []; + const fn = (args, opts) => { + calls.push({ args, opts }); + const out = respond(args, opts); + return out === undefined ? "" : out; + }; + return { fn, calls, pushes: () => calls.filter((c) => c.args[0] === "push") }; +} + +function fakeCore() { + const core = { outputs: {}, failed: null, logs: [], summaryText: "" }; + core.setOutput = (k, v) => { core.outputs[k] = v; }; + core.setFailed = (m) => { core.failed = m; }; + core.info = (m) => core.logs.push(m); + let buffer = ""; + core.summary = { + addRaw(text) { buffer += text; return this; }, + async write() { core.summaryText = buffer; }, + }; + return core; +} + +const CONFIG = { + enabled: true, + github_repo: "keras-team/keras-hub", + default_branch: "master", + release_branch_pattern: "^r(\\d+)\\.(\\d+)$", + tag_prefix: "v", + version_files: [VERSION_FILE], + packages: ["keras-hub", "keras-nlp"], + import_names: ["keras_hub", "keras_nlp"], + build_setup_command: "pip install -r requirements.txt", + build_command: "python pip_build.py", + dist_dirs: ["dist/", "keras_nlp/dist/"], + dev_suffix_on_master: ".dev0", + smoke_snippet: "import keras_hub, keras_nlp", + backends: ["tensorflow", "jax", "torch"], + legacy_publish_workflow: "publish-to-pypi.yml", +}; + +/** Temp root holding release/repos/keras-hub.json (with overrides). */ +function configRoot(overrides = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "release-cfg-")); + fs.mkdirSync(path.join(root, "release", "repos"), { recursive: true }); + fs.writeFileSync(path.join(root, "release", "repos", "keras-hub.json"), + JSON.stringify({ ...CONFIG, ...overrides }, null, 2)); + return root; +} + +const context = (over = {}) => ({ + repo: { owner: "keras-team", repo: "shared-workflows" }, + ref: "refs/heads/main", runId: 1, serverUrl: "https://github.com", ...over, +}); + +function panelEnv(over = {}) { + return { + RELEASE_REPOSITORY: "keras-hub", RELEASE_CHANNEL: "stable", + RELEASE_KIND: "patch", RELEASE_BRANCH: "r0.33", RELEASE_COMMIT_SHA: "", + RELEASE_VERSION: "", RELEASE_CHERRY_PICKS: "", RELEASE_BUMP_MASTER: "false", + RELEASE_MODE: "auto", RELEASE_OVERRIDE_GENERATED_TESTS: "false", + RELEASE_DRY_RUN: "false", RELEASE_APP_SLUG: SLUG, + RELEASE_TRIGGERING_ACTOR: "alice", RELEASE_TARGET_OWNER: "keras-team", + RELEASE_TARGET_NAME: "keras-hub", GITHUB_RUN_ATTEMPT: "1", + RELEASE_APP_TOKEN: "test-token", RUNNER_TEMP: os.tmpdir(), + ...over, + }; +} + +const releaseObj = (id, tag, sha, extra = {}) => ({ + id, tag_name: tag, draft: false, prerelease: false, target_commitish: sha, + author: { login: BOT }, ...extra, +}); + +/** A bot bump PR into r0.33. */ +function bumpPr(number, v, extra = {}) { + return { + number, state: "closed", merged_at: null, merge_commit_sha: null, + merged_by: null, head: { ref: `release-tool/bump-${v}`, + repo: { full_name: "keras-team/keras-hub" } }, + base: { ref: "r0.33" }, user: { login: BOT }, labels: [], body: "", + ...extra, + }; +} + +/** r0.33 at b0 (0.33.0, tagged and released); master at 0.34.0.dev0. */ +function baseFixture() { + return { + branches: { "master": SHA.master, "r0.33": SHA.b0 }, + files: { + [`${SHA.master}:${VERSION_FILE}`]: versionText("0.34.0.dev0"), + [`${SHA.b0}:${VERSION_FILE}`]: versionText("0.33.0"), + }, + refs: [{ ref: "refs/tags/v0.33.0", object: { type: "commit", sha: SHA.b0 } }], + releases: [releaseObj(1, "v0.33.0", SHA.b0)], + pulls: [{ + number: 101, state: "closed", merged: true, merged_at: "2026-09-01T00:00:00Z", + merge_commit_sha: SHA.m1, commits: 1, title: "Fix *thing* ", + head: { ref: "fix", repo: { full_name: "bob/keras-hub" } }, + base: { ref: "master" }, user: { login: "bob" }, labels: [], body: "", + }], + commits: { + [SHA.m1]: { parents: [SHA.p0], message: "Fix *thing* (#101)" }, + [SHA.master]: { parents: [SHA.m1], message: "Tip" }, + }, + history: [ + { sha: SHA.master, parents: [SHA.m1] }, + { sha: SHA.m1, parents: [SHA.p0] }, + { sha: SHA.p0, parents: [] }, + ], + compares: { + "r0.33...master": { status: "diverged", + commits: [{ sha: SHA.m1, message: "Fix *thing* (#101)" }] }, + "master...r0.33": { status: "diverged", commits: [] }, + }, + permissions: { alice: "write" }, + workflows: { "publish-to-pypi.yml": { status: 200, state: "disabled_manually" } }, + }; +} + +module.exports = { + BOT, CONFIG, SHA, SLUG, VERSION_FILE, WRITE_METHODS, + baseFixture, bumpPr, configRoot, context, fakeCore, fakeGit, fakeGithub, + httpError, panelEnv, releaseObj, versionText, +}; diff --git a/scripts/release/validate_publish.js b/scripts/release/validate_publish.js new file mode 100644 index 0000000..8d51fc8 --- /dev/null +++ b/scripts/release/validate_publish.js @@ -0,0 +1,136 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * publish.yml `validate` job (env `release-read`; read-only App token). + * + * - Resolves the actor: the human triggering actor, or `requested_by` only + * when the trigger is the release App's own bot; other bots are refused. + * - The actor must have write on the target repo. + * - The tag is dereferenced to its commit (annotated tags included); that + * sha must be reachable from a release branch, a published non-draft + * release must exist for the tag (at that sha when the release records a + * sha), and every version file at the sha must equal the tag's version. + * Outputs `sha` and `version` for the build job. + */ + +const { RefusalError } = require("./lib/errors"); +const { loadConfig } = require("./lib/config"); +const pep440 = require("./lib/pep440"); +const { branchXY } = require("./lib/version"); +const { hasWrite, resolvePublishActor } = require("./lib/guards"); +const { SHA40 } = require("./lib/detect"); +const { Gh } = require("./lib/gh"); +const S = require("./lib/summary"); +const plan = require("./plan"); + +/** Release branches, the tag's own X.Y line first. */ +function linesFirst(branches, pattern, v) { + const same = (b) => { + const xy = branchXY(b, pattern); + return xy.x === v.x && xy.y === v.y; + }; + return [...branches.filter(same), ...branches.filter((b) => !same(b))]; +} + +async function validate({ gh, config, actor, tag }) { + if (!hasWrite(await gh.permission(actor))) { + throw new RefusalError(`${S.escapeMd(actor)} does not have write on ` + + `${gh.fullName}.`, "ask someone with write on the target repo to publish."); + } + const prefix = config.tag_prefix; + const version = tag.startsWith(prefix) ? tag.slice(prefix.length) : ""; + let parsed; + try { + parsed = pep440.parse(version); + } catch (err) { + throw new RefusalError(`Tag "${S.escapeMd(tag)}" is not ${prefix}X.Y.Z or ` + + `${prefix}X.Y.Z.devN.`, "pass an existing release tag."); + } + const sha = await gh.resolveTag(tag); + if (!sha) { + throw new RefusalError(`Tag ${S.escapeMd(tag)} does not exist on ${gh.fullName}.`, + "publish the GitHub release first (run the Release panel)."); + } + const release = (await gh.listReleases()) + .find((r) => r.tag_name === tag && !r.draft); + if (!release) { + throw new RefusalError(`No published (non-draft) release for ${S.escapeMd(tag)}.`, + "publish the release through the Release panel, then dispatch again."); + } + if (SHA40.test(release.target_commitish || "") && release.target_commitish !== sha) { + throw new RefusalError(`Tag ${S.escapeMd(tag)} points at ${sha}, but its ` + + `release was made at ${release.target_commitish}: the tag moved.`, + "restore the tag to the release's commit (rollback team), then dispatch again."); + } + let branch = null; + const branches = await gh.releaseBranches(config.release_branch_pattern); + for (const b of linesFirst(branches, config.release_branch_pattern, parsed)) { + if (await gh.isAncestor(sha, b)) { + branch = b; + break; + } + } + if (!branch) { + throw new RefusalError(`${sha} (tag ${S.escapeMd(tag)}) is not reachable ` + + "from any release branch.", "only commits on a release branch can be published."); + } + const atSha = await gh.versionAt(sha, config.version_files); + if (atSha !== version) { + throw new RefusalError(`Version files at ${sha} read ${atSha}, not ${version}.`, + "the tag is on the wrong commit; fix it before publishing."); + } + return { sha, version, branch }; +} + +async function main({ github, context, core, env = process.env, + rootDir = plan.ROOT_DIR }) { + const report = new S.Report("Publish validate"); + report.setHeader(null, "validate"); + try { + if (context.ref !== "refs/heads/main") { + throw new RefusalError(`publish.yml ran from ${S.escapeMd(context.ref)}; ` + + "only main may use the App key.", "start the release from the panel on main."); + } + const appSlug = plan.requireAppSlug(env); + const actor = resolvePublishActor({ + triggeringActor: String(env.PUBLISH_TRIGGERING_ACTOR || ""), + requestedBy: String(env.PUBLISH_REQUESTED_BY || ""), appSlug, + }); + const repository = String(env.PUBLISH_REPOSITORY || ""); + plan.assertKnownRepository(repository); + const config = loadConfig(repository, rootDir); + const [owner, repo] = plan.resolveTarget(config, context, + env.PUBLISH_TARGET_OWNER, env.PUBLISH_TARGET_NAME); + const gh = new Gh({ github, owner, repo }); + const tag = String(env.PUBLISH_TAG || ""); + const result = await validate({ gh, config, actor, tag }); + report.setHeader(result.version, "validate"); + core.setOutput("sha", result.sha); + core.setOutput("version", result.version); + report.add(`${S.escapeMd(tag)} -> \`${result.sha}\` on ` + + `${S.escapeMd(result.branch)}; published release found; version files ` + + `read ${result.version}; actor ${S.escapeMd(actor)} has write.`); + report.setNext("build, smoke and generated tests run next; upload follows."); + } catch (err) { + plan.failWith(core, report, err); + } + await S.writeSummary(core, report); +} + +module.exports = { main, validate }; diff --git a/scripts/release/validate_publish.test.js b/scripts/release/validate_publish.test.js new file mode 100644 index 0000000..462ebc9 --- /dev/null +++ b/scripts/release/validate_publish.test.js @@ -0,0 +1,124 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); +const T = require("./test_support"); +const validatePublish = require("./validate_publish"); + +const { SHA } = T; +const ROOT = T.configRoot(); + +/** v0.33.1 is an annotated tag -> ms on r0.33, with a published release. */ +function fixture() { + return { + branches: { "master": SHA.master, "r0.32": SHA.other, "r0.33": SHA.ms }, + files: { [`${SHA.ms}:${T.VERSION_FILE}`]: T.versionText("0.33.1") }, + refs: [{ ref: "refs/tags/v0.33.1", object: { type: "tag", sha: SHA.tagobj } }], + tagObjects: { [SHA.tagobj]: { object: { type: "commit", sha: SHA.ms } } }, + releases: [T.releaseObj(3, "v0.33.1", SHA.ms)], + compares: { [`${SHA.ms}...r0.33`]: { status: "identical", commits: [] } }, + permissions: { alice: "write", mallory: "none" }, + }; +} + +async function run(fx = fixture(), env = {}, ctx = {}) { + const gh = T.fakeGithub(fx); + const core = T.fakeCore(); + await validatePublish.main({ + github: gh.github, context: T.context(ctx), core, rootDir: ROOT, + env: { + RELEASE_APP_SLUG: T.SLUG, PUBLISH_TRIGGERING_ACTOR: "alice", + PUBLISH_REQUESTED_BY: "", PUBLISH_REPOSITORY: "keras-hub", + PUBLISH_TAG: "v0.33.1", PUBLISH_TARGET_OWNER: "keras-team", + PUBLISH_TARGET_NAME: "keras-hub", ...env, + }, + }); + return { gh, core }; +} + +describe("validate_publish", () => { + it("dereferences the annotated tag and outputs the commit sha", async () => { + const { gh, core } = await run(); + assert.equal(core.failed, null); + assert.equal(core.outputs.sha, SHA.ms); + assert.notEqual(core.outputs.sha, SHA.tagobj); + assert.equal(core.outputs.version, "0.33.1"); + assert.deepEqual(gh.writes(), []); + }); + + it("evaluates a human dispatcher as themselves, ignoring requested_by", async () => { + const { core } = await run(fixture(), { PUBLISH_REQUESTED_BY: "mallory" }); + assert.equal(core.failed, null); + const denied = await run(fixture(), { + PUBLISH_TRIGGERING_ACTOR: "mallory", PUBLISH_REQUESTED_BY: "alice", + }); + assert.match(denied.core.failed, /does not have write/); + }); + + it("uses requested_by when the App bot dispatched", async () => { + const { core } = await run(fixture(), { + PUBLISH_TRIGGERING_ACTOR: T.BOT, PUBLISH_REQUESTED_BY: "mallory", + }); + assert.match(core.failed, /mallory does not have write/); + }); + + it("refuses github-actions[bot] and an empty slug", async () => { + assert.ok((await run(fixture(), { PUBLISH_TRIGGERING_ACTOR: "github-actions[bot]" })).core.failed); + assert.match((await run(fixture(), { RELEASE_APP_SLUG: "" })).core.failed, /RELEASE_APP_SLUG/); + }); + + it("refuses when the App token was minted for a different repo", async () => { + const { gh, core } = await run(fixture(), { PUBLISH_TARGET_OWNER: "someone" }); + assert.match(core.failed, /scoped to/); + assert.equal(gh.calls.length, 0); + }); + + it("refuses to run from any ref but main, before any API call", async () => { + const { gh, core } = await run(fixture(), {}, { ref: "refs/heads/feature" }); + assert.match(core.failed, /only main may use the App key/); + assert.equal(gh.calls.length, 0); + }); + + const negatives = { + "sha not reachable from a release branch": (fx) => { + fx.compares[`${SHA.ms}...r0.33`] = { status: "diverged", commits: [] }; + }, + "release is a draft": (fx) => { fx.releases[0].draft = true; }, + "no release at all": (fx) => { fx.releases = []; }, + "tag moved off the release's sha": (fx) => { fx.releases[0].target_commitish = SHA.b0; }, + "version file differs from the tag": (fx) => { + fx.files[`${SHA.ms}:${T.VERSION_FILE}`] = T.versionText("0.33.0"); + }, + "tag does not exist": (fx) => { fx.refs = []; }, + }; + for (const [name, mutate] of Object.entries(negatives)) { + it(`refuses when the ${name}`, async () => { + const fx = fixture(); + mutate(fx); + const { core } = await run(fx); + assert.ok(core.failed, name); + assert.equal(core.outputs.sha, undefined); + }); + } + + it("refuses a tag that is not vX.Y.Z[.devN]", async () => { + const { core } = await run(fixture(), { PUBLISH_TAG: "0.33.1" }); + assert.match(core.failed, /is not v/); + }); +}); diff --git a/scripts/release/workflows.test.js b/scripts/release/workflows.test.js new file mode 100644 index 0000000..e72ace6 --- /dev/null +++ b/scripts/release/workflows.test.js @@ -0,0 +1,580 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * YAML-bound tests for release.yml, publish.yml and test.yml (plan section 7). + * + * Every check reads the real workflow files through yaml_lite.js; gate + * conditions are the literal `if:` strings evaluated by expr.js. There is no + * separately maintained JS mirror of any workflow logic. + */ + +const test = require("node:test"); +const assert = require("node:assert/strict"); +const path = require("node:path"); + +const yaml = require("./yaml_lite.js"); +const { evaluate, evaluateJobIf, ExprError } = require("./expr.js"); + +const WORKFLOW_DIR = path.resolve(__dirname, "..", "..", ".github", "workflows"); +const load = (name) => yaml.parseFile(path.join(WORKFLOW_DIR, name)); +const release = load("release.yml"); +const publish = load("publish.yml"); +const tests = load("test.yml"); +const RELEASE_WORKFLOWS = { "release.yml": release, "publish.yml": publish }; +const ALL_WORKFLOWS = { ...RELEASE_WORKFLOWS, "test.yml": tests }; + +const CHECKOUT = "actions/checkout"; +const APP_TOKEN = "actions/create-github-app-token"; + +// Plan section 5, verbatim. +const UPLOAD_GATE = + "always() && github.repository == 'keras-team/shared-workflows' && " + + "github.ref == 'refs/heads/main' && needs.validate.result == 'success' && " + + "needs.build.result == 'success' && needs.smoke.result == 'success' && " + + "(needs.generated-tests.result == 'success' || " + + "(inputs.override_generated_tests == true && " + + "needs.generated-tests.result == 'failure'))"; +// Plan section 4, verbatim. +const DISPATCH_IF = "inputs.dry_run == false && needs.act.outputs.published == 'true'"; +const GENERATED_TESTS_IF = "always() && needs.build.result == 'success'"; + +// Plan section 4 "Field labels", verbatim, in form order. +const RELEASE_INPUTS = [ + ["repository", "choice", "Which repo to release (must be enabled in release/repos/)."], + ["channel", "choice", "dev = pre-release X.Y.Z.devN (GitHub pre-release); stable = final release."], + ["kind", "choice", "major = new X.Y line on a release branch (patch component 0); patch = X.Y.Z fix on an existing release branch."], + ["release_branch", "string", "Release branch, e.g. r0.33. Always required; used for the same-branch lock."], + ["commit_sha", "string", "Major with a new branch only: the master commit to cut the branch from."], + ["version", "string", "Optional. Leave blank to use the computed version (the dry run shows it). Type one only to resume or re-publish a specific version."], + ["cherry_picks", "string", "Master PR numbers (recommended, e.g. #3101) or commit SHAs, any order; applied in master merge order. Allowed for patch, and for major when the release branch already exists (e.g. fixes before X.Y.0)."], + ["bump_master", "boolean", "Also open a PR bumping master to the next dev version."], + ["release_mode", "choice", "auto: after you merge the version PR and run the panel again, tags, publishes the GitHub release and uploads to PyPI in one go. draft: creates a draft release with notes for you to review (no tag yet); run the panel again to publish, which creates the tag and uploads. The bot never merges the PR in either mode."], + ["override_generated_tests", "boolean", "Upload even if the Gemini-generated tests fail (e.g. a wrong generated test or a Gemini outage). The smoke install/import test always blocks."], + ["dry_run", "boolean", "Show what would happen and change nothing. Untick to act."], +]; + +// Jobs that hold a secret or an OIDC token. No cache restore is allowed here. +const PRIVILEGED = { + "release.yml/act": "release", + "release.yml/dispatch": "release", + "publish.yml/validate": "release-read", + "publish.yml/generate": "gemini", + "publish.yml/upload": "pypi", +}; + +// Plan sections 4-5: owner, repositories and permission-* of every App token. +const TOKEN_SPECS = { + "release.yml/act": { + owner: "${{ needs.plan.outputs.target_owner }}", + repositories: "${{ needs.plan.outputs.target_name }}", + real: { contents: "write", "pull-requests": "write", workflows: "write", actions: "read", metadata: "read" }, + // Dry run: contents write for draft visibility, the rest read; `workflows` + // has no read level on GitHub, so it is omitted. + dry: { contents: "write", "pull-requests": "read", actions: "read", metadata: "read" }, + }, + "release.yml/dispatch": { + owner: "${{ github.repository_owner }}", + repositories: "shared-workflows", + real: { actions: "write" }, + dry: { actions: "write" }, + }, + "publish.yml/validate": { + owner: "${{ steps.target.outputs.owner }}", + repositories: "${{ steps.target.outputs.name }}", + real: { metadata: "read", contents: "read" }, + dry: { metadata: "read", contents: "read" }, + }, +}; + +function jobs(wf) { + return Object.entries(wf.jobs); +} + +function allJobs(workflows = ALL_WORKFLOWS) { + const out = []; + for (const [file, wf] of Object.entries(workflows)) { + for (const [name, job] of jobs(wf)) out.push({ id: `${file}/${name}`, file, name, job }); + } + return out; +} + +function allSteps(workflows = ALL_WORKFLOWS) { + const out = []; + for (const j of allJobs(workflows)) { + (j.job.steps || []).forEach((step, index) => out.push({ ...j, step, index })); + } + return out; +} + +function action(step) { + return typeof step.uses === "string" ? step.uses.split("@")[0] : null; +} + +function envName(job) { + if (job.environment === undefined || job.environment === null) return null; + return typeof job.environment === "string" ? job.environment : job.environment.name; +} + +function gateContext(needsList, { results = {}, override = false, repository, ref } = {}) { + const needs = {}; + for (const n of needsList) needs[n] = { result: results[n] || "success" }; + return { + github: { + repository: repository || "keras-team/shared-workflows", + ref: ref || "refs/heads/main", + }, + inputs: { override_generated_tests: override }, + needs, + }; +} + +// ---------------------------------------------------------------- helpers --- + +test("yaml_lite reads the workflow subset and rejects what it cannot read", () => { + const doc = yaml.parse([ + "a: 'it''s' # comment", + 'b: "x: y"', + "c: [one, 'two']", + "d: |", + " line 1", + " indented", + "e: >-", + " folded", + " text", + "f:", + "- 1", + "- k: v", + " j: w", + "g: {}", + "h: plain # trailing", + "", + ].join("\n")); + assert.deepEqual(doc, { + a: "it's", + b: "x: y", + c: ["one", "two"], + d: "line 1\n indented\n", + e: "folded text", + f: ["1", { k: "v", j: "w" }], + g: {}, + h: "plain", + }); + assert.throws(() => yaml.parse("a: 1\na: 2\n"), /duplicate key/); + assert.throws(() => yaml.parse("a: &x 1\n"), /anchors/); + assert.throws(() => yaml.parse("a: {b: 1}\n"), /flow mappings/); + assert.throws(() => yaml.parse("a: b\n c\n"), /multi-line plain/); + assert.throws(() => yaml.parse("a:\n\tb: 1\n"), /tab/); +}); + +test("expr evaluates like the Actions runner", () => { + const ctx = { inputs: { dry_run: true }, needs: { a: { result: "success" } }, github: { ref: "refs/heads/main" } }; + assert.equal(evaluate("${{ inputs.dry_run && 'read' || 'write' }}", ctx), "read"); + assert.equal(evaluate("inputs.dry_run == false", ctx), false); + assert.equal(evaluate("github.ref == 'REFS/HEADS/MAIN'", ctx), true); // case-insensitive + assert.equal(evaluate("'1' == 1", ctx), true); // loose equality + assert.equal(evaluate("'true' == true", ctx), false); // NaN != 1 + assert.equal(evaluate("!(needs.a.result != 'success')", ctx), true); + assert.equal(evaluate("needs.missing.result", ctx), null); + assert.throws(() => evaluate("input.dry_run", ctx), ExprError); + assert.throws(() => evaluate("contains(inputs.x, 'y')", ctx), ExprError); + // Implicit success(): a failed need blocks an if: without a status call. + const failed = { ...ctx, needs: { a: { result: "failure" } } }; + assert.equal(evaluateJobIf("true", failed), false); + assert.equal(evaluateJobIf("always() && true", failed), true); +}); + +// ------------------------------------------------------ triggers / inputs --- + +test("both release workflows are workflow_dispatch only with contents: read", () => { + for (const [file, wf] of Object.entries(RELEASE_WORKFLOWS)) { + assert.deepEqual(Object.keys(wf.on), ["workflow_dispatch"], file); + assert.deepEqual(wf.permissions, { contents: "read" }, file); + assert.equal(wf.concurrency, undefined, `${file} must not have workflow-level concurrency`); + } +}); + +test("release.yml has exactly the 11 plan inputs with verbatim labels", () => { + const inputs = release.on.workflow_dispatch.inputs; + assert.deepEqual(Object.keys(inputs), RELEASE_INPUTS.map(([n]) => n)); + for (const [name, type, description] of RELEASE_INPUTS) { + assert.equal(inputs[name].type, type, `${name} type`); + assert.equal(inputs[name].description, description, `${name} description`); + } + assert.deepEqual(inputs.repository.options, ["keras", "keras-hub", "keras-rs", "kinetic"]); + assert.deepEqual(inputs.channel.options, ["dev", "stable"]); + assert.deepEqual(inputs.kind.options, ["major", "patch"]); + assert.deepEqual([...inputs.release_mode.options].sort(), ["auto", "draft"]); + assert.equal(inputs.release_branch.required, "true"); + assert.equal(inputs.version.required, "false"); + assert.equal(inputs.dry_run.default, "true"); + assert.equal(inputs.bump_master.default, "false"); + assert.equal(inputs.override_generated_tests.default, "false"); + assert.equal(inputs.action, undefined, "there is no action input"); +}); + +test("publish.yml inputs and both run-names", () => { + const inputs = publish.on.workflow_dispatch.inputs; + assert.deepEqual(Object.keys(inputs), ["repository", "tag", "override_generated_tests", "requested_by"]); + assert.equal(inputs.repository.type, "choice"); + assert.equal(inputs.tag.type, "string"); + assert.equal(inputs.override_generated_tests.type, "boolean"); + assert.equal(inputs.override_generated_tests.default, "false"); + assert.equal(inputs.requested_by.type, "string"); + assert.equal(inputs.requested_by.required, "false"); + assert.equal(publish["run-name"], "publish ${{ inputs.repository }} ${{ inputs.tag }}"); + assert.equal( + release["run-name"], + "release ${{ inputs.repository }} ${{ inputs.release_branch }} ${{ inputs.kind }} ${{ inputs.version || 'auto' }}", + ); +}); + +// ------------------------------------------- environments / permissions --- + +test("environments: only act/dispatch release, validate release-read, generate gemini, upload pypi", () => { + const actual = {}; + for (const { id, job } of allJobs()) { + const env = envName(job); + if (env !== null) actual[id] = env; + } + assert.deepEqual(actual, PRIVILEGED); +}); + +test("id-token: write only on upload; every other job-level permission is read", () => { + for (const { id, job } of allJobs()) { + for (const [scope, level] of Object.entries(job.permissions || {})) { + if (scope === "id-token") { + assert.equal(id, "publish.yml/upload", `${id} must not request id-token`); + assert.equal(level, "write"); + } else { + assert.equal(level, "read", `${id} ${scope}`); + } + } + } + assert.deepEqual(publish.jobs.upload.permissions, { "id-token": "write", contents: "read" }); + assert.deepEqual(release.jobs.plan.permissions, { actions: "read", contents: "read" }); + for (const wf of Object.values(ALL_WORKFLOWS)) { + assert.equal((wf.permissions || {})["id-token"], undefined); + } +}); + +test("secrets appear only in environment jobs, and only the expected ones", () => { + const expected = { + "release.yml/act": ["RELEASE_APP_CLIENT_ID", "RELEASE_APP_PRIVATE_KEY"], + "release.yml/dispatch": ["RELEASE_APP_CLIENT_ID", "RELEASE_APP_PRIVATE_KEY"], + "publish.yml/validate": ["RELEASE_APP_CLIENT_ID", "RELEASE_APP_PRIVATE_KEY"], + "publish.yml/generate": ["GEMINI_API"], + }; + for (const { id, job } of allJobs()) { + const used = [...new Set((JSON.stringify(job).match(/secrets\.[A-Za-z0-9_]+/g) || []).map((s) => s.slice(8)))].sort(); + assert.deepEqual(used, expected[id] || [], id); + } + for (const [file, wf] of Object.entries(ALL_WORKFLOWS)) { + assert.ok(!JSON.stringify(wf.env || {}).includes("secrets."), `${file} workflow env`); + } + // GEMINI_API only in the step that runs the generator. + const geminiSteps = allSteps().filter(({ step }) => JSON.stringify(step).includes("secrets.GEMINI_API")); + assert.equal(geminiSteps.length, 1); + assert.equal(geminiSteps[0].step.run, "python -I sw/scripts/release/py/generate_test.py"); + assert.deepEqual(Object.keys(geminiSteps[0].step.env).filter((k) => k.includes("GEMINI")), ["GEMINI_API_KEY"]); +}); + +test("App tokens never leave their job", () => { + for (const { id, job } of allJobs()) { + assert.ok(!JSON.stringify(job.outputs || {}).includes("app-token"), `${id} outputs`); + for (const step of job.steps || []) { + if (action(step) === APP_TOKEN) { + assert.equal((step.with || {})["skip-token-revoke"], undefined, `${id} must keep token revocation`); + } + } + } +}); + +test("each create-github-app-token step has the plan's owner, repositories and permissions", () => { + const tokenSteps = allSteps().filter(({ step }) => action(step) === APP_TOKEN); + assert.deepEqual(tokenSteps.map((s) => s.id).sort(), Object.keys(TOKEN_SPECS).sort()); + for (const { id, step } of tokenSteps) { + const spec = TOKEN_SPECS[id]; + const w = step.with; + assert.equal(w.owner, spec.owner, `${id} owner`); + assert.equal(w.repositories, spec.repositories, `${id} repositories`); + for (const dryRun of [true, false]) { + const perms = {}; + for (const [key, raw] of Object.entries(w)) { + if (!key.startsWith("permission-")) continue; + const value = raw.includes("${{") ? evaluate(raw, { inputs: { dry_run: dryRun } }) : raw; + if (value !== "") perms[key.slice("permission-".length)] = value; + } + assert.deepEqual(perms, dryRun ? spec.dry : spec.real, `${id} dry_run=${dryRun}`); + } + } +}); + +// ------------------------------------------------------------- gates --- + +test("dispatch.if is exactly the plan's condition and behaves", () => { + const job = release.jobs.dispatch; + assert.equal(job.if, DISPATCH_IF); + const ctx = (dryRun, published, actResult = "success") => ({ + inputs: { dry_run: dryRun }, + needs: { act: { result: actResult, outputs: { published } } }, + }); + assert.equal(evaluateJobIf(job.if, ctx(false, "true")), true); + assert.equal(evaluateJobIf(job.if, ctx(true, "true")), false); + assert.equal(evaluateJobIf(job.if, ctx(false, "false")), false); + assert.equal(evaluateJobIf(job.if, ctx(false, "")), false); + assert.equal(evaluateJobIf(job.if, ctx(false, "true", "failure")), false); + assert.equal(job.needs, "act"); +}); + +test("upload.if is the plan's gate string, verbatim", () => { + assert.equal(publish.jobs.upload.if, UPLOAD_GATE); +}); + +test("upload gate truth table, evaluated on the literal YAML string", () => { + const job = publish.jobs.upload; + const needs = job.needs; + for (const n of ["validate", "build", "smoke", "generated-tests"]) { + assert.ok(needs.includes(n), `upload needs ${n}`); + } + const rows = [ + ["all green", {}, false, true], + ["override + generated-tests failure", { "generated-tests": "failure" }, true, true], + // generate failing fails generated-tests at its first step. + ["override + generate failure", { "generated-tests": "failure" }, true, true], + ["override + smoke failure", { smoke: "failure", "generated-tests": "failure" }, true, false], + ["override + smoke failure, tests green", { smoke: "failure" }, true, false], + ["no override + generated-tests failure", { "generated-tests": "failure" }, false, false], + ["generated-tests skipped", { "generated-tests": "skipped" }, false, false], + ["generated-tests cancelled", { "generated-tests": "cancelled" }, false, false], + ["build cancelled", { build: "cancelled", smoke: "skipped", "generated-tests": "skipped" }, false, false], + ["validate failure", { validate: "failure", build: "skipped", smoke: "skipped", "generated-tests": "skipped" }, false, false], + ["validate failure, rest green", { validate: "failure" }, false, false], + ["override + build failure", { build: "failure", smoke: "skipped", "generated-tests": "skipped" }, true, false], + ["override + validate failure", { validate: "failure", "generated-tests": "failure" }, true, false], + ["override + generated-tests cancelled", { "generated-tests": "cancelled" }, true, false], + ["override + generated-tests skipped", { "generated-tests": "skipped" }, true, false], + ["smoke skipped", { smoke: "skipped" }, false, false], + ["override + smoke skipped", { smoke: "skipped", "generated-tests": "failure" }, true, false], + ["smoke cancelled", { smoke: "cancelled" }, false, false], + ]; + for (const [name, results, override, expected] of rows) { + const ctx = gateContext(needs, { results, override }); + assert.equal(evaluateJobIf(job.if, ctx), expected, name); + } + // Repo and ref guards: a fork or a non-main ref never uploads. + assert.equal(evaluateJobIf(job.if, gateContext(needs, { repository: "laxmareddyp/shared-workflows" })), false); + assert.equal(evaluateJobIf(job.if, gateContext(needs, { ref: "refs/heads/feature" })), false); + assert.equal(evaluateJobIf(job.if, gateContext(needs, { ref: "refs/tags/main" })), false); +}); + +test("generated-tests runs after a successful build even when generate failed, and fails first", () => { + const job = publish.jobs["generated-tests"]; + assert.equal(job.if, GENERATED_TESTS_IF); + const ctx = (build, generate) => ({ + needs: { resolve: { result: "success" }, build: { result: build }, generate: { result: generate } }, + }); + assert.equal(evaluateJobIf(job.if, ctx("success", "failure")), true); + assert.equal(evaluateJobIf(job.if, ctx("success", "success")), true); + assert.equal(evaluateJobIf(job.if, ctx("failure", "success")), false); + const first = job.steps[0]; + assert.equal(first.env.GENERATE_RESULT, "${{ needs.generate.result }}"); + assert.match(first.run, /\[ "\$GENERATE_RESULT" != "success" \]/); + assert.match(first.run, /exit 1/); + assert.ok(job.needs.includes("generate") && job.needs.includes("build")); +}); + +// ----------------------------------------------------- step hygiene --- + +test("no ${{ inputs.* }} or ${{ github.event.* }} inside run: or script:", () => { + const bad = /\$\{\{[^}]*\b(inputs\.|github\.event\.)/; + for (const { id, step } of allSteps()) { + if (typeof step.run === "string") assert.doesNotMatch(step.run, bad, `${id} run: ${step.name}`); + const script = (step.with || {}).script; + if (typeof script === "string") assert.doesNotMatch(script, bad, `${id} script: ${step.name}`); + } +}); + +test("no ${{ }} expression of any kind inside run: or script: in the release workflows", () => { + for (const { id, step } of allSteps(RELEASE_WORKFLOWS)) { + for (const body of [step.run, (step.with || {}).script]) { + if (typeof body === "string") assert.ok(!body.includes("${{"), `${id}: ${step.name}`); + } + } +}); + +test("persist-credentials: false on every checkout", () => { + const checkouts = allSteps().filter(({ step }) => action(step) === CHECKOUT); + assert.ok(checkouts.length >= 10); + for (const { id, step } of checkouts) { + assert.equal((step.with || {})["persist-credentials"], "false", `${id}: ${step.name}`); + } +}); + +test("act has no job-level concurrency", () => { + assert.equal(release.jobs.act.concurrency, undefined); + for (const { id, job } of allJobs(RELEASE_WORKFLOWS)) { + assert.equal(job.concurrency, undefined, id); + } +}); + +test("no cache restore in privileged jobs", () => { + for (const id of Object.keys(PRIVILEGED)) { + const [file, name] = id.split("/"); + for (const step of ALL_WORKFLOWS[file].jobs[name].steps) { + const a = action(step) || ""; + assert.ok(!a.startsWith("actions/cache"), `${id} uses ${a}`); + if (/^actions\/setup-/.test(a)) { + const w = step.with || {}; + assert.equal(w.cache, undefined, `${id} ${a} cache:`); + assert.equal(w["cache-dependency-path"], undefined, `${id} ${a} cache-dependency-path:`); + } + } + } +}); + +test("act and dispatch refuse re-runs in their first step", () => { + for (const name of ["act", "dispatch"]) { + const first = release.jobs[name].steps[0]; + assert.equal(first.if, undefined, `${name} guard must always run`); + assert.match(first.run, /if \[ "\$GITHUB_RUN_ATTEMPT" != "1" \]; then/); + assert.match(first.run, /exit 1/); + } +}); + +test("scripts are entered through require() with inputs from env", () => { + const cases = [ + [release.jobs.plan, "plan", "./scripts/release/plan.js"], + [release.jobs.act, "act", "./scripts/release/act.js"], + [publish.jobs.validate, "validate", "./scripts/release/validate_publish.js"], + ]; + for (const [job, id, file] of cases) { + const step = job.steps.find((s) => s.id === id); + assert.ok(step, id); + assert.ok(step.with.script.includes(`require('${file}')`), `${id} requires ${file}`); + } + const dispatchStep = release.jobs.dispatch.steps.find((s) => (s.with || {}).script); + assert.ok(dispatchStep.with.script.includes("require('./scripts/release/dispatch.js')")); + assert.equal(dispatchStep.with["github-token"], "${{ steps.app-token.outputs.token }}"); + assert.equal(release.jobs.act.steps.find((s) => s.id === "act").with["github-token"], "${{ steps.app-token.outputs.token }}"); + // Inputs are wired once, at workflow level, through env. + for (const [name] of RELEASE_INPUTS) { + assert.ok(Object.values(release.env).includes(`\${{ inputs.${name} }}`), `release env carries ${name}`); + } +}); + +test("dispatch passes the tag and the triggering actor through env", () => { + const step = release.jobs.dispatch.steps.find((s) => (s.with || {}).script); + assert.equal(step.env.RELEASE_TAG, "${{ needs.act.outputs.tag }}"); + assert.equal(step.env.RELEASE_TRIGGERING_ACTOR, "${{ github.triggering_actor }}"); +}); + +test("generate checks out shared-workflows without a repository override", () => { + const job = publish.jobs.generate; + const checkouts = job.steps.filter((s) => action(s) === CHECKOUT); + const sw = checkouts.find((s) => s.with.path === "sw"); + assert.ok(sw); + assert.equal(sw.with.repository, undefined); + assert.equal(sw.with.ref, undefined); + const target = checkouts.find((s) => s.with.path === "target"); + assert.equal(target.with["fetch-depth"], "0"); + assert.equal(target.with["fetch-tags"], "true"); + assert.equal(target.with.ref, "${{ needs.validate.outputs.sha }}"); +}); + +test("build runs target code without secrets and records digests", () => { + const job = publish.jobs.build; + assert.equal(envName(job), null); + const checkout = job.steps.find((s) => action(s) === CHECKOUT); + assert.equal(checkout.with.ref, "${{ needs.validate.outputs.sha }}"); + assert.equal(checkout.with["fetch-depth"], "1"); + const names = job.steps.map((s) => s.name); + assert.ok(names.indexOf("Build setup") < names.indexOf("Build")); + assert.ok(names.indexOf("Build") < names.findIndex((n) => n.startsWith("Record sha256"))); + assert.equal(job.outputs.digests, "${{ steps.digests.outputs.digests }}"); +}); + +test("smoke and generated-tests: no checkout, one pip command for all wheels, no dist overwrite", () => { + for (const name of ["smoke", "generated-tests"]) { + const job = publish.jobs[name]; + assert.ok(!job.steps.some((s) => action(s) === CHECKOUT), `${name} has no checkout`); + const install = job.steps.filter((s) => typeof s.run === "string" && s.run.includes('"${wheels[@]}"')); + assert.equal(install.length, 1, name); + assert.equal(install[0].run.split('"${wheels[@]}"').length, 2, `${name}: one pip command installs every wheel`); + assert.equal(job.strategy.matrix.backend, "${{ fromJSON(needs.resolve.outputs.backends) }}"); + } + const smokeRun = publish.jobs.smoke.steps.find((s) => (s.run || "").includes("site-packages")).run; + assert.match(smokeRun, /"site-packages" not in path/); + const distUploads = allSteps({ "publish.yml": publish }).filter( + ({ step }) => action(step) === "actions/upload-artifact" && step.with.name === "dist", + ); + assert.deepEqual(distUploads.map((s) => s.name), ["build"]); +}); + +test("upload: sha256 recheck, then validator, then split, then keras-hub first with skip-existing", () => { + const job = publish.jobs.upload; + const idx = (pred, label) => { + const i = job.steps.findIndex(pred); + assert.ok(i >= 0, label); + return i; + }; + const digests = idx((s) => (s.run || "").includes("validate_dist.py digests"), "digest recheck"); + const meta = idx((s) => (s.run || "").includes("validate_dist.py metadata"), "validator"); + const split = idx((s) => (s.run || "").includes("split_existing.py"), "split"); + const pypa = job.steps + .map((s, i) => [s, i]) + .filter(([s]) => action(s) === "pypa/gh-action-pypi-publish"); + assert.equal(pypa.length, 2); + assert.ok(digests < meta && meta < split && split < pypa[0][1] && pypa[0][1] < pypa[1][1]); + assert.equal(pypa[0][0].with["packages-dir"], "${{ steps.split.outputs.upload_dir_0 }}"); + assert.equal(pypa[1][0].with["packages-dir"], "${{ steps.split.outputs.upload_dir_1 }}"); + for (const [s] of pypa) assert.equal(s.with["skip-existing"], "true"); + assert.ok(job.steps[meta].run.includes('--tag "$PUBLISH_TAG"'), "Version is checked against the tag"); + assert.equal(job.steps[digests].env.DIGESTS, "${{ needs.build.outputs.digests }}"); +}); + +test("post-publish digest-checks with the upload manifest", () => { + const step = publish.jobs["post-publish"].steps.find((s) => (s.run || "").includes("post_publish.py")); + assert.equal(step.env.MANIFEST, "${{ needs.upload.outputs.manifest }}"); + assert.equal(step.env.DIGESTS, "${{ needs.build.outputs.digests }}"); + assert.ok(publish.jobs["post-publish"].needs.includes("upload")); +}); + +test("report jobs always run and hold no secrets", () => { + for (const wf of Object.values(RELEASE_WORKFLOWS)) { + assert.equal(wf.jobs.report.if, "always()"); + assert.equal(envName(wf.jobs.report), null); + } +}); + +test("test.yml keeps unit-tests unchanged and adds pytest and zizmor", () => { + assert.deepEqual(Object.keys(tests.jobs), ["unit-tests", "pytest", "zizmor"]); + assert.deepEqual(tests.jobs["unit-tests"], { + "runs-on": "ubuntu-latest", + steps: [ + { + name: "Check out repository", + uses: "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + with: { "persist-credentials": "false" }, + }, + { name: "Run unit tests", run: "node --test 'scripts/**/*.test.js'" }, + ], + }); + assert.ok(tests.jobs.pytest.steps.some((s) => (s.run || "").includes("pytest -q scripts/release/py"))); + assert.ok(tests.jobs.zizmor.steps.some((s) => (s.run || "").startsWith("zizmor "))); +}); diff --git a/scripts/release/yaml_lite.js b/scripts/release/yaml_lite.js new file mode 100644 index 0000000..76599c1 --- /dev/null +++ b/scripts/release/yaml_lite.js @@ -0,0 +1,344 @@ +/** + * @license + * Copyright 2026 The Keras Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ============================================================================= + */ + +/** + * Tiny, strict reader for the YAML subset used by the release workflows. + * + * Test helper only (zero npm deps). Supported: block mappings, block + * sequences (including `- key: value` items and compact sequences under a + * key), plain / single-quoted / double-quoted scalars, literal and folded + * block scalars (`|`, `|-`, `|+`, `>`, `>-`, `>+`), flow sequences of scalars + * (`[a, 'b']`), the empty flow mapping `{}`, and `#` comments. + * + * Every scalar is returned as a string (no bool/number typing), so tests + * compare against the literal text in the workflow. Anything outside the + * subset (tabs, anchors, aliases, tags, flow mappings, multi-line plain + * scalars, duplicate keys) throws, so a test can never silently read a + * construct it does not understand. + */ + +const fs = require("node:fs"); + +class YamlLiteError extends Error {} + +function fail(lineNo, message) { + throw new YamlLiteError(`yaml_lite: line ${lineNo + 1}: ${message}`); +} + +function isSkippable(line) { + const t = line.trim(); + return t === "" || t.startsWith("#"); +} + +function isSeqItem(content) { + return content === "-" || content.startsWith("- "); +} + +// Reads a quoted scalar starting at s[0]. Returns { value, end } where `end` +// is the index just past the closing quote. +function readQuoted(s, lineNo) { + const q = s[0]; + let out = ""; + let i = 1; + while (i < s.length) { + const c = s[i]; + if (q === "'") { + if (c === "'") { + if (s[i + 1] === "'") { + out += "'"; + i += 2; + continue; + } + return { value: out, end: i + 1 }; + } + out += c; + i += 1; + continue; + } + if (c === "\\") { + const n = s[i + 1]; + const map = { n: "\n", t: "\t", '"': '"', "\\": "\\", "/": "/", r: "\r", 0: "\0" }; + if (!(n in map)) fail(lineNo, `unsupported escape \\${n}`); + out += map[n]; + i += 2; + continue; + } + if (c === '"') return { value: out, end: i + 1 }; + out += c; + i += 1; + } + return fail(lineNo, "unterminated quoted scalar"); +} + +function assertOnlyComment(rest, lineNo) { + const t = rest.trim(); + if (t !== "" && !t.startsWith("#")) { + fail(lineNo, `unexpected text after scalar: ${JSON.stringify(t)}`); + } +} + +function stripPlainComment(s) { + // A comment starts at " #" (or "#" at the very start) outside quotes. + const idx = s.search(/(^|\s)#/); + return (idx === -1 ? s : s.slice(0, idx)).trimEnd(); +} + +function splitFlowItems(inner, lineNo) { + const items = []; + let cur = ""; + let i = 0; + while (i < inner.length) { + const c = inner[i]; + if (c === "'" || c === '"') { + const { end } = readQuoted(inner.slice(i), lineNo); + cur += inner.slice(i, i + end); + i += end; + continue; + } + if (c === "[" || c === "{") fail(lineNo, "nested flow collections are not supported"); + if (c === ",") { + items.push(cur); + cur = ""; + i += 1; + continue; + } + cur += c; + i += 1; + } + items.push(cur); + if (items.length === 1 && items[0].trim() === "") return []; + return items.map((it) => { + const t = it.trim(); + if (t === "") fail(lineNo, "empty flow sequence item"); + return parseScalar(t, lineNo); + }); +} + +function parseScalar(raw, lineNo) { + const s = raw.trim(); + if (s === "") return null; + const c = s[0]; + if (c === "'" || c === '"') { + const { value, end } = readQuoted(s, lineNo); + assertOnlyComment(s.slice(end), lineNo); + return value; + } + if (c === "[") { + const body = stripPlainComment(s); + if (!body.endsWith("]")) fail(lineNo, "unterminated flow sequence"); + return splitFlowItems(body.slice(1, -1), lineNo); + } + if (c === "{") { + if (stripPlainComment(s) === "{}") return {}; + return fail(lineNo, "flow mappings are not supported"); + } + if (c === "&" || c === "*" || c === "!") { + return fail(lineNo, "anchors, aliases and tags are not supported"); + } + if (c === "@" || c === "`") fail(lineNo, `reserved indicator ${c}`); + const value = stripPlainComment(s); + if (value === "") return null; + return value; +} + +// Splits `key: rest` at the first `:` followed by space/end, outside quotes. +// Returns null when the text is not a mapping entry. +function splitKey(content, lineNo) { + if (content[0] === "'" || content[0] === '"') { + const { value, end } = readQuoted(content, lineNo); + const after = content.slice(end); + if (after === ":" || after.startsWith(": ")) { + return { key: value, rest: after.slice(1) }; + } + return null; + } + if (content.startsWith("#")) return null; + const m = /:(\s|$)/.exec(content); + if (!m) return null; + const key = content.slice(0, m.index); + if (key.includes(" #")) return null; + if (/^[[{&*!|>]/.test(key)) return null; + return { key: key.trimEnd(), rest: content.slice(m.index + 1) }; +} + +class Parser { + constructor(text) { + this.lines = text.replace(/\r\n/g, "\n").split("\n"); + this.i = 0; + } + + indentOf(lineNo) { + const line = this.lines[lineNo]; + const m = /^( *)/.exec(line); + if (line[m[1].length] === "\t") fail(lineNo, "tab indentation is not supported"); + return m[1].length; + } + + skip() { + while (this.i < this.lines.length && isSkippable(this.lines[this.i])) this.i += 1; + } + + atEnd() { + this.skip(); + return this.i >= this.lines.length; + } + + parseNode(minIndent) { + if (this.atEnd()) return null; + const ind = this.indentOf(this.i); + if (ind < minIndent) return null; + const content = this.lines[this.i].slice(ind); + if (isSeqItem(content)) return this.parseSeq(ind); + return this.parseMap(ind); + } + + parseMap(indent) { + const obj = {}; + for (;;) { + if (this.atEnd()) break; + const ind = this.indentOf(this.i); + if (ind < indent) break; + if (ind > indent) fail(this.i, "unexpected indentation"); + const content = this.lines[this.i].slice(ind); + if (isSeqItem(content)) break; + const entry = splitKey(content, this.i); + if (!entry) fail(this.i, `expected "key: value", got ${JSON.stringify(content)}`); + if (Object.prototype.hasOwnProperty.call(obj, entry.key)) { + fail(this.i, `duplicate key ${JSON.stringify(entry.key)}`); + } + const lineNo = this.i; + this.i += 1; + obj[entry.key] = this.parseValue(entry.rest, indent, lineNo); + } + return obj; + } + + parseValue(rest, parentIndent, lineNo) { + const trimmed = rest.trim(); + const header = stripPlainComment(trimmed); + if (header === "") { + if (this.atEnd()) return null; + const ind = this.indentOf(this.i); + const content = this.lines[this.i].slice(ind); + if (ind > parentIndent) return this.parseNode(ind); + if (ind === parentIndent && isSeqItem(content)) return this.parseSeq(ind); + return null; + } + if (/^[|>][-+]?$/.test(header)) return this.parseBlockScalar(header, parentIndent); + const value = parseScalar(trimmed, lineNo); + // A plain scalar continued on a more-indented line is not supported. + if (!this.atEnd() && this.indentOf(this.i) > parentIndent) { + fail(this.i, "multi-line plain scalars are not supported"); + } + return value; + } + + parseBlockScalar(header, parentIndent) { + const style = header[0]; + const chomp = header[1] || ""; + const body = []; + let blockIndent = null; + while (this.i < this.lines.length) { + const line = this.lines[this.i]; + if (line.trim() === "") { + body.push(""); + this.i += 1; + continue; + } + const ind = this.indentOf(this.i); + if (ind <= parentIndent) break; + if (blockIndent === null) blockIndent = ind; + if (ind < blockIndent) fail(this.i, "block scalar line is less indented than its first line"); + body.push(line.slice(blockIndent)); + this.i += 1; + } + // Trailing blank lines belong to chomping, not content. + let trailing = 0; + while (body.length && body[body.length - 1] === "") { + body.pop(); + trailing += 1; + } + let text; + if (style === "|") { + text = body.join("\n"); + } else { + text = ""; + for (let k = 0; k < body.length; k += 1) { + const cur = body[k]; + if (k === 0) { + text = cur; + continue; + } + const prev = body[k - 1]; + if (cur === "") text += "\n"; + else if (prev === "" || /^\s/.test(cur) || /^\s/.test(prev)) text += (prev === "" ? "" : "\n") + cur; + else text += " " + cur; + } + } + if (body.length === 0) return ""; + if (chomp === "-") return text; + if (chomp === "+") return text + "\n".repeat(trailing + 1); + return text + "\n"; + } + + parseSeq(indent) { + const arr = []; + for (;;) { + if (this.atEnd()) break; + const ind = this.indentOf(this.i); + if (ind < indent) break; + if (ind > indent) fail(this.i, "unexpected indentation in sequence"); + const content = this.lines[this.i].slice(ind); + if (!isSeqItem(content)) break; + const after = content === "-" ? "" : content.slice(2); + const afterTrim = after.trimStart(); + const itemIndent = ind + 2 + (after.length - afterTrim.length); + if (afterTrim === "" || afterTrim.startsWith("#")) { + this.i += 1; + arr.push(this.parseNode(ind + 1)); + continue; + } + if (isSeqItem(afterTrim)) fail(this.i, "nested inline sequences are not supported"); + if (splitKey(afterTrim, this.i)) { + // Re-read this line as the first key of a mapping at itemIndent. + this.lines[this.i] = " ".repeat(itemIndent) + afterTrim; + arr.push(this.parseMap(itemIndent)); + continue; + } + const lineNo = this.i; + this.i += 1; + arr.push(this.parseValue(afterTrim, ind, lineNo)); + } + return arr; + } +} + +function parse(text) { + const p = new Parser(text); + if (p.atEnd()) return null; + if (p.indentOf(p.i) !== 0) fail(p.i, "document must start at column 0"); + const doc = p.parseNode(0); + if (!p.atEnd()) fail(p.i, "trailing content that could not be parsed"); + return doc; +} + +function parseFile(path) { + return parse(fs.readFileSync(path, "utf8")); +} + +module.exports = { parse, parseFile, YamlLiteError };