diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e1bfb24038..ec0419dbfe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -30,7 +30,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}-${{ inputs.build_type || 'branch' }} - cancel-in-progress: true + cancel-in-progress: ${{ github.ref_type != 'tag' }} permissions: {} @@ -78,8 +78,7 @@ jobs: packages: read pull-requests: read uses: ./.github/workflows/matrix-job.yaml - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. + # Keep CUDA builds separate; the release gather job retains their native classifiers. with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -96,8 +95,7 @@ jobs: packages: read pull-requests: read uses: ./.github/workflows/matrix-job.yaml - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. + # Build against the matching cuvs-java artifact from this workflow run. with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -148,6 +146,75 @@ jobs: script: "ci/build_lucene.sh" artifact-name-prefix: "cuvs-lucene" file_to_upload: "java/cuvs-lucene/target/" + java-maven-gather: + if: ${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + needs: [java-build, lucene-build] + runs-on: linux-amd64-cpu4 + permissions: + contents: read + actions: read + strategy: + fail-fast: false + matrix: + module: [cuvs-java, cuvs-lucene] + steps: + - name: Checkout release source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ inputs.sha || github.sha }} + persist-credentials: false + - name: Download module builds from this run + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: ${{ matrix.module }}-cuda* + path: ${{ runner.temp }}/java-builds + - name: Assemble Maven repository + env: + MODULE: ${{ matrix.module }} + run: | + python3 java/ci/assemble_maven_repo.py \ + --module "${MODULE}" --tag "${GITHUB_REF_NAME}" \ + --artifacts-dir "${RUNNER_TEMP}/java-builds" \ + --output-dir "${RUNNER_TEMP}/maven-repo" + - name: Upload assembled Maven repository + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ matrix.module }}_maven_repo + path: ${{ runner.temp }}/maven-repo + if-no-files-found: error + java-publish: + needs: java-maven-gather + permissions: + actions: read + contents: read + uses: rapidsai/shared-workflows/.github/workflows/maven-publish.yaml@main + secrets: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + MAVEN_DEPLOY_TOKEN: ${{ secrets.MAVEN_DEPLOY_TOKEN }} + with: + artifact-name: cuvs-java_maven_repo + source-git-sha: ${{ inputs.sha || github.sha }} + profile-id: ${{ vars.MAVEN_STAGING_PROFILE_ID || 'com.nvidia.cuvs' }} + docs-project: cuvs-java + lucene-publish: + # The Lucene release depends on the matching published Java coordinate. + # Documentation is dispatched separately from release-tracking. + if: ${{ !cancelled() && needs.java-publish.outputs.deployment-state == 'PUBLISHED' }} + needs: [java-maven-gather, java-publish] + permissions: + actions: read + contents: read + uses: rapidsai/shared-workflows/.github/workflows/maven-publish.yaml@main + secrets: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + MAVEN_DEPLOY_TOKEN: ${{ secrets.MAVEN_DEPLOY_TOKEN }} + with: + artifact-name: cuvs-lucene_maven_repo + source-git-sha: ${{ inputs.sha || github.sha }} + profile-id: ${{ vars.MAVEN_STAGING_PROFILE_ID || 'com.nvidia.cuvs' }} + docs-project: cuvs-lucene python-build: needs: [build-details, cpp-build] permissions: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02bf88d226..f4e4bb0ded 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,6 +30,11 @@ repos: - id: ruff-check args: [--fix] - id: ruff-format + - repo: https://github.com/PyCQA/bandit + rev: 1.8.6 + hooks: + - id: bandit + exclude: (^|/)test_[^/]*[.]py$ - repo: https://github.com/pre-commit/mirrors-mypy rev: 'v1.19.1' hooks: diff --git a/java/ci/RELEASE.md b/java/ci/RELEASE.md new file mode 100644 index 0000000000..ecfcfa6ccf --- /dev/null +++ b/java/ci/RELEASE.md @@ -0,0 +1,47 @@ +# Java and Lucene Maven releases + +Release tags build both modules using the existing CUDA matrix. The release +gather job downloads each module's artifacts from the same workflow run, keeps +the CUDA builds in separate directories, and creates one Maven repository tree +per coordinate: + +- `com.nvidia.cuvs:cuvs-java` +- `com.nvidia.cuvs.lucene:cuvs-lucene` + +`assemble_maven_repo.py` takes the unclassified jar, sources, javadoc, and POM +from the highest numeric CUDA version in the matrix. This preserves the existing +higher-CUDA selection for common release artifacts and avoids merging +same-named jars from separate builds. Java retains each native architecture/CUDA +classifier. Lucene retains its attached `jar-with-dependencies` assembly when +present. Test jars and other Maven target directory contents are excluded. + +Every input POM must match the release tag after normalizing a zero-padded patch +number. Lucene's POM must depend on the same Java version. Conflicting native +classifiers, missing required artifacts, invalid JARs, and unexpected build +directories fail gathering before publication. + +Both modules use the shared Maven publisher with automatic publication enabled. +Lucene waits for the Java publisher's `PUBLISHED` output, so its dependency is +released first. Each published module emits a completed-publication receipt. +The standalone dispatcher in release-tracking starts the registered docs project +in `rapidsai/docs`; docs failures do not prevent Lucene publication. +Branch and nightly builds keep their existing build behavior; these newly added +publication jobs run on release tags. + +## Configuration and rollout + +Use the existing `GPG_PRIVATE_KEY`, `GPG_PASSPHRASE`, `MAVEN_DEPLOY_USERNAME`, and +`MAVEN_DEPLOY_TOKEN` conventions. The Sonatype token must authorize both Maven +coordinates. The common staging profile defaults to `com.nvidia.cuvs`; set +`MAVEN_STAGING_PROFILE_ID` to the actual verified parent namespace if different, +for example `com.nvidia`. Confirm the namespace and account access during rollout. + +App credentials remain in release-tracking's existing `release-dispatch` +environment. Deploy the standalone docs dispatcher, `cuvs-java` and `cuvs-lucene` +docs registrations, and the shared workflow receipt/output contract before +enabling this caller on the release branch. Only one system should publish these +coordinates for a release; retire the legacy manual publishing path at cutover. + +The local checks exercise Maven assembly and docs configuration without compiling +Java or CUDA. The complete matrix build, Sonatype namespace authorization, App +dispatch, and S3 upload still require validation in the release environment. diff --git a/java/ci/assemble_maven_repo.py b/java/ci/assemble_maven_repo.py new file mode 100644 index 0000000000..08dc50906c --- /dev/null +++ b/java/ci/assemble_maven_repo.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Gather cuVS Java build outputs into one Maven coordinate per module.""" + +import argparse +import filecmp +import re +import shutil +import xml.etree.ElementTree as ET # nosec B405: DTDs are rejected below. +import zipfile +from pathlib import Path + +GROUPS = { + "cuvs-java": "com.nvidia.cuvs", + "cuvs-lucene": "com.nvidia.cuvs.lucene", +} +NS = {"m": "http://maven.apache.org/POM/4.0.0"} + + +def read_pom(pom): + """Read direct POM coordinates without accepting entity declarations.""" + text = pom.read_text() + if "" + "com.nvidia.cuvs" + "cuvs-java" + f"{dependency}" + "" + ) + (build / "pom.xml").write_text( + '' + f"{GROUPS[module]}" + f"{module}{version}" + f"{dependencies}" + ) + for suffix in (".jar", "-sources.jar", "-javadoc.jar", "-tests.jar"): + make_jar(build / f"{module}-{version}{suffix}", cuda) + if module == "cuvs-java": + make_jar( + build / f"{module}-{version}-x86_64-cuda{cuda.split('.')[0]}.jar", + cuda, + ) + else: + make_jar(build / f"{module}-{version}-jar-with-dependencies.jar", cuda) + (build / "unrelated.txt").write_text("not a Maven artifact") + return build + + +def test_native_classifiers_and_explicit_primary_selection(tmp_path): + """Preserve both CUDA classifiers while selecting one common artifact.""" + builds = tmp_path / "builds" + make_build(builds, "cuvs-java", "12.9.2") + primary = make_build(builds, "cuvs-java", "13.3.0") + destination = assemble( + "cuvs-java", "v26.12.00", builds, tmp_path / "maven" + ) + assert destination.relative_to(tmp_path / "maven").as_posix() == ( + "com/nvidia/cuvs/cuvs-java/26.12.0" + ) + assert (destination / "cuvs-java-26.12.0.jar").read_bytes() == ( + primary / "cuvs-java-26.12.0.jar" + ).read_bytes() + assert {path.name for path in destination.iterdir()} == { + "cuvs-java-26.12.0.pom", + "cuvs-java-26.12.0.jar", + "cuvs-java-26.12.0-sources.jar", + "cuvs-java-26.12.0-javadoc.jar", + "cuvs-java-26.12.0-x86_64-cuda12.jar", + "cuvs-java-26.12.0-x86_64-cuda13.jar", + } + + +def test_lucene_layout_and_dependency_assembly(tmp_path): + """Keep Lucene in its own coordinate with its attached assembly jar.""" + builds = tmp_path / "builds" + make_build(builds, "cuvs-lucene", "12.9.2") + make_build(builds, "cuvs-lucene", "13.3.0") + destination = assemble( + "cuvs-lucene", "v26.12.0", builds, tmp_path / "maven" + ) + assert destination.relative_to(tmp_path / "maven").as_posix() == ( + "com/nvidia/cuvs/lucene/cuvs-lucene/26.12.0" + ) + assert (destination / "cuvs-lucene-26.12.0.pom").is_file() + assert ( + destination / "cuvs-lucene-26.12.0-jar-with-dependencies.jar" + ).is_file() + assert not (destination / "cuvs-lucene-26.12.0-tests.jar").exists() + + +@pytest.mark.parametrize("module", GROUPS) +def test_pom_tag_mismatch_is_rejected(tmp_path, module): + """Fail before creating output when a build belongs to another version.""" + builds = tmp_path / "builds" + make_build(builds, module, "13.3.0", version="26.10.0") + with pytest.raises(ValueError, match="POM does not match"): + assemble(module, "v26.12.0", builds, tmp_path / "maven") + assert not (tmp_path / "maven").exists() + + +def test_native_classifier_collision_is_rejected(tmp_path): + """Reject byte-different jars targeting the same native coordinate.""" + builds = tmp_path / "builds" + make_build(builds, "cuvs-java", "13.2.0") + make_build(builds, "cuvs-java", "13.3.0") + with pytest.raises(ValueError, match="Conflicting native classifier"): + assemble("cuvs-java", "v26.12.0", builds, tmp_path / "maven") + + +@pytest.mark.parametrize( + "suffix", ["-sources.jar", "-javadoc.jar", "-x86_64-cuda13.jar"] +) +def test_missing_release_artifact_is_rejected(tmp_path, suffix): + """Require the complete publication set before any upload.""" + builds = tmp_path / "builds" + primary = make_build(builds, "cuvs-java", "13.3.0") + (primary / f"cuvs-java-26.12.0{suffix}").unlink() + with pytest.raises(ValueError): + assemble("cuvs-java", "v26.12.0", builds, tmp_path / "maven") + assert not (tmp_path / "maven").exists() + + +def test_lucene_dependency_version_must_match(tmp_path): + """Reject a Lucene release that points at a different Java version.""" + builds = tmp_path / "builds" + make_build(builds, "cuvs-lucene", "13.3.0", dependency="26.10.0") + with pytest.raises(ValueError, match="matching Java release"): + assemble("cuvs-lucene", "v26.12.0", builds, tmp_path / "maven") + + +def test_pom_entity_declaration_is_rejected(tmp_path): + """Reject XML entity declarations before the XML parser is called.""" + builds = tmp_path / "builds" + primary = make_build(builds, "cuvs-java", "13.3.0") + pom = primary / "pom.xml" + pom.write_text( + ']>' + pom.read_text() + ) + with pytest.raises(ValueError, match="entity declarations"): + assemble("cuvs-java", "v26.12.0", builds, tmp_path / "maven") diff --git a/pyproject.toml b/pyproject.toml index ac3fc1e4fa..d3b2743d40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ exclude = [ ] [tool.ruff.lint] +extend-select = ["B"] ignore = [ # whitespace before : "E203",