From 8b087652030f0bf433948da7cff32d2b5557a78d Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Mon, 10 Aug 2026 10:10:59 +0000 Subject: [PATCH 1/8] Run acceptance tests against a FIPS build in CI Co-authored-by: Isaac --- .github/workflows/push.yml | 50 +++++++++++++++++++++++++++++++++++ acceptance/fips_test.go | 53 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 acceptance/fips_test.go diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index b4c5238326b..9c86e8a0341 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -195,6 +195,56 @@ jobs: exit 1 fi + # Go picks its cryptographic module at build time via GOFIPS140, so the module + # a binary uses is a property of the build and not of the code. The regular + # test job builds without it, which leaves the FIPS module unexercised: FIPS + # mode narrows the cipher suites the TLS client offers, and nothing else here + # would notice if that broke a handshake or a hashing call. Build the CLI with + # the validated module and run the acceptance suite against it. + # + # Linux only: the module is platform-independent, so a second OS would spend + # the wall-clock time to re-test the same crypto. + test-fips: + needs: + - cleanups + - testmask + + if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }} + name: "task test (linux, fips)" + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + defaults: + run: + shell: bash + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup build environment + uses: ./.github/actions/setup-build-environment + with: + cache-key: test-fips + + - name: Run tests + env: + # Pinned to a frozen module version that has completed CMVP validation. + # "latest" also enables FIPS mode, but tracks the in-tree source and so + # moves with every Go release, leaving no fixed artifact to cite. + GOFIPS140: v1.0.0 + ENVFILTER: DATABRICKS_BUNDLE_ENGINE=direct + run: go tool -modfile=tools/task/go.mod task test + + - name: Summarize failed tests + if: ${{ failure() }} + run: uv run tools/summarize_failed_tests.py test-output.json | tee -a "$GITHUB_STEP_SUMMARY" + test-exp-aitools: needs: - cleanups diff --git a/acceptance/fips_test.go b/acceptance/fips_test.go new file mode 100644 index 00000000000..20bd1889edc --- /dev/null +++ b/acceptance/fips_test.go @@ -0,0 +1,53 @@ +package acceptance_test + +import ( + "os" + "os/exec" + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +// approvedFIPSModule is the frozen Go Cryptographic Module version that has +// completed CMVP validation. GOFIPS140=latest also enables FIPS mode, but tracks +// the in-tree source and moves with every Go release, so it leaves no fixed +// artifact to cite; only a frozen version is accepted here. +const approvedFIPSModule = "v1.0.0" + +// TestCLIBuiltWithFIPSModule asserts that the binary the acceptance suite just +// exercised really was built against the validated module. +// +// Without this, the FIPS CI job would still pass if GOFIPS140 silently stopped +// reaching the compiler -- the suite would run happily against an ordinary +// build and report that FIPS "works". The setting is only observable in the +// binary's own build info, so read it back from there. +// +// Skipped unless GOFIPS140 is set, so the default build is unaffected. +func TestCLIBuiltWithFIPSModule(t *testing.T) { + if os.Getenv("GOFIPS140") == "" { + t.Skip("not a FIPS build") + } + + cwd, err := os.Getwd() + require.NoError(t, err) + + execPath := BuildCLI(t, getBuildDir(t, cwd, runtime.GOOS, runtime.GOARCH), "", runtime.GOOS, runtime.GOARCH) + + out, err := exec.Command("go", "version", "-m", execPath).Output() + require.NoError(t, err) + buildInfo := string(out) + + // The stamped value carries a hash suffix (v1.0.0-) that moves with the + // toolchain, so match the version prefix rather than the whole string. + require.Contains(t, buildInfo, "GOFIPS140="+approvedFIPSModule, + "binary was not built against the approved FIPS module") + + // GOFIPS140 links the module and defaults FIPS mode on. Both matter: a binary + // that links the module but leaves the mode off behaves like a plain build. + require.Contains(t, buildInfo, "DefaultGODEBUG=fips140=on", + "FIPS module is linked but FIPS mode is not enabled by default") + + require.NotContains(t, buildInfo, "GOFIPS140=latest", + "binary was built with an unvalidated module version") +} From 4c33fa271135220c0b2a4162ad249bf3d016ea53 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 13 Aug 2026 11:22:26 +0000 Subject: [PATCH 2/8] Correct the FIPS job comment: it does not cover TLS --- .github/workflows/push.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 9c86e8a0341..860adf9b865 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -197,13 +197,20 @@ jobs: # Go picks its cryptographic module at build time via GOFIPS140, so the module # a binary uses is a property of the build and not of the code. The regular - # test job builds without it, which leaves the FIPS module unexercised: FIPS - # mode narrows the cipher suites the TLS client offers, and nothing else here - # would notice if that broke a handshake or a hashing call. Build the CLI with - # the validated module and run the acceptance suite against it. + # test job builds without it, which leaves the frozen module unexercised: in + # FIPS mode every test binary runs the module's integrity check and + # known-answer self-tests at init, crypto/rand becomes a NIST DRBG, and + # generated keys get pairwise consistency tests. Build against the validated + # module so a module that cannot load or self-test on our toolchain fails here + # rather than for a user. # - # Linux only: the module is platform-independent, so a second OS would spend - # the wall-clock time to re-test the same crypto. + # This job does not cover TLS. The suite talks to local plaintext test servers, + # so the cipher suite narrowing FIPS mode applies to crypto/tls is only + # exercised by the integration tests, which reach a real workspace. + # + # Linux only as a cost trade-off, not because the other platforms are + # equivalent: the integrity self-check is object-format specific (ELF, Mach-O, + # PE) and the module carries per-platform assembly. test-fips: needs: - cleanups From 89280376760e3fb72a4d01a3a374e90ef2c9ada8 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 13 Aug 2026 12:03:17 +0000 Subject: [PATCH 3/8] Run the existing test matrix against the FIPS module instead of a separate job --- .github/workflows/push.yml | 66 ++++++-------------------------------- 1 file changed, 9 insertions(+), 57 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 860adf9b865..6d7fb924668 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -155,6 +155,15 @@ jobs: - name: Run tests env: ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} + # Go picks its cryptographic module at build time, so which module a binary + # uses is a property of the build and not of the code. Released binaries are + # built with this same value, so testing without it would exercise a + # configuration we do not ship. Setting it here rather than in a separate job + # covers every OS in the matrix, which matters because the module's integrity + # self-check is written by object-format specific linker code (ELF, Mach-O, + # PE) and carries per-platform assembly. Pinned to a frozen version: only + # v1.0.0 has a CMVP certificate, and "latest" tracks the in-tree source. + GOFIPS140: v1.0.0 # On pull requests, run only a subset of acceptance subtests on the slower # windows/macOS cells so neither lands on the critical path to merge. An # empty percentage disables subsetting, so Linux PRs and push-to-main run @@ -195,63 +204,6 @@ jobs: exit 1 fi - # Go picks its cryptographic module at build time via GOFIPS140, so the module - # a binary uses is a property of the build and not of the code. The regular - # test job builds without it, which leaves the frozen module unexercised: in - # FIPS mode every test binary runs the module's integrity check and - # known-answer self-tests at init, crypto/rand becomes a NIST DRBG, and - # generated keys get pairwise consistency tests. Build against the validated - # module so a module that cannot load or self-test on our toolchain fails here - # rather than for a user. - # - # This job does not cover TLS. The suite talks to local plaintext test servers, - # so the cipher suite narrowing FIPS mode applies to crypto/tls is only - # exercised by the integration tests, which reach a real workspace. - # - # Linux only as a cost trade-off, not because the other platforms are - # equivalent: the integrity self-check is object-format specific (ELF, Mach-O, - # PE) and the module carries per-platform assembly. - test-fips: - needs: - - cleanups - - testmask - - if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }} - name: "task test (linux, fips)" - runs-on: - group: databricks-protected-runner-group-large - labels: linux-ubuntu-latest-large - - defaults: - run: - shell: bash - - permissions: - id-token: write - contents: read - - steps: - - name: Checkout repository and submodules - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Setup build environment - uses: ./.github/actions/setup-build-environment - with: - cache-key: test-fips - - - name: Run tests - env: - # Pinned to a frozen module version that has completed CMVP validation. - # "latest" also enables FIPS mode, but tracks the in-tree source and so - # moves with every Go release, leaving no fixed artifact to cite. - GOFIPS140: v1.0.0 - ENVFILTER: DATABRICKS_BUNDLE_ENGINE=direct - run: go tool -modfile=tools/task/go.mod task test - - - name: Summarize failed tests - if: ${{ failure() }} - run: uv run tools/summarize_failed_tests.py test-output.json | tee -a "$GITHUB_STEP_SUMMARY" - test-exp-aitools: needs: - cleanups From 78c35f01f9e652b64a7cebf360683e42c1a8af28 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 13 Aug 2026 12:14:55 +0000 Subject: [PATCH 4/8] Trim comments --- .github/workflows/push.yml | 10 ++-------- acceptance/fips_test.go | 20 +++++--------------- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6d7fb924668..5ed1a1c8566 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -155,14 +155,8 @@ jobs: - name: Run tests env: ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} - # Go picks its cryptographic module at build time, so which module a binary - # uses is a property of the build and not of the code. Released binaries are - # built with this same value, so testing without it would exercise a - # configuration we do not ship. Setting it here rather than in a separate job - # covers every OS in the matrix, which matters because the module's integrity - # self-check is written by object-format specific linker code (ELF, Mach-O, - # PE) and carries per-platform assembly. Pinned to a frozen version: only - # v1.0.0 has a CMVP certificate, and "latest" tracks the in-tree source. + # Pinned to a frozen version: only v1.0.0 has a CMVP certificate, and + # "latest" tracks the in-tree source. GOFIPS140: v1.0.0 # On pull requests, run only a subset of acceptance subtests on the slower # windows/macOS cells so neither lands on the critical path to merge. An diff --git a/acceptance/fips_test.go b/acceptance/fips_test.go index 20bd1889edc..2366b5ca349 100644 --- a/acceptance/fips_test.go +++ b/acceptance/fips_test.go @@ -9,22 +9,14 @@ import ( "github.com/stretchr/testify/require" ) -// approvedFIPSModule is the frozen Go Cryptographic Module version that has -// completed CMVP validation. GOFIPS140=latest also enables FIPS mode, but tracks -// the in-tree source and moves with every Go release, so it leaves no fixed -// artifact to cite; only a frozen version is accepted here. const approvedFIPSModule = "v1.0.0" -// TestCLIBuiltWithFIPSModule asserts that the binary the acceptance suite just -// exercised really was built against the validated module. -// -// Without this, the FIPS CI job would still pass if GOFIPS140 silently stopped -// reaching the compiler -- the suite would run happily against an ordinary -// build and report that FIPS "works". The setting is only observable in the -// binary's own build info, so read it back from there. -// -// Skipped unless GOFIPS140 is set, so the default build is unaffected. +// TestCLIBuiltWithFIPSModule asserts that the CLI was built against the validated +// cryptographic module. Without it, CI would still pass if GOFIPS140 stopped reaching +// the compiler: the suite would run against an ordinary build and report that FIPS works. func TestCLIBuiltWithFIPSModule(t *testing.T) { + // Required: the integration suite runs this package too (task integration passes + // ./acceptance), and it runs from eng-dev-ecosystem, which does not set GOFIPS140. if os.Getenv("GOFIPS140") == "" { t.Skip("not a FIPS build") } @@ -43,8 +35,6 @@ func TestCLIBuiltWithFIPSModule(t *testing.T) { require.Contains(t, buildInfo, "GOFIPS140="+approvedFIPSModule, "binary was not built against the approved FIPS module") - // GOFIPS140 links the module and defaults FIPS mode on. Both matter: a binary - // that links the module but leaves the mode off behaves like a plain build. require.Contains(t, buildInfo, "DefaultGODEBUG=fips140=on", "FIPS module is linked but FIPS mode is not enabled by default") From 9f1ce2286e91f06075e401627fc088b5f4a5e0bd Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 13 Aug 2026 18:47:24 +0000 Subject: [PATCH 5/8] Move GOFIPS140 to Taskfile so every task invocation gets it --- .github/workflows/push.yml | 3 --- Taskfile.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5ed1a1c8566..b4c5238326b 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -155,9 +155,6 @@ jobs: - name: Run tests env: ENVFILTER: DATABRICKS_BUNDLE_ENGINE=${{ matrix.deployment }} - # Pinned to a frozen version: only v1.0.0 has a CMVP certificate, and - # "latest" tracks the in-tree source. - GOFIPS140: v1.0.0 # On pull requests, run only a subset of acceptance subtests on the slower # windows/macOS cells so neither lands on the critical path to merge. An # empty percentage disables subsetting, so Linux PRs and push-to-main run diff --git a/Taskfile.yml b/Taskfile.yml index 0a2030181f6..8ed24ad0f60 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -20,6 +20,9 @@ vars: EMBED_SOURCES: sh: 'uv run -p ">=3.11" --no-project python tools/list_embeds.py' +env: + GOFIPS140: v1.0.0 + # pydabs-* tasks live in python/Taskfile.yml so `task pydabs-foo` works when # run from python/. Flattened so they keep their `pydabs-` names at the root. includes: From 63a1c61f3a12bf95180067bad787f57f3c2143ca Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Thu, 13 Aug 2026 19:20:51 +0000 Subject: [PATCH 6/8] Replace the FIPS Go test with an acceptance test --- acceptance/fips/out.test.toml | 2 ++ acceptance/fips/output.txt | 2 ++ acceptance/fips/script | 1 + acceptance/fips/test.toml | 2 ++ acceptance/fips_test.go | 43 ----------------------------------- 5 files changed, 7 insertions(+), 43 deletions(-) create mode 100644 acceptance/fips/out.test.toml create mode 100644 acceptance/fips/output.txt create mode 100644 acceptance/fips/script create mode 100644 acceptance/fips/test.toml delete mode 100644 acceptance/fips_test.go diff --git a/acceptance/fips/out.test.toml b/acceptance/fips/out.test.toml new file mode 100644 index 00000000000..c502b28221b --- /dev/null +++ b/acceptance/fips/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/fips/output.txt b/acceptance/fips/output.txt new file mode 100644 index 00000000000..5eb61e9e8a3 --- /dev/null +++ b/acceptance/fips/output.txt @@ -0,0 +1,2 @@ + +>>> go version -m [CLI] diff --git a/acceptance/fips/script b/acceptance/fips/script new file mode 100644 index 00000000000..e05da665f1b --- /dev/null +++ b/acceptance/fips/script @@ -0,0 +1 @@ +trace go version -m $CLI | contains.py 'GOFIPS140=v1.0.0' 'DefaultGODEBUG=fips140=on' > LOG.buildinfo diff --git a/acceptance/fips/test.toml b/acceptance/fips/test.toml new file mode 100644 index 00000000000..36537d78501 --- /dev/null +++ b/acceptance/fips/test.toml @@ -0,0 +1,2 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +Cloud = true diff --git a/acceptance/fips_test.go b/acceptance/fips_test.go deleted file mode 100644 index 2366b5ca349..00000000000 --- a/acceptance/fips_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package acceptance_test - -import ( - "os" - "os/exec" - "runtime" - "testing" - - "github.com/stretchr/testify/require" -) - -const approvedFIPSModule = "v1.0.0" - -// TestCLIBuiltWithFIPSModule asserts that the CLI was built against the validated -// cryptographic module. Without it, CI would still pass if GOFIPS140 stopped reaching -// the compiler: the suite would run against an ordinary build and report that FIPS works. -func TestCLIBuiltWithFIPSModule(t *testing.T) { - // Required: the integration suite runs this package too (task integration passes - // ./acceptance), and it runs from eng-dev-ecosystem, which does not set GOFIPS140. - if os.Getenv("GOFIPS140") == "" { - t.Skip("not a FIPS build") - } - - cwd, err := os.Getwd() - require.NoError(t, err) - - execPath := BuildCLI(t, getBuildDir(t, cwd, runtime.GOOS, runtime.GOARCH), "", runtime.GOOS, runtime.GOARCH) - - out, err := exec.Command("go", "version", "-m", execPath).Output() - require.NoError(t, err) - buildInfo := string(out) - - // The stamped value carries a hash suffix (v1.0.0-) that moves with the - // toolchain, so match the version prefix rather than the whole string. - require.Contains(t, buildInfo, "GOFIPS140="+approvedFIPSModule, - "binary was not built against the approved FIPS module") - - require.Contains(t, buildInfo, "DefaultGODEBUG=fips140=on", - "FIPS module is linked but FIPS mode is not enabled by default") - - require.NotContains(t, buildInfo, "GOFIPS140=latest", - "binary was built with an unvalidated module version") -} From 190faac2d65b4287a6194827f348b88fd5f1b82c Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 14 Aug 2026 11:11:58 +0000 Subject: [PATCH 7/8] Commit the FIPS build values to the acceptance golden --- acceptance/fips/output.txt | 3 +++ acceptance/fips/script | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/acceptance/fips/output.txt b/acceptance/fips/output.txt index 5eb61e9e8a3..910a5b8e062 100644 --- a/acceptance/fips/output.txt +++ b/acceptance/fips/output.txt @@ -1,2 +1,5 @@ >>> go version -m [CLI] + build -tags=fips140v1.0 + build DefaultGODEBUG=fips140=on + build GOFIPS140=v1.0.0-c2097c7c diff --git a/acceptance/fips/script b/acceptance/fips/script index e05da665f1b..4307e5ecfa9 100644 --- a/acceptance/fips/script +++ b/acceptance/fips/script @@ -1 +1 @@ -trace go version -m $CLI | contains.py 'GOFIPS140=v1.0.0' 'DefaultGODEBUG=fips140=on' > LOG.buildinfo +trace go version -m $CLI | grep -i fips140 From 3b0ca3256516cde147e8218b402b46b70dde701b Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 14 Aug 2026 12:24:28 +0000 Subject: [PATCH 8/8] Assert FIPS values explicitly as well as committing them --- acceptance/fips/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/fips/script b/acceptance/fips/script index 4307e5ecfa9..a87c3f5e01b 100644 --- a/acceptance/fips/script +++ b/acceptance/fips/script @@ -1 +1 @@ -trace go version -m $CLI | grep -i fips140 +trace go version -m $CLI | grep -i fips140 | contains.py 'GOFIPS140=v1.0.0' 'DefaultGODEBUG=fips140=on'