diff --git a/.github/workflows/ca-trust.yml b/.github/workflows/ca-trust.yml new file mode 100644 index 0000000..c585bcd --- /dev/null +++ b/.github/workflows/ca-trust.yml @@ -0,0 +1,19 @@ +name: CA trust +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + trust: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Regression tests with public CA fixtures + run: node --test scripts/ca-trust.test.mjs + - run: docker build --pull --target base -t ca-voterr . + - run: docker run --rm --entrypoint sh -v "$PWD/scripts:/checks:ro" ca-voterr /checks/check-ca-trust.sh /checks/ca-fixtures diff --git a/Dockerfile b/Dockerfile index f8e43f2..6e2d81c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,13 @@ ARG RUBY_VERSION=3.3.12 FROM registry.docker.com/library/ruby:$RUBY_VERSION-slim as base +# Apply the October 2026 CA distrust before Bundler or app HTTPS requests. +RUN apt-get update -qq && \ + apt-get install --no-install-recommends -y ca-certificates openssl && \ + rm -rf /var/lib/apt/lists/* +COPY scripts/distrust-compromised-cas.sh /usr/local/bin/distrust-compromised-cas +RUN sh /usr/local/bin/distrust-compromised-cas && update-ca-certificates --fresh + # Rails app lives here WORKDIR /rails @@ -60,3 +67,4 @@ ENTRYPOINT ["/rails/bin/docker-entrypoint"] # Start the server by default, this can be overwritten at runtime EXPOSE 3000 CMD ["./bin/rails", "server"] + diff --git a/docs/ca-trust-incident.md b/docs/ca-trust-incident.md new file mode 100644 index 0000000..337cf3b --- /dev/null +++ b/docs/ca-trust-incident.md @@ -0,0 +1,51 @@ +# October 2026 eMudhra CA incident + +The reviewed outbound clients use normal TLS verification: Plex uses Faraday and TMDB uses Net::HTTP. Normal verification +is insufficient when its root store still permits the compromised CA hierarchy. +This change updates trust in images; it does not establish whether any running +deployment was attacked. + +The build disables these trust anchors by exact SHA-256 fingerprint: +- emSign ECC Root CA - G3: `86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b` +- emSign Root TLS CA - G3, self-signed: `7dd78d5f4f13459a83dff9abbba62edbaf6f2d102bf257fd712f4d9f2746ed8d` +- emSign Root TLS CA - G3, cross-signed: `3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd` +- AAA Certificate Services: `d7a7a0fb5d7e2731d771e9484ebcdef71d5f0c3e0a2948782bc83ee0ea699ef4` + +AAA is a retired root with an alternate signing path to the compromised +intermediates. Leaving it in an older distro bundle can defeat removal of just +the eMudhra root. Other emSign roots and USERTrust/Sectigo roots are preserved. +Valid sites that still depend on the removed hierarchies can stop connecting; +their operators must move to an unaffected chain. Do not bypass verification. + +The script scans distribution and locally imported CA sources, disables matching +configuration entries, and moves blocked certificates outside the *.crt import +glob. `update-ca-certificates --fresh` then regenerates both the bundle and +hashed directory, so clients cannot fall back to an old hashed certificate. +A malformed source certificate fails the build. Re-running is safe. + +## Deployment + +The shared Ruby base applies distrust before Bundler installs and is inherited +by the final runtime. Rebuild with `docker build --pull --no-cache -t voterr:patched .` +and deploy the resulting image through the normal hosting workflow. + +Existing containers do not change when the PR is merged. Rebuild and redeploy. +For native development, update the host/runtime CA store separately. Custom +application CA options, mounted bundles, and trust-related environment overrides +must be audited separately; this build does not rewrite external files. + +## Validation + +Run `node --test scripts/ca-trust.test.mjs` (Node 22+, OpenSSL on PATH). +Fixtures are public CA certificates with no private keys. Tests cover renamed +and duplicate blocked certificates, local imports, idempotence, preservation of +ISRG Root X1, actual OpenSSL rejection, and failure on corrupt input. +CI also runs `scripts/check-ca-trust.sh` against built image trust stores. +A package vulnerability scan alone does not prove that an application rejects +a compromised chain. + +## References + +- [Mozilla incident and alternate cross-signs](https://bugzilla.mozilla.org/show_bug.cgi?id=2083449) +- [eMudhra certificate repository and fingerprints](https://repository.emsign.com/) +- [Historical Mozilla-derived AAA certificate fingerprint](https://android.googlesource.com/platform/system/ca-certificates/+/5a9e1c65f159e4a8461bf646aa116001e2844516/google/roots.pem) diff --git a/scripts/ca-fixtures/aaa.pem b/scripts/ca-fixtures/aaa.pem new file mode 100644 index 0000000..33c71ba --- /dev/null +++ b/scripts/ca-fixtures/aaa.pem @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- diff --git a/scripts/ca-fixtures/emsign.pem b/scripts/ca-fixtures/emsign.pem new file mode 100644 index 0000000..63637d8 --- /dev/null +++ b/scripts/ca-fixtures/emsign.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQGEwJJTjET +MBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGlt +aXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMwHhcNMTgwMjE4MTgzMDAw +WhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTEl +MCMGA1UEChMcZU11ZGhyYSBUZWNobm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWdu +IEVDQyBSb290IENBIC0gRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3i +AhKAnjlfSU2fySU0WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4P +PwT01xySfvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjOPQQD +AwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWBCUfvO6wIBHxcmbHt +RwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD+JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- diff --git a/scripts/ca-fixtures/isrg.pem b/scripts/ca-fixtures/isrg.pem new file mode 100644 index 0000000..c0c7e62 --- /dev/null +++ b/scripts/ca-fixtures/isrg.pem @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAwTzELMAkG +A1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2VhcmNoIEdyb3VwMRUw +EwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4WhcNMzUwNjA0MTEwNDM4WjBP +MQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJuZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3Jv +dXAxFTATBgNVBAMTDElTUkcgUm9vdCBYMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC +ggIBAK3oJHP0FDfzm54rVygch77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj +/RQSa78f0uoxmyF+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7i +S4+3mX6UA5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW +T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyHB5T0Y3Hs +LuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UCB5iPNgiV5+I3lg02 +dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUvKBds0pjBqAlkd25HN7rOrFle +aJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWnOlFuhjuefXKnEgV4We0+UXgVCwOPjdAv +BbI+e0ocS3MFEvzG6uBQE3xDk3SzynTnjh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymC +zLq9gwQbooMDQaHWBfEbwrbwqHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC +1CLQJ13hef4Y53CIrU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIB +BjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq +hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZLubhzEFnT +IZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ3BebYhtF8GaV0nxv +wuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KKNFtY2PwByVS5uCbMiogziUwt +hDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5ORAzI4JMPJ+GslWYHb4phowim57iaztX +OoJwTdwJx4nLCgdNbOhdjsnvzqvHu7UrTkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIu +vtd7u+Nxe5AW0wdeRlN8NwdCjNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1N +bdWhscdCb+ZAJzVcoyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4k +qKOJ2qxq4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA +mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57demyPxgcY +xn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= +-----END CERTIFICATE----- diff --git a/scripts/ca-trust.test.mjs b/scripts/ca-trust.test.mjs new file mode 100644 index 0000000..e9e1023 --- /dev/null +++ b/scripts/ca-trust.test.mjs @@ -0,0 +1,64 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, existsSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; + +const here = fileURLToPath(new URL(".", import.meta.url)); +const script = join(here, "distrust-compromised-cas.sh"); +const fixture = (name) => readFileSync(join(here, "ca-fixtures", name + ".pem"), "utf8"); +const run = (cmd, args) => spawnSync(cmd, args, { encoding: "utf8" }); + +test("distrusts renamed G3/AAA anchors, preserves unrelated roots, and is idempotent", () => { + const root = mkdtempSync(join(tmpdir(), "ca-distrust-")); + try { + const ca = join(root, "usr/share/ca-certificates/mozilla"); + const local = join(root, "usr/local/share/ca-certificates"); + mkdirSync(ca, { recursive: true }); + mkdirSync(local, { recursive: true }); + mkdirSync(join(root, "etc")); + writeFileSync(join(ca, "renamed.crt"), fixture("emsign")); + writeFileSync(join(ca, "keep.crt"), fixture("isrg")); + writeFileSync(join(local, "legacy-cross-sign-root.crt"), fixture("aaa")); + // An imported duplicate must not restore trust. + writeFileSync(join(local, "duplicate.crt"), fixture("emsign")); + writeFileSync(join(root, "etc/ca-certificates.conf"), "# preserve comments\nmozilla/renamed.crt\nmozilla/keep.crt\n"); + for (let i = 0; i < 2; i++) { + const result = run("sh", [script, root]); + assert.equal(result.status, 0, result.stderr); + } + assert.equal(existsSync(join(ca, "renamed.crt")), false); + assert.equal(existsSync(join(local, "legacy-cross-sign-root.crt")), false); + assert.equal(existsSync(join(local, "duplicate.crt")), false); + assert.equal(readFileSync(join(ca, "keep.crt"), "utf8"), fixture("isrg")); + assert.equal(readFileSync(join(root, "etc/ca-certificates.conf"), "utf8"), + "# preserve comments\n!mozilla/renamed.crt\nmozilla/keep.crt\n"); + + const bundle = join(root, "filtered.pem"); + writeFileSync(bundle, readdirSync(ca).filter(p => p.endsWith(".crt")).map(p => readFileSync(join(ca, p), "utf8")).join("\n")); + // -trusted isolates the fixture anchors, including on OpenSSL 3 where + // -CAfile/-CApath alone can still consult the default CAstore. + const args = ["verify", "-trusted", bundle]; + assert.equal(run("openssl", [...args, join(here, "ca-fixtures/isrg.pem")]).status, 0); + for (const name of ["emsign", "aaa"]) { + const result = run("openssl", [...args, join(here, "ca-fixtures", name + ".pem")]); + assert.notEqual(result.status, 0, name + " must be distrusted: " + result.stdout + result.stderr); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("fails on a malformed CA instead of silently leaving it trusted", () => { + const root = mkdtempSync(join(tmpdir(), "ca-invalid-")); + try { + const ca = join(root, "usr/share/ca-certificates"); + mkdirSync(ca, { recursive: true }); + writeFileSync(join(ca, "bad.crt"), "not a certificate"); + assert.notEqual(run("sh", [script, root]).status, 0); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/check-ca-trust.sh b/scripts/check-ca-trust.sh new file mode 100644 index 0000000..ee148ff --- /dev/null +++ b/scripts/check-ca-trust.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Check the actual regenerated bundle AND hashed CA directory in a built image. +set -eu +fixtures=${1:?usage: check-ca-trust.sh fixture-directory} +openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt -CApath /etc/ssl/certs "$fixtures/isrg.pem" +for cert in emsign aaa; do + openssl x509 -in "$fixtures/$cert.pem" -noout >/dev/null + if openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt -CApath /etc/ssl/certs "$fixtures/$cert.pem"; then + echo "ERROR: blocked $cert anchor is still trusted" >&2 + exit 1 + fi +done +if command -v node >/dev/null 2>&1; then + node -e 'if (!(process.env.NODE_OPTIONS || "").split(/\s+/).includes("--use-openssl-ca")) throw new Error("Node must use the filtered OpenSSL trust store")' +fi diff --git a/scripts/distrust-compromised-cas.sh b/scripts/distrust-compromised-cas.sh new file mode 100644 index 0000000..60bcff5 --- /dev/null +++ b/scripts/distrust-compromised-cas.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Disable the eMudhra G3 hierarchies and the retired AAA cross-signing root. +# Incident: https://bugzilla.mozilla.org/show_bug.cgi?id=2083449 +# Fingerprints: https://repository.emsign.com/ and Mozilla's historical root store. +# Run before update-ca-certificates --fresh. No network access is required. +set -eu + +root=${1:-/} +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +: > "$work/disabled" + +for dir in "$root/usr/share/ca-certificates" "$root/usr/local/share/ca-certificates"; do + [ ! -d "$dir" ] || find "$dir" -type f -name '*.crt' >> "$work/files" +done +[ -f "$work/files" ] || { echo "No CA source directories found" >&2; exit 1; } + +while IFS= read -r cert; do + # Do not hide openssl errors behind a pipeline: a corrupt CA must fail the build. + fingerprint=$(openssl x509 -in "$cert" -noout -sha256 -fingerprint) + fingerprint=$(printf '%s' "$fingerprint" | cut -d= -f2 | tr -d ':' | tr 'A-F' 'a-f') + case "$fingerprint" in + # emSign ECC G3, TLS G3 (both variants), and AAA Certificate Services. + 86a1ecba089c4a8d3bbe2734c612ba341d813e043cf9e8a862cd5c57a36bbe6b|\ + 7dd78d5f4f13459a83dff9abbba62edbaf6f2d102bf257fd712f4d9f2746ed8d|\ + 3094025792039a68af6e553686410bdab7c0d4a36b9084cba64202e35d6ebbdd|\ + d7a7a0fb5d7e2731d771e9484ebcdef71d5f0c3e0a2948782bc83ee0ea699ef4) + relative=${cert#"$root/usr/share/ca-certificates/"} + printf '%s\n' "$relative" >> "$work/disabled" + # Keep the public cert for inspection, outside the *.crt import glob. + mv "$cert" "$cert.distrusted" + echo "Distrusted CA: $fingerprint" >&2 + ;; + esac +done < "$work/files" + +conf="$root/etc/ca-certificates.conf" +if [ -f "$conf" ]; then + awk 'FILENAME == ARGV[1] { blocked[$0] = 1; next } + { if ($0 in blocked) print "!" $0; else print }' \ + "$work/disabled" "$conf" > "$work/config" + cat "$work/config" > "$conf" +fi