From 8c4bd8f5e7f964c8eedf1cd4de69d78c601cc759 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Tue, 29 Sep 2026 12:21:41 +0000 Subject: [PATCH 1/2] fix: distribute native execution installers on demand --- CONTRIBUTING.md | 13 +- apps/docs/content/docs/self-hosted-native.mdx | 21 +++- apps/docs/content/guide-sources.json | 4 +- deploy/distribution/Dockerfile | 1 + deploy/install/configuration.py | 17 ++- deploy/install/install.py | 2 + deploy/install/native_installers.py | 56 +++++++++ deploy/install/test_install.py | 33 +++++ deploy/install/test_native_installers.py | 49 ++++++++ docs/maintainers.md | 10 ++ docs/self-hosted-native.md | 21 +++- scripts/build-core-distribution.sh | 8 +- scripts/core-distribution-manifest.py | 43 ++++++- scripts/core-distribution-manifest.test.py | 33 +++++ scripts/promote-qualified-release.py | 30 ++++- scripts/promote-qualified-release.test.py | 22 +++- scripts/publish-core-release.py | 14 +++ scripts/publish-core-release.test.py | 28 ++++- .../nativeinstaller/assets/bootstrap.sh | 2 +- .../nativeinstaller/bootstrap_test.go | 115 ++++++++++++++++++ .../internal/nativeinstaller/catalog.go | 29 ++++- .../internal/nativeinstaller/catalog_test.go | 58 +++++++++ 22 files changed, 575 insertions(+), 34 deletions(-) create mode 100644 deploy/install/native_installers.py create mode 100644 deploy/install/test_native_installers.py create mode 100644 services/agents-api/internal/nativeinstaller/bootstrap_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9f5be9aff..51b691ba7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3804,7 +3804,18 @@ Existing, rotated or revoked credentials are never replaced by onboarding. Machi bootstrap routes use this grant, not an Environment ID as authentication. Public artifact routes contain no credentials. Native bundles must match the Core source revision and Runtime wire version. Core release qualification consumes the same -three-platform native CI artifacts and includes them in its distribution. +three-platform native CI artifacts and publishes them as independent, source-qualified +Release assets. Core images and default control-service archives carry only their +small catalog (build, protocol, platform, checksum and versioned HTTPS URL), never +native execution archives. The public artifact route redirects a requested platform +to its catalog URL without proxying or caching it; clients verify the Core-provided +checksum before extraction. Installation grants are sent only to Core, never to +artifact hosts. Explicit offline distributions include one copy of each native +archive outside the Core image. The Core installer retains this directory privately +and mounts it read-only for container Core, or points native Core at the same files. +Core verifies local archives before serving; missing online archives redirect, while +corrupt local content fails closed. Neither installation nor repair downloads native +execution payloads; Session bootstrap requests only the current machine's platform. `make check-distribution` exercises catalog assembly with manifests larger than Node's default subprocess output buffer; catalog reads allow up to 64 MiB. Bootstrap scripts own platform download/extraction only; installation, startup, diff --git a/apps/docs/content/docs/self-hosted-native.mdx b/apps/docs/content/docs/self-hosted-native.mdx index 0e4c134bc..e21f3adb3 100644 --- a/apps/docs/content/docs/self-hosted-native.mdx +++ b/apps/docs/content/docs/self-hosted-native.mdx @@ -81,12 +81,21 @@ to resume; completed components and credentials are retained and an existing daemon is reused. After authentication failures, check the Environment credential in Core. Do not remove the workspace or Session history to retry. -Qualified release distributions contain Linux amd64, macOS arm64 and Windows -amd64 installers. Core serves these matched artifacts directly, including in a -private repository deployment. Unsupported platforms fail explicitly. Operators -running a standalone Core binary can set `OAC_NATIVE_INSTALLER_DIR` to its matched -`native-installers` directory. Without qualified artifacts, Session responses -report installation unavailable instead of selecting another version. +Qualified releases publish separate Linux amd64, macOS arm64 and Windows amd64 +installers. Default Core installation carries only their version and checksum +catalog. The one-command bootstrap downloads only the current platform from the +fixed Release URL and verifies its checksum before extraction. It requires access +to that public download host; installation credentials stay on Core. Unsupported +platforms fail explicitly, and download failure never selects a different version. + +The explicit offline Core archive carries one copy of each installer outside the +Core image. Installing it retains the files in the installation's private +`native-installers` directory and makes them available through Core's same public +artifact endpoint. No external download is needed for these archives. Standalone +Core operators can set `OAC_NATIVE_INSTALLER_DIR` to the matched catalog directory, +with optional locally supplied archives. A corrupt local archive refuses startup; +missing catalog metadata disables one-command installation. Model access and +capability dependencies may still require networking. ## Manual distribution installation diff --git a/apps/docs/content/guide-sources.json b/apps/docs/content/guide-sources.json index 742468321..77515784f 100644 --- a/apps/docs/content/guide-sources.json +++ b/apps/docs/content/guide-sources.json @@ -16,7 +16,7 @@ "contracts/agents-api/environments.md": "4c4b9423b22919d6aa6bd98d96422e1a8d610d3442d4cf7731a575bd31fd43f0", "docs/getting-started/nodes.md": "13e1f1e867d3e361ff860960ceacc2dd5806e64f4176cd33f5e49f9a9f4d57e4", "docs/getting-started/self-hosted.md": "4591e05bfa7657f647bedecd9db07b7086acd94b5548511c88f7e5e3567fdb11", - "docs/self-hosted-native.md": "a3b470461285f0aa9507b262ec8edf11e02f6683328d05206849debe1360b77f", + "docs/self-hosted-native.md": "1d08e46b8406411f0efeab5940cbda907cd55afb8aec159f5233d2c0ef9d3d54", "apps/web/public/onboarding/monitor-en.webp": "29dc220cb1250c7016b7c4bf7f30e9510b07c07b814a48c3aa9303b2d76f20b2", "docs/web/README.md": "47159689b2488f0a94a1af8488bcf98b465e0cca4003d64a4699d7b4db24e086", "docs/api/web-management.md": "ec9a583c0a94845082ec31c7220ddcfb9aff7147c63fb36fa4c87463570f1b8c", @@ -45,7 +45,7 @@ "content/docs/environments-and-files.mdx": "fa7062118cc8f13ddbd627f9990744a2002b7ca988e0b203df7ef3fd3996d9e4", "content/docs/hosted-providers.mdx": "06f0b13b053965bbdd1bfb0a7e85d19b50987841e0512fa8331fc9a67ff71fe0", "content/docs/self-hosted-execution.mdx": "161a9000634b4075a55960556322a1b535d8157ba0a4abbeaa84c85c7292ad33", - "content/docs/self-hosted-native.mdx": "4893e7effe8a395856611da2216e7b7a9f27b8f7de6812fbc9c5ebea0326de34", + "content/docs/self-hosted-native.mdx": "b2c5b2bb2080b5338bbaa3469f196555be31089ee713d2b9f1be8d47fa7e29cc", "public/images/source/apps/web/public/onboarding/monitor-en.webp": "29dc220cb1250c7016b7c4bf7f30e9510b07c07b814a48c3aa9303b2d76f20b2", "content/docs/console.mdx": "a930ce36035de74f0c2bef71bed067fc2287ba1c70a485758bf100d484f6adfd", "content/docs/admin-api.mdx": "39f5e5373145f8d7e24e2880216fe59ac1c7fc3e43a768be0504936c0b4221fb", diff --git a/deploy/distribution/Dockerfile b/deploy/distribution/Dockerfile index 3483197c1..3a89e1cc6 100644 --- a/deploy/distribution/Dockerfile +++ b/deploy/distribution/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates COPY --chmod=0555 bin/oac-core bin/oac-core-migrate bin/oac-core-device bin/oac-core-environment-key /usr/local/bin/ COPY e2b/ /opt/oac/e2b/ +# Only the small version/checksum catalog, never native execution archives. COPY native-installers/ /opt/oac/native-installers/ ENV OAC_ADDR=:8091 diff --git a/deploy/install/configuration.py b/deploy/install/configuration.py index 948ac7dc2..deec56c45 100644 --- a/deploy/install/configuration.py +++ b/deploy/install/configuration.py @@ -183,8 +183,8 @@ def core_environment(root, config, state): } if native: result["OAC_E2B_PROVIDER_BIN"] = str(root / "native/e2b/oac-e2b-provider") - if (root / "native/native-installers/catalog.json").is_file(): - result["OAC_NATIVE_INSTALLER_DIR"] = str(root / "native/native-installers") + if (root / "native-installers/catalog.json").is_file(): + result["OAC_NATIVE_INSTALLER_DIR"] = str(root / "native-installers") if native else "/opt/oac/native-installers" if core["oauth_trusted_origins"]: result["OAC_OAUTH_TRUSTED_ORIGINS"] = ",".join(core["oauth_trusted_origins"]) if core["runtime_history"] is not None: @@ -234,6 +234,8 @@ def compose_config(root, config, state): services["database"]["ports"] = [f'127.0.0.1:{config["ports"]["database"]}:5432'] else: mounts = [bind(root / "secrets" / name, f"{RUN}/{name}") for name in ("credential.key", "database.password")] + if (root / "native-installers/catalog.json").is_file(): + mounts.append(bind(root / "native-installers", "/opt/oac/native-installers")) mounts += [bind(root / "generated" / name, f"{RUN}/{name}") for name in ("core-key-digests.json", "settings.json")] if config["core"]["runtime_history"] is not None: mounts.append(bind(root / "generated/runtime-history.json", f"{RUN}/runtime-history.json")) @@ -287,6 +289,16 @@ def __init__(self, files, services, unit): self.files, self.services, self.unit = files, services, unit +def native_installer_inputs(root): + directory = root / "native-installers" + catalog = directory / "catalog.json" + if not catalog.is_file(): + return None + # Archives are immutable and verified on installation/startup. Adding an + # offline archive must restart Core so its local availability map refreshes. + return [sha256(catalog.read_bytes()), sorted(p.name for p in directory.glob("*.tar.gz") if p.is_file())] + + def render(root, config, state, applied_at): root = Path(root) mode, native = config["mode"], config.get("native_core", False) @@ -307,6 +319,7 @@ def render(root, config, state, applied_at): core_settings = [item for item in settings["settings"] if "core" in item["restarts"]] external["core"] = json.dumps({ "core-key-digests.json": sha256(files["core-key-digests.json"]), + "native-installers": native_installer_inputs(root), "settings": sha256(json.dumps([settings["path"], settings["apply_command"], core_settings], sort_keys=True)), "runtime-history.json": sha256(files.get("runtime-history.json", "")), "credential.key": secrets["credential.key"], "database.password": secrets["database.password"], diff --git a/deploy/install/install.py b/deploy/install/install.py index 73aca17f7..d5be367be 100644 --- a/deploy/install/install.py +++ b/deploy/install/install.py @@ -27,6 +27,7 @@ import configuration from configuration import valid_core_origin import rename +import native_installers import native_service import oac_cli import sandbox_setup @@ -528,6 +529,7 @@ def finish(root, bundle, manifest, fresh=False, selection=None): step("Preparing service files") prepare_node_payload(root, state, bundle) native_service.prepare(root, state, bundle) + native_installers.prepare(root, state, bundle) install_oac(root, bundle) retry = f"rerun ./install.sh --install-dir {root}" try: diff --git a/deploy/install/native_installers.py b/deploy/install/native_installers.py new file mode 100644 index 000000000..bbc018206 --- /dev/null +++ b/deploy/install/native_installers.py @@ -0,0 +1,56 @@ +"""Retain matched catalog metadata and explicitly supplied offline installers.""" + +import json +import os +from pathlib import Path +import re +import shutil +import tempfile + + +from distribution import digest + + +def prepare(root, state, bundle): + if state["mode"] == "web-only": + return + source, target = Path(bundle) / "native-installers", Path(root) / "native-installers" + if not source.exists(): + return + if source.is_symlink() or not source.is_dir() or target.is_symlink(): + raise RuntimeError("Invalid native installer directory") + catalog_path = source / "catalog.json" + if catalog_path.is_symlink(): + raise RuntimeError("Invalid native installer catalog") + catalog = json.loads(catalog_path.read_text()) + if catalog["version"] != state["source_commit"]: + raise RuntimeError("Native installer catalog does not match Core") + expected = {"catalog.json": digest(catalog_path)} + for platform, artifact in catalog["artifacts"].items(): + if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)", platform): + raise RuntimeError("Invalid native installer platform") + expected[platform + ".tar.gz"] = artifact["sha256"] + # Validate both directories before filling any missing files. Offline content + # already installed stays available when repairing with a thin bundle. + for directory in (source, target): + if not directory.exists(): + continue + for path in directory.iterdir(): + if (path.name not in expected or path.is_symlink() or not path.is_file() + or digest(path) != expected[path.name]): + raise RuntimeError("Native installer files differ; preserve them and inspect the distribution") + target.mkdir(mode=0o700, exist_ok=True) + for name in expected: + incoming, installed = source / name, target / name + if not incoming.exists() or installed.exists(): + continue + descriptor, temporary = tempfile.mkstemp(prefix=".native-installer-", dir=root) + try: + with os.fdopen(descriptor, "wb") as outgoing, incoming.open("rb") as stream: + shutil.copyfileobj(stream, outgoing) + outgoing.flush() + os.fsync(outgoing.fileno()) + os.replace(temporary, installed) + finally: + if os.path.exists(temporary): + os.unlink(temporary) diff --git a/deploy/install/test_install.py b/deploy/install/test_install.py index 0390f40ba..eb4ef2848 100644 --- a/deploy/install/test_install.py +++ b/deploy/install/test_install.py @@ -56,6 +56,39 @@ def key_file(self, contents="synthetic-existing-core-key-0123456789", mode=0o600 path.chmod(mode) return path + def test_catalog_installed_for_container_and_native_core(self): + import hashlib + native = self.bundle / "native-installers" + native.mkdir() + raw = b"offline native archive" + catalog = {"version": self.manifest["source_commit"], "artifacts": { + "linux-amd64": {"sha256": hashlib.sha256(raw).hexdigest()}}} + (native / "catalog.json").write_text(json.dumps(catalog)) + # The installer checks the outer checksum inventory before copying. + with (self.bundle / "SHA256SUMS").open("a") as sums: + for path in native.iterdir(): + sums.write(distribution.digest(path) + " " + str(path.relative_to(self.bundle)) + "\n") + self.install() + first = self.document("generated/compose.json")["services"]["core"]["labels"] + (native / "linux-amd64.tar.gz").write_bytes(raw) + self.install() + second = self.document("generated/compose.json")["services"]["core"]["labels"] + self.assertNotEqual(first, second) + (native / "linux-amd64.tar.gz").unlink() + self.install() + self.assertEqual(second, self.document("generated/compose.json")["services"]["core"]["labels"]) + installed = self.root / "native-installers" + self.assertEqual((installed / "linux-amd64.tar.gz").read_bytes(), raw) + compose = self.document("generated/compose.json") + mount = next(m for m in compose["services"]["core"]["volumes"] if m["target"] == "/opt/oac/native-installers") + self.assertTrue(mount["read_only"]) + self.assertEqual(mount["source"], str(installed)) + config = self.document("config.json") + config["native_core"] = True + config["ports"]["database"] = 5432 + environment = install.configuration.core_environment(self.root, config, self.document("state.json")) + self.assertEqual(environment["OAC_NATIVE_INSTALLER_DIR"], str(installed)) + def test_host_check_accepts_current_account_including_root(self): for uid in (0, 1000): with self.subTest(uid=uid), mock.patch.object(install.os, "getuid", return_value=uid), \ diff --git a/deploy/install/test_native_installers.py b/deploy/install/test_native_installers.py new file mode 100644 index 000000000..ca1c4237f --- /dev/null +++ b/deploy/install/test_native_installers.py @@ -0,0 +1,49 @@ +"""Online/offline catalog retention shared by container and native Core.""" +import json +from pathlib import Path +import tempfile +import unittest + +import native_installers +from distribution import digest + + +class NativeInstallersTests(unittest.TestCase): + def test_online_install_offline_addition_and_repair_preserve_verified_content(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + root, bundle = base / "root", base / "bundle" + root.mkdir() + source = bundle / "native-installers" + source.mkdir(parents=True) + archive = source / "linux-amd64.tar.gz" + archive.write_bytes(b"qualified") + payload_hash = digest(archive) + archive.unlink() + catalog = {"version": "a" * 40, "artifacts": {"linux-amd64": {"sha256": payload_hash}}} + (source / "catalog.json").write_text(json.dumps(catalog)) + state = {"mode": "core-only", "source_commit": "a" * 40} + native_installers.prepare(root, state, bundle) + self.assertEqual([p.name for p in (root / "native-installers").iterdir()], ["catalog.json"]) + archive.write_bytes(b"qualified") + native_installers.prepare(root, state, bundle) + installed = root / "native-installers/linux-amd64.tar.gz" + self.assertEqual(installed.read_bytes(), b"qualified") + self.assertEqual(installed.stat().st_mode & 0o777, 0o600) + archive.unlink() + native_installers.prepare(root, state, bundle) + self.assertEqual(installed.read_bytes(), b"qualified") + installed.write_bytes(b"corrupt") + with self.assertRaisesRegex(RuntimeError, "files differ"): + native_installers.prepare(root, state, bundle) + self.assertEqual(installed.read_bytes(), b"corrupt") + + def test_foreign_catalog_refuses_writes(self): + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + source = base / "bundle/native-installers" + source.mkdir(parents=True) + (source / "catalog.json").write_text(json.dumps({"version": "foreign", "artifacts": {}})) + with self.assertRaisesRegex(RuntimeError, "does not match"): + native_installers.prepare(base / "root", {"mode": "all", "source_commit": "a" * 40}, base / "bundle") + self.assertFalse((base / "root").exists()) diff --git a/docs/maintainers.md b/docs/maintainers.md index 6effd091a..b787d3ba6 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -29,6 +29,16 @@ release base must select offline mode. Nodes obtain bootstrap metadata from the console that generated their command. The console serves local artifacts or redirects missing ones to the pinned HTTPS release URL. Published assets download anonymously. +Native daemon/Harness installers are separate `oac-native--.tar.gz` +Release assets with checksum files. The default Core archive and image carry only +`native-installers/catalog.json`; Session bootstrap downloads the machine's platform +on demand. The explicit offline archive includes these installers once, outside the +Core image. Core installation retains them locally for the same bootstrap endpoint. +The tag workflow assembles the catalog from matching native CI outputs. For a local +build with native onboarding, first run `scripts/build-native-catalog.mjs INPUT OUTPUT` +and set `OAC_NATIVE_INSTALLER_BUILD_DIR=OUTPUT`; missing or foreign native assets +prevent release publication. No Release or registry download occurs when Core starts. + A bundle carries a fixed set of docs (the build lists them). Links between them stay relative; every other relative link is rewritten to the same file on GitHub at the bundle's commit, and the build fails if a link or anchor does not resolve. diff --git a/docs/self-hosted-native.md b/docs/self-hosted-native.md index 4d7d183da..00ab3b335 100644 --- a/docs/self-hosted-native.md +++ b/docs/self-hosted-native.md @@ -78,12 +78,21 @@ to resume; completed components and credentials are retained and an existing daemon is reused. After authentication failures, check the Environment credential in Core. Do not remove the workspace or Session history to retry. -Qualified release distributions contain Linux amd64, macOS arm64 and Windows -amd64 installers. Core serves these matched artifacts directly, including in a -private repository deployment. Unsupported platforms fail explicitly. Operators -running a standalone Core binary can set `OAC_NATIVE_INSTALLER_DIR` to its matched -`native-installers` directory. Without qualified artifacts, Session responses -report installation unavailable instead of selecting another version. +Qualified releases publish separate Linux amd64, macOS arm64 and Windows amd64 +installers. Default Core installation carries only their version and checksum +catalog. The one-command bootstrap downloads only the current platform from the +fixed Release URL and verifies its checksum before extraction. It requires access +to that public download host; installation credentials stay on Core. Unsupported +platforms fail explicitly, and download failure never selects a different version. + +The explicit offline Core archive carries one copy of each installer outside the +Core image. Installing it retains the files in the installation's private +`native-installers` directory and makes them available through Core's same public +artifact endpoint. No external download is needed for these archives. Standalone +Core operators can set `OAC_NATIVE_INSTALLER_DIR` to the matched catalog directory, +with optional locally supplied archives. A corrupt local archive refuses startup; +missing catalog metadata disables one-command installation. Model access and +capability dependencies may still require networking. ## Manual distribution installation diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index 2424fceed..dfa95e625 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -99,7 +99,7 @@ if [[ "$(go env GOVERSION)" != "$required_go" ]]; then exit 1 fi for file in install.sh install.py install_output.py install_display.py node_output.py configuration.py config_model.py config.schema.json oac_cli.py convert.py rename.py \ - native_service.py node_install.py node_spec.py node_generations.py sandbox_setup.py distribution.py \ + native_service.py native_installers.py node_install.py node_spec.py node_generations.py sandbox_setup.py distribution.py \ model_provider_sessions.py; do cp "deploy/install/$file" "$bundle/$file" done @@ -139,8 +139,8 @@ tar -xzf "$stage/e2b-build/oac-e2b-provider-linux-amd64.tar.gz" \ cp -R "$stage/core/e2b" "$bundle/native/e2b" mkdir -p "$stage/core/native-installers" if [[ -n "${OAC_NATIVE_INSTALLER_BUILD_DIR:-}" ]]; then - cp -R "$OAC_NATIVE_INSTALLER_BUILD_DIR/." "$stage/core/native-installers/" - cp -R "$stage/core/native-installers" "$bundle/native/" + python3 scripts/core-distribution-manifest.py native-catalog "$bundle" "$stage" "$revision" \ + "$OAC_NATIVE_INSTALLER_BUILD_DIR" "$release_base_url" fi cp deploy/distribution/Dockerfile "$stage/core/Dockerfile" build_image core "$stage/core" @@ -248,10 +248,12 @@ if [[ "$offline" == 1 ]]; then mkdir "$bundle/artifacts" # Hard links keep the optional archive from requiring another Runtime-sized copy. for asset in "$stage/artifacts/"*; do ln "$asset" "$bundle/artifacts/"; done + python3 scripts/core-distribution-manifest.py native-offline "$bundle" "$stage" python3 scripts/core-distribution-manifest.py archive "$bundle" "$source_epoch" offline offline_name="${archive_name%.tar.gz}-offline.tar.gz" mv "$stage/$offline_name" "$stage/$offline_name.sha256" "$output_dir/" printf 'Offline Core distribution: %s/%s\n' "$output_dir" "$offline_name" fi mv "$stage/artifacts/"* "$output_dir/" +if [[ -d "$stage/native-artifacts" ]]; then mv "$stage/native-artifacts/"* "$output_dir/"; fi mv "$bundle" "$output_dir/" diff --git a/scripts/core-distribution-manifest.py b/scripts/core-distribution-manifest.py index 54e56552f..ab974cf81 100644 --- a/scripts/core-distribution-manifest.py +++ b/scripts/core-distribution-manifest.py @@ -207,6 +207,43 @@ def release_base(value): return value.rstrip("/") +def native_catalog(bundle, stage, revision, source, artifact_base_url=""): + """Ship only catalog metadata in Core; publish native archives independently.""" + bundle, stage, source = pathlib.Path(bundle), pathlib.Path(stage), pathlib.Path(source) + base = release_base(artifact_base_url) + catalog = json.loads((source / "catalog.json").read_text()) + if catalog["version"] != revision or not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("Native catalog must match the distribution revision") + if not catalog["artifacts"]: + raise ValueError("Native catalog has no qualified artifacts") + assets = stage / "native-artifacts" + assets.mkdir() + for platform, entry in catalog["artifacts"].items(): + if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)", platform): + raise ValueError("Invalid native installer platform") + archive = source / (platform + ".tar.gz") + if archive.is_symlink() or sha256(archive) != entry["sha256"]: + raise ValueError("Native installer checksum mismatch") + filename = f"oac-native-{revision}-{platform}.tar.gz" + shutil.copyfile(archive, assets / filename) + (assets / (filename + ".sha256")).write_text(entry["sha256"] + " " + filename + "\n") + catalog["artifacts"][platform] = {"sha256": entry["sha256"], **({"url": base + "/" + filename} if base else {})} + for directory in (stage / "core/native-installers", bundle / "native-installers"): + directory.mkdir(parents=True, exist_ok=True) + (directory / "catalog.json").write_text(json.dumps(catalog, indent=2) + "\n") + + +def native_offline(bundle, stage): + bundle, stage = pathlib.Path(bundle), pathlib.Path(stage) + path = bundle / "native-installers/catalog.json" + if not path.exists(): + return + catalog = json.loads(path.read_text()) + for platform in catalog["artifacts"]: + source = stage / "native-artifacts" / f"oac-native-{catalog['version']}-{platform}.tar.gz" + os.link(source, path.parent / (platform + ".tar.gz")) + + def package_artifacts(bundle, stage, revision): """Move optional payload out of Core; the manifest owns every asset digest.""" assets = stage / "artifacts" @@ -295,9 +332,11 @@ def manifest(bundle, stage, revision, source_tree, artifact_base_url="", offline def checksums(bundle): bundle = pathlib.Path(bundle) - # Optional payload hashes are already authenticated by manifest.json. + # Optional payload hashes are authenticated by manifest.json and the native + # catalog. Thin/offline metadata stays identical for same-version repair. members = sorted(path for path in bundle.rglob("*") if path.is_file() and path.relative_to(bundle).parts[0] != "artifacts" + and not (path.parent == bundle / "native-installers" and path.name.endswith(".tar.gz")) and path.name != "SHA256SUMS") (bundle / "SHA256SUMS").write_text("".join(sha256(path) + " " + path.relative_to(bundle).as_posix() + "\n" for path in members)) @@ -513,7 +552,7 @@ def check(image, link): if __name__ == "__main__": commands = {"extract-runtime": extract_runtime, "verify-runtime": verify_runtime, "verify-image": verify_image, "built-image": built_image, "manifest": manifest, "archive": archive, "bootstraps": bootstraps, - "release-base": release_base, "docs": docs} + "release-base": release_base, "docs": docs, "native-catalog": native_catalog, "native-offline": native_offline} try: commands[sys.argv[1]](*sys.argv[2:]) except (KeyError, TypeError): diff --git a/scripts/core-distribution-manifest.test.py b/scripts/core-distribution-manifest.test.py index ae5ad9d89..de063ce7f 100644 --- a/scripts/core-distribution-manifest.test.py +++ b/scripts/core-distribution-manifest.test.py @@ -50,6 +50,39 @@ def descriptor(data, media_type, label): class DistributionTests(unittest.TestCase): + def test_native_payload_is_independent_and_offline_has_one_copy(self): + source = self.stage / "qualified-native" + source.mkdir() + catalog = {"version": REVISION, "protocol_version": "fixture", "artifacts": {}} + for platform in ("linux-amd64", "darwin-arm64", "windows-amd64"): + raw = ("native:" + platform).encode() + (source / (platform + ".tar.gz")).write_bytes(raw) + catalog["artifacts"][platform] = {"sha256": hashlib.sha256(raw).hexdigest()} + (source / "catalog.json").write_text(json.dumps(catalog)) + distribution.native_catalog(self.bundle, self.stage, REVISION, source, RELEASE_BASE) + self.assertEqual([p.name for p in (self.stage / "core/native-installers").iterdir()], ["catalog.json"]) + self.assertEqual([p.name for p in (self.bundle / "native-installers").iterdir()], ["catalog.json"]) + distribution.archive(self.bundle, "1") + with tarfile.open(self.bundle.with_name(self.bundle.name + ".tar.gz")) as archive: + self.assertFalse(any("native-installers/" in p.name and p.name.endswith(".tar.gz") for p in archive.getmembers())) + distribution.checksums(self.bundle) + sums = (self.bundle / "SHA256SUMS").read_bytes() + distribution.native_offline(self.bundle, self.stage) + distribution.checksums(self.bundle) + self.assertEqual((self.bundle / "SHA256SUMS").read_bytes(), sums) + distribution.archive(self.bundle, "1", "offline") + with tarfile.open(self.bundle.with_name(self.bundle.name + "-offline.tar.gz")) as archive: + native = [p for p in archive.getmembers() if "native-installers/" in p.name and p.name.endswith(".tar.gz")] + self.assertEqual(len(native), 3) + for member in native: + platform = pathlib.Path(member.name).name.removesuffix(".tar.gz") + self.assertEqual(archive.extractfile(member).read(), (source / (platform + ".tar.gz")).read_bytes()) + metadata = json.loads((self.bundle / "native-installers/catalog.json").read_text()) + for platform, entry in metadata["artifacts"].items(): + filename = f"oac-native-{REVISION}-{platform}.tar.gz" + self.assertEqual(entry["url"], RELEASE_BASE + "/" + filename) + self.assertEqual(distribution.sha256(self.stage / "native-artifacts" / filename), entry["sha256"]) + def setUp(self): self.temporary = tempfile.TemporaryDirectory() self.addCleanup(self.temporary.cleanup) diff --git a/scripts/promote-qualified-release.py b/scripts/promote-qualified-release.py index 27b83824f..c1151c05f 100644 --- a/scripts/promote-qualified-release.py +++ b/scripts/promote-qualified-release.py @@ -82,9 +82,10 @@ def canonical(value): return json.dumps(value, sort_keys=True, separators=(",", ":")) -def verify_archive(path, offline): +def verify_archive(path, offline, native=None): """Read without extraction; verify every ordinary member and optional payload.""" hashes, metadata, sums = {}, None, None + catalog = {} with tarfile.open(path, "r|gz") as archive: for member in archive: parts = pathlib.PurePosixPath(member.name).parts @@ -93,15 +94,19 @@ def verify_archive(path, offline): raise ValueError("Invalid archive member") relative = "/".join(parts[1:]) stream = archive.extractfile(member) - if relative in ("manifest.json", "SHA256SUMS"): + if relative in ("manifest.json", "SHA256SUMS", "native-installers/catalog.json"): if member.size > 1024 * 1024: raise ValueError("Oversized archive metadata") raw = stream.read() hashes[member.name] = hashlib.sha256(raw).hexdigest() if relative == "manifest.json": metadata = json.loads(raw) - else: + elif relative == "SHA256SUMS": sums = raw.decode() + else: + catalog = json.loads(raw) + if native is not None: + native.update(catalog) else: hashes[member.name] = digest(stream) if metadata is None or sums is None: @@ -135,6 +140,9 @@ def verify_archive(path, offline): raise ValueError("Invalid Runtime asset name") if offline: expected["artifacts/" + name] = entry["sha256"] + if offline: + for platform, entry in catalog.get("artifacts", {}).items(): + expected["native-installers/" + platform + ".tar.gz"] = entry["sha256"] actual = {name[len(STEM) + 1:]: value for name, value in hashes.items() if name != STEM + "/SHA256SUMS"} if actual != expected: @@ -144,8 +152,9 @@ def verify_archive(path, offline): def inspect_candidate(directory): files = {p.name: file_identity(p) for p in directory.iterdir()} - metadata = verify_archive(directory / (STEM + "-offline.tar.gz"), True) - if verify_archive(directory / (STEM + ".tar.gz"), False) != metadata: + native, online_native = {}, {} + metadata = verify_archive(directory / (STEM + "-offline.tar.gz"), True, native) + if verify_archive(directory / (STEM + ".tar.gz"), False, online_native) != metadata or native != online_native: raise ValueError("Thin and offline manifests differ") expected = set() for suffix in (".tar.gz", "-offline.tar.gz"): @@ -158,6 +167,17 @@ def inspect_candidate(directory): expected.add(name) if files.get(name) != {key: entry[key] for key in ("sha256", "size")}: raise ValueError("Runtime asset checksum/size mismatch") + if native: + if native["version"] != SOURCE: + raise ValueError("Native catalog source mismatch") + for platform, entry in native["artifacts"].items(): + if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)", platform): + raise ValueError("Invalid native platform") + name = f"oac-native-{SOURCE}-{platform}.tar.gz" + expected.update((name, name + ".sha256")) + if (entry.get("url") != BASE + "/" + name or files.get(name, {}).get("sha256") != entry["sha256"] + or (directory / (name + ".sha256")).read_text() != entry["sha256"] + " " + name + "\n"): + raise ValueError("Native asset URL/checksum mismatch") if set(files) != expected: raise ValueError("Candidate asset set mismatch; provide only generated flat files") return metadata, files diff --git a/scripts/promote-qualified-release.test.py b/scripts/promote-qualified-release.test.py index 1b59820fe..9fdb082f4 100644 --- a/scripts/promote-qualified-release.test.py +++ b/scripts/promote-qualified-release.test.py @@ -53,8 +53,14 @@ def setUp(self): def archives(self): for offline in (False, True): data = {"manifest.json": json.dumps(self.metadata).encode(), "source.tar.gz": b"source fixture"} + if hasattr(self, "native"): + data["native-installers/catalog.json"] = json.dumps(self.native).encode() + if offline: + for platform in self.native["artifacts"]: + data["native-installers/" + platform + ".tar.gz"] = b"native" data["SHA256SUMS"] = "".join( - hashlib.sha256(value).hexdigest() + " " + key + "\n" for key, value in data.items()).encode() + hashlib.sha256(value).hexdigest() + " " + key + "\n" for key, value in data.items() + if not (key.startswith("native-installers/") and key.endswith(".tar.gz"))).encode() if offline: data.update({"artifacts/" + entry["filename"]: (self.assets / entry["filename"]).read_bytes() for entry in self.metadata["artifacts"].values()}) @@ -67,6 +73,20 @@ def archives(self): checksum = promotion.file_identity(self.assets / name)["sha256"] (self.assets / (name + ".sha256")).write_text(checksum + " " + name + "\n") + def test_independent_native_assets_remain_in_qualification_inventory(self): + name = f"oac-native-{promotion.SOURCE}-linux-amd64.tar.gz" + digest = hashlib.sha256(b"native").hexdigest() + (self.assets / name).write_bytes(b"native") + (self.assets / (name + ".sha256")).write_text(digest + " " + name + "\n") + self.native = {"version": promotion.SOURCE, "artifacts": {"linux-amd64": { + "sha256": digest, "url": promotion.BASE + "/" + name}}} + self.archives() + _, inventory = promotion.inspect_candidate(self.assets) + self.assertIn(name, inventory) + (self.assets / name).unlink() + with self.assertRaisesRegex(ValueError, "Native asset"): + promotion.inspect_candidate(self.assets) + def test_full_asset_inventory_and_archive_contents(self): metadata, inventory = promotion.inspect_candidate(self.assets) self.assertEqual(metadata, self.metadata) diff --git a/scripts/publish-core-release.py b/scripts/publish-core-release.py index c0ae53978..34ec1b9cb 100644 --- a/scripts/publish-core-release.py +++ b/scripts/publish-core-release.py @@ -8,6 +8,7 @@ import pathlib import re import subprocess +import tarfile from urllib.parse import quote spec = importlib.util.spec_from_file_location( @@ -81,6 +82,19 @@ def publish(assets, repository, revision, tag, mode): if checksum.read_text() != distribution.sha256(archive) + " " + archive.name + "\n": raise ValueError("Distribution archive checksum mismatch") + # Native archives are independent assets, authenticated by the catalog in + # the checked control archive. Refuse incomplete transfers before creating a draft. + with tarfile.open(assets / (stem + ".tar.gz"), "r:gz") as archive: + catalog = json.load(archive.extractfile(stem + "/native-installers/catalog.json")) + if catalog["version"] != revision or not catalog["artifacts"]: + raise ValueError("Native installer catalog does not match the release") + for platform, entry in catalog["artifacts"].items(): + if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)", platform): + raise ValueError("Invalid native installer platform") + path = assets / f"oac-native-{revision}-{platform}.tar.gz" + if distribution.sha256(path) != entry["sha256"]: + raise ValueError("Native installer checksum mismatch") + refuse_existing(repository, tag) if mode == "publish": verify_tag(repository, tag, revision) diff --git a/scripts/publish-core-release.test.py b/scripts/publish-core-release.test.py index 9e830565b..127d0ea9d 100644 --- a/scripts/publish-core-release.test.py +++ b/scripts/publish-core-release.test.py @@ -3,6 +3,8 @@ import hashlib import importlib.util import json +import io +import tarfile import pathlib import subprocess import tempfile @@ -28,12 +30,36 @@ def setUp(self): (self.assets / name).write_bytes(b"archive fixture") (self.assets / (name + ".sha256")).write_text( hashlib.sha256(b"archive fixture").hexdigest() + " " + name + "\n") + catalog = {"version": self.revision, "artifacts": {}} + for platform in ("linux-amd64", "darwin-arm64", "windows-amd64"): + name = f"oac-native-{self.revision}-{platform}.tar.gz" + (self.assets / name).write_bytes(b"native archive") + catalog["artifacts"][platform] = {"sha256": hashlib.sha256(b"native archive").hexdigest()} + with tarfile.open(self.assets / (self.stem + ".tar.gz"), "w:gz") as archive: + raw = json.dumps(catalog).encode() + member = tarfile.TarInfo(self.stem + "/native-installers/catalog.json") + member.size = len(raw) + archive.addfile(member, io.BytesIO(raw)) + path = self.assets / (self.stem + ".tar.gz") + path.with_name(path.name + ".sha256").write_text(publisher.distribution.sha256(path) + " " + path.name + "\n") self.release = None self.existing = [] stack = contextlib.ExitStack() self.addCleanup(stack.close) self.api = stack.enter_context(mock.patch.object(publisher, "api", side_effect=self.response)) + def test_missing_native_asset_refuses_release_creation(self): + (self.assets / f"oac-native-{self.revision}-windows-amd64.tar.gz").unlink() + with self.assertRaises(FileNotFoundError): + self.publish() + self.assertEqual(self.writes(), []) + + def test_corrupt_native_asset_refuses_release_creation(self): + (self.assets / f"oac-native-{self.revision}-linux-amd64.tar.gz").write_bytes(b"corrupt") + with self.assertRaisesRegex(ValueError, "Native installer checksum"): + self.publish() + self.assertEqual(self.writes(), []) + def response(self, repository, endpoint, *args): if endpoint.startswith("git/"): return {"object": {"type": "commit", "sha": self.revision}} @@ -75,7 +101,7 @@ def test_version_tag_publishes_complete_fixed_id(self): self.publish() self.assertFalse(self.release["draft"]) self.assertFalse(self.release["prerelease"]) - self.assertEqual(len(self.release["assets"]), 6) + self.assertEqual(len(self.release["assets"]), 9) self.assertEqual(self.api.call_args.args[1:], ("releases/7", "--method", "PATCH", "-F", "draft=false")) diff --git a/services/agents-api/internal/nativeinstaller/assets/bootstrap.sh b/services/agents-api/internal/nativeinstaller/assets/bootstrap.sh index ef27d2f61..f97f98929 100644 --- a/services/agents-api/internal/nativeinstaller/assets/bootstrap.sh +++ b/services/agents-api/internal/nativeinstaller/assets/bootstrap.sh @@ -10,7 +10,7 @@ work=$(mktemp -d) trap 'rm -rf "$work"' EXIT echo 'Downloading the installer matched to Core...' curl -fsS "$base/$os-$arch.sha256" -o "$work/checksum" || { echo 'This Core has no qualified installer for this platform.' >&2; exit 1; } -curl -fsS "$base/$os-$arch.tar.gz" -o "$work/bundle.tar.gz" +curl -fsSL --proto-redir =https "$base/$os-$arch.tar.gz" -o "$work/bundle.tar.gz" if command -v sha256sum >/dev/null; then actual=$(sha256sum "$work/bundle.tar.gz"); actual=${actual%% *} else diff --git a/services/agents-api/internal/nativeinstaller/bootstrap_test.go b/services/agents-api/internal/nativeinstaller/bootstrap_test.go new file mode 100644 index 000000000..5bdfa6820 --- /dev/null +++ b/services/agents-api/internal/nativeinstaller/bootstrap_test.go @@ -0,0 +1,115 @@ +package nativeinstaller + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// Exercise the shipped script against two actual TLS origins. The downloaded +// executable is a fixture so this test cannot install software or call a model. +func TestBootstrapDownloadsVerifiedPlatformAcrossHTTPSRedirect(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX bootstrap") + } + for _, tool := range []string{"bash", "curl", "tar"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skip(tool + " unavailable") + } + } + var archive bytes.Buffer + gz := gzip.NewWriter(&archive) + tw := tar.NewWriter(gz) + payload := []byte("#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$OAC_BOOTSTRAP_TEST_RESULT\"\n") + if err := tw.WriteHeader(&tar.Header{Name: "oac-daemon", Mode: 0700, Size: int64(len(payload))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(payload); err != nil { + t.Fatal(err) + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + hash := sha256.Sum256(archive.Bytes()) + platform := runtime.GOOS + "-" + runtime.GOARCH + requests := 0 + artifact := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.Header.Get("Authorization") != "" || r.Header.Get("Cookie") != "" || strings.Contains(r.URL.String(), "private-grant") { + t.Error("credential reached artifact origin") + } + if r.URL.Path != "/oac-native-build-"+platform+".tar.gz" { + t.Error("wrong platform requested") + } + _, _ = w.Write(archive.Bytes()) + })) + defer artifact.Close() + dir := t.TempDir() + ca := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: artifact.Certificate().Raw}), 0600); err != nil { + t.Fatal(err) + } + for _, valid := range []bool{true, false} { + t.Run(fmt.Sprint(valid), func(t *testing.T) { + core := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".sha256") { + checksum := hex.EncodeToString(hash[:]) + if !valid { + checksum = strings.Repeat("0", 64) + } + fmt.Fprintln(w, checksum) + return + } + if r.URL.Path != "/api/v1/agent-daemon/install/build/"+platform+".tar.gz" { + t.Error("wrong request: " + r.URL.Path) + } + http.Redirect(w, r, artifact.URL+"/oac-native-build-"+platform+".tar.gz", http.StatusTemporaryRedirect) + })) + defer core.Close() + result := filepath.Join(t.TempDir(), "result") + command := exec.Command("bash", "assets/bootstrap.sh", core.URL+"/api/v1/agent-daemon/install/build", "private-grant", "--harness", "codex") + command.Env = append(os.Environ(), "CURL_CA_BUNDLE="+ca, "OAC_BOOTSTRAP_TEST_RESULT="+result, "NO_PROXY=127.0.0.1", "no_proxy=127.0.0.1") + output, err := command.CombinedOutput() + if valid { + if err != nil { + t.Fatalf("%v: %s", err, output) + } + args, err := os.ReadFile(result) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(args), core.URL+"/api/v1/agent-daemon/installation\n--authorization\nprivate-grant\n--harness\ncodex") { + t.Fatalf("incorrect install handoff: %s", args) + } + } else { + if err == nil || !strings.Contains(string(output), "checksum mismatch") { + t.Fatalf("expected checksum failure: %v: %s", err, output) + } + if _, err := os.Stat(result); !os.IsNotExist(err) { + t.Fatal("executed corrupt download") + } + } + if strings.Contains(string(output), "private-grant") { + t.Fatal("authorization in output") + } + }) + } + if requests != 2 { + t.Fatalf("got %d platform downloads", requests) + } +} diff --git a/services/agents-api/internal/nativeinstaller/catalog.go b/services/agents-api/internal/nativeinstaller/catalog.go index 0e7ab9efe..bd13dd2dd 100644 --- a/services/agents-api/internal/nativeinstaller/catalog.go +++ b/services/agents-api/internal/nativeinstaller/catalog.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path/filepath" "regexp" @@ -24,6 +25,7 @@ var bootstrap embed.FS type Artifact struct { SHA256 string `json:"sha256"` + URL string `json:"url,omitempty"` } type Catalog struct { @@ -31,12 +33,15 @@ type Catalog struct { ProtocolVersion string `json:"protocol_version"` Artifacts map[string]Artifact `json:"artifacts"` directory string + local map[string]bool } +var checksum = regexp.MustCompile(`^[0-9a-f]{64}$`) + var platformName = regexp.MustCompile(`^(linux|darwin|windows)-(amd64|arm64)$`) -// Load refuses a different build or protocol, and verifies all archives once -// before exposing them. The distribution directory is immutable while serving. +// Load checks the matched catalog without downloading execution payloads. Local +// offline archives are verified once; the directory stays immutable while serving. func Load(directory, version string) (*Catalog, error) { raw, err := os.ReadFile(filepath.Join(directory, "catalog.json")) if err != nil { @@ -46,14 +51,26 @@ func Load(directory, version string) (*Catalog, error) { if json.Unmarshal(raw, &c) != nil || c.Version != version || !proto.VersionCompatible(c.ProtocolVersion) || len(c.Artifacts) == 0 { return nil, errors.New("native installer catalog does not match this Core build and protocol") } + c.local = make(map[string]bool) for platform, artifact := range c.Artifacts { - if !platformName.MatchString(platform) { + if !platformName.MatchString(platform) || !checksum.MatchString(artifact.SHA256) { return nil, errors.New("invalid native installer platform") } + if artifact.URL != "" { + u, err := url.Parse(artifact.URL) + filename := "oac-native-" + c.Version + "-" + platform + ".tar.gz" + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || !strings.HasSuffix(u.Path, "/"+filename) || strings.Contains(strings.ToLower(u.Path), "/latest/") { + return nil, errors.New("invalid versioned native installer URL") + } + } file, err := os.Open(filepath.Join(directory, platform+".tar.gz")) + if errors.Is(err, os.ErrNotExist) && artifact.URL != "" { + continue + } if err != nil { return nil, err } + c.local[platform] = true hash := sha256.New() _, err = io.Copy(hash, file) file.Close() @@ -96,7 +113,11 @@ func (c *Catalog) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.NotFound(w, r) return } - http.ServeFile(w, r, filepath.Join(c.directory, name)) + if c.local[platform] { + http.ServeFile(w, r, filepath.Join(c.directory, name)) + return + } + http.Redirect(w, r, artifact.URL, http.StatusTemporaryRedirect) } func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", "'\"'\"'") + "'" } diff --git a/services/agents-api/internal/nativeinstaller/catalog_test.go b/services/agents-api/internal/nativeinstaller/catalog_test.go index e74d5dec2..9b56892c2 100644 --- a/services/agents-api/internal/nativeinstaller/catalog_test.go +++ b/services/agents-api/internal/nativeinstaller/catalog_test.go @@ -50,3 +50,61 @@ func TestCatalogRequiresMatchedImmutableArtifacts(t *testing.T) { t.Fatal("accepted corrupt archive") } } + +func TestOnlineCatalogRedirectsOnlyDeclaredMatchedArchives(t *testing.T) { + dir := t.TempDir() + payload := []byte("qualified archive") + sum := sha256.Sum256(payload) + manifest := Catalog{Version: "build", ProtocolVersion: proto.Version, Artifacts: map[string]Artifact{ + "linux-amd64": {SHA256: hex.EncodeToString(sum[:]), URL: "https://downloads.example/v1/oac-native-build-linux-amd64.tar.gz"}, + }} + save := func() { + t.Helper() + raw, _ := json.Marshal(manifest) + if err := os.WriteFile(filepath.Join(dir, "catalog.json"), raw, 0600); err != nil { + t.Fatal(err) + } + } + save() + catalog, err := Load(dir, "build") + if err != nil { + t.Fatal(err) + } + for _, method := range []string{"GET", "HEAD"} { + w := httptest.NewRecorder() + catalog.ServeHTTP(w, httptest.NewRequest(method, "/api/v1/agent-daemon/install/build/linux-amd64.tar.gz", nil)) + if w.Code != 307 || w.Header().Get("Location") != manifest.Artifacts["linux-amd64"].URL { + t.Fatalf("redirect: %v", w) + } + } + for _, bad := range []string{"http://downloads.example/v1/oac-native-build-linux-amd64.tar.gz", "https://user:secret@downloads.example/v1/oac-native-build-linux-amd64.tar.gz", "https://downloads.example/latest/oac-native-build-linux-amd64.tar.gz", "https://downloads.example/v1/foreign.tar.gz"} { + previous := manifest.Artifacts["linux-amd64"] + modified := previous + modified.URL = bad + manifest.Artifacts["linux-amd64"] = modified + save() + if _, err := Load(dir, "build"); err == nil { + t.Fatalf("accepted %s", bad) + } + manifest.Artifacts["linux-amd64"] = previous + } + save() + if err := os.WriteFile(filepath.Join(dir, "linux-amd64.tar.gz"), payload, 0600); err != nil { + t.Fatal(err) + } + catalog, err = Load(dir, "build") + if err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + catalog.ServeHTTP(w, httptest.NewRequest("GET", "/api/v1/agent-daemon/install/build/linux-amd64.tar.gz", nil)) + if w.Code != 200 || w.Body.String() != string(payload) || w.Header().Get("Location") != "" { + t.Fatal("offline archive not served locally") + } + if err := os.WriteFile(filepath.Join(dir, "linux-amd64.tar.gz"), []byte("corrupt"), 0600); err != nil { + t.Fatal(err) + } + if _, err := Load(dir, "build"); err == nil { + t.Fatal("corruption must not fall back to online download") + } +} From efa1f5767b60e5a227bd5851cc80dd144f45df7d Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Tue, 29 Sep 2026 12:23:22 +0000 Subject: [PATCH 2/2] test: verify native release checksum sidecars --- scripts/publish-core-release.py | 3 ++- scripts/publish-core-release.test.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/publish-core-release.py b/scripts/publish-core-release.py index 34ec1b9cb..0d80b0e72 100644 --- a/scripts/publish-core-release.py +++ b/scripts/publish-core-release.py @@ -92,7 +92,8 @@ def publish(assets, repository, revision, tag, mode): if not re.fullmatch(r"(linux|darwin|windows)-(amd64|arm64)", platform): raise ValueError("Invalid native installer platform") path = assets / f"oac-native-{revision}-{platform}.tar.gz" - if distribution.sha256(path) != entry["sha256"]: + if (distribution.sha256(path) != entry["sha256"] + or path.with_name(path.name + ".sha256").read_text() != entry["sha256"] + " " + path.name + "\n"): raise ValueError("Native installer checksum mismatch") refuse_existing(repository, tag) diff --git a/scripts/publish-core-release.test.py b/scripts/publish-core-release.test.py index 127d0ea9d..6ba0ee32b 100644 --- a/scripts/publish-core-release.test.py +++ b/scripts/publish-core-release.test.py @@ -34,7 +34,9 @@ def setUp(self): for platform in ("linux-amd64", "darwin-arm64", "windows-amd64"): name = f"oac-native-{self.revision}-{platform}.tar.gz" (self.assets / name).write_bytes(b"native archive") - catalog["artifacts"][platform] = {"sha256": hashlib.sha256(b"native archive").hexdigest()} + checksum = hashlib.sha256(b"native archive").hexdigest() + catalog["artifacts"][platform] = {"sha256": checksum} + (self.assets / (name + ".sha256")).write_text(checksum + " " + name + "\n") with tarfile.open(self.assets / (self.stem + ".tar.gz"), "w:gz") as archive: raw = json.dumps(catalog).encode() member = tarfile.TarInfo(self.stem + "/native-installers/catalog.json") @@ -101,7 +103,7 @@ def test_version_tag_publishes_complete_fixed_id(self): self.publish() self.assertFalse(self.release["draft"]) self.assertFalse(self.release["prerelease"]) - self.assertEqual(len(self.release["assets"]), 9) + self.assertEqual(len(self.release["assets"]), 12) self.assertEqual(self.api.call_args.args[1:], ("releases/7", "--method", "PATCH", "-F", "draft=false"))