From 5e4f31f83cfb34d52e9aee2453455852d22f7f1b Mon Sep 17 00:00:00 2001 From: "Jayper.Z" Date: Tue, 21 Jul 2026 11:05:56 +0800 Subject: [PATCH] fix: verify production VM smoke on Pages --- scripts/verify-pages-releases.py | 55 +++++++++++++++------- tests/test_pages_policy.sh | 57 ++++++++++++++++++----- tests/test_pages_provenance.py | 79 ++++++++++++++++++++++++++++---- 3 files changed, 155 insertions(+), 36 deletions(-) diff --git a/scripts/verify-pages-releases.py b/scripts/verify-pages-releases.py index 5afb18f..8442b02 100755 --- a/scripts/verify-pages-releases.py +++ b/scripts/verify-pages-releases.py @@ -91,13 +91,15 @@ }, VM_CONTRACT_V2: { "status", "target", "release_contract", "vm_only", "not_ax9000_firmware", - "hardware_validation", "nss_validation", "raw_bios_file", "raw_bios_qemu", + "hardware_validation", "nss_validation", "exact_release_image", "serial_labels", + "https", "http_redirect", "runtime_evidence", "production_runtime", + "raw_bios_persistence", "vmdk_import_persistence", "ssh_port_probe", "ssh", + "authorized_keys", "dropbear_enabled", "dropbear_running", "http_redirect_status", + "https_status", "auth_challenge", "http_host_port", "https_host_port", + "ssh_host_port", "serial_log", "ssh_probe_log", "raw_bios_file", "raw_bios_qemu", "iso_bios_file", "iso_bios_qemu", "iso_efi_file", "iso_efi_qemu", "vmdk_bios_file", "vmdk_bios_qemu", "vmdk_efi_file", "vmdk_efi_qemu", - "esxi_validation", "exact_release_image", "serial_labels", "http", - "ssh_runtime_evidence", "ssh_port_probe", "ssh", "authorized_keys", - "dropbear_enabled", "dropbear_running", "http_status", "auth_challenge", - "http_host_port", "ssh_host_port", "serial_log", "ssh_probe_log", + "esxi_validation", }, } VM_PUBLISHED_VARIANTS = ",".join(VM_VARIANTS) @@ -838,8 +840,6 @@ def verify_vm_candidate(gh: Path, candidate: tuple[int, str, str, str, int, dict "nss_validation": "false", "exact_release_image": "true", "serial_labels": "PASS", - "http": "PASS", - "ssh_runtime_evidence": "PASS", "ssh_port_probe": "PASS", "ssh": "DISABLED_BY_DEFAULT", "authorized_keys": "ABSENT", @@ -847,24 +847,47 @@ def verify_vm_candidate(gh: Path, candidate: tuple[int, str, str, str, int, dict "dropbear_running": "NO", } if contract_version == VM_CONTRACT_V1: - expected_smoke["qemu_boot"] = "PASS" + expected_smoke.update({ + "qemu_boot": "PASS", + "http": "PASS", + "ssh_runtime_evidence": "PASS", + }) else: expected_smoke.update({ "release_contract": "vm-x86_64/v2", + "https": "PASS", + "http_redirect": "PASS", + "runtime_evidence": "PASS", + "production_runtime": "PASS", + "raw_bios_persistence": "PASS", + "vmdk_import_persistence": "PASS", "esxi_validation": "not-tested", **{f"{variant}_file": names[variant] for variant in VM_VARIANTS}, **{f"{variant}_qemu": "runtime-pass" for variant in VM_VARIANTS}, }) if any(smoke[key] != value for key, value in expected_smoke.items()): raise VerificationError("VM smoke report does not exactly prove the release safety contract") - if contract_version == VM_CONTRACT_V1 and Path(smoke["image"]).name != names["image"]: - raise VerificationError("VM smoke report did not test the exact published image") - if (smoke["http_status"], smoke["auth_challenge"]) not in {("200", "false"), ("403", "true")}: - raise VerificationError("VM smoke report contains an invalid LuCI HTTP result") - http_port = parse_vm_host_port(smoke["http_host_port"], "HTTP host port") - ssh_port = parse_vm_host_port(smoke["ssh_host_port"], "SSH host port") - if http_port == ssh_port: - raise VerificationError("VM smoke report reuses the same HTTP and SSH host port") + if contract_version == VM_CONTRACT_V1: + if Path(smoke["image"]).name != names["image"]: + raise VerificationError("VM smoke report did not test the exact published image") + if (smoke["http_status"], smoke["auth_challenge"]) not in {("200", "false"), ("403", "true")}: + raise VerificationError("VM smoke report contains an invalid LuCI HTTP result") + ports = { + "http": parse_vm_host_port(smoke["http_host_port"], "HTTP host port"), + "ssh": parse_vm_host_port(smoke["ssh_host_port"], "SSH host port"), + } + else: + if smoke["http_redirect_status"] not in {"301", "302", "307", "308"}: + raise VerificationError("VM smoke report contains an invalid HTTP-to-HTTPS redirect") + if (smoke["https_status"], smoke["auth_challenge"]) not in {("200", "false"), ("403", "true")}: + raise VerificationError("VM smoke report contains an invalid LuCI HTTPS/authentication result") + ports = { + "http": parse_vm_host_port(smoke["http_host_port"], "HTTP host port"), + "https": parse_vm_host_port(smoke["https_host_port"], "HTTPS host port"), + "ssh": parse_vm_host_port(smoke["ssh_host_port"], "SSH host port"), + } + if len(set(ports.values())) != len(ports): + raise VerificationError("VM smoke report reuses a host port") require_vm_result_path(smoke["serial_log"], "serial.log", "serial log path", contract_version) require_vm_result_path(smoke["ssh_probe_log"], "ssh-port-probe.txt", "SSH probe log path", contract_version) diff --git a/tests/test_pages_policy.sh b/tests/test_pages_policy.sh index 9cb4287..c4a6b27 100755 --- a/tests/test_pages_policy.sh +++ b/tests/test_pages_policy.sh @@ -101,20 +101,53 @@ if grep -Fq 'href="https://github.com/tifycloud/NexaWrt/actions/workflows/vm-rel fi grep -Fq 'release.not_ax9000_firmware !== true' "$SITE/app.js" grep -Fq 'VM_ARTIFACT_LABEL_KEYS' "$PROOF_VERIFIER" -grep -Fq 'VM_SMOKE_REPORT_KEYS' "$PROOF_VERIFIER" grep -Fq '"SSH_AUTHORIZED_KEYS": "absent"' "$PROOF_VERIFIER" -grep -Fq '"dropbear_enabled": "NO"' "$PROOF_VERIFIER" -grep -Fq '"dropbear_running": "NO"' "$PROOF_VERIFIER" -grep -Fq '"ssh_runtime_evidence": "PASS"' "$PROOF_VERIFIER" -grep -Fq '"ssh_port_probe": "PASS"' "$PROOF_VERIFIER" -grep -Fq 'expected_smoke["qemu_boot"] = "PASS"' "$PROOF_VERIFIER" -grep -Fq '"release_contract": "vm-x86_64/v2"' "$PROOF_VERIFIER" -grep -Fq '"esxi_validation": "not-tested"' "$PROOF_VERIFIER" grep -Fq '"PUBLISHED_VARIANTS": VM_PUBLISHED_VARIANTS' "$PROOF_VERIFIER" -grep -Fq 'parse_vm_host_port(smoke["http_host_port"]' "$PROOF_VERIFIER" -grep -Fq 'parse_vm_host_port(smoke["ssh_host_port"]' "$PROOF_VERIFIER" -grep -Fq 'require_vm_result_path(smoke["serial_log"], "serial.log"' "$PROOF_VERIFIER" -grep -Fq 'require_vm_result_path(smoke["ssh_probe_log"], "ssh-port-probe.txt"' "$PROOF_VERIFIER" + +# Validate the exported VM smoke contracts instead of depending on a particular +# Python assignment or formatting shape. The provenance test executed above +# exercises the corresponding strict values, status pairs, ports, and paths. +python3 - "$ROOT_DIR" <<'PYCONTRACT' +import importlib.util +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +scripts = root / "scripts" +sys.path.insert(0, str(scripts)) +spec = importlib.util.spec_from_file_location("verify_pages_releases", scripts / "verify-pages-releases.py") +module = importlib.util.module_from_spec(spec) +if spec.loader is None: + raise SystemExit("Pages verifier module loader is unavailable") +spec.loader.exec_module(module) + +expected_v1 = { + "status", "target", "image", "vm_only", "not_ax9000_firmware", + "hardware_validation", "nss_validation", "exact_release_image", "qemu_boot", + "serial_labels", "http", "ssh_runtime_evidence", "ssh_port_probe", "ssh", + "authorized_keys", "dropbear_enabled", "dropbear_running", "http_status", + "auth_challenge", "http_host_port", "ssh_host_port", "serial_log", "ssh_probe_log", +} +expected_v2 = { + "status", "target", "release_contract", "vm_only", "not_ax9000_firmware", + "hardware_validation", "nss_validation", "exact_release_image", "serial_labels", + "https", "http_redirect", "runtime_evidence", "production_runtime", + "raw_bios_persistence", "vmdk_import_persistence", "ssh_port_probe", "ssh", + "authorized_keys", "dropbear_enabled", "dropbear_running", "http_redirect_status", + "https_status", "auth_challenge", "http_host_port", "https_host_port", + "ssh_host_port", "serial_log", "ssh_probe_log", "raw_bios_file", "raw_bios_qemu", + "iso_bios_file", "iso_bios_qemu", "iso_efi_file", "iso_efi_qemu", + "vmdk_bios_file", "vmdk_bios_qemu", "vmdk_efi_file", "vmdk_efi_qemu", + "esxi_validation", +} +contracts = module.VM_SMOKE_REPORT_KEYS +if set(contracts) != {module.VM_CONTRACT_V1, module.VM_CONTRACT_V2}: + raise SystemExit(f"unexpected VM smoke contract versions: {sorted(contracts)!r}") +if contracts[module.VM_CONTRACT_V1] != expected_v1: + raise SystemExit(f"v1 VM smoke contract changed: {sorted(contracts[module.VM_CONTRACT_V1] ^ expected_v1)!r}") +if contracts[module.VM_CONTRACT_V2] != expected_v2: + raise SystemExit(f"v2 VM smoke contract changed: {sorted(contracts[module.VM_CONTRACT_V2] ^ expected_v2)!r}") +PYCONTRACT grep -Fq 'identity["id"] != asset_id' "$GENERATOR" if grep -Eiq ']+(password|secret|token|key)' "$SITE/index.html"; then echo 'secret-bearing configuration field found in site UI' >&2 diff --git a/tests/test_pages_provenance.py b/tests/test_pages_provenance.py index d4b12fd..4fd98c4 100755 --- a/tests/test_pages_provenance.py +++ b/tests/test_pages_provenance.py @@ -196,22 +196,25 @@ def vm_evidence_payloads(version: str = "v0.1.0-rc.1", *, "nss_validation": "false", "exact_release_image": "true", "serial_labels": "PASS", - "http": "PASS", - "ssh_runtime_evidence": "PASS", "ssh_port_probe": "PASS", "ssh": "DISABLED_BY_DEFAULT", "authorized_keys": "ABSENT", "dropbear_enabled": "NO", "dropbear_running": "NO", - "http_status": "200", - "auth_challenge": "false", - "http_host_port": "18080", - "ssh_host_port": "18022", "serial_log": str(result_dir / "serial.log"), "ssh_probe_log": str(result_dir / "ssh-port-probe.txt"), } if contract_version == module.VM_CONTRACT_V1: - smoke.update({"image": names["image"], "qemu_boot": "PASS"}) + smoke.update({ + "image": names["image"], + "qemu_boot": "PASS", + "http": "PASS", + "ssh_runtime_evidence": "PASS", + "http_status": "200", + "auth_challenge": "false", + "http_host_port": "18080", + "ssh_host_port": "18022", + }) image_keys = ["image"] else: labels.update({ @@ -223,6 +226,18 @@ def vm_evidence_payloads(version: str = "v0.1.0-rc.1", *, }) smoke.update({ "release_contract": "vm-x86_64/v2", + "https": "PASS", + "http_redirect": "PASS", + "runtime_evidence": "PASS", + "production_runtime": "PASS", + "raw_bios_persistence": "PASS", + "vmdk_import_persistence": "PASS", + "http_redirect_status": "307", + "https_status": "403", + "auth_challenge": "true", + "http_host_port": "18080", + "https_host_port": "18443", + "ssh_host_port": "18022", "esxi_validation": "not-tested", **{f"{variant}_file": names[variant] for variant in module.VM_VARIANTS}, **{f"{variant}_qemu": "runtime-pass" for variant in module.VM_VARIANTS}, @@ -688,13 +703,37 @@ def fake_attestation(gh: Path, subject: Path, bundle: Path, tag: str, digest: st ), "v2 artifact labels accepted an unknown extra key", ) - for key in ("release_contract", "raw_bios_file", "iso_bios_qemu", "vmdk_efi_file", "esxi_validation"): + assert len(module.VM_SMOKE_REPORT_KEYS[module.VM_CONTRACT_V1]) == 23 + assert len(module.VM_SMOKE_REPORT_KEYS[module.VM_CONTRACT_V2]) == 39 + for key in ( + "release_contract", "https", "http_redirect", "runtime_evidence", "production_runtime", + "raw_bios_persistence", "vmdk_import_persistence", "http_redirect_status", "https_status", + "auth_challenge", "https_host_port", "raw_bios_file", "iso_bios_qemu", "vmdk_efi_file", + "esxi_validation", + ): expect_verification_error( lambda key=key: verify_vm_fixture( vm_evidence_payloads(contract_version=2, missing_smoke=key), contract_version=2, ), f"v2 smoke report accepted missing exact key: {key}", ) + expect_verification_error( + lambda: verify_vm_fixture( + vm_evidence_payloads(contract_version=2, smoke_updates={"UNSUPPORTED_SMOKE": "PASS"}), + contract_version=2, + ), + "v2 smoke report accepted an unknown extra key", + ) + for field in ( + "https", "http_redirect", "runtime_evidence", "production_runtime", + "raw_bios_persistence", "vmdk_import_persistence", + ): + expect_verification_error( + lambda field=field: verify_vm_fixture( + vm_evidence_payloads(contract_version=2, smoke_updates={field: "FAIL"}), contract_version=2, + ), + f"v2 smoke report accepted failed production evidence: {field}", + ) expect_verification_error( lambda: verify_vm_fixture( vm_evidence_payloads(contract_version=2, smoke_updates={"iso_efi_qemu": "boot-pass"}), @@ -702,6 +741,30 @@ def fake_attestation(gh: Path, subject: Path, bundle: Path, tag: str, digest: st ), "v2 smoke report accepted less than a runtime pass", ) + for updates in ( + {"http_redirect_status": "200"}, + {"https_status": "403", "auth_challenge": "false"}, + {"https_status": "200", "auth_challenge": "true"}, + ): + expect_verification_error( + lambda updates=updates: verify_vm_fixture( + vm_evidence_payloads(contract_version=2, smoke_updates=updates), contract_version=2, + ), + f"v2 smoke report accepted invalid HTTPS/redirect/auth evidence: {updates!r}", + ) + for field, value in ( + ("https_host_port", "443"), + ("https_host_port", "65536"), + ("https_host_port", "018443"), + ("https_host_port", "18080"), + ("ssh_host_port", "18443"), + ): + expect_verification_error( + lambda field=field, value=value: verify_vm_fixture( + vm_evidence_payloads(contract_version=2, smoke_updates={field: value}), contract_version=2, + ), + f"v2 smoke report accepted invalid or reused host port: {field}={value}", + ) expect_verification_error( lambda: verify_vm_fixture( vm_evidence_payloads(contract_version=2, smoke_updates={"vmdk_bios_file": "other.vmdk"}),