Skip to content

fix: verify production VM smoke on Pages #14

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tifycloud merged 1 commit into from
Jul 21, 2026
Merged

fix: verify production VM smoke on Pages #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 39 additions & 16 deletions scripts/verify-pages-releases.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,15 @@
},
VM_CONTRACT_V2: {
"status", "target", "release_contract", "vm_only", "not_ax9000_firmware",
"hardware_validation", "nss_validation", "raw_bios_file", "raw_bios_qemu",
"hardware_validation", "nss_validation", "exact_release_image", "serial_labels",
"https", "http_redirect", "runtime_evidence", "production_runtime",
"raw_bios_persistence", "vmdk_import_persistence", "ssh_port_probe", "ssh",
"authorized_keys", "dropbear_enabled", "dropbear_running", "http_redirect_status",
"https_status", "auth_challenge", "http_host_port", "https_host_port",
"ssh_host_port", "serial_log", "ssh_probe_log", "raw_bios_file", "raw_bios_qemu",
"iso_bios_file", "iso_bios_qemu", "iso_efi_file", "iso_efi_qemu",
"vmdk_bios_file", "vmdk_bios_qemu", "vmdk_efi_file", "vmdk_efi_qemu",
"esxi_validation", "exact_release_image", "serial_labels", "http",
"ssh_runtime_evidence", "ssh_port_probe", "ssh", "authorized_keys",
"dropbear_enabled", "dropbear_running", "http_status", "auth_challenge",
"http_host_port", "ssh_host_port", "serial_log", "ssh_probe_log",
"esxi_validation",
},
}
VM_PUBLISHED_VARIANTS = ",".join(VM_VARIANTS)
Expand Down Expand Up @@ -838,33 +840,54 @@ def verify_vm_candidate(gh: Path, candidate: tuple[int, str, str, str, int, dict
"nss_validation": "false",
"exact_release_image": "true",
"serial_labels": "PASS",
"http": "PASS",
"ssh_runtime_evidence": "PASS",
"ssh_port_probe": "PASS",
"ssh": "DISABLED_BY_DEFAULT",
"authorized_keys": "ABSENT",
"dropbear_enabled": "NO",
"dropbear_running": "NO",
}
if contract_version == VM_CONTRACT_V1:
expected_smoke["qemu_boot"] = "PASS"
expected_smoke.update({
"qemu_boot": "PASS",
"http": "PASS",
"ssh_runtime_evidence": "PASS",
})
else:
expected_smoke.update({
"release_contract": "vm-x86_64/v2",
"https": "PASS",
"http_redirect": "PASS",
"runtime_evidence": "PASS",
"production_runtime": "PASS",
"raw_bios_persistence": "PASS",
"vmdk_import_persistence": "PASS",
"esxi_validation": "not-tested",
**{f"{variant}_file": names[variant] for variant in VM_VARIANTS},
**{f"{variant}_qemu": "runtime-pass" for variant in VM_VARIANTS},
})
if any(smoke[key] != value for key, value in expected_smoke.items()):
raise VerificationError("VM smoke report does not exactly prove the release safety contract")
if contract_version == VM_CONTRACT_V1 and Path(smoke["image"]).name != names["image"]:
raise VerificationError("VM smoke report did not test the exact published image")
if (smoke["http_status"], smoke["auth_challenge"]) not in {("200", "false"), ("403", "true")}:
raise VerificationError("VM smoke report contains an invalid LuCI HTTP result")
http_port = parse_vm_host_port(smoke["http_host_port"], "HTTP host port")
ssh_port = parse_vm_host_port(smoke["ssh_host_port"], "SSH host port")
if http_port == ssh_port:
raise VerificationError("VM smoke report reuses the same HTTP and SSH host port")
if contract_version == VM_CONTRACT_V1:
if Path(smoke["image"]).name != names["image"]:
raise VerificationError("VM smoke report did not test the exact published image")
if (smoke["http_status"], smoke["auth_challenge"]) not in {("200", "false"), ("403", "true")}:
raise VerificationError("VM smoke report contains an invalid LuCI HTTP result")
ports = {
"http": parse_vm_host_port(smoke["http_host_port"], "HTTP host port"),
"ssh": parse_vm_host_port(smoke["ssh_host_port"], "SSH host port"),
}
else:
if smoke["http_redirect_status"] not in {"301", "302", "307", "308"}:
raise VerificationError("VM smoke report contains an invalid HTTP-to-HTTPS redirect")
if (smoke["https_status"], smoke["auth_challenge"]) not in {("200", "false"), ("403", "true")}:
raise VerificationError("VM smoke report contains an invalid LuCI HTTPS/authentication result")
ports = {
"http": parse_vm_host_port(smoke["http_host_port"], "HTTP host port"),
"https": parse_vm_host_port(smoke["https_host_port"], "HTTPS host port"),
"ssh": parse_vm_host_port(smoke["ssh_host_port"], "SSH host port"),
}
if len(set(ports.values())) != len(ports):
raise VerificationError("VM smoke report reuses a host port")
require_vm_result_path(smoke["serial_log"], "serial.log", "serial log path", contract_version)
require_vm_result_path(smoke["ssh_probe_log"], "ssh-port-probe.txt", "SSH probe log path", contract_version)

Expand Down
57 changes: 45 additions & 12 deletions tests/test_pages_policy.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,20 +101,53 @@ if grep -Fq 'href="https://github.com/tifycloud/NexaWrt/actions/workflows/vm-rel
fi
grep -Fq 'release.not_ax9000_firmware !== true' "$SITE/app.js"
grep -Fq 'VM_ARTIFACT_LABEL_KEYS' "$PROOF_VERIFIER"
grep -Fq 'VM_SMOKE_REPORT_KEYS' "$PROOF_VERIFIER"
grep -Fq '"SSH_AUTHORIZED_KEYS": "absent"' "$PROOF_VERIFIER"
grep -Fq '"dropbear_enabled": "NO"' "$PROOF_VERIFIER"
grep -Fq '"dropbear_running": "NO"' "$PROOF_VERIFIER"
grep -Fq '"ssh_runtime_evidence": "PASS"' "$PROOF_VERIFIER"
grep -Fq '"ssh_port_probe": "PASS"' "$PROOF_VERIFIER"
grep -Fq 'expected_smoke["qemu_boot"] = "PASS"' "$PROOF_VERIFIER"
grep -Fq '"release_contract": "vm-x86_64/v2"' "$PROOF_VERIFIER"
grep -Fq '"esxi_validation": "not-tested"' "$PROOF_VERIFIER"
grep -Fq '"PUBLISHED_VARIANTS": VM_PUBLISHED_VARIANTS' "$PROOF_VERIFIER"
grep -Fq 'parse_vm_host_port(smoke["http_host_port"]' "$PROOF_VERIFIER"
grep -Fq 'parse_vm_host_port(smoke["ssh_host_port"]' "$PROOF_VERIFIER"
grep -Fq 'require_vm_result_path(smoke["serial_log"], "serial.log"' "$PROOF_VERIFIER"
grep -Fq 'require_vm_result_path(smoke["ssh_probe_log"], "ssh-port-probe.txt"' "$PROOF_VERIFIER"

# Validate the exported VM smoke contracts instead of depending on a particular
# Python assignment or formatting shape. The provenance test executed above
# exercises the corresponding strict values, status pairs, ports, and paths.
python3 - "$ROOT_DIR" <<'PYCONTRACT'
import importlib.util
import sys
from pathlib import Path

root = Path(sys.argv[1])
scripts = root / "scripts"
sys.path.insert(0, str(scripts))
spec = importlib.util.spec_from_file_location("verify_pages_releases", scripts / "verify-pages-releases.py")
module = importlib.util.module_from_spec(spec)
if spec.loader is None:
raise SystemExit("Pages verifier module loader is unavailable")
spec.loader.exec_module(module)

expected_v1 = {
"status", "target", "image", "vm_only", "not_ax9000_firmware",
"hardware_validation", "nss_validation", "exact_release_image", "qemu_boot",
"serial_labels", "http", "ssh_runtime_evidence", "ssh_port_probe", "ssh",
"authorized_keys", "dropbear_enabled", "dropbear_running", "http_status",
"auth_challenge", "http_host_port", "ssh_host_port", "serial_log", "ssh_probe_log",
}
expected_v2 = {
"status", "target", "release_contract", "vm_only", "not_ax9000_firmware",
"hardware_validation", "nss_validation", "exact_release_image", "serial_labels",
"https", "http_redirect", "runtime_evidence", "production_runtime",
"raw_bios_persistence", "vmdk_import_persistence", "ssh_port_probe", "ssh",
"authorized_keys", "dropbear_enabled", "dropbear_running", "http_redirect_status",
"https_status", "auth_challenge", "http_host_port", "https_host_port",
"ssh_host_port", "serial_log", "ssh_probe_log", "raw_bios_file", "raw_bios_qemu",
"iso_bios_file", "iso_bios_qemu", "iso_efi_file", "iso_efi_qemu",
"vmdk_bios_file", "vmdk_bios_qemu", "vmdk_efi_file", "vmdk_efi_qemu",
"esxi_validation",
}
contracts = module.VM_SMOKE_REPORT_KEYS
if set(contracts) != {module.VM_CONTRACT_V1, module.VM_CONTRACT_V2}:
raise SystemExit(f"unexpected VM smoke contract versions: {sorted(contracts)!r}")
if contracts[module.VM_CONTRACT_V1] != expected_v1:
raise SystemExit(f"v1 VM smoke contract changed: {sorted(contracts[module.VM_CONTRACT_V1] ^ expected_v1)!r}")
if contracts[module.VM_CONTRACT_V2] != expected_v2:
raise SystemExit(f"v2 VM smoke contract changed: {sorted(contracts[module.VM_CONTRACT_V2] ^ expected_v2)!r}")
PYCONTRACT
grep -Fq 'identity["id"] != asset_id' "$GENERATOR"
if grep -Eiq '<input[^>]+(password|secret|token|key)' "$SITE/index.html"; then
echo 'secret-bearing configuration field found in site UI' >&2
Expand Down
79 changes: 71 additions & 8 deletions tests/test_pages_provenance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,22 +196,25 @@ def vm_evidence_payloads(version: str = "v0.1.0-rc.1", *,
"nss_validation": "false",
"exact_release_image": "true",
"serial_labels": "PASS",
"http": "PASS",
"ssh_runtime_evidence": "PASS",
"ssh_port_probe": "PASS",
"ssh": "DISABLED_BY_DEFAULT",
"authorized_keys": "ABSENT",
"dropbear_enabled": "NO",
"dropbear_running": "NO",
"http_status": "200",
"auth_challenge": "false",
"http_host_port": "18080",
"ssh_host_port": "18022",
"serial_log": str(result_dir / "serial.log"),
"ssh_probe_log": str(result_dir / "ssh-port-probe.txt"),
}
if contract_version == module.VM_CONTRACT_V1:
smoke.update({"image": names["image"], "qemu_boot": "PASS"})
smoke.update({
"image": names["image"],
"qemu_boot": "PASS",
"http": "PASS",
"ssh_runtime_evidence": "PASS",
"http_status": "200",
"auth_challenge": "false",
"http_host_port": "18080",
"ssh_host_port": "18022",
})
image_keys = ["image"]
else:
labels.update({
Expand All @@ -223,6 +226,18 @@ def vm_evidence_payloads(version: str = "v0.1.0-rc.1", *,
})
smoke.update({
"release_contract": "vm-x86_64/v2",
"https": "PASS",
"http_redirect": "PASS",
"runtime_evidence": "PASS",
"production_runtime": "PASS",
"raw_bios_persistence": "PASS",
"vmdk_import_persistence": "PASS",
"http_redirect_status": "307",
"https_status": "403",
"auth_challenge": "true",
"http_host_port": "18080",
"https_host_port": "18443",
"ssh_host_port": "18022",
"esxi_validation": "not-tested",
**{f"{variant}_file": names[variant] for variant in module.VM_VARIANTS},
**{f"{variant}_qemu": "runtime-pass" for variant in module.VM_VARIANTS},
Expand Down Expand Up @@ -688,20 +703,68 @@ def fake_attestation(gh: Path, subject: Path, bundle: Path, tag: str, digest: st
),
"v2 artifact labels accepted an unknown extra key",
)
for key in ("release_contract", "raw_bios_file", "iso_bios_qemu", "vmdk_efi_file", "esxi_validation"):
assert len(module.VM_SMOKE_REPORT_KEYS[module.VM_CONTRACT_V1]) == 23
assert len(module.VM_SMOKE_REPORT_KEYS[module.VM_CONTRACT_V2]) == 39
for key in (
"release_contract", "https", "http_redirect", "runtime_evidence", "production_runtime",
"raw_bios_persistence", "vmdk_import_persistence", "http_redirect_status", "https_status",
"auth_challenge", "https_host_port", "raw_bios_file", "iso_bios_qemu", "vmdk_efi_file",
"esxi_validation",
):
expect_verification_error(
lambda key=key: verify_vm_fixture(
vm_evidence_payloads(contract_version=2, missing_smoke=key), contract_version=2,
),
f"v2 smoke report accepted missing exact key: {key}",
)
expect_verification_error(
lambda: verify_vm_fixture(
vm_evidence_payloads(contract_version=2, smoke_updates={"UNSUPPORTED_SMOKE": "PASS"}),
contract_version=2,
),
"v2 smoke report accepted an unknown extra key",
)
for field in (
"https", "http_redirect", "runtime_evidence", "production_runtime",
"raw_bios_persistence", "vmdk_import_persistence",
):
expect_verification_error(
lambda field=field: verify_vm_fixture(
vm_evidence_payloads(contract_version=2, smoke_updates={field: "FAIL"}), contract_version=2,
),
f"v2 smoke report accepted failed production evidence: {field}",
)
expect_verification_error(
lambda: verify_vm_fixture(
vm_evidence_payloads(contract_version=2, smoke_updates={"iso_efi_qemu": "boot-pass"}),
contract_version=2,
),
"v2 smoke report accepted less than a runtime pass",
)
for updates in (
{"http_redirect_status": "200"},
{"https_status": "403", "auth_challenge": "false"},
{"https_status": "200", "auth_challenge": "true"},
):
expect_verification_error(
lambda updates=updates: verify_vm_fixture(
vm_evidence_payloads(contract_version=2, smoke_updates=updates), contract_version=2,
),
f"v2 smoke report accepted invalid HTTPS/redirect/auth evidence: {updates!r}",
)
for field, value in (
("https_host_port", "443"),
("https_host_port", "65536"),
("https_host_port", "018443"),
("https_host_port", "18080"),
("ssh_host_port", "18443"),
):
expect_verification_error(
lambda field=field, value=value: verify_vm_fixture(
vm_evidence_payloads(contract_version=2, smoke_updates={field: value}), contract_version=2,
),
f"v2 smoke report accepted invalid or reused host port: {field}={value}",
)
expect_verification_error(
lambda: verify_vm_fixture(
vm_evidence_payloads(contract_version=2, smoke_updates={"vmdk_bios_file": "other.vmdk"}),
Expand Down
Loading