diff --git a/.github/workflows/custom-build.yml b/.github/workflows/custom-build.yml index 52209a6..ce58641 100644 --- a/.github/workflows/custom-build.yml +++ b/.github/workflows/custom-build.yml @@ -97,6 +97,32 @@ jobs: "$REQUESTED_CATALOG_VERSION" \ "$REQUESTED_REQUEST_HASH" + - name: Audit custom build artifact before upload + shell: bash + env: + BUILT_ARTIFACT_DIR: ${{ steps.build.outputs.artifact_dir }} + BUILT_MANIFEST: ${{ steps.build.outputs.manifest }} + BUILT_REQUEST_HASH: ${{ steps.build.outputs.request_hash }} + run: | + set -euo pipefail + test -n "$BUILT_ARTIFACT_DIR" + test -n "$BUILT_MANIFEST" + test "$BUILT_REQUEST_HASH" = "$REQUESTED_REQUEST_HASH" + test -f "$BUILT_MANIFEST" + test ! -L "$BUILT_MANIFEST" + test -s "$BUILT_MANIFEST" + test -f "$BUILT_ARTIFACT_DIR/SHA256SUMS" + test ! -L "$BUILT_ARTIFACT_DIR/SHA256SUMS" + test -s "$BUILT_ARTIFACT_DIR/SHA256SUMS" + python3 ./scripts/custom-artifacts.py audit \ + --artifact-dir "$BUILT_ARTIFACT_DIR" \ + --manifest "$BUILT_MANIFEST" \ + --expected-commit "$GITHUB_SHA" \ + --expected-target "$REQUESTED_TARGET" \ + --expected-flavor "$REQUESTED_FLAVOR" \ + --expected-request-hash "$BUILT_REQUEST_HASH" + (cd "$BUILT_ARTIFACT_DIR" && sha256sum --check --strict SHA256SUMS) + - name: Upload custom build artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: @@ -105,3 +131,4 @@ jobs: if-no-files-found: error retention-days: 14 compression-level: 0 + include-hidden-files: true diff --git a/scripts/custom-artifacts.py b/scripts/custom-artifacts.py new file mode 100755 index 0000000..619cd4b --- /dev/null +++ b/scripts/custom-artifacts.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +"""Create and audit NexaWrt custom-build manifests and checksum inventories.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import pathlib +import platform +import re +import shutil +import stat +import subprocess +import sys +from typing import Any + +MANIFEST_NAME = "custom-build-manifest.json" +CHECKSUMS_NAME = "SHA256SUMS" +SHA256_RE = re.compile(r"[0-9a-f]{64}") +COMMIT_RE = re.compile(r"[0-9a-f]{40}") +CHECKSUM_LINE_RE = re.compile(r"([0-9a-f]{64}) (.+)") + + +class AuditError(RuntimeError): + """Raised when an artifact tree violates the custom-build contract.""" + + +def digest(path: pathlib.Path) -> str: + hasher = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def lexical_absolute(raw: str) -> pathlib.Path: + return pathlib.Path(os.path.abspath(raw)) + + +def require_directory(raw: str) -> pathlib.Path: + path = lexical_absolute(raw) + try: + file_stat = path.lstat() + except FileNotFoundError as exc: + raise AuditError(f"artifact directory is missing: {path}") from exc + if stat.S_ISLNK(file_stat.st_mode) or not stat.S_ISDIR(file_stat.st_mode): + raise AuditError(f"artifact directory is not a real directory: {path}") + return path.resolve(strict=True) + + +def require_regular_file( + path: pathlib.Path, + *, + description: str, + nonempty: bool = True, +) -> pathlib.Path: + try: + file_stat = path.lstat() + except FileNotFoundError as exc: + raise AuditError(f"{description} is missing: {path}") from exc + if stat.S_ISLNK(file_stat.st_mode) or not stat.S_ISREG(file_stat.st_mode): + raise AuditError(f"{description} must be a regular non-symlink file: {path}") + if nonempty and file_stat.st_size == 0: + raise AuditError(f"{description} must not be empty: {path}") + return path + + +def require_direct_child(raw: str, artifact_dir: pathlib.Path, expected_name: str) -> pathlib.Path: + candidate = lexical_absolute(raw) + try: + parent = candidate.parent.resolve(strict=True) + except OSError as exc: + raise AuditError(f"{expected_name} parent is unavailable: {candidate.parent}") from exc + if candidate.name != expected_name or parent != artifact_dir: + raise AuditError(f"{expected_name} path escapes or does not match the artifact directory") + return artifact_dir / expected_name + + +def safe_relative_name(path: pathlib.Path, artifact_dir: pathlib.Path) -> str: + try: + relative = path.relative_to(artifact_dir) + except ValueError as exc: + raise AuditError(f"artifact path escapes its root: {path}") from exc + name = relative.as_posix() + if ( + not relative.parts + or relative.is_absolute() + or any(part in {"", ".", ".."} for part in relative.parts) + or any(character in name for character in ("\\", "\n", "\r", "\0")) + ): + raise AuditError(f"unsafe artifact path: {name!r}") + return name + + +def inventory_tree(artifact_dir: pathlib.Path) -> dict[str, pathlib.Path]: + files: dict[str, pathlib.Path] = {} + for current_raw, directory_names, file_names in os.walk(artifact_dir, followlinks=False): + current = pathlib.Path(current_raw) + for name in list(directory_names): + path = current / name + file_stat = path.lstat() + if stat.S_ISLNK(file_stat.st_mode): + raise AuditError(f"artifact tree contains a symlink: {path}") + if not stat.S_ISDIR(file_stat.st_mode): + raise AuditError(f"artifact tree contains a non-directory entry: {path}") + safe_relative_name(path, artifact_dir) + for name in file_names: + path = current / name + file_stat = path.lstat() + if stat.S_ISLNK(file_stat.st_mode): + raise AuditError(f"artifact tree contains a symlink: {path}") + if not stat.S_ISREG(file_stat.st_mode): + raise AuditError(f"artifact tree contains a non-regular file: {path}") + relative = safe_relative_name(path, artifact_dir) + if relative in files: + raise AuditError(f"duplicate artifact path: {relative}") + files[relative] = path + return files + + +def read_json(path: pathlib.Path, description: str) -> Any: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise AuditError(f"{description} is not valid UTF-8 JSON: {path}") from exc + + +def command_version(command: str, *arguments: str) -> str | None: + executable = shutil.which(command) + if executable is None: + return None + try: + result = subprocess.run( + [executable, *arguments], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=15, + ) + except (OSError, subprocess.SubprocessError): + return None + output = result.stdout.strip().splitlines() + return output[0][:500] if output else None + + +def dpkg_versions() -> dict[str, str]: + if shutil.which("dpkg-query") is None: + return {} + packages = [ + "build-essential", + "clang", + "gcc", + "g++", + "make", + "libc6-dev", + "python3", + "git", + "rsync", + "zstd", + ] + versions: dict[str, str] = {} + for package in packages: + try: + result = subprocess.run( + ["dpkg-query", "-W", "-f=${Status}\t${Version}\n", package], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=15, + ) + except (OSError, subprocess.SubprocessError): + continue + fields = result.stdout.strip().split("\t", 1) + if result.returncode == 0 and len(fields) == 2 and fields[0] == "install ok installed": + versions[package] = fields[1][:200] + return versions + + +def os_release() -> dict[str, str]: + path = pathlib.Path("/etc/os-release") + if not path.is_file() or path.is_symlink(): + return {} + allowed = {"ID", "VERSION_ID", "PRETTY_NAME"} + values: dict[str, str] = {} + for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): + key, separator, value = line.partition("=") + if separator and key in allowed: + values[key.lower()] = value.strip().strip('"')[:300] + return values + + +def build_environment() -> dict[str, object]: + release = os_release() + tools = { + "bash": command_version("bash", "--version"), + "clang": command_version("clang", "--version"), + "gcc": command_version("gcc", "--version"), + "git": command_version("git", "--version"), + "make": command_version("make", "--version"), + "python3": command_version("python3", "--version"), + "tar": command_version("tar", "--version"), + "zstd": command_version("zstd", "--version"), + } + return { + "scope": "informational host metadata; not a reproducible-build guarantee", + "runner": { + "provider": "github-actions" if os.environ.get("GITHUB_ACTIONS") == "true" else "local", + "name": os.environ.get("RUNNER_NAME", "local"), + "image_os": os.environ.get("ImageOS") or release.get("pretty_name") or platform.system(), + "image_version": os.environ.get("ImageVersion") or release.get("version_id") or "unknown", + "os": os.environ.get("RUNNER_OS", platform.system()), + "arch": os.environ.get("RUNNER_ARCH", platform.machine()), + }, + "host": { + "platform": platform.platform(), + "os_release": release, + }, + "dpkg_packages": dpkg_versions(), + "tools": {name: version for name, version in tools.items() if version is not None}, + } + + +def validate_expected_values(commit: str, target: str, flavor: str, request_hash: str) -> None: + if COMMIT_RE.fullmatch(commit) is None: + raise AuditError("expected commit must be a full lowercase Git object ID") + if target not in {"x86_64", "xiaomi_ax9000"}: + raise AuditError(f"unsupported expected target: {target}") + if flavor not in {"official", "nss"}: + raise AuditError(f"unsupported expected flavor: {flavor}") + if flavor == "nss" and target != "xiaomi_ax9000": + raise AuditError("nss flavor is valid only for xiaomi_ax9000") + if SHA256_RE.fullmatch(request_hash) is None: + raise AuditError("expected request hash must be a full lowercase SHA256 value") + + +def validate_request( + request: Any, + *, + expected_target: str, + expected_flavor: str, + expected_request_hash: str, +) -> dict[str, Any]: + if not isinstance(request, dict): + raise AuditError("normalized request must be a JSON object") + target = request.get("target") + if not isinstance(target, dict) or target.get("id") != expected_target: + raise AuditError("normalized request target does not match the expected target") + if request.get("flavor") != expected_flavor: + raise AuditError("normalized request flavor does not match the expected flavor") + if request.get("request_hash") != expected_request_hash: + raise AuditError("normalized request hash does not match the expected request hash") + for field in ("catalog_version", "resolved_components", "packages"): + if field not in request: + raise AuditError(f"normalized request is missing {field}") + components = request["resolved_components"] + packages = request["packages"] + if not isinstance(components, list) or not components or any(not isinstance(item, str) or not item for item in components): + raise AuditError("normalized request components are empty or invalid") + if not isinstance(packages, list) or not packages or any(not isinstance(item, str) or not item for item in packages): + raise AuditError("normalized request packages are empty or invalid") + if len(packages) != len(set(packages)): + raise AuditError("normalized request packages contain duplicates") + return request + + +def actual_packages(artifact_dir: pathlib.Path, target: str) -> list[str]: + if target == "x86_64": + record = require_regular_file( + artifact_dir / "custom-imagebuilder-packages.json", + description="final ImageBuilder package record", + ) + packages = read_json(record, "final ImageBuilder package record") + else: + config = require_regular_file(artifact_dir / "config.buildinfo", description="config.buildinfo") + prefix = "CONFIG_PACKAGE_" + suffix = "=y" + packages = sorted( + { + line[len(prefix) : -len(suffix)] + for line in config.read_text(encoding="utf-8").splitlines() + if line.startswith(prefix) and line.endswith(suffix) + } + ) + if ( + not isinstance(packages, list) + or not packages + or any(not isinstance(package, str) or not package for package in packages) + or len(packages) != len(set(packages)) + ): + raise AuditError("final package set is empty, duplicated, or invalid") + return packages + + +def write_atomic(path: pathlib.Path, content: str, description: str) -> None: + if os.path.lexists(path): + require_regular_file(path, description=description, nonempty=False) + temporary = path.with_name(f".{path.name}.tmp") + if os.path.lexists(temporary): + raise AuditError(f"refusing pre-existing temporary path: {temporary}") + try: + with temporary.open("x", encoding="utf-8", newline="\n") as stream: + stream.write(content) + os.replace(temporary, path) + finally: + try: + temporary.unlink() + except FileNotFoundError: + pass + + +def parse_checksums(path: pathlib.Path, artifact_dir: pathlib.Path) -> dict[str, str]: + checksums: dict[str, str] = {} + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + match = CHECKSUM_LINE_RE.fullmatch(line) + if match is None: + raise AuditError(f"invalid SHA256SUMS line {line_number}") + expected_digest, relative_raw = match.groups() + relative = pathlib.PurePosixPath(relative_raw) + if ( + relative.is_absolute() + or not relative.parts + or any(part in {"", ".", ".."} for part in relative.parts) + or any(character in relative_raw for character in ("\\", "\n", "\r", "\0")) + ): + raise AuditError(f"unsafe SHA256SUMS path on line {line_number}: {relative_raw!r}") + normalized = relative.as_posix() + if normalized in checksums: + raise AuditError(f"duplicate SHA256SUMS path: {normalized}") + candidate = artifact_dir.joinpath(*relative.parts) + try: + candidate.relative_to(artifact_dir) + except ValueError as exc: + raise AuditError(f"SHA256SUMS path escapes artifact directory: {normalized}") from exc + require_regular_file(candidate, description=f"checksummed artifact {normalized}", nonempty=False) + checksums[normalized] = expected_digest + if not checksums: + raise AuditError("SHA256SUMS contains no entries") + return checksums + + +def validate_manifest_artifacts(manifest: dict[str, Any], files: dict[str, pathlib.Path]) -> None: + records = manifest.get("artifacts") + if not isinstance(records, list) or not records: + raise AuditError("manifest artifacts list is empty or invalid") + expected_names = set(files) - {MANIFEST_NAME, CHECKSUMS_NAME} + seen: set[str] = set() + for record in records: + if not isinstance(record, dict): + raise AuditError("manifest artifact record is not an object") + name = record.get("name") + checksum = record.get("sha256") + size = record.get("size") + if not isinstance(name, str) or name not in expected_names or name in seen: + raise AuditError(f"manifest artifact name is missing, duplicated, or unexpected: {name!r}") + if not isinstance(checksum, str) or SHA256_RE.fullmatch(checksum) is None: + raise AuditError(f"manifest artifact checksum is invalid: {name}") + if not isinstance(size, int) or size < 0: + raise AuditError(f"manifest artifact size is invalid: {name}") + path = files[name] + if path.stat().st_size != size or digest(path) != checksum: + raise AuditError(f"manifest artifact metadata does not match: {name}") + seen.add(name) + if seen != expected_names: + raise AuditError(f"manifest artifact inventory mismatch; missing={sorted(expected_names - seen)}") + + +def audit( + *, + artifact_dir_raw: str, + manifest_raw: str, + expected_commit: str, + expected_target: str, + expected_flavor: str, + expected_request_hash: str, +) -> None: + validate_expected_values(expected_commit, expected_target, expected_flavor, expected_request_hash) + artifact_dir = require_directory(artifact_dir_raw) + manifest_path = require_direct_child(manifest_raw, artifact_dir, MANIFEST_NAME) + checksums_path = artifact_dir / CHECKSUMS_NAME + require_regular_file(manifest_path, description="custom build manifest") + require_regular_file(checksums_path, description="SHA256SUMS") + files = inventory_tree(artifact_dir) + manifest = read_json(manifest_path, "custom build manifest") + if not isinstance(manifest, dict): + raise AuditError("custom build manifest must be a JSON object") + expected_fields = { + "request_hash": expected_request_hash, + "target": expected_target, + "flavor": expected_flavor, + "commit": expected_commit, + } + for field, expected in expected_fields.items(): + if manifest.get(field) != expected: + raise AuditError(f"manifest {field} does not match the expected value") + validate_manifest_artifacts(manifest, files) + checksums = parse_checksums(checksums_path, artifact_dir) + expected_checksum_names = set(files) - {CHECKSUMS_NAME} + if set(checksums) != expected_checksum_names: + missing = sorted(expected_checksum_names - set(checksums)) + extra = sorted(set(checksums) - expected_checksum_names) + raise AuditError(f"SHA256SUMS inventory mismatch; missing={missing}, extra={extra}") + if MANIFEST_NAME not in checksums: + raise AuditError("SHA256SUMS does not cover custom-build-manifest.json") + for name, expected_digest in checksums.items(): + if digest(files[name]) != expected_digest: + raise AuditError(f"SHA256SUMS digest mismatch: {name}") + + +def finalize(args: argparse.Namespace) -> None: + validate_expected_values(args.expected_commit, args.expected_target, args.expected_flavor, args.expected_request_hash) + artifact_dir = require_directory(args.artifact_dir) + request_path = lexical_absolute(args.request_file) + require_regular_file(request_path, description="normalized request") + request = validate_request( + read_json(request_path, "normalized request"), + expected_target=args.expected_target, + expected_flavor=args.expected_flavor, + expected_request_hash=args.expected_request_hash, + ) + manifest_path = artifact_dir / MANIFEST_NAME + checksums_path = artifact_dir / CHECKSUMS_NAME + request_artifact_path = artifact_dir / "custom-request.json" + inventory_tree(artifact_dir) + write_atomic( + request_artifact_path, + json.dumps(request, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + "custom request artifact", + ) + packages = actual_packages(artifact_dir, args.expected_target) + missing_resolved = sorted(set(request["packages"]) - set(packages)) + if missing_resolved: + raise AuditError(f"final package set omitted resolved packages: {missing_resolved}") + + files_before_manifest = inventory_tree(artifact_dir) + artifacts = [ + {"name": name, "sha256": digest(path), "size": path.stat().st_size} + for name, path in sorted(files_before_manifest.items()) + if name not in {MANIFEST_NAME, CHECKSUMS_NAME} + ] + if not artifacts: + raise AuditError("artifact directory is empty") + manifest = { + "schema_version": 1, + "project": "NexaWrt", + "commit": args.expected_commit, + "request_hash": args.expected_request_hash, + "catalog_version": request["catalog_version"], + "target": args.expected_target, + "flavor": args.expected_flavor, + "build_environment": build_environment(), + "resolved_components": request["resolved_components"], + "resolved_packages": request["packages"], + "packages": packages, + "artifacts": artifacts, + } + write_atomic( + manifest_path, + json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + "custom build manifest", + ) + + files_for_checksums = inventory_tree(artifact_dir) + checksum_lines = [ + f"{digest(path)} {name}" + for name, path in sorted(files_for_checksums.items()) + if name != CHECKSUMS_NAME + ] + write_atomic(checksums_path, "\n".join(checksum_lines) + "\n", "SHA256SUMS") + audit( + artifact_dir_raw=str(artifact_dir), + manifest_raw=str(manifest_path), + expected_commit=args.expected_commit, + expected_target=args.expected_target, + expected_flavor=args.expected_flavor, + expected_request_hash=args.expected_request_hash, + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + for command in ("finalize", "audit"): + subparser = subparsers.add_parser(command) + subparser.add_argument("--artifact-dir", required=True) + subparser.add_argument("--expected-commit", required=True) + subparser.add_argument("--expected-target", required=True) + subparser.add_argument("--expected-flavor", required=True) + subparser.add_argument("--expected-request-hash", required=True) + if command == "finalize": + subparser.add_argument("--request-file", required=True) + else: + subparser.add_argument("--manifest", required=True) + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + try: + if args.command == "finalize": + finalize(args) + else: + audit( + artifact_dir_raw=args.artifact_dir, + manifest_raw=args.manifest, + expected_commit=args.expected_commit, + expected_target=args.expected_target, + expected_flavor=args.expected_flavor, + expected_request_hash=args.expected_request_hash, + ) + except (AuditError, OSError, UnicodeError, ValueError, KeyError) as exc: + print(f"custom artifact audit failed: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/custom-build.sh b/scripts/custom-build.sh index 087dc2e..cb15591 100755 --- a/scripts/custom-build.sh +++ b/scripts/custom-build.sh @@ -3,6 +3,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" RESOLVER="$ROOT_DIR/scripts/resolve-components.py" +CUSTOM_ARTIFACTS="$ROOT_DIR/scripts/custom-artifacts.py" fail() { printf 'custom build refused: %s\n' "$*" >&2 @@ -48,6 +49,7 @@ if [[ -n "$COMPONENTS" ]]; then fail "non-empty components must contain only comma-separated catalog component IDs" fi [[ -f "$RESOLVER" && ! -L "$RESOLVER" ]] || fail "component resolver is missing or unsafe" +[[ -f "$CUSTOM_ARTIFACTS" && ! -L "$CUSTOM_ARTIFACTS" ]] || fail "custom artifact auditor is missing or unsafe" RESOLVER_ARGS=(--target "$TARGET" --flavor "$FLAVOR") if [[ -n "$COMPONENTS" ]]; then @@ -182,198 +184,34 @@ case "$TARGET" in esac [[ -d "$ARTIFACT_DIR" && ! -L "$ARTIFACT_DIR" ]] || fail "build did not produce a safe artifact directory" -cp "$REQUEST_FILE" "$ARTIFACT_DIR/custom-request.json" MANIFEST_PATH="$ARTIFACT_DIR/custom-build-manifest.json" -python3 - "$ARTIFACT_DIR" "$REQUEST_FILE" "$PROJECT_COMMIT" "$FLAVOR" <<'PY' || - fail "unable to create custom build manifest" -import hashlib -import json -import os -import pathlib -import platform -import shutil -import stat -import subprocess -import sys - -artifact_dir = pathlib.Path(sys.argv[1]).resolve(strict=True) -request_path = pathlib.Path(sys.argv[2]).resolve(strict=True) -project_commit = sys.argv[3] -flavor = sys.argv[4] -request = json.loads(request_path.read_text(encoding="utf-8")) -manifest_path = artifact_dir / "custom-build-manifest.json" -checksums_path = artifact_dir / "SHA256SUMS" - - -def digest(path: pathlib.Path) -> str: - hasher = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - hasher.update(chunk) - return hasher.hexdigest() - - -def command_version(command: str, *arguments: str) -> str | None: - executable = shutil.which(command) - if executable is None: - return None - try: - result = subprocess.run( - [executable, *arguments], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=15, - ) - except (OSError, subprocess.SubprocessError): - return None - output = result.stdout.strip().splitlines() - return output[0][:500] if output else None - - -def dpkg_versions() -> dict[str, str]: - if shutil.which("dpkg-query") is None: - return {} - packages = [ - "build-essential", "clang", "gcc", "g++", "make", "libc6-dev", - "python3", "git", "rsync", "zstd", - ] - versions: dict[str, str] = {} - for package in packages: - try: - result = subprocess.run( - ["dpkg-query", "-W", "-f=${Status}\t${Version}\n", package], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - timeout=15, - ) - except (OSError, subprocess.SubprocessError): - continue - fields = result.stdout.strip().split("\t", 1) - if result.returncode == 0 and len(fields) == 2 and fields[0] == "install ok installed": - versions[package] = fields[1][:200] - return versions - - -def os_release() -> dict[str, str]: - path = pathlib.Path("/etc/os-release") - if not path.is_file() or path.is_symlink(): - return {} - allowed = {"ID", "VERSION_ID", "PRETTY_NAME"} - values: dict[str, str] = {} - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - key, separator, value = line.partition("=") - if separator and key in allowed: - values[key.lower()] = value.strip().strip('"')[:300] - return values - - -def build_environment() -> dict[str, object]: - release = os_release() - tools = { - "bash": command_version("bash", "--version"), - "clang": command_version("clang", "--version"), - "gcc": command_version("gcc", "--version"), - "git": command_version("git", "--version"), - "make": command_version("make", "--version"), - "python3": command_version("python3", "--version"), - "tar": command_version("tar", "--version"), - "zstd": command_version("zstd", "--version"), - } - return { - "scope": "informational host metadata; not a reproducible-build guarantee", - "runner": { - "provider": "github-actions" if os.environ.get("GITHUB_ACTIONS") == "true" else "local", - "name": os.environ.get("RUNNER_NAME", "local"), - "image_os": os.environ.get("ImageOS") or release.get("pretty_name") or platform.system(), - "image_version": os.environ.get("ImageVersion") or release.get("version_id") or "unknown", - "os": os.environ.get("RUNNER_OS", platform.system()), - "arch": os.environ.get("RUNNER_ARCH", platform.machine()), - }, - "host": { - "platform": platform.platform(), - "os_release": release, - }, - "dpkg_packages": dpkg_versions(), - "tools": {name: version for name, version in tools.items() if version is not None}, - } - - -artifacts = [] -for path in sorted(artifact_dir.rglob("*")): - if path == manifest_path or path == checksums_path: - continue - file_stat = path.lstat() - if stat.S_ISLNK(file_stat.st_mode): - raise SystemExit(f"artifact tree contains a symlink: {path}") - if not stat.S_ISREG(file_stat.st_mode): - continue - relative = path.relative_to(artifact_dir).as_posix() - artifacts.append({"name": relative, "sha256": digest(path), "size": file_stat.st_size}) -if not artifacts: - raise SystemExit("artifact directory is empty") - -target = request["target"]["id"] -if request.get("flavor") != flavor: - raise SystemExit("manifest flavor does not match the normalized request") -resolved_packages = request["packages"] -if target == "x86_64": - package_record = artifact_dir / "custom-imagebuilder-packages.json" - if not package_record.is_file() or package_record.is_symlink(): - raise SystemExit("x86 custom build is missing its final ImageBuilder package record") - actual_packages = json.loads(package_record.read_text(encoding="utf-8")) -else: - config_buildinfo = artifact_dir / "config.buildinfo" - if not config_buildinfo.is_file() or config_buildinfo.is_symlink(): - raise SystemExit("AX9000 custom build is missing config.buildinfo") - prefix = "CONFIG_PACKAGE_" - suffix = "=y" - actual_packages = sorted({ - line[len(prefix):-len(suffix)] - for line in config_buildinfo.read_text(encoding="utf-8").splitlines() - if line.startswith(prefix) and line.endswith(suffix) - }) -if ( - not isinstance(actual_packages, list) - or not actual_packages - or any(not isinstance(package, str) or not package for package in actual_packages) - or len(actual_packages) != len(set(actual_packages)) -): - raise SystemExit("final package set is empty, duplicated, or invalid") -missing_resolved = sorted(set(resolved_packages) - set(actual_packages)) -if missing_resolved: - raise SystemExit(f"final package set omitted resolved packages: {missing_resolved}") - -manifest = { - "schema_version": 1, - "project": "NexaWrt", - "commit": project_commit, - "request_hash": request["request_hash"], - "catalog_version": request["catalog_version"], - "target": target, - "flavor": flavor, - "build_environment": build_environment(), - "resolved_components": request["resolved_components"], - "resolved_packages": resolved_packages, - "packages": actual_packages, - "artifacts": artifacts, -} -temporary = manifest_path.with_suffix(".json.tmp") -temporary.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8") -os.replace(temporary, manifest_path) - -checksum_lines = [] -for path in sorted(artifact_dir.rglob("*")): - if path == checksums_path or not path.is_file(): - continue - if path.is_symlink(): - raise SystemExit(f"artifact tree contains a symlink: {path}") - checksum_lines.append(f"{digest(path)} {path.relative_to(artifact_dir).as_posix()}") -checksums_path.write_text("\n".join(checksum_lines) + "\n", encoding="utf-8") -PY +CHECKSUMS_PATH="$ARTIFACT_DIR/SHA256SUMS" +if ! python3 "$CUSTOM_ARTIFACTS" finalize \ + --artifact-dir "$ARTIFACT_DIR" \ + --request-file "$REQUEST_FILE" \ + --expected-commit "$PROJECT_COMMIT" \ + --expected-target "$TARGET" \ + --expected-flavor "$FLAVOR" \ + --expected-request-hash "$REQUEST_HASH"; then + fail "unable to create and audit custom build manifest" +fi +[[ -f "$MANIFEST_PATH" && ! -L "$MANIFEST_PATH" && -s "$MANIFEST_PATH" ]] || + fail "custom build manifest is missing, empty, symlinked, or not a regular file" +[[ -f "$CHECKSUMS_PATH" && ! -L "$CHECKSUMS_PATH" && -s "$CHECKSUMS_PATH" ]] || + fail "SHA256SUMS is missing, empty, symlinked, or not a regular file" +if ! python3 "$CUSTOM_ARTIFACTS" audit \ + --artifact-dir "$ARTIFACT_DIR" \ + --manifest "$MANIFEST_PATH" \ + --expected-commit "$PROJECT_COMMIT" \ + --expected-target "$TARGET" \ + --expected-flavor "$FLAVOR" \ + --expected-request-hash "$REQUEST_HASH"; then + fail "custom build artifact postcondition audit failed" +fi +command -v sha256sum >/dev/null 2>&1 || fail "sha256sum is required to verify custom build artifacts" +if ! (cd "$ARTIFACT_DIR" && sha256sum --check --strict SHA256SUMS); then + fail "custom build SHA256SUMS verification failed" +fi ARTIFACT_NAME="NexaWrt-custom-${TARGET}-${FLAVOR}-${REQUEST_HASH:0:12}" printf 'Custom build artifacts: %s\n' "$ARTIFACT_DIR" diff --git a/scripts/test-vm-release.sh b/scripts/test-vm-release.sh index 9966572..69256df 100755 --- a/scripts/test-vm-release.sh +++ b/scripts/test-vm-release.sh @@ -164,6 +164,22 @@ raise SystemExit(0 if all(value in text for value in required) else 1) PY } +serial_runtime_failure_reason() { + local serial_log="$1" + [[ -s "$serial_log" ]] || return 1 + python3 - "$serial_log" <<'PY_FAILURE' +import pathlib +import re +import sys + +text = pathlib.Path(sys.argv[1]).read_bytes().decode("utf-8", errors="replace").replace("\r", "") +reasons = re.findall(r"(?m)^NEXAWRT_VM_PRODUCTION_RUNTIME_FAILED=([a-z0-9_]+)$", text) +if not reasons: + raise SystemExit(1) +print(reasons[-1]) +PY_FAILURE +} + probe_luci_https() { set +e current_https_status="$(curl --silent --show-error --insecure \ @@ -411,6 +427,11 @@ run_variant() { fail "QEMU exited before $variant runtime validation completed" return 1 fi + if runtime_failure_reason="$(serial_runtime_failure_reason "$CURRENT_SERIAL_LOG")"; then + dump_variant_diagnostics "$variant" + fail "$variant production runtime gate reported failure: $runtime_failure_reason" + return 1 + fi if [[ "$serial_labels_result" != PASS ]] && serial_has_release_labels "$CURRENT_SERIAL_LOG"; then serial_labels_result=PASS fi diff --git a/tests/test_custom_artifacts.py b/tests/test_custom_artifacts.py new file mode 100755 index 0000000..ab24992 --- /dev/null +++ b/tests/test_custom_artifacts.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Small regression fixtures for custom artifact finalization and auditing.""" + +from __future__ import annotations + +import hashlib +import json +import os +import pathlib +import re +import shutil +import subprocess +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +TOOL = ROOT / "scripts" / "custom-artifacts.py" +CUSTOM_BUILD = ROOT / "scripts" / "custom-build.sh" +COMMIT = "1" * 40 +REQUEST_HASH = "a" * 64 + + +class CustomArtifactTests(unittest.TestCase): + def setUp(self) -> None: + self.temporary = tempfile.TemporaryDirectory(prefix="nexawrt-custom-artifacts-") + self.root = pathlib.Path(self.temporary.name) + self.artifact_dir = self.root / "artifacts" + self.artifact_dir.mkdir() + (self.artifact_dir / "firmware.img.gz").write_bytes(b"fixture firmware\n") + (self.artifact_dir / "build.log").write_text("fixture build\n", encoding="utf-8") + (self.artifact_dir / "custom-imagebuilder-packages.json").write_text( + json.dumps(["base-files", "luci"]) + "\n", + encoding="utf-8", + ) + self.request_path = self.root / "request.json" + self.request_path.write_text( + json.dumps( + { + "catalog_version": "2026.07.20", + "flavor": "official", + "packages": ["base-files", "luci"], + "request_hash": REQUEST_HASH, + "resolved_components": ["web-ui"], + "target": {"id": "x86_64"}, + } + ) + + "\n", + encoding="utf-8", + ) + + def tearDown(self) -> None: + self.temporary.cleanup() + + def command(self, operation: str, **overrides: str) -> list[str]: + values = { + "artifact_dir": str(self.artifact_dir), + "expected_commit": COMMIT, + "expected_target": "x86_64", + "expected_flavor": "official", + "expected_request_hash": REQUEST_HASH, + } + values.update(overrides) + command = ["python3", str(TOOL), operation] + for key, value in values.items(): + command.extend([f"--{key.replace('_', '-')}", value]) + if operation == "finalize": + command.extend(["--request-file", str(self.request_path)]) + else: + command.extend(["--manifest", str(self.artifact_dir / "custom-build-manifest.json")]) + return command + + def finalize(self) -> None: + subprocess.run(self.command("finalize"), check=True, capture_output=True, text=True) + + def assert_audit_rejected(self, **overrides: str) -> None: + result = subprocess.run(self.command("audit", **overrides), check=False, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_finalize_and_audit_small_fixture(self) -> None: + self.finalize() + manifest_path = self.artifact_dir / "custom-build-manifest.json" + checksums_path = self.artifact_dir / "SHA256SUMS" + self.assertTrue(manifest_path.is_file() and not manifest_path.is_symlink() and manifest_path.stat().st_size > 0) + self.assertTrue(checksums_path.is_file() and not checksums_path.is_symlink() and checksums_path.stat().st_size > 0) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + self.assertEqual(manifest["request_hash"], REQUEST_HASH) + self.assertEqual(manifest["target"], "x86_64") + self.assertEqual(manifest["flavor"], "official") + self.assertEqual(manifest["commit"], COMMIT) + self.assertIn("custom-build-manifest.json", checksums_path.read_text(encoding="utf-8")) + subprocess.run(self.command("audit"), check=True, capture_output=True, text=True) + subprocess.run( + ["sha256sum", "--check", "--strict", "SHA256SUMS"], + cwd=self.artifact_dir, + check=True, + capture_output=True, + text=True, + ) + + def test_missing_or_symlinked_manifest_is_rejected(self) -> None: + self.finalize() + manifest_path = self.artifact_dir / "custom-build-manifest.json" + manifest_path.unlink() + self.assert_audit_rejected() + outside = self.root / "outside-manifest.json" + outside.write_text("{}\n", encoding="utf-8") + manifest_path.symlink_to(outside) + self.assert_audit_rejected() + + def test_checksum_traversal_and_missing_manifest_entry_are_rejected(self) -> None: + self.finalize() + checksums_path = self.artifact_dir / "SHA256SUMS" + lines = checksums_path.read_text(encoding="utf-8").splitlines() + checksums_path.write_text( + "\n".join(line for line in lines if not line.endswith(" custom-build-manifest.json")) + "\n", + encoding="utf-8", + ) + self.assert_audit_rejected() + outside = self.root / "outside.bin" + outside.write_bytes(b"outside\n") + outside_hash = hashlib.sha256(outside.read_bytes()).hexdigest() + checksums_path.write_text(f"{outside_hash} ../outside.bin\n", encoding="utf-8") + self.assert_audit_rejected() + + def test_corruption_and_identity_mismatch_are_rejected(self) -> None: + self.finalize() + self.assert_audit_rejected(expected_request_hash="b" * 64) + self.assert_audit_rejected(expected_commit="2" * 40) + with (self.artifact_dir / "firmware.img.gz").open("ab") as stream: + stream.write(b"corruption\n") + self.assert_audit_rejected() + + def test_generator_failure_is_not_hidden_by_heredoc_control_flow(self) -> None: + source = CUSTOM_BUILD.read_text(encoding="utf-8") + self.assertIsNone( + re.search(r"<<\s*['\"]?[A-Za-z0-9_]+['\"]?\s*\|\|\s*\n\s*fail\b", source), + "a fail handler after a heredoc operator is consumed as heredoc input", + ) + self.assertIn('if ! python3 "$CUSTOM_ARTIFACTS" finalize', source) + self.request_path.write_text("{not-json\n", encoding="utf-8") + result = subprocess.run(self.command("finalize"), check=False, capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertFalse((self.artifact_dir / "custom-build-manifest.json").exists()) + self.assertFalse((self.artifact_dir / "SHA256SUMS").exists()) + + def test_manifest_argument_cannot_escape_artifact_directory(self) -> None: + self.finalize() + outside = self.root / "outside-manifest.json" + shutil.copyfile(self.artifact_dir / "custom-build-manifest.json", outside) + result = subprocess.run( + self.command("audit")[:-2] + ["--manifest", str(outside)], + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_custom_build_policy.sh b/tests/test_custom_build_policy.sh index 73dab93..55ebdcb 100755 --- a/tests/test_custom_build_policy.sh +++ b/tests/test_custom_build_policy.sh @@ -5,6 +5,7 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" WORKFLOW="$ROOT_DIR/.github/workflows/custom-build.yml" CUSTOM_BUILD="$ROOT_DIR/scripts/custom-build.sh" +CUSTOM_ARTIFACTS="$ROOT_DIR/scripts/custom-artifacts.py" PREPARE="$ROOT_DIR/scripts/prepare.sh" VM_BUILD="$ROOT_DIR/scripts/build-vm-image.sh" @@ -19,10 +20,12 @@ require_fixed() { grep -Fq -- "$text" "$file" || fail "missing required policy text in ${file#"$ROOT_DIR"/}: $text" } -for file in "$WORKFLOW" "$CUSTOM_BUILD" "$PREPARE" "$VM_BUILD"; do +for file in "$WORKFLOW" "$CUSTOM_BUILD" "$CUSTOM_ARTIFACTS" "$PREPARE" "$VM_BUILD"; do [[ -f "$file" && ! -L "$file" ]] || fail "required file is missing or symlinked: ${file#"$ROOT_DIR"/}" done [[ -x "$CUSTOM_BUILD" ]] || fail "scripts/custom-build.sh is not executable" +[[ -x "$CUSTOM_ARTIFACTS" ]] || fail "scripts/custom-artifacts.py is not executable" +python3 -m py_compile "$CUSTOM_ARTIFACTS" bash -n "$CUSTOM_BUILD" bash -n "$PREPARE" bash -n "$VM_BUILD" @@ -71,8 +74,15 @@ require_fixed "$WORKFLOW" 'REQUESTED_CATALOG_VERSION: ${{ inputs.catalog_version require_fixed "$WORKFLOW" 'REQUESTED_REQUEST_HASH: ${{ inputs.request_hash }}' require_fixed "$WORKFLOW" '[[ "$REQUESTED_REQUEST_HASH" =~ ^[0-9a-f]{64}$ ]]' require_fixed "$WORKFLOW" './scripts/custom-build.sh' +require_fixed "$WORKFLOW" 'Audit custom build artifact before upload' +require_fixed "$WORKFLOW" 'BUILT_ARTIFACT_DIR: ${{ steps.build.outputs.artifact_dir }}' +require_fixed "$WORKFLOW" 'BUILT_MANIFEST: ${{ steps.build.outputs.manifest }}' +require_fixed "$WORKFLOW" 'BUILT_REQUEST_HASH: ${{ steps.build.outputs.request_hash }}' +require_fixed "$WORKFLOW" './scripts/custom-artifacts.py audit' +require_fixed "$WORKFLOW" 'sha256sum --check --strict SHA256SUMS' require_fixed "$WORKFLOW" 'if-no-files-found: error' require_fixed "$WORKFLOW" 'compression-level: 0' +require_fixed "$WORKFLOW" 'include-hidden-files: true' require_fixed "$CUSTOM_BUILD" 'scripts/resolve-components.py' require_fixed "$CUSTOM_BUILD" 'An empty component' @@ -89,12 +99,19 @@ require_fixed "$CUSTOM_BUILD" '"$ROOT_DIR/scripts/build-vm-image.sh" x86-64 cust require_fixed "$CUSTOM_BUILD" 'NEXAWRT_COMPONENT_TARGET=xiaomi_ax9000' require_fixed "$CUSTOM_BUILD" 'DIST_DIR_OVERRIDE="$ARTIFACT_DIR"' require_fixed "$CUSTOM_BUILD" 'DIST_NSS_DIR_OVERRIDE="$ARTIFACT_DIR"' +require_fixed "$CUSTOM_BUILD" 'if ! python3 "$CUSTOM_ARTIFACTS" finalize' +require_fixed "$CUSTOM_BUILD" 'if ! python3 "$CUSTOM_ARTIFACTS" audit' +require_fixed "$CUSTOM_BUILD" '[[ -f "$MANIFEST_PATH" && ! -L "$MANIFEST_PATH" && -s "$MANIFEST_PATH" ]]' +require_fixed "$CUSTOM_BUILD" '[[ -f "$CHECKSUMS_PATH" && ! -L "$CHECKSUMS_PATH" && -s "$CHECKSUMS_PATH" ]]' +require_fixed "$CUSTOM_BUILD" 'sha256sum --check --strict SHA256SUMS' for manifest_field in commit catalog_version request_hash resolved_components resolved_packages packages sha256 build_environment runner image_os image_version os arch dpkg_packages tools scope; do - require_fixed "$CUSTOM_BUILD" "\"$manifest_field\"" + require_fixed "$CUSTOM_ARTIFACTS" "\"$manifest_field\"" +done +for file in "$CUSTOM_BUILD" "$CUSTOM_ARTIFACTS"; do + if grep -Eq '(^|[^[:alnum:]_])(eval|bash[[:space:]]+-c|sh[[:space:]]+-c)([^[:alnum:]_]|$)' "$file"; then + fail "custom artifact chain contains a shell evaluation primitive in ${file#"$ROOT_DIR"/}" + fi done -if grep -Eq '(^|[^[:alnum:]_])(eval|bash[[:space:]]+-c|sh[[:space:]]+-c)([^[:alnum:]_]|$)' "$CUSTOM_BUILD"; then - fail "custom build script contains a shell evaluation primitive" -fi require_fixed "$PREPARE" 'NEXAWRT_COMPONENT_TARGET must be xiaomi_ax9000' require_fixed "$PREPARE" 'NEXAWRT_COMPONENT_FLAVOR must be official or nss' @@ -125,7 +142,7 @@ require_fixed "$VM_BUILD" ' dropbear' require_fixed "$VM_BUILD" 'VM_PACKAGES+=("$custom_package")' require_fixed "$VM_BUILD" 'custom-imagebuilder-packages.json' require_fixed "$VM_BUILD" '"PACKAGES=$PACKAGE_LIST"' -for file in "$WORKFLOW" "$CUSTOM_BUILD" "$PREPARE" "$VM_BUILD"; do +for file in "$WORKFLOW" "$CUSTOM_BUILD" "$CUSTOM_ARTIFACTS" "$PREPARE" "$VM_BUILD"; do if grep -Fq -- '--no-defaults' "$file"; then fail "custom build chain exposes --no-defaults in ${file#"$ROOT_DIR"/}" fi diff --git a/tests/test_static.sh b/tests/test_static.sh index 1a7ca0d..f8ea7cd 100755 --- a/tests/test_static.sh +++ b/tests/test_static.sh @@ -7,6 +7,7 @@ unset GITHUB_ACTIONS GITHUB_REF GITHUB_REPOSITORY GITHUB_RUN_ATTEMPT \ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" python3 "$ROOT_DIR/tests/test_component_catalog.py" "$ROOT_DIR/tests/test_custom_build_policy.sh" +python3 "$ROOT_DIR/tests/test_custom_artifacts.py" # Always exercise both static policies. If a source path is supplied, validate # it using the caller-selected flavor after the repository-only checks. @@ -63,6 +64,7 @@ echo 'flavor policy: official default, isolated nss work tree, and official-only "$ROOT_DIR/tests/test_browser_build_policy.sh" "$ROOT_DIR/tests/test_vm_policy.sh" "$ROOT_DIR/tests/test_vm_release_policy.sh" +"$ROOT_DIR/tests/test_vm_runtime_gate.sh" # Includes strict identity-matched, idempotent stable-draft recovery coverage. "$ROOT_DIR/tests/test_vm_promotion_policy.sh" "$ROOT_DIR/tests/test_pages_policy.sh" diff --git a/tests/test_vm_runtime_gate.sh b/tests/test_vm_runtime_gate.sh new file mode 100755 index 0000000..f11ab44 --- /dev/null +++ b/tests/test_vm_runtime_gate.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GATE_SCRIPT="$ROOT_DIR/vm-files-release/etc/init.d/nexawrt-runtime-gate" +DEFAULTS_SCRIPT="$ROOT_DIR/vm-files-release/etc/uci-defaults/10-vm-release" +VM_TEST_SCRIPT="$ROOT_DIR/scripts/test-vm-release.sh" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +fail() { + printf 'test_vm_runtime_gate: %s\n' "$*" >&2 + exit 1 +} + +ETC_ROOT="$TMP_DIR/etc" +SYS_ROOT="$TMP_DIR/sys" +FAKE_BIN="$TMP_DIR/bin" +EVIDENCE_DIR="$TMP_DIR/evidence" +CONSOLE="$TMP_DIR/console" +mkdir -p "$ETC_ROOT/init.d" "$SYS_ROOT/class/net/eth1" "$FAKE_BIN" "$EVIDENCE_DIR" +printf '%s\n' 'root:x:20000:0:99999:7:::' >"$ETC_ROOT/shadow" +printf '%s\n' '0123456789abcdef0123456789abcdef' >"$ETC_ROOT/nexawrt-install-id" + +cat >"$ETC_ROOT/init.d/firewall" <<'SH' +#!/bin/sh +[ "${1:-}" = enabled ] +SH +cat >"$ETC_ROOT/init.d/dropbear" <<'SH' +#!/bin/sh +exit 1 +SH +cat >"$FAKE_BIN/nft" <<'SH' +#!/bin/sh +[ "${NEXAWRT_TEST_NFT:-pass}" = pass ] +SH +cat >"$FAKE_BIN/ip" <<'SH' +#!/bin/sh +printf '%s\n' '2: eth0: ' ' inet 192.168.8.1/24 scope global eth0' +SH +cat >"$FAKE_BIN/uci" <<'SH' +#!/bin/sh +key="${3:-}" +case "$key" in + uhttpd.main.redirect_https) printf '%s\n' 1 ;; + network.lan.device) printf '%s\n' eth0 ;; + network.lan.proto) printf '%s\n' static ;; + dhcp.lan.ignore) printf '%s\n' 0 ;; + network.wan.device) printf '%s\n' eth1 ;; + network.wan.proto) printf '%s\n' dhcp ;; + *) exit 1 ;; +esac +SH +cat >"$FAKE_BIN/pidof" <<'SH' +#!/bin/sh +exit 1 +SH +cat >"$FAKE_BIN/sleep" <<'SH' +#!/bin/sh +exit 0 +SH +chmod 0755 "$ETC_ROOT/init.d/firewall" "$ETC_ROOT/init.d/dropbear" "$FAKE_BIN"/* + +run_gate() { + PATH="$FAKE_BIN:$PATH" \ + NEXAWRT_RUNTIME_CONSOLE="$CONSOLE" \ + NEXAWRT_RUNTIME_ETC_ROOT="$ETC_ROOT" \ + NEXAWRT_RUNTIME_SYS_ROOT="$SYS_ROOT" \ + NEXAWRT_RUNTIME_EVIDENCE_DIR="$EVIDENCE_DIR" \ + NEXAWRT_RUNTIME_MAX_ATTEMPTS=2 \ + NEXAWRT_RUNTIME_RETRY_DELAY=0 \ + sh -c '. "$1"; run' runtime-test "$GATE_SCRIPT" +} + +run_gate +for expected in \ + NEXAWRT_VM_PRODUCTION_RUNTIME_V1_BEGIN \ + network_mode=router \ + lan_device=eth0 \ + lan_address=192.168.8.1 \ + wan_device=eth1 \ + https_redirect=PASS \ + firewall_enabled=PASS \ + nftables_fw4=PASS \ + lan_dhcp=PASS \ + root_password=UNIQUE_FIRST_BOOT_VALUE \ + ssh_disabled=PASS \ + NEXAWRT_VM_PRODUCTION_RUNTIME_V1_END; do + grep -Fxq "$expected" "$CONSOLE" || fail "success evidence missing: $expected" +done +cmp -s "$CONSOLE" "$EVIDENCE_DIR/production-runtime.evidence" || + fail 'console and atomic evidence file differ' + +: >"$CONSOLE" +if NEXAWRT_TEST_NFT=fail run_gate; then + fail 'gate accepted a missing fw4 nftables table' +fi +grep -Fxq 'NEXAWRT_VM_PRODUCTION_RUNTIME_FAILED=nftables_fw4_missing' "$CONSOLE" || + fail 'failure reason was not emitted to the console' +grep -Fxq 'NEXAWRT_VM_PRODUCTION_RUNTIME_FAILED=nftables_fw4_missing' \ + "$EVIDENCE_DIR/production-runtime.failed" || fail 'failure evidence file missing' + +grep -Fxq 'USE_PROCD=1' "$GATE_SCRIPT" || fail 'runtime gate must use the procd rc.common path' +grep -Fxq "EXTRA_COMMANDS='run'" "$GATE_SCRIPT" || fail 'runtime gate run action is not registered with rc.common' + +PROCD_LOG="$TMP_DIR/procd.log" +PROCD_LOG="$PROCD_LOG" sh -c ' + initscript=/etc/init.d/nexawrt-runtime-gate + procd_open_instance() { printf "open:%s\n" "$1" >>"$PROCD_LOG"; } + procd_set_param() { printf "param:%s\n" "$*" >>"$PROCD_LOG"; } + procd_close_instance() { printf "close\n" >>"$PROCD_LOG"; } + . "$1" + start_service +' procd-test "$GATE_SCRIPT" +grep -Fxq 'open:gate' "$PROCD_LOG" || fail 'procd gate instance was not opened' +grep -Fxq 'param:command /etc/init.d/nexawrt-runtime-gate run' "$PROCD_LOG" || + fail 'procd worker command is not the explicit run action' +grep -Fxq 'close' "$PROCD_LOG" || fail 'procd gate instance was not closed' + +grep -Fq '/etc/init.d/nexawrt-runtime-gate start >/dev/null 2>&1 || {' "$DEFAULTS_SCRIPT" || + fail 'first boot does not register the gate with procd' +if grep -Eq 'nexawrt-runtime-gate start.*[[:space:]]&[[:space:]]*$' "$DEFAULTS_SCRIPT"; then + fail 'unmanaged background gate launch remains in uci-defaults' +fi +grep -Fq '/etc/init.d/nexawrt-runtime-gate enable' "$DEFAULTS_SCRIPT" || + fail 'runtime gate is not enabled for subsequent boots' +grep -Fq 'NEXAWRT_VM_PRODUCTION_RUNTIME_FAILED=procd_registration' "$DEFAULTS_SCRIPT" || + fail 'procd registration failure is not emitted to the console' +grep -Fq 'production runtime gate reported failure' "$VM_TEST_SCRIPT" || + fail 'QEMU validator does not fail fast on a runtime gate failure marker' + +printf '%s\n' 'VM runtime gate tests passed.' diff --git a/vm-files-release/etc/init.d/nexawrt-runtime-gate b/vm-files-release/etc/init.d/nexawrt-runtime-gate index a93acab..acced05 100755 --- a/vm-files-release/etc/init.d/nexawrt-runtime-gate +++ b/vm-files-release/etc/init.d/nexawrt-runtime-gate @@ -1,48 +1,105 @@ #!/bin/sh /etc/rc.common -# shellcheck disable=SC2034 # rc.common consumes START and USE_PROCD. +# shellcheck disable=SC2034 # rc.common consumes START, USE_PROCD and EXTRA_COMMANDS. START=99 -USE_PROCD=0 +USE_PROCD=1 +EXTRA_COMMANDS='run' +EXTRA_HELP='\trun Execute the production runtime checks\n' + +CONSOLE="${NEXAWRT_RUNTIME_CONSOLE:-/dev/console}" +ETC_ROOT="${NEXAWRT_RUNTIME_ETC_ROOT:-/etc}" +SYS_ROOT="${NEXAWRT_RUNTIME_SYS_ROOT:-/sys}" +EVIDENCE_DIR="${NEXAWRT_RUNTIME_EVIDENCE_DIR:-/tmp/nexawrt-vm-release}" +MAX_ATTEMPTS="${NEXAWRT_RUNTIME_MAX_ATTEMPTS:-120}" +RETRY_DELAY="${NEXAWRT_RUNTIME_RETRY_DELAY:-1}" + +case "$MAX_ATTEMPTS" in + ''|*[!0-9]*) MAX_ATTEMPTS=120 ;; +esac +case "$RETRY_DELAY" in + ''|*[!0-9]*) RETRY_DELAY=1 ;; +esac emit_failure() { - printf 'NEXAWRT_VM_PRODUCTION_RUNTIME_FAILED=%s\n' "$1" >/dev/console + reason="$1" + mkdir -p "$EVIDENCE_DIR" 2>/dev/null || true + printf 'NEXAWRT_VM_PRODUCTION_RUNTIME_FAILED=%s\n' "$reason" >"$CONSOLE" + printf 'NEXAWRT_VM_PRODUCTION_RUNTIME_FAILED=%s\n' "$reason" \ + >"$EVIDENCE_DIR/production-runtime.failed" 2>/dev/null || true return 1 } -start() { - attempt=0 - while [ "$attempt" -lt 20 ]; do - attempt=$((attempt + 1)) - if /etc/init.d/firewall enabled >/dev/null 2>&1 && - nft list table inet fw4 >/dev/null 2>&1 && - ip -4 addr show dev eth0 | grep -q 'inet 192\.168\.8\.1/24' && - [ "$(uci -q get uhttpd.main.redirect_https)" = 1 ] && - [ "$(uci -q get network.lan.device)" = eth0 ] && - [ "$(uci -q get network.lan.proto)" = static ] && - [ "$(uci -q get dhcp.lan.ignore || printf 1)" = 0 ] && - ! /etc/init.d/dropbear enabled >/dev/null 2>&1 && - ! pidof dropbear >/dev/null 2>&1 && - grep -Eq '^root:[^!*:]' /etc/shadow; then - wan_device='absent' - network_mode='management-only' - if [ -e /sys/class/net/eth1 ]; then - [ "$(uci -q get network.wan.device)" = eth1 ] || { - emit_failure wan_binding - return 1 - } - [ "$(uci -q get network.wan.proto)" = dhcp ] || { - emit_failure wan_protocol - return 1 - } - wan_device='eth1' - network_mode='router' - fi - installation_id="$(cat /etc/nexawrt-install-id 2>/dev/null || true)" - [ "${#installation_id}" -eq 32 ] || { - emit_failure installation_id - return 1 - } - cat >/dev/console </dev/null 2>&1; then + runtime_failure='firewall_not_enabled' + return 1 + fi + if ! nft list table inet fw4 >/dev/null 2>&1; then + runtime_failure='nftables_fw4_missing' + return 1 + fi + if ! ip -4 addr show dev eth0 | grep -q 'inet 192\.168\.8\.1/24'; then + runtime_failure='lan_address_not_ready' + return 1 + fi + if [ "$(uci -q get uhttpd.main.redirect_https 2>/dev/null || true)" != 1 ]; then + runtime_failure='https_redirect_disabled' + return 1 + fi + if [ "$(uci -q get network.lan.device 2>/dev/null || true)" != eth0 ]; then + runtime_failure='lan_binding' + return 1 + fi + if [ "$(uci -q get network.lan.proto 2>/dev/null || true)" != static ]; then + runtime_failure='lan_protocol' + return 1 + fi + if [ "$(uci -q get dhcp.lan.ignore 2>/dev/null || true)" != 0 ]; then + runtime_failure='lan_dhcp_disabled' + return 1 + fi + if "$ETC_ROOT/init.d/dropbear" enabled >/dev/null 2>&1; then + runtime_failure='dropbear_enabled' + return 1 + fi + if pidof dropbear >/dev/null 2>&1; then + runtime_failure='dropbear_running' + return 1 + fi + if ! grep -Eq '^root:[^!*:]' "$ETC_ROOT/shadow"; then + runtime_failure='root_password_unset' + return 1 + fi + + wan_device='absent' + network_mode='management-only' + if [ -e "$SYS_ROOT/class/net/eth1" ]; then + if [ "$(uci -q get network.wan.device 2>/dev/null || true)" != eth1 ]; then + runtime_failure='wan_binding' + return 1 + fi + if [ "$(uci -q get network.wan.proto 2>/dev/null || true)" != dhcp ]; then + runtime_failure='wan_protocol' + return 1 + fi + wan_device='eth1' + network_mode='router' + fi + + installation_id="$(cat "$ETC_ROOT/nexawrt-install-id" 2>/dev/null || true)" + if [ "${#installation_id}" -ne 32 ]; then + runtime_failure='installation_id' + return 1 + fi + return 0 +} + +emit_success() { + evidence_tmp="$EVIDENCE_DIR/production-runtime.evidence.tmp.$$" + evidence_file="$EVIDENCE_DIR/production-runtime.evidence" + umask 077 + mkdir -p "$EVIDENCE_DIR" || return 1 + cat >"$evidence_tmp" <"$CONSOLE" +} + +run() { + attempt=0 + runtime_failure='runtime_not_ready' + while [ "$attempt" -lt "$MAX_ATTEMPTS" ]; do + attempt=$((attempt + 1)) + if check_runtime_state; then + emit_success || emit_failure evidence_write + return $? fi - sleep 1 + [ "$attempt" -lt "$MAX_ATTEMPTS" ] && sleep "$RETRY_DELAY" done - emit_failure timeout + emit_failure "$runtime_failure" +} + +start_service() { + # uci-defaults runs inside the early boot service. Registering the worker with + # procd lets it survive that caller and wait independently for network/firewall. + # The enabled S99 service registers the same one-shot check on every later boot. + procd_open_instance gate + # shellcheck disable=SC2154 # rc.common defines initscript before start_service. + procd_set_param command "$initscript" run + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param term_timeout 5 + procd_close_instance } diff --git a/vm-files-release/etc/uci-defaults/10-vm-release b/vm-files-release/etc/uci-defaults/10-vm-release index b71f428..273a216 100755 --- a/vm-files-release/etc/uci-defaults/10-vm-release +++ b/vm-files-release/etc/uci-defaults/10-vm-release @@ -137,10 +137,13 @@ EOF_EVIDENCE mv "$EVIDENCE_TMP" "$EVIDENCE_FILE" cat "$EVIDENCE_FILE" >/dev/console -# uci-defaults runs before late init services. Start the production gate in the -# background so the first boot is checked after firewall and HTTPS are ready; the -# enabled rc.d entry repeats the same check on every later boot. -( /etc/init.d/nexawrt-runtime-gate start ) >/dev/null 2>&1 & +# uci-defaults runs before late init services. Ask procd to own the one-shot +# runtime worker so it survives this early-boot process and waits for network and +# firewall readiness. The enabled S99 service repeats the check on later boots. +/etc/init.d/nexawrt-runtime-gate start >/dev/null 2>&1 || { + printf '%s\n' 'NEXAWRT_VM_PRODUCTION_RUNTIME_FAILED=procd_registration' >/dev/console + exit 1 +} cat >/dev/console <