From 1cbaf658a8d21e8d1c2ccabba0b172958f29f368 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 02:52:07 +0200 Subject: [PATCH 01/31] lock(vold): trace pinned source and seed a bounded 67-project dependency probe --- locks/lineage-vold-source-probe-20261016.json | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 locks/lineage-vold-source-probe-20261016.json diff --git a/locks/lineage-vold-source-probe-20261016.json b/locks/lineage-vold-source-probe-20261016.json new file mode 100644 index 00000000..8b5c08d7 --- /dev/null +++ b/locks/lineage-vold-source-probe-20261016.json @@ -0,0 +1,135 @@ +{ + "schema_version": 1, + "kind": "unmodified-vold-provisional-source-probe", + "manifest_sha256": "d4ded1ca84de399497b68d7ac006808059b6be9cf76725029420a3495a2e5468", + "base_build_id": "lineage-24.0-20261016-nightly-pdx235-signed", + "source_revision": "11d8982f824d52e8122c7aa732207ba66b8f23c8", + "source_evidence": [ + "https://github.com/LineageOS/android_system_vold/blob/11d8982f824d52e8122c7aa732207ba66b8f23c8/Android.bp", + "https://github.com/LineageOS/android_device_sony_pdx235/blob/0559de69466e1e9af69b121985f592c4f4609283/device.mk", + "https://github.com/LineageOS/android_device_sony_sm6375-common/blob/d1d80af1ad9032cb092ff6d85db437709cc54e28/common.mk" + ], + "candidate_groups": { + "core_build_toolchain": [ + "android", + "art", + "bionic", + "build/make", + "build/soong", + "build/blueprint", + "build/release", + "build/pesto", + "libcore", + "prebuilts/clang/host/linux-x86", + "prebuilts/build-tools", + "prebuilts/go/linux-x86", + "prebuilts/rust", + "prebuilts/jdk/jdk21", + "prebuilts/bazel/common", + "prebuilts/ndk" + ], + "vold_direct_and_native": [ + "system/vold", + "system/core", + "system/fs/fs_mgr", + "system/fs/casefolding_remover", + "system/security", + "system/sepolicy", + "system/logging", + "system/extras", + "system/memory/libmeminfo", + "system/memory/libion", + "system/memory/libwrapfd", + "frameworks/base", + "frameworks/native", + "frameworks/av", + "frameworks/hardware/interfaces", + "bootable/libbootloader", + "bootable/recovery", + "hardware/interfaces", + "hardware/libhardware", + "external/avb", + "external/boringssl", + "external/e2fsprogs", + "external/f2fs-tools", + "external/selinux", + "external/protobuf", + "external/zlib", + "external/icu", + "external/libcxx", + "external/libcap", + "external/expat", + "external/dtc", + "external/sqlite", + "external/googletest", + "external/fmtlib" + ], + "lineage_sony_product": [ + "device/sony/pdx235", + "device/sony/sm6375-common", + "device/lineage/sepolicy", + "device/qcom/sepolicy_vndr/legacy-um", + "hardware/lineage/interfaces", + "hardware/sony", + "hardware/qcom-caf/common", + "hardware/qcom-caf/sm8350/audio", + "hardware/qcom-caf/wlan", + "vendor/lineage", + "vendor/qcom/opensource/commonsys-intf/display", + "vendor/qcom/opensource/commonsys/display", + "vendor/qcom/opensource/display", + "tools/extract-utils" + ], + "module_service_api": [ + "packages/modules/adb", + "packages/modules/Connectivity", + "packages/modules/ArtPrebuilt" + ] + }, + "direct_module_to_project_hints": { + "android.security.maintenance-ndk": "system/security", + "libavb": "external/avb", + "libfs_mgr": "system/fs/fs_mgr", + "libdm": "system/fs/fs_mgr", + "libcasefoldingremover": "system/fs/casefolding_remover", + "libbootloader_message": "bootable/libbootloader", + "libvold_binder": "system/vold", + "vold_flags_c_lib": "system/vold", + "libvold": "system/vold", + "android.system.vold-V1-cpp": "system/vold", + "libincremental_aidl-cpp": "frameworks/base", + "libbinder": "frameworks/native", + "libbinder_ndk": "frameworks/native", + "libcutils": "system/core", + "libutils": "system/core", + "libbase": "system/core", + "libselinux": "external/selinux", + "libcrypto": "external/boringssl", + "liblog": "system/logging", + "liblogwrap": "system/core", + "libhardware": "hardware/libhardware", + "libhardware_legacy": "hardware/libhardware", + "libgsi": "system/fs/fs_mgr", + "libext2_uuid": "external/e2fsprogs", + "libext4_utils": "system/extras", + "libf2fs_sparseblock": "system/extras", + "libfec": "system/fs/fs_mgr", + "libfs_avb": "system/fs/fs_mgr", + "libmeminfo_headers": "system/memory/libmeminfo", + "libboot_control_client": "hardware/interfaces", + "android.hardware.boot@1.0": "hardware/interfaces", + "android.hardware.boot-V1-ndk": "hardware/interfaces", + "libkeymint_support": "system/security", + "libsysutils": "system/core", + "libincfs": "system/incremental_delivery", + "fuseMedia.bpf": "system/bpf", + "mke2fs": "external/e2fsprogs" + }, + "known_unpinned_vendor_inputs": [ + "vendor/sony/pdx235/pdx235-vendor.mk", + "vendor/sony/pdx235/BoardConfigVendor.mk", + "vendor/sony/sm6375-common/sm6375-common-vendor.mk", + "vendor/sony/sm6375-common/BoardConfigVendor.mk" + ], + "qualification_note": "Initial hints, not a complete Soong source closure. Repo sync and build errors must discover additional immutable manifest projects; missing proprietary vendor inputs cannot be fabricated." +} From 0508281cf7df2b7ffb9a733aca5e2fef3d8f7f5b Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 02:54:06 +0200 Subject: [PATCH 02/31] feat(vold): derive pinned subset plan and diagnose missing build dependencies --- src/ci/vold_source_closure.py | 329 ++++++++++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 src/ci/vold_source_closure.py diff --git a/src/ci/vold_source_closure.py b/src/ci/vold_source_closure.py new file mode 100644 index 00000000..e36e32ad --- /dev/null +++ b/src/ci/vold_source_closure.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Conservative, auditable source-dependency probing for unmodified pdx235 vold. + +A seed is NOT an authoritative Soong closure. Derive candidate projects from +locked source and build diagnostics, then verify exact Git HEADs. Missing +proprietary inputs and unknown module providers are reported, not fabricated. +Never loosen the byte-for-byte comparison or call the timeout patch. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re +import sys +from typing import Any +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parents[2] +SEED = ROOT / "locks/lineage-vold-source-probe-20261016.json" +BASELINE = ROOT / "locks/lineage-vold-baseline-20261016.json" +PATH_RE = re.compile(r"^[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)+$") +PATH_IN_LOG = re.compile( + r"(? tuple[dict, dict[str, str], dict]: + seed = json.loads(SEED.read_text()) + baseline = json.loads(BASELINE.read_text()) + raw = (ROOT / baseline["manifest"]["path"]).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != baseline["manifest"]["sha256"] or seed.get("manifest_sha256") != digest: + raise ClosureError("source subset plan does not match the immutable 1,243-project manifest") + if seed.get("base_build_id") != baseline.get("base_build_id") or seed.get("schema_version") != 1: + raise ClosureError("source subset plan has an unexpected target or schema") + try: + xml = ET.fromstring(raw) + except ET.ParseError as exc: + raise ClosureError("pinned project manifest cannot be parsed") from exc + projects: dict[str, str] = {} + for element in xml.findall("project"): + path, revision = element.get("path"), element.get("revision") + if not path or not PATH_RE.fullmatch(path) and "/" in path or not revision or not re.fullmatch("[0-9a-f]{40}", revision): + # Root-level paths are valid in official Android manifests. + if not path or not re.fullmatch(r"[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)*", path) or not revision or not re.fullmatch("[0-9a-f]{40}", revision): + raise ClosureError("manifest contains an invalid project identity") + if path in projects: + raise ClosureError("source manifest contains duplicate project paths") + projects[path] = revision + if len(projects) != 1243: + raise ClosureError("pinned source manifest does not contain exactly 1,243 projects") + groups = seed.get("candidate_groups") + if not isinstance(groups, dict) or not groups: + raise ClosureError("source seed has no reviewed dependency groups") + for title, selected in groups.items(): + if not isinstance(title, str) or not isinstance(selected, list) or not selected: + raise ClosureError("seed group is empty or malformed") + if any(not isinstance(path, str) or path not in projects for path in selected): + raise ClosureError(f"seed group {title} references non-pinned projects") + if seed.get("direct_module_to_project_hints") is None or any( + path not in projects for path in seed["direct_module_to_project_hints"].values() + ): + raise ClosureError("module provider hints are not bound to immutable manifest projects") + for vendor in seed["known_unpinned_vendor_inputs"]: + if not vendor.startswith("vendor/sony/"): + raise ClosureError("unexpected unreviewed vendor-input exception") + if any(vendor == p or vendor.startswith(p + "/") for p in projects): + raise ClosureError("vendor-input exception unexpectedly became manifest-pinned") + return seed, projects, baseline + + +def initial_projects(seed: dict) -> list[str]: + return sorted({path for group in seed["candidate_groups"].values() for path in group}) + + +def validate_selection(selection: dict, projects: dict[str, str], seed: dict) -> list[str]: + if not isinstance(selection, dict) or selection.get("schema_version") != 1: + raise ClosureError("invalid dependency probe selection") + if selection.get("manifest_sha256") != seed["manifest_sha256"]: + raise ClosureError("dependency selection has the wrong source revision") + paths = selection.get("selected_projects") + if not isinstance(paths, list) or not paths or len(paths) > len(projects): + raise ClosureError("dependency selection contains an invalid project list") + if len(paths) != len(set(paths)) or any( + not isinstance(path, str) or path not in projects for path in paths + ): + raise ClosureError("dependency selection contains unknown, unpinned or duplicate projects") + if not set(initial_projects(seed)).issubset(paths): + raise ClosureError("dependency selection may not omit reviewed seed projects") + return sorted(paths) + + +def selection_record(paths: list[str], seed: dict) -> dict: + return { + "schema_version": 1, + "kind": "unmodified-vold-provisional-source-selection", + "manifest_sha256": seed["manifest_sha256"], + "selected_projects": sorted(paths), + "warning": "Not complete or qualified. Must run the real unmodified build and pass a byte-exact reference check.", + } + + +def owner(path: str, projects: dict[str, str]) -> str | None: + clean = path.strip(".,:;()[]{}'\"") + # A file in system/fs/fs_mgr belongs to that project, not 'system'. + matches = [root for root in projects if clean == root or clean.startswith(root + "/")] + return max(matches, key=len) if matches else None + + +def missing_dependencies( + text: str, selected: list[str], projects: dict[str, str], seed: dict +) -> dict: + pinned_missing: set[str] = set() + unpinned: set[str] = set() + unresolved_modules: set[str] = set() + modules: set[str] = set() + hints = seed["direct_module_to_project_hints"] + for line in text.splitlines(): + low = line.lower() + if not any(token in low for token in ERROR_MARKERS): + continue + for pattern in (MISSING_MODULE, QUOTED_MODULE): + modules.update(pattern.findall(line)) + for match in PATH_IN_LOG.finditer(line): + path = match[1] + project = owner(path, projects) + if project is not None and project not in selected: + pinned_missing.add(project) + elif project is None and path.startswith(("vendor/sony/", "device/sony/")): + unpinned.add(path) + for module in modules: + mapped = hints.get(module) + if mapped is None: + unresolved_modules.add(module) + elif mapped not in selected: + pinned_missing.add(mapped) + return { + "proposed_project_additions": sorted(pinned_missing), + "unpinned_missing_paths": sorted(unpinned), + "modules_without_verified_owner": sorted(unresolved_modules), + "modules_observed": sorted(modules), + "statically_unpinned_vendor_inputs": seed["known_unpinned_vendor_inputs"], + } + + +def iter_blueprint_blocks(raw: str): + """Extract balanced module declarations, without assuming nested blocks are flat.""" + for candidate in BLUEPRINT_BLOCK.finditer(raw): + start = candidate.end() + depth = 1 + quote = False + escape = False + index = start + while index < len(raw) and depth: + char = raw[index] + if quote: + if escape: + escape = False + elif char == "\\": + escape = True + elif char == '"': + quote = False + else: + if char == '"': + quote = True + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + index += 1 + if depth == 0: + body = raw[start:index - 1] + name_match = BLUEPRINT_NAME.search(body) + if name_match: + yield name_match[1], body + + +def blueprint_module_graph(source_root: Path, selected: list[str]) -> dict: + """Best-effort static Soong graph of synced projects only. + + Does NOT evaluate Soong product conditionals or generated modules. + """ + if source_root.is_symlink(): + raise ClosureError("unsafe symbolic-link source checkout") + source_root = source_root.resolve(strict=True) + found: dict[str, list[dict]] = {} + count_files = 0 + for project in selected: + directory = source_root / project + if directory.is_symlink() or not directory.is_dir(): + continue + for bp in directory.rglob("Android.bp"): + if bp.is_symlink() or not bp.is_file() or bp.stat().st_size > 4_000_000: + continue + count_files += 1 + if count_files > 12000: + raise ClosureError("source contains unexpectedly many Blueprint files") + try: + raw = bp.read_text(errors="replace") + except OSError as exc: + raise ClosureError("unable to read checked-out module metadata") from exc + for name, body in iter_blueprint_blocks(raw): + deps = set() + for field, values in BP_ARRAY.findall(body): + if field in BP_DEP_FIELDS: + deps.update(x.lstrip(":") for x in BP_ITEM.findall(values)) + if field in ("srcs", "generated_sources"): + deps.update(x[1:] for x in BP_ITEM.findall(values) if x.startswith(":")) + found.setdefault(name, []).append({ + "path": bp.relative_to(source_root).as_posix(), + "deps": sorted(deps), + }) + reachable = set() + stack = ["vold"] + unresolved = set() + ambiguous = set() + while stack: + current = stack.pop() + if current in reachable: + continue + reachable.add(current) + matches = found.get(current, []) + if not matches: + unresolved.add(current) + if len(matches) > 1: + ambiguous.add(current) + for match in matches: + stack.extend(d for d in match["deps"] if d not in reachable) + return { + "kind": "partial-static-soong-graph", + "scanned_android_bp_files": count_files, + "indexed_module_names": len(found), + "reachable_module_names": len(reachable), + "reachable_module_sample": sorted(reachable)[:160], + "unresolved_module_names": sorted(unresolved), + "ambiguous_module_names": sorted(ambiguous), + "warning": "Over-approximation: product-selected Soong modules, generated dependencies and missing unsynced module providers still need a real Soong evaluation.", + } + + +def write_json(path: Path, report: dict) -> None: + if path.is_symlink(): + raise ClosureError("refusing to replace a symbolic-link source report") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + cmd_plan = sub.add_parser("plan", help="generate a small immutable-manifest selection without downloading") + cmd_plan.add_argument("--output", type=Path, required=True) + cmd_index = sub.add_parser("index", help="index checked-out Android.bp module references") + cmd_index.add_argument("--selection", type=Path, required=True) + cmd_index.add_argument("--source-root", type=Path, required=True) + cmd_index.add_argument("--output", type=Path, required=True) + cmd_diag = sub.add_parser("diagnose", help="map build errors to additional pinned paths") + cmd_diag.add_argument("--selection", type=Path, required=True) + cmd_diag.add_argument("--log", type=Path, required=True) + cmd_diag.add_argument("--output", type=Path, required=True) + cmd_diag.add_argument("--next-selection", type=Path) + args = parser.parse_args() + try: + seed, projects, baseline = verified_contract() + if args.action == "plan": + selected = initial_projects(seed) + write_json(args.output, selection_record(selected, seed)) + print(json.dumps({ + "selected_projects": len(selected), + "total_manifest_projects": len(projects), + "source_ota_sha256": baseline["source_ota_sha256"], + "known_external_vendor_inputs": seed["known_unpinned_vendor_inputs"], + "selection_path": str(args.output), + }, sort_keys=True)) + else: + selected = validate_selection(json.loads(args.selection.read_text()), projects, seed) + if args.action == "index": + report = blueprint_module_graph(args.source_root, selected) + else: + report = missing_dependencies(args.log.read_text(errors="replace"), selected, projects, seed) + if args.next_selection is not None: + additions = report["proposed_project_additions"] + if len(additions) > 30: + raise ClosureError("too many automatic additions: require independent review") + if additions: + write_json(args.next_selection, selection_record(sorted(set(selected + additions)), seed)) + report["next_selection"] = str(args.next_selection) + else: + report["next_selection"] = None + write_json(args.output, report) + print(json.dumps(report, sort_keys=True)) + except (OSError, ValueError, KeyError, TypeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From c3110340b77c2943d5c78432505e9c29f9c30f01 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 02:55:27 +0200 Subject: [PATCH 03/31] feat(vold): support verified subset sync and incremental pinned dependency checkout --- src/ci/build_original_vold.py | 122 ++++++++++++++++++++++++---------- 1 file changed, 88 insertions(+), 34 deletions(-) diff --git a/src/ci/build_original_vold.py b/src/ci/build_original_vold.py index c12e5982..9971c10a 100644 --- a/src/ci/build_original_vold.py +++ b/src/ci/build_original_vold.py @@ -82,18 +82,46 @@ def command(args: list[str], root: Path, **kwargs: object) -> subprocess.Complet return subprocess.run(args, cwd=root, check=True, **kwargs) -def verify_checkout(source: Path, manifest: bytes, *, clean: bool = False) -> None: +def verified_selection(selection_file: Path | None, manifest: bytes) -> list[str] | None: + if selection_file is None: + return None + if selection_file.is_symlink() or not selection_file.is_file(): + raise BuildError("dependency selection must be a real checked-in or generated JSON file") + from vold_source_closure import verified_contract, validate_selection + seed, projects, _baseline = verified_contract() + if hashlib.sha256(manifest).hexdigest() != seed["manifest_sha256"]: + raise BuildError("dependency selection does not match the authenticated manifest") + try: + selection = json.loads(selection_file.read_bytes()) + except (ValueError, UnicodeError) as exc: + raise BuildError("dependency selection is not valid JSON") from exc + return validate_selection(selection, projects, seed) + + +def verify_checkout( + source: Path, manifest: bytes, *, clean: bool = False, + selected: list[str] | None = None, +) -> None: repo = shutil.which("repo") if repo is None: raise BuildError("repo is required to verify complete source revisions") - result = subprocess.run( - [repo, "manifest", "-r"], cwd=source, check=True, capture_output=True - ) expected = project_map(manifest) - if project_map(result.stdout) != expected: - raise BuildError("source checkout does not match the 1,243 pinned revisions") + active_manifest = source / ".repo/manifest.xml" + if active_manifest.is_dir() or not active_manifest.exists() or active_manifest.read_bytes() != manifest: + raise BuildError("repo selected manifest differs from the exact official snapshot") + if selected is None: + # The legacy full-checkout gate remains unchanged for fully synced trees. + result = subprocess.run( + [repo, "manifest", "-r"], cwd=source, check=True, capture_output=True + ) + if project_map(result.stdout) != expected: + raise BuildError("complete source checkout does not match pinned revisions") + else: + if not selected or any(relpath not in expected for relpath in selected): + raise BuildError("unreviewed project outside the immutable manifest") if clean: - for relpath, (_name, revision) in sorted(expected.items()): + to_verify = expected if selected is None else {p: expected[p] for p in selected} + for relpath, (_name, revision) in sorted(to_verify.items()): path = source / relpath if path.is_symlink() or not path.is_dir() or not path.resolve().is_relative_to(source): raise BuildError(f"source project is missing or unsafe: {relpath}") @@ -108,53 +136,65 @@ def verify_checkout(source: Path, manifest: bytes, *, clean: bool = False) -> No raise BuildError(f"source project has unreviewed local content: {relpath}") -def sync(source: Path, lock: dict, manifest: bytes, jobs: int) -> None: - if source.is_symlink() or (source.exists() and ( - not source.is_dir() or any(source.iterdir()) - )): - raise BuildError("source directory must be new or empty and not a symlink") +def sync( + source: Path, lock: dict, manifest: bytes, jobs: int, + selected: list[str] | None = None, resume: bool = False, +) -> None: + if source.is_symlink() or source.exists() and not source.is_dir(): + raise BuildError("source directory must be a real directory") + if resume: + if not (source / ".repo/manifest.xml").is_file(): + raise BuildError("resuming a subset requires the existing pinned repo checkout") + if (source / ".repo/manifest.xml").read_bytes() != manifest: + raise BuildError("resuming checkout with a different manifest is forbidden") + elif source.exists() and any(source.iterdir()): + raise BuildError("source directory must be new or empty; use --resume to add pinned dependencies") repo = shutil.which("repo") if repo is None: raise BuildError("repo tool is unavailable on source build runner") source.mkdir(parents=True, exist_ok=True) - manifest_commit = lock["projects"]["manifest"]["revision"] - command([ + if not resume: + manifest_commit = lock["projects"]["manifest"]["revision"] + command([ repo, "init", "-u", "https://github.com/LineageOS/android.git", "-b", manifest_commit - ], source) - # Do not overwrite .repo/manifest.xml: Repo manages this file, and + ], source) + # Do not overwrite .repo/manifest.xml: Repo manages this file, and # recent versions regenerate it during subsequent sync/init operations. # Instead install the immutable flattened manifest in Repo's manifest # checkout and formally select it with -m, as supported by Repo. - manifest_name = "modos-vold-original-20261016.xml" - manifest_dir = source / ".repo/manifests" - if manifest_dir.is_symlink() or not manifest_dir.is_dir(): - raise BuildError("repo manifests checkout is unavailable or unsafe") - pinned_manifest = manifest_dir / manifest_name - if pinned_manifest.exists() or pinned_manifest.is_symlink(): - raise BuildError("preexisting untrusted local pinned manifest") - pinned_manifest.write_bytes(manifest) - command([ - repo, "init", "-u", "https://github.com/LineageOS/android.git", - "-b", manifest_commit, "-m", manifest_name - ], source) + manifest_name = "modos-vold-original-20261016.xml" + manifest_dir = source / ".repo/manifests" + if manifest_dir.is_symlink() or not manifest_dir.is_dir(): + raise BuildError("repo manifests checkout is unavailable or unsafe") + pinned_manifest = manifest_dir / manifest_name + if pinned_manifest.exists() or pinned_manifest.is_symlink(): + raise BuildError("preexisting untrusted local pinned manifest") + pinned_manifest.write_bytes(manifest) + command([ + repo, "init", "-u", "https://github.com/LineageOS/android.git", + "-b", manifest_commit, "-m", manifest_name + ], source) active_manifest = source / ".repo/manifest.xml" if not active_manifest.exists() or active_manifest.is_dir(): raise BuildError("repo did not activate the immutable full manifest") if active_manifest.read_bytes() != manifest: raise BuildError("repo's active manifest differs from the pinned source") - command([repo, "sync", "--no-clone-bundle", f"-j{jobs}"], source) - verify_checkout(source, manifest, clean=True) + # Explicit project-path sync prevents all 1,243 projects being fetched. + # On repeated probes, keep the same pinned workspace and fetch only the + # new members; repo does not need to re-download already pinned projects. + command([repo, "sync", "--no-clone-bundle", f"-j{jobs}", *(selected or [])], source) + verify_checkout(source, manifest, clean=True, selected=selected) def build_baseline( source: Path, dest: Path, reference: Path, lock: dict, manifest: bytes, - jobs: int + jobs: int, selected: list[str] | None = None, ) -> dict: if source.is_symlink() or not (source / ".repo/manifest.xml").exists(): raise BuildError("baseline source directory is missing or unsafe") source = source.resolve(strict=True) - verify_checkout(source, manifest, clean=True) + verify_checkout(source, manifest, clean=True, selected=selected) official = read_vold(reference) if dest.is_symlink() or (dest.exists() and ( not dest.is_dir() or any(dest.iterdir()) @@ -179,6 +219,14 @@ def build_baseline( raise BuildError("source build did not produce an executable vold") built = read_vold(artifact) report = comparison(official, built, lock) + if selected is not None: + report["selected_source_projects"] = len(selected) + report["selected_source_paths_sha256"] = hashlib.sha256( + ("\\n".join(sorted(selected)) + "\\n").encode() + ).hexdigest() + report["source_checkout_scope"] = "manifest-pinned-subset" + else: + report["source_checkout_scope"] = "complete-manifest" write_receipt(dest / "baseline-report.json", report) require_identical(report) return report @@ -191,6 +239,9 @@ def main() -> int: opt = sub.add_parser(name) opt.add_argument("--source-root", type=Path, required=True) opt.add_argument("--jobs", type=int, default=4) + opt.add_argument("--projects-file", type=Path, help="optional validated subset plan; never weakens the binary equality gate") + if name == "sync": + opt.add_argument("--resume", action="store_true", help="fetch missing approved projects within the same checkout") if name == "compare": opt.add_argument("--reference-vold", type=Path, required=True) opt.add_argument("--output-dir", type=Path, required=True) @@ -199,12 +250,15 @@ def main() -> int: parser.error("--jobs must be between 1 and 16") try: lock, manifest = load_contract() + selected = verified_selection(args.projects_file, manifest) if args.mode == "sync": - sync(args.source_root, lock, manifest, args.jobs) + if args.resume and selected is None: + raise BuildError("full-manifest checkout does not support incremental resume") + sync(args.source_root, lock, manifest, args.jobs, selected, args.resume) else: report = build_baseline( args.source_root, args.output_dir, args.reference_vold, - lock, manifest, args.jobs, + lock, manifest, args.jobs, selected, ) print(json.dumps(report, sort_keys=True)) except (OSError, BuildError, BaselineError, subprocess.CalledProcessError) as exc: From 6d375b78dea83db7874550224aee980828201bb6 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 02:56:00 +0200 Subject: [PATCH 04/31] fix(vold): preserve init indentation and canonical selection digest --- src/ci/build_original_vold.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ci/build_original_vold.py b/src/ci/build_original_vold.py index 9971c10a..a8458452 100644 --- a/src/ci/build_original_vold.py +++ b/src/ci/build_original_vold.py @@ -156,13 +156,13 @@ def sync( if not resume: manifest_commit = lock["projects"]["manifest"]["revision"] command([ - repo, "init", "-u", "https://github.com/LineageOS/android.git", - "-b", manifest_commit + repo, "init", "-u", "https://github.com/LineageOS/android.git", + "-b", manifest_commit, ], source) # Do not overwrite .repo/manifest.xml: Repo manages this file, and - # recent versions regenerate it during subsequent sync/init operations. - # Instead install the immutable flattened manifest in Repo's manifest - # checkout and formally select it with -m, as supported by Repo. + # recent versions regenerate it during subsequent sync/init operations. + # Instead install the immutable flattened manifest in Repo's manifest + # checkout and formally select it with -m, as supported by Repo. manifest_name = "modos-vold-original-20261016.xml" manifest_dir = source / ".repo/manifests" if manifest_dir.is_symlink() or not manifest_dir.is_dir(): @@ -222,7 +222,7 @@ def build_baseline( if selected is not None: report["selected_source_projects"] = len(selected) report["selected_source_paths_sha256"] = hashlib.sha256( - ("\\n".join(sorted(selected)) + "\\n").encode() + ("\n".join(sorted(selected)) + "\n").encode() ).hexdigest() report["source_checkout_scope"] = "manifest-pinned-subset" else: From 884f8906a381d5dd0965378437511a9aec8df7e5 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 02:57:06 +0200 Subject: [PATCH 05/31] feat(vold): add compile-only probe with no false binary-reproducibility claim --- src/ci/build_original_vold.py | 45 +++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/src/ci/build_original_vold.py b/src/ci/build_original_vold.py index a8458452..71f8c1e7 100644 --- a/src/ci/build_original_vold.py +++ b/src/ci/build_original_vold.py @@ -187,24 +187,19 @@ def sync( verify_checkout(source, manifest, clean=True, selected=selected) -def build_baseline( - source: Path, dest: Path, reference: Path, lock: dict, manifest: bytes, - jobs: int, selected: list[str] | None = None, -) -> dict: +def compile_original_vold( + source: Path, lock: dict, manifest: bytes, jobs: int, + selected: list[str] | None = None, +) -> Path: + """Do not apply or import the patch. A successful compile is NOT proof.""" if source.is_symlink() or not (source / ".repo/manifest.xml").exists(): raise BuildError("baseline source directory is missing or unsafe") source = source.resolve(strict=True) verify_checkout(source, manifest, clean=True, selected=selected) - official = read_vold(reference) - if dest.is_symlink() or (dest.exists() and ( - not dest.is_dir() or any(dest.iterdir()) - )): - raise BuildError("baseline output directory must be new or empty") - dest.mkdir(parents=True, exist_ok=True) artifact = source / lock["build"]["artifact"] if artifact.is_symlink(): raise BuildError("Android vold build artifact cannot be a symbolic link") - # Never mistake an existing build's vold for a result of this build. + # An earlier trial must not be mistaken for the current compiler result. if artifact.exists(): if not artifact.is_file(): raise BuildError("unexpected prior vold output type") @@ -215,14 +210,29 @@ def build_baseline( f"m -j{jobs} vold" ) command(["bash", "-euc", script], source) - if not artifact.exists() or not artifact.stat().st_mode & 0o111: - raise BuildError("source build did not produce an executable vold") + if artifact.is_symlink() or not artifact.is_file() or not artifact.stat().st_mode & 0o111: + raise BuildError("source build did not produce a regular executable vold") + return artifact + + +def build_baseline( + source: Path, dest: Path, reference: Path, lock: dict, manifest: bytes, + jobs: int, selected: list[str] | None = None, +) -> dict: + """A build is only *qualified* if every byte matches the official OTA.""" + official = read_vold(reference) + if dest.is_symlink() or (dest.exists() and ( + not dest.is_dir() or any(dest.iterdir()) + )): + raise BuildError("baseline output directory must be new or empty") + dest.mkdir(parents=True, exist_ok=True) + artifact = compile_original_vold(source, lock, manifest, jobs, selected) built = read_vold(artifact) report = comparison(official, built, lock) if selected is not None: report["selected_source_projects"] = len(selected) report["selected_source_paths_sha256"] = hashlib.sha256( - ("\n".join(sorted(selected)) + "\n").encode() + ("\\n".join(sorted(selected)) + "\\n").encode() ).hexdigest() report["source_checkout_scope"] = "manifest-pinned-subset" else: @@ -235,7 +245,7 @@ def build_baseline( def main() -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="mode", required=True) - for name in ("sync", "compare"): + for name in ("sync", "probe", "compare"): opt = sub.add_parser(name) opt.add_argument("--source-root", type=Path, required=True) opt.add_argument("--jobs", type=int, default=4) @@ -255,6 +265,11 @@ def main() -> int: if args.resume and selected is None: raise BuildError("full-manifest checkout does not support incremental resume") sync(args.source_root, lock, manifest, args.jobs, selected, args.resume) + elif args.mode == "probe": + if selected is None: + raise BuildError("compilation probe requires an explicit pinned subset selection") + compile_original_vold(args.source_root, lock, manifest, args.jobs, selected) + print("Dependency probe compiled; this is NOT byte-identical proof or release approval.") else: report = build_baseline( args.source_root, args.output_dir, args.reference_vold, From 1fcbf3edf5da3b6a77f33c8a97b6f43d2d3715f7 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 02:58:26 +0200 Subject: [PATCH 06/31] feat(vold): bounded iterative source closure using one reusable pinned checkout --- src/ci/probe_original_vold.sh | 91 +++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/ci/probe_original_vold.sh diff --git a/src/ci/probe_original_vold.sh b/src/ci/probe_original_vold.sh new file mode 100644 index 00000000..553e718a --- /dev/null +++ b/src/ci/probe_original_vold.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later +# Same-workspace, immutable-revision probing: sync only selected projects, +# reuse downloaded Git objects, map errors to the pinned manifest, expand. +# Never apply the timeout patch or publish an OTA. +set -euo pipefail + +: "${VOLD_WORK:?isolated scratch workspace is required}" +: "${VOLD_PROBE_STAGE:?probe or compare stage is required}" +: "${VOLD_SOURCE_REF:?absolute original vold path is required for compare}" + +[[ "${VOLD_PROBE_STAGE}" == probe || "${VOLD_PROBE_STAGE}" == compare ]] || { + echo 'Error: unsupported unmodified vold dependency stage' >&2 + exit 1 +} +[[ "${VOLD_WORK}" == /* && -d "${VOLD_WORK}" && ! -L "${VOLD_WORK}" ]] || { + echo 'Error: isolated scratch directory is missing or unsafe' >&2 + exit 1 +} +[[ "${VOLD_PROBE_STAGE}" != compare || ( -s "${VOLD_SOURCE_REF}" && ! -L "${VOLD_SOURCE_REF}" ) ]] || { + echo 'Error: authenticated original vold reference is required' >&2 + exit 1 +} +jobs="${VOLD_PROBE_JOBS:-4}" +[[ "${jobs}" =~ ^[1-9][0-9]?$ ]] && (( jobs <= 16 )) || { + echo 'Error: probe concurrency must be between 1 and 16' >&2 + exit 1 +} + +selection="${VOLD_WORK}/source-selection.json" +python3 src/ci/vold_source_closure.py plan --output "${selection}" +python3 src/ci/build_original_vold.py sync \ + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" + +# Most missing dependencies are reported during Soong setup before the +# expensive target compile. Resolve their exact pinned project paths once +# and reuse the same checkout and Android out directory on each iteration. +# Unknown module providers must stop for review; never turn on missing +# dependency flags or attempt an unverified project URL. +for iteration in 1 2 3; do + python3 src/ci/vold_source_closure.py index \ + --source-root "${VOLD_WORK}/source" \ + --selection "${selection}" \ + --output "${VOLD_WORK}/source-index-${iteration}.json" || + echo "::warning::Partial Soong index could not be collected on pass ${iteration}" + + log="${VOLD_WORK}/vold-build-pass-${iteration}.log" + if [[ "${VOLD_PROBE_STAGE}" == probe ]]; then + build_cmd=(python3 src/ci/build_original_vold.py probe + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}") + else + build_cmd=(python3 src/ci/build_original_vold.py compare + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" + --reference-vold "${VOLD_SOURCE_REF}" + --output-dir "${VOLD_WORK}/baseline-${iteration}" --jobs "${jobs}") + fi + + if "${build_cmd[@]}" >"${log}" 2>&1; then + tail -n 15 -- "${log}" + if [[ "${VOLD_PROBE_STAGE}" == probe ]]; then + echo '::notice::Unmodified vold compiled, but no authenticated binary equality has been established.' + echo "Compile-only probe completed; next stage MUST use authenticated compare." >"${VOLD_WORK}/probe-only.txt" + else + echo '::notice::Unmodified vold is byte-identical to the authenticated reference.' + fi + exit 0 + fi + + echo "::warning::Pinned source probe pass ${iteration} failed. Last diagnostic lines:" + tail -n 55 -- "${log}" + next="${VOLD_WORK}/selection-next.json" + rm -f -- "${next}" + python3 src/ci/vold_source_closure.py diagnose \ + --selection "${selection}" --log "${log}" \ + --output "${VOLD_WORK}/dependency-diagnosis-${iteration}.json" \ + --next-selection "${next}" + + if [[ ! -f "${next}" || "${iteration}" == 3 ]]; then + echo '::error::No automatically identifiable missing pinned projects remain, or expansion budget exhausted.' + echo '::error::Review diagnostic JSON for unknown Soong modules, missing vendor inputs or real build errors.' + exit 1 + fi + mv -- "${next}" "${selection}" + echo "::notice::Expanding the same pinned checkout on pass $((iteration + 1))." + python3 src/ci/build_original_vold.py sync --resume \ + --source-root "${VOLD_WORK}/source" \ + --projects-file "${selection}" --jobs "${jobs}" +done + +echo '::error::Source probing exceeded the bounded three-pass limit' >&2 +exit 1 From a53cb4ecdeb1ae39406ce9a39960e2801c2770b7 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 02:59:17 +0200 Subject: [PATCH 07/31] ci(vold): add hosted graph-plan stage and bounded subset build probes --- .github/workflows/vold-baseline.yml | 92 ++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 28 deletions(-) diff --git a/.github/workflows/vold-baseline.yml b/.github/workflows/vold-baseline.yml index e7899bb1..110db9e8 100644 --- a/.github/workflows/vold-baseline.yml +++ b/.github/workflows/vold-baseline.yml @@ -4,6 +4,16 @@ name: Qualify original LineageOS pdx235 vold (no patch) # once this *baseline-only* workflow is merged to the default branch. on: workflow_dispatch: + inputs: + mode: + description: Static plan, source-only compile probe (not proof), or authenticated byte comparison + type: choice + required: true + default: plan + options: + - plan + - probe + - compare permissions: contents: read @@ -13,7 +23,42 @@ concurrency: cancel-in-progress: false jobs: + dependency_plan: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Checkout immutable manifest and source probe + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Trace the initial pinned 67-project source selection + shell: bash + run: | + set -euo pipefail + python3 src/ci/vold_source_closure.py plan --output "${RUNNER_TEMP}/vold-source-plan.json" + python3 - <<'PY' + import json + import os + from pathlib import Path + info = json.loads((Path(os.environ["RUNNER_TEMP"]) / "vold-source-plan.json").read_text()) + assert info["schema_version"] == 1 + count = len(info["selected_projects"]) + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as summary: + summary.write(f"### Provisional vold source graph: {count}/1243 manifest projects\n") + summary.write("This is a source probe, **not** a complete Soong closure or a reproduced binary.\n") + summary.write("A real build also requires the missing Sony vendor configuration and a verified Android toolchain.\n") + PY + - name: Upload static source dependency selection + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: vold-source-provisional-plan + path: ${{ runner.temp }}/vold-source-plan.json + retention-days: 14 + prerequisites: + needs: dependency_plan + if: inputs.mode != 'plan' runs-on: ubuntu-24.04 permissions: contents: read @@ -37,6 +82,7 @@ jobs: } unmodified_vold: + name: ${{ inputs.mode == 'probe' && 'Source compile probe — NOT QUALIFIED' || 'Authentic byte-for-byte original vold check' }} needs: prerequisites runs-on: [self-hosted, linux, x64, modos-lineage-large] timeout-minutes: 360 @@ -68,8 +114,8 @@ jobs: } path="$(realpath -e -- "$path")" free="$(df -PB1 "$path" | awk 'END {print $4}')" - (( free >= 250000000000 )) || { - echo '::error::At least 250 GB free space is required before syncing the complete 1,243-project tree' + (( free >= 100000000000 )) || { + echo '::error::The 67-project subset probe requires at least 100 GB as a conservative starting allowance (not a proven maximum)' exit 1 } workspace="$(mktemp -d "$path/modos-original-vold.XXXXXXXX")" @@ -77,6 +123,7 @@ jobs: echo "VOLD_WORK=$workspace" >> "$GITHUB_ENV" - name: Fetch only the immutable official October 16 source OTA + if: inputs.mode == 'compare' shell: bash run: | set -euo pipefail @@ -96,45 +143,34 @@ jobs: --output "$VOLD_WORK/source.zip" "$url" - name: Authenticate official publisher and extract original vold + if: inputs.mode == 'compare' shell: bash run: | set -euo pipefail bash src/ci/extract_original_vold.sh "$VOLD_WORK/source.zip" "$VOLD_WORK/reference" - - name: Synchronize the exact 1,243-project original manifest - shell: bash - run: | - set -euo pipefail - python3 src/ci/build_original_vold.py sync \ - --source-root "$VOLD_WORK/source" --jobs 4 - - name: Build unmodified vold and require byte-exact equality + - name: Probe missing project dependencies and compare original when selected shell: bash + env: + VOLD_PROBE_STAGE: ${{ inputs.mode }} + VOLD_SOURCE_REF: ${{ env.VOLD_WORK }}/reference/original-vold + VOLD_PROBE_JOBS: '4' run: | set -euo pipefail - python3 src/ci/build_original_vold.py compare \ - --source-root "$VOLD_WORK/source" \ - --reference-vold "$VOLD_WORK/reference/original-vold" \ - --output-dir "$VOLD_WORK/baseline" --jobs 4 - python3 - <<'PY' - import json - import os - from pathlib import Path - p = Path(os.environ['VOLD_WORK']) / 'baseline' / 'baseline-report.json' - data = json.loads(p.read_text()) - assert data['byte_identical'] is True - with Path(os.environ['GITHUB_STEP_SUMMARY']).open('a') as report: - report.write('## Unmodified vold reproduced byte-for-byte\n') - report.write(f"Source OTA SHA-256: `{data['official_ota_sha256']}`\n\n") - report.write(f"Original vold SHA-256: `{data['reference_sha256']}`\n") - PY + bash src/ci/probe_original_vold.sh - - name: Retain small comparison receipt only + - name: Upload only small dependency diagnostics or comparison receipt if: always() && env.VOLD_WORK != '' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: original-pdx235-vold-reproducibility-report - path: ${{ env.VOLD_WORK }}/baseline/baseline-report.json + name: vold-probe-${{ inputs.mode }}-diagnostics + path: | + ${{ env.VOLD_WORK }}/baseline-*/baseline-report.json + ${{ env.VOLD_WORK }}/source-selection.json + ${{ env.VOLD_WORK }}/source-index-*.json + ${{ env.VOLD_WORK }}/dependency-diagnosis-*.json + ${{ env.VOLD_WORK }}/probe-only.txt if-no-files-found: warn retention-days: 14 From dd2d5c6e280a0339cae0c523d0d8777af899ad34 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:00:06 +0200 Subject: [PATCH 08/31] test(vold): update baseline guard for explicit unmodified graph/compile probe --- tests/vold_baseline_ci_test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/vold_baseline_ci_test.py b/tests/vold_baseline_ci_test.py index 9f22afc3..4e711916 100644 --- a/tests/vold_baseline_ci_test.py +++ b/tests/vold_baseline_ci_test.py @@ -166,17 +166,20 @@ def test_workflow_is_opt_in_readonly_original_only(self): self.assertNotIn(" schedule:", flow) self.assertIn("runs-on: [self-hosted, linux, x64, modos-lineage-large]", flow) self.assertIn("MODOS_VOLD_BASELINE_RUNNER_READY", flow) - self.assertIn("free >= 250000000000", flow) + self.assertIn("free >= 100000000000", flow) self.assertIn("contents: read", flow) self.assertIn("baseline-report.json", flow) self.assertIn("rm -rf -- \"$VOLD_WORK\"", flow) - self.assertIn("src/ci/build_original_vold.py compare", flow) + self.assertIn("src/ci/probe_original_vold.sh", flow) + probe = (ROOT / "src/ci/probe_original_vold.sh").read_text() + self.assertIn("src/ci/build_original_vold.py compare", probe) + self.assertIn("src/ci/build_original_vold.py probe", probe) self.assertIn("source_care_map_require_lineageos_pdx235_publisher", auth) self.assertIn("avbroot ota verify", auth) self.assertIn("avbroot ota extract", auth) self.assertIn("comparison(official, built, lock)", original) self.assertIn("verify_checkout(source, manifest, clean=True)", original) - for text in (flow, original, auth): + for text in (flow, original, auth, probe): self.assertNotIn("git apply", text) self.assertNotIn("ota patch", text) self.assertNotIn("force-publish", text) From d013cabdf21f36d88c8afc870a3efa65cfac3acb Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:00:57 +0200 Subject: [PATCH 09/31] test(vold): exercise immutable 67-project seeds, graph, error-driven expansion, safe reuse --- tests/vold_source_closure_test.py | 161 ++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 tests/vold_source_closure_test.py diff --git a/tests/vold_source_closure_test.py b/tests/vold_source_closure_test.py new file mode 100644 index 00000000..7492f929 --- /dev/null +++ b/tests/vold_source_closure_test.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Regression tests for pin-bound, iterative unmodified vold dependency scans.""" +from __future__ import annotations + +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src/ci")) + +import build_original_vold as original # noqa: E402 +import vold_source_closure as closure # noqa: E402 + + +class VoldSourceClosureTest(unittest.TestCase): + def setUp(self): + self.seed, self.manifest, self.baseline = closure.verified_contract() + self.initial = closure.initial_projects(self.seed) + + def test_pinned_67_project_bootstrap_is_strictly_smaller_than_full_manifest(self): + self.assertEqual(len(self.manifest), 1243) + self.assertEqual(len(self.initial), 67) + self.assertEqual(len(self.initial), len(set(self.initial))) + self.assertTrue({"system/vold", "build/soong", "frameworks/base", "device/sony/pdx235"}.issubset(self.initial)) + self.assertTrue(set(self.initial).issubset(self.manifest)) + self.assertEqual(self.seed["manifest_sha256"], self.baseline["manifest"]["sha256"]) + self.assertIn("vendor/sony/pdx235/pdx235-vendor.mk", self.seed["known_unpinned_vendor_inputs"]) + self.assertNotIn("vendor/sony/pdx235", self.manifest) + + def test_invalid_or_unpinned_expansions_fail_closed(self): + correct = closure.selection_record(self.initial, self.seed) + self.assertEqual(closure.validate_selection(correct, self.manifest, self.seed), self.initial) + for mutate in [ + lambda x: x.update(manifest_sha256="0" * 64), + lambda x: x.update(selected_projects=["system/vold"]), + lambda x: x.update(selected_projects=x["selected_projects"] + ["vendor/sony/pdx235"]), + lambda x: x.update(selected_projects=x["selected_projects"] * 2), + ]: + selection = dict(correct) + mutate(selection) + with self.assertRaises(closure.ClosureError): + closure.validate_selection(selection, self.manifest, self.seed) + + def test_diagnostics_identify_pinned_source_vs_unpinned_vendor(self): + log = ( + 'error: module "vold" depends on undefined module "fuseMedia.bpf"\n' + 'error: source file system/bpf/fuse_media.c: No such file or directory\n' + 'error: vendor/sony/pdx235/pdx235-vendor.mk missing\n' + 'error: module "vold" depends on undefined module "unsupported_vendor_blob"\n' + ) + report = closure.missing_dependencies(log, self.initial, self.manifest, self.seed) + self.assertIn("system/bpf", report["proposed_project_additions"]) + self.assertIn("vendor/sony/pdx235/pdx235-vendor.mk", report["unpinned_missing_paths"]) + self.assertIn("unsupported_vendor_blob", report["modules_without_verified_owner"]) + self.assertNotIn("vendor/sony/pdx235", report["proposed_project_additions"]) + self.assertEqual(closure.owner("system/fs/fs_mgr/fs_mgr/Android.bp", self.manifest), "system/fs/fs_mgr") + self.assertIsNone(closure.owner("vendor/sony/pdx235/pdx235-vendor.mk", self.manifest)) + + def test_static_soong_modules_trace_defaults_and_transitive_references(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bp = root / "system/vold/Android.bp" + bp.parent.mkdir(parents=True) + bp.write_text( + 'cc_defaults {\n name: "vold_default",\n shared_libs: ["libbase"],\n}\n' + 'cc_binary {\n name: "vold",\n defaults: ["vold_default"],\n' + 'static_libs: ["libvold"],\n srcs: [":vold_aidl"],\n' + 'arch: { arm64: { shared_libs: ["libcrypto"], }, },\n}\n' + 'cc_library_static {\n name: "libvold",\n static_libs: ["libdm"],\n}\n' + 'filegroup {\n name: "vold_aidl",\n srcs: ["binder/IVold.aidl"],\n}\n' + ) + report = closure.blueprint_module_graph(root, self.initial) + self.assertEqual(report["scanned_android_bp_files"], 1) + self.assertGreaterEqual(report["reachable_module_names"], 6) + self.assertTrue({"libcrypto", "libbase", "libdm"}.issubset(report["unresolved_module_names"])) + self.assertNotIn("vold_aidl", report["unresolved_module_names"]) + + def test_incremental_sync_reuses_checkout_and_syncs_only_pinned_projects(self): + manifest_bytes = (ROOT / self.baseline["manifest"]["path"]).read_bytes() + source_ref = self.baseline["projects"]["manifest"]["revision"] + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "source" + log = [] + + def fake_command(argv, cwd, **kwargs): + log.append(argv) + repo_dir = cwd / ".repo" + if argv[1] == "init" and "-m" not in argv: + (repo_dir / "manifests").mkdir(parents=True) + (repo_dir / "manifest.xml").write_text("prior default") + if argv[1] == "init" and "-m" in argv: + (repo_dir / "manifest.xml").write_bytes( + (repo_dir / "manifests" / argv[-1]).read_bytes() + ) + + with ( + patch.object(original.shutil, "which", return_value="/usr/bin/repo"), + patch.object(original, "command", side_effect=fake_command), + patch.object(original, "verify_checkout"), + ): + original.sync(source, self.baseline, manifest_bytes, 2, self.initial) + self.assertEqual([a[1] for a in log], ["init", "init", "sync"]) + self.assertEqual(log[-1][-len(self.initial):], self.initial) + self.assertIn(source_ref, log[0]) + log.clear() + more = sorted(set(self.initial) | {"system/bpf"}) + original.sync(source, self.baseline, manifest_bytes, 2, more, resume=True) + self.assertEqual(len(log), 1) + self.assertEqual(log[0][1], "sync") + self.assertIn("system/bpf", log[0]) + + def test_selections_are_checked_against_unchanged_manifest(self): + raw = (ROOT / self.baseline["manifest"]["path"]).read_bytes() + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + (source / ".repo").mkdir() + (source / ".repo/manifest.xml").write_bytes(raw) + first = self.initial[:2] + revisions = original.project_map(raw) + + def mocked_command(argv, cwd, **kwargs): + result = revisions[cwd.relative_to(source).as_posix()][1] if argv[1] == "rev-parse" else "" + return type("Result", (), {"stdout": result})() + + for project in first: + (source / project).mkdir(parents=True) + with ( + patch.object(original.shutil, "which", return_value="/usr/bin/repo"), + patch.object(original, "command", side_effect=mocked_command), + patch.object(original.subprocess, "run", + side_effect=AssertionError("subset should never request a full repo manifest -r")), + ): + original.verify_checkout(source, raw, clean=True, selected=first) + (source / ".repo/manifest.xml").write_text("tampered") + with self.assertRaises(original.BuildError): + original.verify_checkout(source, raw, clean=True, selected=first) + + def test_probe_remains_independent_of_published_vold_patch(self): + script = (ROOT / "src/ci/probe_original_vold.sh").read_text() + workflow = (ROOT / ".github/workflows/vold-baseline.yml").read_text() + builder = (ROOT / "src/ci/build_original_vold.py").read_text() + self.assertIn("repo" if False else "sync --resume", script) + self.assertIn("for iteration in 1 2 3", script) + self.assertIn("vold_source_closure.py diagnose", script) + self.assertIn("dependency_plan:", workflow) + self.assertIn("default: plan", workflow) + self.assertIn("Source compile probe — NOT QUALIFIED", workflow) + self.assertIn("probe", builder) + for source in (script, workflow, builder): + self.assertNotIn("git apply", source) + self.assertNotIn("force-publish", source) + self.assertNotIn("SOONG_ALLOW_MISSING_DEPENDENCIES", source) + self.assertNotIn("publish: true", source) + + +if __name__ == "__main__": + unittest.main() From cf8aa7b92725758bad31a963adc3a55907df8698 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:01:17 +0200 Subject: [PATCH 10/31] ci(vold): validate static source graph and incremental bounded source checkout --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c93ab3b..bb5dcd18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,9 @@ jobs: - name: Test unmodified original vold CI-only baseline run: python3 tests/vold_baseline_ci_test.py + - name: Test bounded original Vold source dependency resolver + run: python3 tests/vold_source_closure_test.py + - name: Test pdx235 Custota metadata preparation run: bash tests/pdx235_custota_metadata_test.sh From 3e0f5b5163956339dddbf3819645354bec483701 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:03:23 +0200 Subject: [PATCH 11/31] fix(vold): add direct BPF/IncFS Soong project dependencies before first sync --- locks/lineage-vold-source-probe-20261016.json | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/locks/lineage-vold-source-probe-20261016.json b/locks/lineage-vold-source-probe-20261016.json index 8b5c08d7..77b5f6dc 100644 --- a/locks/lineage-vold-source-probe-20261016.json +++ b/locks/lineage-vold-source-probe-20261016.json @@ -29,40 +29,43 @@ "prebuilts/ndk" ], "vold_direct_and_native": [ - "system/vold", - "system/core", - "system/fs/fs_mgr", - "system/fs/casefolding_remover", - "system/security", - "system/sepolicy", - "system/logging", - "system/extras", - "system/memory/libmeminfo", - "system/memory/libion", - "system/memory/libwrapfd", - "frameworks/base", - "frameworks/native", - "frameworks/av", - "frameworks/hardware/interfaces", "bootable/libbootloader", "bootable/recovery", - "hardware/interfaces", - "hardware/libhardware", "external/avb", "external/boringssl", + "external/dtc", "external/e2fsprogs", + "external/expat", "external/f2fs-tools", - "external/selinux", - "external/protobuf", - "external/zlib", + "external/fmtlib", + "external/googletest", "external/icu", - "external/libcxx", "external/libcap", - "external/expat", - "external/dtc", + "external/libcxx", + "external/protobuf", + "external/selinux", "external/sqlite", - "external/googletest", - "external/fmtlib" + "external/zlib", + "frameworks/av", + "frameworks/base", + "frameworks/hardware/interfaces", + "frameworks/native", + "hardware/interfaces", + "hardware/libhardware", + "system/bpf", + "system/bpfprogs", + "system/core", + "system/extras", + "system/fs/casefolding_remover", + "system/fs/fs_mgr", + "system/incremental_delivery", + "system/logging", + "system/memory/libion", + "system/memory/libmeminfo", + "system/memory/libwrapfd", + "system/security", + "system/sepolicy", + "system/vold" ], "lineage_sony_product": [ "device/sony/pdx235", @@ -119,11 +122,13 @@ "libboot_control_client": "hardware/interfaces", "android.hardware.boot@1.0": "hardware/interfaces", "android.hardware.boot-V1-ndk": "hardware/interfaces", - "libkeymint_support": "system/security", + "libkeymint_support": "hardware/interfaces", "libsysutils": "system/core", "libincfs": "system/incremental_delivery", - "fuseMedia.bpf": "system/bpf", - "mke2fs": "external/e2fsprogs" + "fuseMedia.bpf": "system/bpfprogs", + "mke2fs": "external/e2fsprogs", + "libfec_rs": "system/extras", + "libasync_safe": "bionic" }, "known_unpinned_vendor_inputs": [ "vendor/sony/pdx235/pdx235-vendor.mk", From 6a3753346a42b9f03e69d62f48ffc49f0ffdd4e6 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:03:44 +0200 Subject: [PATCH 12/31] test(vold): cover all 70 directly traced projects and late missing-path expansion --- .github/workflows/vold-baseline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vold-baseline.yml b/.github/workflows/vold-baseline.yml index 110db9e8..ce9413de 100644 --- a/.github/workflows/vold-baseline.yml +++ b/.github/workflows/vold-baseline.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v7 with: persist-credentials: false - - name: Trace the initial pinned 67-project source selection + - name: Trace the initial pinned 70-project source selection shell: bash run: | set -euo pipefail @@ -115,7 +115,7 @@ jobs: path="$(realpath -e -- "$path")" free="$(df -PB1 "$path" | awk 'END {print $4}')" (( free >= 100000000000 )) || { - echo '::error::The 67-project subset probe requires at least 100 GB as a conservative starting allowance (not a proven maximum)' + echo '::error::The 70-project subset probe requires at least 100 GB as a conservative starting allowance (not a proven maximum)' exit 1 } workspace="$(mktemp -d "$path/modos-original-vold.XXXXXXXX")" From 2b7e6af0ad057efad1fffc50e21f52be0438113a Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:03:59 +0200 Subject: [PATCH 13/31] test(vold): trace 70 direct source projects and verify late expansion --- tests/vold_source_closure_test.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/vold_source_closure_test.py b/tests/vold_source_closure_test.py index 7492f929..134812ae 100644 --- a/tests/vold_source_closure_test.py +++ b/tests/vold_source_closure_test.py @@ -21,9 +21,9 @@ def setUp(self): self.seed, self.manifest, self.baseline = closure.verified_contract() self.initial = closure.initial_projects(self.seed) - def test_pinned_67_project_bootstrap_is_strictly_smaller_than_full_manifest(self): + def test_pinned_70_project_bootstrap_is_strictly_smaller_than_full_manifest(self): self.assertEqual(len(self.manifest), 1243) - self.assertEqual(len(self.initial), 67) + self.assertEqual(len(self.initial), 70) self.assertEqual(len(self.initial), len(set(self.initial))) self.assertTrue({"system/vold", "build/soong", "frameworks/base", "device/sony/pdx235"}.issubset(self.initial)) self.assertTrue(set(self.initial).issubset(self.manifest)) @@ -48,12 +48,12 @@ def test_invalid_or_unpinned_expansions_fail_closed(self): def test_diagnostics_identify_pinned_source_vs_unpinned_vendor(self): log = ( 'error: module "vold" depends on undefined module "fuseMedia.bpf"\n' - 'error: source file system/bpf/fuse_media.c: No such file or directory\n' + 'error: source file system/update_engine/Android.bp: No such file or directory\n' 'error: vendor/sony/pdx235/pdx235-vendor.mk missing\n' 'error: module "vold" depends on undefined module "unsupported_vendor_blob"\n' ) report = closure.missing_dependencies(log, self.initial, self.manifest, self.seed) - self.assertIn("system/bpf", report["proposed_project_additions"]) + self.assertIn("system/update_engine", report["proposed_project_additions"]) self.assertIn("vendor/sony/pdx235/pdx235-vendor.mk", report["unpinned_missing_paths"]) self.assertIn("unsupported_vendor_blob", report["modules_without_verified_owner"]) self.assertNotIn("vendor/sony/pdx235", report["proposed_project_additions"]) @@ -107,11 +107,11 @@ def fake_command(argv, cwd, **kwargs): self.assertEqual(log[-1][-len(self.initial):], self.initial) self.assertIn(source_ref, log[0]) log.clear() - more = sorted(set(self.initial) | {"system/bpf"}) + more = sorted(set(self.initial) | {"system/update_engine"}) original.sync(source, self.baseline, manifest_bytes, 2, more, resume=True) self.assertEqual(len(log), 1) self.assertEqual(log[0][1], "sync") - self.assertIn("system/bpf", log[0]) + self.assertIn("system/update_engine", log[0]) def test_selections_are_checked_against_unchanged_manifest(self): raw = (ROOT / self.baseline["manifest"]["path"]).read_bytes() @@ -143,7 +143,7 @@ def test_probe_remains_independent_of_published_vold_patch(self): script = (ROOT / "src/ci/probe_original_vold.sh").read_text() workflow = (ROOT / ".github/workflows/vold-baseline.yml").read_text() builder = (ROOT / "src/ci/build_original_vold.py").read_text() - self.assertIn("repo" if False else "sync --resume", script) + self.assertIn("sync --resume", script) self.assertIn("for iteration in 1 2 3", script) self.assertIn("vold_source_closure.py diagnose", script) self.assertIn("dependency_plan:", workflow) From 335165982420dca84fbd380e75f118f10873c5a9 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:04:49 +0200 Subject: [PATCH 14/31] docs(vold): document traced 70-project graph and three-pass pinned checkout --- docs/vold-baseline-ci.md | 150 ++++++++++++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 35 deletions(-) diff --git a/docs/vold-baseline-ci.md b/docs/vold-baseline-ci.md index 189361f4..2b916eb2 100644 --- a/docs/vold-baseline-ci.md +++ b/docs/vold-baseline-ci.md @@ -37,41 +37,121 @@ the reference and built SHA-256s, byte counts and first differing offset. Only `byte_identical: true` is success. The workflow fails on a mismatch and uploads a diagnostic JSON receipt, rather than bypassing the check. -## Runner provisioning (required before first real run) - -The GitHub Actions workflow -[`vold-baseline.yml`](../.github/workflows/vold-baseline.yml) is -manual-only (`workflow_dispatch`) and runs on the repository's `main` -branch. A regular 14 GB GitHub-hosted Ubuntu runner is too small. - -The repository owner must provision a **trusted, isolated Linux x64 -self-hosted runner**, labeled `modos-lineage-large`, with a dedicated -scratch filesystem offering at least **250 GB free** and sufficient CPU/RAM -for an Android 17 `m vold` build. It needs LineageOS/Android build -prerequisites, the `repo` tool, Git, Python 3, curl, OpenSSL, unzip, -`debugfs`, and a verified `avbroot` binary supporting `ota verify` and -`ota extract`. Install and verify these dependencies through the host's -normal trusted administration process, not from an unreviewed workflow PR. - -Configure two repository variables: - -- `MODOS_LINEAGE_LARGE_SCRATCH` — absolute path to that real, dedicated - scratch directory on the self-hosted runner. -- `MODOS_VOLD_BASELINE_RUNNER_READY=true` — set only when the owner has - confirmed the runner is isolated, sufficiently provisioned and trusted. - -The workflow's GitHub token has **read-only** repository permission and does -not receive signing keys or publishing secrets. Concurrent baseline jobs -are serialized. A failed preflight stops before allocating the source -workspace; all scratch content is deleted after uploading the small -comparison report. Do not register an untrusted shared self-hosted runner -for this purpose. - -Once the baseline-only PR has passed ordinary CI and been merged, -open **Actions → Qualify original LineageOS pdx235 vold (no patch) → -Run workflow** on `main`. Any missing runner, unavailable pinned OTA, -manifest mismatch, build failure, or byte mismatch is a failure, not an -approval. The full source tree or reference image is never uploaded. +## Dependency-closure strategy (begin with 70 of 1,243 projects) + +The original workflow fetched **all 1,243 pinned projects** before issuing +`m vold`. That is unnecessary for a single module, and it made each trial +expensive. Instead, the dependency-analysis recipe in +[`locks/lineage-vold-source-probe-20261016.json`](../locks/lineage-vold-source-probe-20261016.json) +selects **70 initial projects**. This is a conservative initial dependency +seed, NOT a complete or proven Soong build graph. + +The source reasoning is grounded in three exact upstream revisions: + +- [`system/vold/Android.bp`](https://github.com/LineageOS/android_system_vold/blob/11d8982f824d52e8122c7aa732207ba66b8f23c8/Android.bp): + `vold` links `libvold`, and its defaults include Binder, cryptography, + AVB, fs_mgr, native libc, and system service libraries. The module also + directly requires `mke2fs`, `vold_prepare_subdirs`, and `fuseMedia.bpf`. +- [`device/sony/pdx235/device.mk`](https://github.com/LineageOS/android_device_sony_pdx235/blob/0559de69466e1e9af69b121985f592c4f4609283/device.mk): + the pdx235 build inherits chipset definitions and a generated proprietary + Sony vendor makefile. +- [`device/sony/sm6375-common/common.mk`](https://github.com/LineageOS/android_device_sony_sm6375-common/blob/d1d80af1ad9032cb092ff6d85db437709cc54e28/common.mk): + the common product also includes proprietary `vendor/sony/sm6375-common`. + Two other Sony vendor BoardConfig files are referenced. + +**Important external dependency:** the exact official public manifest does +**not** contain `vendor/sony/pdx235` or +`vendor/sony/sm6375-common`. Do not invent their contents or disable +Soong missing-dependency checks. The product's `extract-files.py` and the +matching authenticated source image can help resolve the original proprietary +inputs, but that derivation requires separate evidence and must not be +treated as publisher-identical build configuration automatically. + +The on-disk dependency resolver +[`src/ci/vold_source_closure.py`](../src/ci/vold_source_closure.py) +verifies the complete original manifest and produces a pinned selection. +After a limited `repo sync `, it scans the available +`Android.bp` definitions for a **partial** transitive module graph. The real +Soong compiler, rather than the static scan, remains authoritative for +conditional/generated modules. Any missing path reported in the compiler +diagnostics is mapped to its **exact existing** project in the official +manifest. Known module names have reviewed initial provider hints; unknown +providers and unpinned vendor paths stop for manual investigation. + +Within a single CI run, +[`src/ci/probe_original_vold.sh`](../src/ci/probe_original_vold.sh) +automatically repeats up to **three** controlled passes. It caches already +synchronized projects in the same working tree, fetches only newly identified +pinned dependencies, and saves small diagnostic JSON documents showing +unknown modules and other blockers. It never enables +`SOONG_ALLOW_MISSING_DEPENDENCIES`, changes the build target, or applies +the exFAT timeout patch. Any unexpected source revision or modified +checkout fails immediately. + +You can test the graph on an ordinary GitHub-hosted runner with **Actions → +Qualify original LineageOS pdx235 vold (no patch) → Run workflow**, +choosing `plan` (the default). This needs **no self-hosted runner or +repository variable** and produces a +`vold-source-provisional-plan` artifact. It does **not** compile or +qualify a binary. + +## Runner provisioning for source compilation + +For `probe` or `compare`, the workflow still requires a trusted, +isolated self-hosted Linux x64 runner labeled `modos-lineage-large`. +Provide a dedicated scratch filesystem with **at least 100 GB free as an +experimental starting allowance**, sufficient CPU/RAM, the Android 17 +build prerequisites, and `repo`, `avbroot`, `debugfs`, `unzip`, +`curl`, `openssl`, `git`, and `python3`. One iteration may require +more storage. The 100 GB threshold is **not an empirically measured maximum +or proof** that the selected dependency graph is complete. + +Set repository Actions variables: + +- `MODOS_LINEAGE_LARGE_SCRATCH` — an absolute, real, isolated scratch + directory on the trusted self-hosted runner. +- `MODOS_VOLD_BASELINE_RUNNER_READY=true` — only when that runner has been + provisioned and independently checked. + +Use `probe` to attempt the smaller pinned source checkout and compile +unmodified `m vold` **without downloading the official source OTA**. +A successfully compiled probe is **not** binary-equality evidence; its +workflow job explicitly labels it as unqualified. It is useful for +discovering missing modules, vendor inputs and toolchains cheaply. + +Use `compare` for a real proof. It first reauthenticates the exact +publisher-signed official 2026-10-16 OTA, extracts the original +`/system/bin/vold`, and invokes the same bounded dependency solver. +**Only** a real `byte_identical: true` report with equal original and +built hashes qualifies this baseline. If the binaries differ, it emits a +diagnostic comparison receipt and fails. + +From a checked-out repository, local inspection is also possible without +compiling or obtaining the proprietary vendor files: + +```sh +python3 src/ci/vold_source_closure.py plan --output /tmp/vold-projects.json +# After a limited repo sync on a trusted build host: +python3 src/ci/vold_source_closure.py index \ + --source-root /path/to/pinned-source \ + --selection /tmp/vold-projects.json \ + --output /tmp/vold-blueprint-graph.json +# After an unsuccessful m vold build: +python3 src/ci/vold_source_closure.py diagnose \ + --selection /tmp/vold-projects.json \ + --log /path/to/build-failure.log \ + --output /tmp/vold-diagnosis.json \ + --next-selection /tmp/vold-projects-expanded.json +``` + +All selection changes must remain part of the exact original manifest. +The build helper supports `sync --projects-file ... --resume` to fetch +additional pinned projects without repeating an entire sync. + +The workflow uses a read-only GitHub token, never receives release or +signing secrets, uploads diagnostics/comparison receipts only, and clears +the large checkout and original OTA after completion. Do not use an +untrusted shared runner. ## After the first authentic success From a428f2b6258aa5155fd5983ffb7e9a01d1e15a6a Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:05:42 +0200 Subject: [PATCH 15/31] fix(vold): bind dynamic isolated scratch reference at shell step execution --- .github/workflows/vold-baseline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/vold-baseline.yml b/.github/workflows/vold-baseline.yml index ce9413de..82c6d532 100644 --- a/.github/workflows/vold-baseline.yml +++ b/.github/workflows/vold-baseline.yml @@ -154,10 +154,10 @@ jobs: shell: bash env: VOLD_PROBE_STAGE: ${{ inputs.mode }} - VOLD_SOURCE_REF: ${{ env.VOLD_WORK }}/reference/original-vold VOLD_PROBE_JOBS: '4' run: | set -euo pipefail + export VOLD_SOURCE_REF="${VOLD_WORK}/reference/original-vold" bash src/ci/probe_original_vold.sh - name: Upload only small dependency diagnostics or comparison receipt From 6fdc9af46f8a369ec25aa1dac6afa65cb8bb7285 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:08:33 +0200 Subject: [PATCH 16/31] test(vold): assert stronger selected-project cleanliness guard --- tests/vold_baseline_ci_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vold_baseline_ci_test.py b/tests/vold_baseline_ci_test.py index 4e711916..77b1ac61 100644 --- a/tests/vold_baseline_ci_test.py +++ b/tests/vold_baseline_ci_test.py @@ -178,7 +178,7 @@ def test_workflow_is_opt_in_readonly_original_only(self): self.assertIn("avbroot ota verify", auth) self.assertIn("avbroot ota extract", auth) self.assertIn("comparison(official, built, lock)", original) - self.assertIn("verify_checkout(source, manifest, clean=True)", original) + self.assertIn("verify_checkout(source, manifest, clean=True, selected=selected)", original) for text in (flow, original, auth, probe): self.assertNotIn("git apply", text) self.assertNotIn("ota patch", text) From 362699ba130397da0c6fefd336700f889d849042 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:08:45 +0200 Subject: [PATCH 17/31] fix(vold): hash canonical newline-delimited selected source paths --- src/ci/build_original_vold.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/build_original_vold.py b/src/ci/build_original_vold.py index 71f8c1e7..6fce04a8 100644 --- a/src/ci/build_original_vold.py +++ b/src/ci/build_original_vold.py @@ -232,7 +232,7 @@ def build_baseline( if selected is not None: report["selected_source_projects"] = len(selected) report["selected_source_paths_sha256"] = hashlib.sha256( - ("\\n".join(sorted(selected)) + "\\n").encode() + ("\n".join(sorted(selected)) + "\n").encode() ).hexdigest() report["source_checkout_scope"] = "manifest-pinned-subset" else: From d4f08546f20160e9aeb9a374fd636bedc5b6b5a1 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:09:44 +0200 Subject: [PATCH 18/31] fix(vold): simplify strict manifest root/revision path validation --- src/ci/vold_source_closure.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ci/vold_source_closure.py b/src/ci/vold_source_closure.py index e36e32ad..df060b86 100644 --- a/src/ci/vold_source_closure.py +++ b/src/ci/vold_source_closure.py @@ -14,14 +14,12 @@ from pathlib import Path import re import sys -from typing import Any import xml.etree.ElementTree as ET ROOT = Path(__file__).resolve().parents[2] SEED = ROOT / "locks/lineage-vold-source-probe-20261016.json" BASELINE = ROOT / "locks/lineage-vold-baseline-20261016.json" -PATH_RE = re.compile(r"^[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)+$") PATH_IN_LOG = re.compile( r"(? tuple[dict, dict[str, str], dict]: projects: dict[str, str] = {} for element in xml.findall("project"): path, revision = element.get("path"), element.get("revision") - if not path or not PATH_RE.fullmatch(path) and "/" in path or not revision or not re.fullmatch("[0-9a-f]{40}", revision): - # Root-level paths are valid in official Android manifests. - if not path or not re.fullmatch(r"[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)*", path) or not revision or not re.fullmatch("[0-9a-f]{40}", revision): - raise ClosureError("manifest contains an invalid project identity") + # One-segment roots (eg. art, bionic) are valid. Any path escaping + # source root or any branch/tag-based revision is not. + if ( + not path + or not re.fullmatch(r"[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)*", path) + or not revision + or not re.fullmatch(r"[0-9a-f]{40}", revision) + ): + raise ClosureError("manifest contains an invalid project identity") if path in projects: raise ClosureError("source manifest contains duplicate project paths") projects[path] = revision From 36f764a9a139a1012908b5ea2ee3d7fd34319ce6 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:11:37 +0200 Subject: [PATCH 19/31] fix(vold): resolve implicit project paths exactly as Repo manifest semantics --- src/ci/vold_source_closure.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ci/vold_source_closure.py b/src/ci/vold_source_closure.py index df060b86..4b39cd68 100644 --- a/src/ci/vold_source_closure.py +++ b/src/ci/vold_source_closure.py @@ -67,7 +67,9 @@ def verified_contract() -> tuple[dict, dict[str, str], dict]: raise ClosureError("pinned project manifest cannot be parsed") from exc projects: dict[str, str] = {} for element in xml.findall("project"): - path, revision = element.get("path"), element.get("revision") + # Repo defaults omitted project paths to the project name. + # The official pinned manifest uses that form for 44 projects. + path, revision = element.get("path", element.get("name")), element.get("revision") # One-segment roots (eg. art, bionic) are valid. Any path escaping # source root or any branch/tag-based revision is not. if ( From 2e6d71484d1f35f415ce88a6a8cbff3514cc35f7 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:11:50 +0200 Subject: [PATCH 20/31] test(vold): cover implicit project path rules in official source manifest --- tests/vold_source_closure_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/vold_source_closure_test.py b/tests/vold_source_closure_test.py index 134812ae..30078ace 100644 --- a/tests/vold_source_closure_test.py +++ b/tests/vold_source_closure_test.py @@ -23,6 +23,9 @@ def setUp(self): def test_pinned_70_project_bootstrap_is_strictly_smaller_than_full_manifest(self): self.assertEqual(len(self.manifest), 1243) + # The official manifest omits path= for dozens of AOSP projects. + self.assertIn("device/common", self.manifest) + self.assertIn("toolchain/pgo-profiles", self.manifest) self.assertEqual(len(self.initial), 70) self.assertEqual(len(self.initial), len(set(self.initial))) self.assertTrue({"system/vold", "build/soong", "frameworks/base", "device/sony/pdx235"}.issubset(self.initial)) From 389b0464d6604a84f295df05b45ae1e8d7c93a06 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:15:53 +0200 Subject: [PATCH 21/31] perf(vold): sync only missing pinned projects on resumed closure iteration --- src/ci/build_original_vold.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/ci/build_original_vold.py b/src/ci/build_original_vold.py index 6fce04a8..bc8c669c 100644 --- a/src/ci/build_original_vold.py +++ b/src/ci/build_original_vold.py @@ -181,9 +181,24 @@ def sync( if active_manifest.read_bytes() != manifest: raise BuildError("repo's active manifest differs from the pinned source") # Explicit project-path sync prevents all 1,243 projects being fetched. - # On repeated probes, keep the same pinned workspace and fetch only the - # new members; repo does not need to re-download already pinned projects. - command([repo, "sync", "--no-clone-bundle", f"-j{jobs}", *(selected or [])], source) + # On resume, preverify existing project HEADs and cleanliness, then fetch + # *only newly selected* missing projects. Preserve the original Soong out + # tree and all previously fetched Git objects across source expansions. + to_sync = selected + if resume: + assert selected is not None + present = [] + missing = [] + for project in selected: + path = source / project + if path.is_symlink() or (path.exists() and not path.is_dir()): + raise BuildError(f"previously synced project path is unsafe: {project}") + (present if path.is_dir() else missing).append(project) + if present: + verify_checkout(source, manifest, clean=True, selected=present) + to_sync = missing + if to_sync is None or to_sync: + command([repo, "sync", "--no-clone-bundle", f"-j{jobs}", *(to_sync or [])], source) verify_checkout(source, manifest, clean=True, selected=selected) From 283875b7f283b92676ab29bb7457a87184f87e93 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:16:19 +0200 Subject: [PATCH 22/31] test(vold): prove resumed sync requests only newly missing project --- tests/vold_source_closure_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/vold_source_closure_test.py b/tests/vold_source_closure_test.py index 30078ace..eae80769 100644 --- a/tests/vold_source_closure_test.py +++ b/tests/vold_source_closure_test.py @@ -99,6 +99,9 @@ def fake_command(argv, cwd, **kwargs): (repo_dir / "manifest.xml").write_bytes( (repo_dir / "manifests" / argv[-1]).read_bytes() ) + if argv[1] == "sync": + for project in argv[4:]: + (cwd / project).mkdir(parents=True, exist_ok=True) with ( patch.object(original.shutil, "which", return_value="/usr/bin/repo"), @@ -114,7 +117,10 @@ def fake_command(argv, cwd, **kwargs): original.sync(source, self.baseline, manifest_bytes, 2, more, resume=True) self.assertEqual(len(log), 1) self.assertEqual(log[0][1], "sync") - self.assertIn("system/update_engine", log[0]) + self.assertEqual(log[0][4:], ["system/update_engine"]) + log.clear() + original.sync(source, self.baseline, manifest_bytes, 2, more, resume=True) + self.assertEqual(log, [], "unchanged pinned checkout should not re-run repo sync") def test_selections_are_checked_against_unchanged_manifest(self): raw = (ROOT / self.baseline["manifest"]["path"]).read_bytes() From d7c83f03e51b2da0b45e221aed73dc6ad98d15b9 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:18:45 +0200 Subject: [PATCH 23/31] feat(vold): generate only pinned Sony makefiles without extracting blobs --- src/ci/prepare_vold_vendor_makefiles.py | 136 ++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 src/ci/prepare_vold_vendor_makefiles.py diff --git a/src/ci/prepare_vold_vendor_makefiles.py b/src/ci/prepare_vold_vendor_makefiles.py new file mode 100644 index 00000000..f3e0d81b --- /dev/null +++ b/src/ci/prepare_vold_vendor_makefiles.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Generate missing Sony product makefiles from pinned public extraction lists. + +This is a diagnostic build-probe convenience, not a proprietary-blob source. +LineageOS extract-utils --regenerate_makefiles explicitly skips all copying, +source image extraction, and blob fixups. The generated makefiles do NOT +establish provenance for missing proprietary binaries or runtime behavior. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +import sys + +from build_original_vold import load_contract, verify_checkout + + +class VendorPreparationError(ValueError): + pass + + +INPUTS = ( + "device/sony/pdx235/extract-files.py", + "device/sony/pdx235/proprietary-files.txt", + "device/sony/sm6375-common/extract-files.py", + "device/sony/sm6375-common/proprietary-files.txt", + "tools/extract-utils/extract_utils/main.py", +) +REQUIRED = ( + "vendor/sony/pdx235/pdx235-vendor.mk", + "vendor/sony/pdx235/BoardConfigVendor.mk", + "vendor/sony/pdx235/Android.bp", + "vendor/sony/sm6375-common/sm6375-common-vendor.mk", + "vendor/sony/sm6375-common/BoardConfigVendor.mk", + "vendor/sony/sm6375-common/Android.bp", +) + + +def safe_file(path: Path, max_bytes: int = 4_000_000) -> bytes: + st = path.lstat() + if not stat.S_ISREG(st.st_mode) or not 0 <= st.st_size <= max_bytes: + raise VendorPreparationError(f"generated makefile or source is unsafe: {path}") + content = path.read_bytes() + if len(content) != st.st_size: + raise VendorPreparationError(f"generated makefile changed while reading: {path}") + return content + + +def report(source: Path, input_hashes: dict[str, str]) -> dict: + outputs = {} + for path in REQUIRED: + outputs[path] = hashlib.sha256(safe_file(source / path)).hexdigest() + sony_root = source / "vendor/sony" + for path in sony_root.rglob("*"): + if path.is_symlink(): + raise VendorPreparationError("generated Sony vendor source contains an unsafe link") + if not path.is_file(): + continue + if path.suffix.lower() in (".so", ".apk", ".jar", ".img", ".bin"): + raise VendorPreparationError("unexpected binary blob from makefiles-only mode") + safe_file(path) + return { + "schema_version": 1, + "kind": "synthetic-probe-vendor-makefiles-not-publisher-provenance", + "method": "pinned device extract-files.py --regenerate_makefiles", + "does_not_extract_blobs": True, + "does_not_qualify_original_vold": True, + "input_sha256": input_hashes, + "generated_makefile_sha256": outputs, + "warning": "Proprietary blobs remain absent. A real build may still fail; never treat these generated makefiles as verified official vendor blobs.", + } + + +def prepare(source: Path, receipt: Path) -> dict: + if source.is_symlink() or not source.is_dir(): + raise VendorPreparationError("original source workspace is missing or linked") + source = source.resolve(strict=True) + lock, manifest = load_contract() + # Probe only already-synced immutable projects; no unpinned tools/scripts. + required_projects = [ + "device/sony/pdx235", "device/sony/sm6375-common", "tools/extract-utils", + ] + verify_checkout(source, manifest, clean=True, selected=required_projects) + hashes = {path: hashlib.sha256(safe_file(source / path)).hexdigest() for path in INPUTS} + sony_root = source / "vendor/sony" + if sony_root.is_symlink() or sony_root.exists() and not sony_root.is_dir(): + raise VendorPreparationError("Sony vendor output root is unsafe") + + if receipt.is_symlink(): + raise VendorPreparationError("vendor makefile receipt must not be a symlink") + if receipt.exists(): + saved = json.loads(safe_file(receipt)) + current = report(source, hashes) + if current != saved: + raise VendorPreparationError("cached generated vendor makefiles differ from the pinned source receipt") + return current + + if sony_root.exists() and any(sony_root.iterdir()): + raise VendorPreparationError("unapproved preexisting Sony vendor workspace") + environment = os.environ.copy() + environment["PYTHONPATH"] = str(source / "tools/extract-utils") + subprocess.run( + [sys.executable, "device/sony/pdx235/extract-files.py", "--regenerate_makefiles"], + cwd=source, + env=environment, + check=True, + ) + current = report(source, hashes) + receipt.parent.mkdir(parents=True, exist_ok=True) + with receipt.open("x", encoding="utf-8") as output: + json.dump(current, output, indent=2, sort_keys=True) + output.write("\n") + return current + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + try: + result = prepare(args.source_root, args.report) + print(json.dumps({"kind": result["kind"], "output_count": len(REQUIRED)}, sort_keys=True)) + except (OSError, KeyError, TypeError, ValueError, subprocess.CalledProcessError) as exc: + print(f"Error: cannot produce pinned Sony makefiles: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 8eeea65d7959125a6209ee9947addc5a341fba1e Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:19:15 +0200 Subject: [PATCH 24/31] perf(vold): regenerate missing public Sony makefiles before first Soong probe --- src/ci/probe_original_vold.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/ci/probe_original_vold.sh b/src/ci/probe_original_vold.sh index 553e718a..1edf1804 100644 --- a/src/ci/probe_original_vold.sh +++ b/src/ci/probe_original_vold.sh @@ -32,6 +32,14 @@ python3 src/ci/vold_source_closure.py plan --output "${selection}" python3 src/ci/build_original_vold.py sync \ --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" +# The pinned Sony public device trees reference proprietary vendor-generated +# mk files absent from the source manifest. Generate ONLY makefile declarations +# using their reviewed --regenerate_makefiles option. This does not download, +# synthesize or validate vendor blobs; absent binaries remain build blockers. +python3 src/ci/prepare_vold_vendor_makefiles.py \ + --source-root "${VOLD_WORK}/source" \ + --report "${VOLD_WORK}/sony-generated-makefiles-report.json" + # Most missing dependencies are reported during Soong setup before the # expensive target compile. Resolve their exact pinned project paths once # and reuse the same checkout and Android out directory on each iteration. From f019d35e54a11a631df7aa73819d9ca0d984a102 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:19:25 +0200 Subject: [PATCH 25/31] ci(vold): retain provenance receipt for generated vendor makefile probe --- .github/workflows/vold-baseline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/vold-baseline.yml b/.github/workflows/vold-baseline.yml index 82c6d532..27aeeba6 100644 --- a/.github/workflows/vold-baseline.yml +++ b/.github/workflows/vold-baseline.yml @@ -168,6 +168,7 @@ jobs: path: | ${{ env.VOLD_WORK }}/baseline-*/baseline-report.json ${{ env.VOLD_WORK }}/source-selection.json + ${{ env.VOLD_WORK }}/sony-generated-makefiles-report.json ${{ env.VOLD_WORK }}/source-index-*.json ${{ env.VOLD_WORK }}/dependency-diagnosis-*.json ${{ env.VOLD_WORK }}/probe-only.txt From 2708e87b20a7b224abb9c9545aa2f09cb7cc2c85 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:20:00 +0200 Subject: [PATCH 26/31] test(vold): verify safe generated Sony makefiles and deny unpinned blobs --- tests/vold_vendor_makefiles_test.py | 97 +++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tests/vold_vendor_makefiles_test.py diff --git a/tests/vold_vendor_makefiles_test.py b/tests/vold_vendor_makefiles_test.py new file mode 100644 index 00000000..0fc5ae4b --- /dev/null +++ b/tests/vold_vendor_makefiles_test.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Ensure Sony makefile regeneration remains a no-blob, unqualified probe.""" +from __future__ import annotations + +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src/ci")) +import prepare_vold_vendor_makefiles as prepare # noqa: E402 + + +class SonyVendorMakefileTest(unittest.TestCase): + def source(self, parent: Path) -> Path: + root = parent / "source" + root.mkdir() + for path in prepare.INPUTS: + file = root / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(f"PINNED TEST CONTENT {path}\n") + return root + + def test_regenerate_makefiles_only_and_compare_cached_receipt(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + report_path = root / "report.json" + calls = [] + + def fake_generate(argv, cwd, env, check): + calls.append(argv) + self.assertTrue(check) + self.assertEqual(cwd, source) + self.assertEqual(argv[0], sys.executable) + self.assertEqual(argv[1:], [ + "device/sony/pdx235/extract-files.py", "--regenerate_makefiles", + ]) + self.assertEqual(env["PYTHONPATH"], str(source / "tools/extract-utils")) + for path in prepare.REQUIRED: + output = source / path + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(f"generated from pinned public list: {path}\n") + + with ( + patch.object(prepare, "verify_checkout"), + patch.object(prepare.subprocess, "run", side_effect=fake_generate), + ): + saved = prepare.prepare(source, report_path) + self.assertEqual(saved["kind"], "synthetic-probe-vendor-makefiles-not-publisher-provenance") + self.assertTrue(saved["does_not_extract_blobs"]) + self.assertTrue(saved["does_not_qualify_original_vold"]) + self.assertEqual(len(saved["generated_makefile_sha256"]), len(prepare.REQUIRED)) + self.assertEqual(len(calls), 1) + self.assertEqual(json.loads(report_path.read_text()), saved) + self.assertEqual(prepare.prepare(source, report_path), saved) + self.assertEqual(len(calls), 1, "reused makefiles must not trigger extraction again") + first = source / prepare.REQUIRED[0] + first.write_text("tampered generated files") + with self.assertRaisesRegex(prepare.VendorPreparationError, "differ"): + prepare.prepare(source, report_path) + + def test_proprietary_elf_or_symlink_are_rejected_even_if_generator_succeeds(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + for path in prepare.REQUIRED: + dest = source / path + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_text("makefile") + (source / "vendor/sony/pdx235/libsecret.so").write_bytes(b"binary") + with self.assertRaisesRegex(prepare.VendorPreparationError, "unexpected binary blob"): + prepare.report(source, {}) + (source / "vendor/sony/pdx235/libsecret.so").unlink() + (source / "vendor/sony/pdx235/untrusted.mk").symlink_to(prepare.REQUIRED[0]) + with self.assertRaisesRegex(prepare.VendorPreparationError, "unsafe link"): + prepare.report(source, {}) + + def test_workflow_is_still_unmodified_and_unqualified(self): + script = (ROOT / "src/ci/probe_original_vold.sh").read_text() + code = (ROOT / "src/ci/prepare_vold_vendor_makefiles.py").read_text() + workflow = (ROOT / ".github/workflows/vold-baseline.yml").read_text() + self.assertIn("src/ci/prepare_vold_vendor_makefiles.py", script) + self.assertIn("sony-generated-makefiles-report.json", workflow) + self.assertIn("--regenerate_makefiles", code) + self.assertIn("does_not_qualify_original_vold", code) + for x in (script, code, workflow): + self.assertNotIn("git apply", x) + self.assertNotIn("ota patch", x) + self.assertNotIn("SOONG_ALLOW_MISSING_DEPENDENCIES", x) + + +if __name__ == "__main__": + unittest.main() From 2f3f4d5145d92f092e38791eab7b26ff7a60adb5 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:20:10 +0200 Subject: [PATCH 27/31] ci(vold): exercise public Sony makefile regeneration without binary extraction --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb5dcd18..96df44aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -105,6 +105,9 @@ jobs: - name: Test bounded original Vold source dependency resolver run: python3 tests/vold_source_closure_test.py + - name: Test Sony vendor makefile-only regeneration guard + run: python3 tests/vold_vendor_makefiles_test.py + - name: Test pdx235 Custota metadata preparation run: bash tests/pdx235_custota_metadata_test.sh From 5d2246136ee15752529fdb5f8dc86c727986eb54 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:20:33 +0200 Subject: [PATCH 28/31] docs(vold): distinguish public Sony makefiles from missing proprietary binaries --- docs/vold-baseline-ci.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/vold-baseline-ci.md b/docs/vold-baseline-ci.md index 2b916eb2..57e6af1a 100644 --- a/docs/vold-baseline-ci.md +++ b/docs/vold-baseline-ci.md @@ -61,11 +61,26 @@ The source reasoning is grounded in three exact upstream revisions: **Important external dependency:** the exact official public manifest does **not** contain `vendor/sony/pdx235` or -`vendor/sony/sm6375-common`. Do not invent their contents or disable -Soong missing-dependency checks. The product's `extract-files.py` and the -matching authenticated source image can help resolve the original proprietary -inputs, but that derivation requires separate evidence and must not be -treated as publisher-identical build configuration automatically. +`vendor/sony/sm6375-common`. Do not invent proprietary blobs or disable +Soong missing-dependency checks. There is a narrower, source-grounded route +for their *makefiles*: the immutable +`device/sony/pdx235/extract-files.py --regenerate_makefiles` uses the +pinned `tools/extract-utils` and both Sony device trees to produce +`pdx235-vendor.mk`, `sm6375-common-vendor.mk`, `BoardConfigVendor.mk`, +and the relevant `Android.bp` declarations **without extracting any +proprietary binaries**. This is confirmed by the pinned extract-utils code: +the `--regenerate_makefiles` path skips source extraction. The probe's +[`prepare_vold_vendor_makefiles.py`](../src/ci/prepare_vold_vendor_makefiles.py) +runs that specific mode once after source sync and records a hash-only +`sony-generated-makefiles-report.json` receipt, reused on subsequent +iterations. It rejects unexpected binary blobs, symlinks and changes. + +Generated public makefile declarations are **not** equivalent to the +original unpublished proprietary vendor files or complete product inputs. +If `m vold` requires actual missing blobs, the build still fails closed. +An independently verified original OTA may supply some blob bytes to the +reviewed extract-utils workflow, but neither runtime compatibility nor +publisher-identity follows merely from regenerating makefiles. The on-disk dependency resolver [`src/ci/vold_source_closure.py`](../src/ci/vold_source_closure.py) From 84240d797c5928fa962508afdc74dcb4377407ca Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:21:31 +0200 Subject: [PATCH 29/31] perf(vold): halt auto-expansion when unpinned vendor or unresolved modules persist --- src/ci/vold_source_closure.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/ci/vold_source_closure.py b/src/ci/vold_source_closure.py index 4b39cd68..8fbff29f 100644 --- a/src/ci/vold_source_closure.py +++ b/src/ci/vold_source_closure.py @@ -273,6 +273,20 @@ def blueprint_module_graph(source_root: Path, selected: list[str]) -> dict: } +def next_projects_after_diagnostic( + selected: list[str], diagnostic: dict, seed: dict, +) -> dict | None: + """Stop expensive source iteration when no safe bounded next step exists.""" + if diagnostic["unpinned_missing_paths"] or diagnostic["modules_without_verified_owner"]: + return None + additions = diagnostic["proposed_project_additions"] + if len(additions) > 30: + raise ClosureError("too many automatic additions: require independent review") + if not additions: + return None + return selection_record(sorted(set(selected + additions)), seed) + + def write_json(path: Path, report: dict) -> None: if path.is_symlink(): raise ClosureError("refusing to replace a symbolic-link source report") @@ -314,14 +328,16 @@ def main() -> int: else: report = missing_dependencies(args.log.read_text(errors="replace"), selected, projects, seed) if args.next_selection is not None: - additions = report["proposed_project_additions"] - if len(additions) > 30: - raise ClosureError("too many automatic additions: require independent review") - if additions: - write_json(args.next_selection, selection_record(sorted(set(selected + additions)), seed)) + expansion = next_projects_after_diagnostic(selected, report, seed) + if expansion is not None: + write_json(args.next_selection, expansion) report["next_selection"] = str(args.next_selection) else: report["next_selection"] = None + report["blocked_on_unresolved_inputs"] = bool( + report["unpinned_missing_paths"] + or report["modules_without_verified_owner"] + ) write_json(args.output, report) print(json.dumps(report, sort_keys=True)) except (OSError, ValueError, KeyError, TypeError) as exc: From 3828b9a484a3c022f2387c84510bbd50cdb8adec Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:21:47 +0200 Subject: [PATCH 30/31] test(vold): block wasted syncs on unresolved Soong modules or Sony inputs --- tests/vold_source_closure_test.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/vold_source_closure_test.py b/tests/vold_source_closure_test.py index eae80769..7ff36bdd 100644 --- a/tests/vold_source_closure_test.py +++ b/tests/vold_source_closure_test.py @@ -63,6 +63,29 @@ def test_diagnostics_identify_pinned_source_vs_unpinned_vendor(self): self.assertEqual(closure.owner("system/fs/fs_mgr/fs_mgr/Android.bp", self.manifest), "system/fs/fs_mgr") self.assertIsNone(closure.owner("vendor/sony/pdx235/pdx235-vendor.mk", self.manifest)) + def test_expand_only_when_all_missing_inputs_have_pinned_owners(self): + good = { + "proposed_project_additions": ["system/update_engine"], + "unpinned_missing_paths": [], + "modules_without_verified_owner": [], + } + next_selection = closure.next_projects_after_diagnostic(self.initial, good, self.seed) + self.assertEqual( + next_selection["selected_projects"], + sorted(set(self.initial) | {"system/update_engine"}), + ) + for name, value in ( + ("unpinned_missing_paths", ["vendor/sony/pdx235/BoardConfigVendor.mk"]), + ("modules_without_verified_owner", ["unknown_genrule"]), + ): + bad = {**good, name: value} + self.assertIsNone( + closure.next_projects_after_diagnostic(self.initial, bad, self.seed) + ) + oversized = {**good, "proposed_project_additions": ["p" + str(i) for i in range(31)]} + with self.assertRaises(closure.ClosureError): + closure.next_projects_after_diagnostic(self.initial, oversized, self.seed) + def test_static_soong_modules_trace_defaults_and_transitive_references(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From 9a36c0bfaf0b3f84e003864939c4150a69ca5cd9 Mon Sep 17 00:00:00 2001 From: 0cwa Date: Thu, 22 Oct 2026 03:43:02 +0200 Subject: [PATCH 31/31] ci(vold): fix pinned Repo setup and diagnose real vendor blockers Verify actual Repo include wrappers and missing-only resume, pin cp2a build setup without automatic source fetches, trace 84 reviewed projects, and stop after three small vendor source projects when proprietary inputs lack provenance. Retain low-cost source plans in normal CI and auditable upstream replay evidence. Keep authentic byte equality and publication gates unchanged. --- .github/workflows/ci.yml | 17 ++ .github/workflows/vold-baseline.yml | 16 +- .../vold-build-setup-review-20261022.json | 36 +++ .../vold-repo-integration-20261022.json | 37 +++ .../vold-vendor-preflight-20261022.json | 43 +++ docs/vold-baseline-ci.md | 183 +++++++++-- locks/lineage-vold-baseline-20261016.json | 9 + locks/lineage-vold-source-probe-20261016.json | 134 ++++++++- src/ci/build_original_vold.py | 189 ++++++++++-- src/ci/prepare_vold_vendor_makefiles.py | 143 ++++----- src/ci/probe_original_vold.sh | 45 ++- src/ci/vold_source_closure.py | 89 +++++- tests/vold_baseline_ci_test.py | 5 +- tests/vold_original_build_test.py | 283 ++++++++++++++++++ tests/vold_probe_orchestration_test.py | 146 +++++++++ tests/vold_source_closure_test.py | 114 ++++++- tests/vold_vendor_makefiles_test.py | 172 +++++++---- 17 files changed, 1435 insertions(+), 226 deletions(-) create mode 100644 docs/evidence/vold-build-setup-review-20261022.json create mode 100644 docs/evidence/vold-repo-integration-20261022.json create mode 100644 docs/evidence/vold-vendor-preflight-20261022.json create mode 100644 tests/vold_original_build_test.py create mode 100644 tests/vold_probe_orchestration_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96df44aa..97668d96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,12 +102,29 @@ jobs: - name: Test unmodified original vold CI-only baseline run: python3 tests/vold_baseline_ci_test.py + - name: Test actual Repo manifest selection and bounded build inputs + run: python3 tests/vold_original_build_test.py + - name: Test bounded original Vold source dependency resolver run: python3 tests/vold_source_closure_test.py - name: Test Sony vendor makefile-only regeneration guard run: python3 tests/vold_vendor_makefiles_test.py + - name: Test original vold probe ordering and bounded expansion + run: python3 tests/vold_probe_orchestration_test.py + + - name: Generate the provisional original vold source plan + run: python3 src/ci/vold_source_closure.py plan --output "${RUNNER_TEMP}/vold-source-plan.json" + + - name: Retain the provisional original vold source plan + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: vold-source-provisional-plan + path: ${{ runner.temp }}/vold-source-plan.json + if-no-files-found: error + retention-days: 14 + - name: Test pdx235 Custota metadata preparation run: bash tests/pdx235_custota_metadata_test.sh diff --git a/.github/workflows/vold-baseline.yml b/.github/workflows/vold-baseline.yml index 27aeeba6..3f2b6871 100644 --- a/.github/workflows/vold-baseline.yml +++ b/.github/workflows/vold-baseline.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v7 with: persist-credentials: false - - name: Trace the initial pinned 70-project source selection + - name: Trace the initial pinned source selection shell: bash run: | set -euo pipefail @@ -47,7 +47,7 @@ jobs: with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as summary: summary.write(f"### Provisional vold source graph: {count}/1243 manifest projects\n") summary.write("This is a source probe, **not** a complete Soong closure or a reproduced binary.\n") - summary.write("A real build also requires the missing Sony vendor configuration and a verified Android toolchain.\n") + summary.write("Sony vendor declarations cannot be regenerated from the public lists alone: the pinned generator reads proprietary ELF inputs.\n") PY - name: Upload static source dependency selection uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 @@ -99,9 +99,15 @@ jobs: - name: Validate isolated host and reserve scratch workspace shell: bash + env: + BASELINE_MODE: ${{ inputs.mode }} run: | set -euo pipefail - for binary in repo avbroot curl unzip openssl sha256sum debugfs git python3 realpath; do + binaries=(repo git python3 realpath) + if [[ "$BASELINE_MODE" == compare ]]; then + binaries+=(avbroot curl unzip openssl sha256sum debugfs) + fi + for binary in "${binaries[@]}"; do command -v "$binary" >/dev/null || { echo "::error::Missing required executable: $binary" exit 1 @@ -115,7 +121,7 @@ jobs: path="$(realpath -e -- "$path")" free="$(df -PB1 "$path" | awk 'END {print $4}')" (( free >= 100000000000 )) || { - echo '::error::The 70-project subset probe requires at least 100 GB as a conservative starting allowance (not a proven maximum)' + echo '::error::The subset probe requires at least 100 GB as a conservative starting allowance (not a proven maximum)' exit 1 } workspace="$(mktemp -d "$path/modos-original-vold.XXXXXXXX")" @@ -168,7 +174,7 @@ jobs: path: | ${{ env.VOLD_WORK }}/baseline-*/baseline-report.json ${{ env.VOLD_WORK }}/source-selection.json - ${{ env.VOLD_WORK }}/sony-generated-makefiles-report.json + ${{ env.VOLD_WORK }}/sony-vendor-input-report.json ${{ env.VOLD_WORK }}/source-index-*.json ${{ env.VOLD_WORK }}/dependency-diagnosis-*.json ${{ env.VOLD_WORK }}/probe-only.txt diff --git a/docs/evidence/vold-build-setup-review-20261022.json b/docs/evidence/vold-build-setup-review-20261022.json new file mode 100644 index 00000000..02f8c468 --- /dev/null +++ b/docs/evidence/vold-build-setup-review-20261022.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "kind": "pinned-lineage-build-setup-source-review", + "date_utc": "2026-10-22", + "source_evidence": [ + "https://github.com/LineageOS/android_build/blob/73e37454cdac3892c961e554524ccc52b1bfcf00/envsetup.sh", + "https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/build/envsetup.sh", + "https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/build/tools/roomservice.py", + "https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/vars/aosp_target_release" + ], + "findings": [ + "The pinned lunch function requires product, release and variant; its legacy syntax rejects lineage_pdx235-userdebug because the release component is absent", + "The pinned LineageOS breakfast helper reads vars/aosp_target_release; that exact file selects cp2a", + "The pinned lunch function invokes roomservice for dependency discovery; roomservice can synchronize projects not selected by the bounded probe", + "The unchanged _lunch_meat function performs product/release/variant configuration without roomservice", + "The pinned envsetup reads optional unset variables, so bash nounset is incompatible with sourcing it unchanged" + ], + "reviewed_configuration": { + "product": "lineage_pdx235", + "release": "cp2a", + "variant": "userdebug", + "precheck": "check_product lineage_pdx235 cp2a", + "setup": "_lunch_meat lineage_pdx235 cp2a userdebug", + "build_target": "m vold", + "release_file_sha256": "258eaaeb1fc743ee5fd3a0ade38905be3bbfe3d2624d7fd477dd966a68d67eb6", + "automatic_kernel_builds": "fail before kernel synchronization or compilation", + "source_files_modified": false + }, + "validation": { + "tests": "tests/vold_original_build_test.py", + "actual_bash_failure_guards_tested": ["upstream setup failure", "automatic kernel-build request"], + "source_workspace_checked_after_successful_build": true, + "android_compilation_attempted": false, + "byte_identity_established": false + } +} diff --git a/docs/evidence/vold-repo-integration-20261022.json b/docs/evidence/vold-repo-integration-20261022.json new file mode 100644 index 00000000..560373de --- /dev/null +++ b/docs/evidence/vold-repo-integration-20261022.json @@ -0,0 +1,37 @@ +{ + "schema_version": 1, + "kind": "local-repo-manifest-and-subset-sync-integration", + "date_utc": "2026-10-22", + "repo_tool": { + "source": "https://github.com/GerritCodeReview/git-repo", + "revision": "d744eb15f38b9351463900babffd732994e3a046", + "manifest_selection_source": "https://github.com/GerritCodeReview/git-repo/blob/d744eb15f38b9351463900babffd732994e3a046/manifest_xml.py" + }, + "fixture": { + "project_source": "Two tiny local Git repositories and a local manifest repository, all created for this integration check", + "project_paths": ["system/vold", "system/update_engine"], + "manifest_revisions": "Immutable commit SHAs generated by the local fixture", + "repo_launcher": "Official main.py executed against the fixture .repo directory; .repo/repo links to the recorded tool checkout", + "android_sources_downloaded": false, + "android_compilation_attempted": false + }, + "verified_steps": [ + "First actual repo init selects the fixture default.xml at the fixture manifest commit", + "The builder installs the flattened manifest into .repo/manifests/modos-vold-original-20261016.xml", + "Second actual repo init -m modos-vold-original-20261016.xml succeeds and writes the modern include wrapper", + "verify_active_manifest accepts the actual wrapper and byte-identical included manifest", + "repo sync --no-clone-bundle -j1 system/vold succeeds and system/update_engine remains absent", + "Resuming with both selected paths issues only repo sync --no-clone-bundle -j1 system/update_engine", + "Resuming the unchanged two-project selection issues no repo sync", + "Real Git HEAD and clean-worktree verification pass for each checked-out fixture project" + ], + "observed_sync_commands": [ + ["sync", "--no-clone-bundle", "-j1", "system/vold"], + ["sync", "--no-clone-bundle", "-j1", "system/update_engine"] + ], + "limitations": [ + "The first sync attempt was blocked before project checkout by the execution sandbox's local multiprocessing-socket restriction; the same fixture was resumed outside that sandbox restriction", + "The local tool wrapper emitted a repo-tool refresh warning; both recorded project sync commands completed with exit status zero", + "This verifies Repo selection and incremental synchronization behavior only. It is not Android source-closure, vendor-provenance, binary-equality, or device-qualification evidence" + ] +} diff --git a/docs/evidence/vold-vendor-preflight-20261022.json b/docs/evidence/vold-vendor-preflight-20261022.json new file mode 100644 index 00000000..8bcd81eb --- /dev/null +++ b/docs/evidence/vold-vendor-preflight-20261022.json @@ -0,0 +1,43 @@ +{ + "byte_identical_vold_qualified": false, + "command": [ + "python3", + "-B", + "device/sony/pdx235/extract-files.py", + "--regenerate_makefiles" + ], + "date": "2026-10-22", + "environment": { + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": "tools/extract-utils" + }, + "exit_status": 1, + "input_sha256": { + "device/sony/pdx235/extract-files.py": "440af4750e0496627003295606935d3a534f2e758ab42adacd3d5da6e255f0e0", + "device/sony/pdx235/proprietary-files.txt": "93da9a7ac7957afde23ac9c59425947a29b410d01fd5b1fcde2efb5ec155855f", + "device/sony/sm6375-common/extract-files.py": "266ac34e070a5755d513979ad74196b4341c363915b0cfaa24e51d4bf49d6ec8", + "device/sony/sm6375-common/proprietary-files.txt": "8056cdd324881ba179fbac2553e77ddc0d1456ade7ef1e37f9630bb5f6c39a29", + "tools/extract-utils/extract_utils/elf.py": "58d3a6db079185d94e2d283211f73fa71f32bb882618ac47b0feb6d2f5e01d31", + "tools/extract-utils/extract_utils/main.py": "3470e7a43a9788f2699c13d627750b7f5fdc448bf364a48a3a7c86f2b931562d", + "tools/extract-utils/extract_utils/makefiles.py": "2a550851c75a2a32d4d294338df3e73f7d95b4418721653224565d27ba6fa111", + "tools/extract-utils/extract_utils/module.py": "1d1f77c66e05a3f3fea1e710caca7842c38c11b76c1a7bb486724826e72f73ef" + }, + "kind": "sony-vendor-regeneration-public-source-replay", + "original_scripts_modified": false, + "partial_outputs": [ + "vendor/sony/pdx235/Android.bp", + "vendor/sony/pdx235/Android.mk", + "vendor/sony/pdx235/BoardConfigVendor.mk", + "vendor/sony/pdx235/pdx235-vendor.mk" + ], + "proprietary_binaries_present": false, + "public_source_revisions": { + "LineageOS/android_device_sony_pdx235": "0559de69466e1e9af69b121985f592c4f4609283", + "LineageOS/android_device_sony_sm6375-common": "d1d80af1ad9032cb092ff6d85db437709cc54e28", + "LineageOS/android_tools_extract-utils": "ff6589c6b0a4a00c592e98940e9e6b6a50c76569" + }, + "python_version": "Python 3.12.14", + "schema_version": 1, + "stderr": "Traceback (most recent call last):\n File \"device/sony/pdx235/extract-files.py\", line 63, in \n utils.run()\n File \"tools/extract-utils/extract_utils/main.py\", line 219, in run\n self.write_makefiles()\n File \"tools/extract-utils/extract_utils/main.py\", line 160, in write_makefiles\n module.write_makefiles(\n File \"tools/extract-utils/extract_utils/module.py\", line 738, in write_makefiles\n proprietary_file.write_makefiles(self, ctx)\n File \"tools/extract-utils/extract_utils/module.py\", line 215, in write_makefiles\n write_product_packages(\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 683, in write_product_packages\n w(write_lib_package, lib64_tree, packages_ctx)\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 626, in w\n return write_packages_group(\n ^^^^^^^^^^^^^^^^^^^^^\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 565, in write_packages_group\n package_name = fn(file, builder, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 382, in write_lib_package\n return write_elfs_package(\n ^^^^^^^^^^^^^^^^^^^\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 266, in write_elfs_package\n machine, bits, libs = get_file_machine_bits_libs(f_path, gen_deps)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"tools/extract-utils/extract_utils/elf.py\", line 43, in get_file_machine_bits_libs\n with open(file_path, 'rb') as f:\n ^^^^^^^^^^^^^^^^^^^^^\nFileNotFoundError: [Errno 2] No such file or directory: 'vendor/sony/pdx235/proprietary/vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so'\n", + "stdout": "Parsing device/sony/pdx235/proprietary-files.txt\nParsing device/sony/sm6375-common/proprietary-files.txt\n" +} diff --git a/docs/vold-baseline-ci.md b/docs/vold-baseline-ci.md index 57e6af1a..59e846ed 100644 --- a/docs/vold-baseline-ci.md +++ b/docs/vold-baseline-ci.md @@ -17,8 +17,20 @@ authenticated original LineageOS OTA. - Complete 1,243-project release manifest: `locks/lineage-pdx235-20261016-build-manifest.xml`. - Source lock: `locks/lineage-vold-baseline-20261016.json`. -- Exact build: `lineage_pdx235-userdebug`, then `m vold`, without - applying any patch or introducing a replacement binary. +- Exact product/variant: `lineage_pdx235` / `userdebug`, release `cp2a`, + then `m vold`, without applying a patch or introducing a replacement binary. + +At the pinned `build/make` revision, the legacy two-component +`lunch lineage_pdx235-userdebug` syntax is rejected. The pinned +[`vendor/lineage/vars/aosp_target_release`](https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/vars/aosp_target_release) +sets `cp2a`; its exact bytes are hash-checked before build setup. The helper +uses the unchanged `_lunch_meat lineage_pdx235 cp2a userdebug` configuration +function after `check_product`, avoiding the `lunch` wrapper's roomservice +downloads. Automatic kernel builds are blocked. Upstream shell setup runs +without nounset because its optional variables are intentionally unset. +The comparison receipt records the actual release and setup invocation. +The pinned setup and roomservice findings are retained in +[`vold-build-setup-review-20261022.json`](evidence/vold-build-setup-review-20261022.json). These pins were recorded from signed ModOS source-verification runs [#39632433468](https://github.com/0cwa/ModOS/actions/runs/39632433468) @@ -29,23 +41,43 @@ match the independently pinned LineageOS official publisher key, the ZIP certificate must be unique, and `avbroot ota verify` must succeed. The original `system.img` is extracted read-only, then its -`/system/bin/vold` is recovered as the reference executable. A full, -clean, exact-revision LineageOS source checkout builds another unmodified +`/system/bin/vold` is recovered as the reference executable. A clean, +exact-revision, dependency-limited LineageOS checkout builds another unmodified `vold` for the **same** pinned product. The resulting report includes the reference and built SHA-256s, byte counts and first differing offset. Only `byte_identical: true` is success. The workflow fails on a mismatch and uploads a diagnostic JSON receipt, rather than bypassing the check. -## Dependency-closure strategy (begin with 70 of 1,243 projects) +## Dependency-closure strategy The original workflow fetched **all 1,243 pinned projects** before issuing `m vold`. That is unnecessary for a single module, and it made each trial expensive. Instead, the dependency-analysis recipe in [`locks/lineage-vold-source-probe-20261016.json`](../locks/lineage-vold-source-probe-20261016.json) -selects **70 initial projects**. This is a conservative initial dependency +selects **84 initial projects** after source tracing corrected the original +70-project seed. The plan artifact records the exact +current selection and manifest digest. This is a conservative dependency seed, NOT a complete or proven Soong build graph. +The additional 14 projects cover verified native/HIDL/AIDL providers, +suspend interfaces, Rust generated flag-library dependencies, and Sony USB +and dataservices namespaces. Three former hints pointed to the wrong project: + +| Module | Verified pinned project | +| --- | --- | +| `libbase` | `system/libbase` | +| `libgsi` | `system/gsid` | +| `libhardware_legacy` | `hardware/libhardware_legacy` | + +The less obvious chain is `vold_flags_c_lib` through Soong's +`cc_aconfig_library` generator to `libaconfig_storage_read_api_cc` and its +Rust bridge/crates. This requires `external/rust/cxx` and +`external/rust/android-crates-io`. Exact-revision evidence for these choices +and the remaining additions is retained in the seed's `source_evidence` +and `reviewed_dependency_chains` fields. This static research saves repeated +failed builds; it does not make the selection complete. + The source reasoning is grounded in three exact upstream revisions: - [`system/vold/Android.bp`](https://github.com/LineageOS/android_system_vold/blob/11d8982f824d52e8122c7aa732207ba66b8f23c8/Android.bp): @@ -59,41 +91,61 @@ The source reasoning is grounded in three exact upstream revisions: the common product also includes proprietary `vendor/sony/sm6375-common`. Two other Sony vendor BoardConfig files are referenced. -**Important external dependency:** the exact official public manifest does -**not** contain `vendor/sony/pdx235` or -`vendor/sony/sm6375-common`. Do not invent proprietary blobs or disable -Soong missing-dependency checks. There is a narrower, source-grounded route -for their *makefiles*: the immutable -`device/sony/pdx235/extract-files.py --regenerate_makefiles` uses the -pinned `tools/extract-utils` and both Sony device trees to produce -`pdx235-vendor.mk`, `sm6375-common-vendor.mk`, `BoardConfigVendor.mk`, -and the relevant `Android.bp` declarations **without extracting any -proprietary binaries**. This is confirmed by the pinned extract-utils code: -the `--regenerate_makefiles` path skips source extraction. The probe's +### Verified proprietary-input blocker + +The exact public manifest does **not** contain `vendor/sony/pdx235` or +`vendor/sony/sm6375-common`. The original assumption that +`--regenerate_makefiles` can generate their declarations using only public +file lists is incorrect at the pinned revisions. This option skips +extraction, but the subsequent +[`write_elfs_package`](https://github.com/LineageOS/android_tools_extract-utils/blob/ff6589c6b0a4a00c592e98940e9e6b6a50c76569/extract_utils/makefiles.py#L256-L271) +still calls +[`get_file_machine_bits_libs`](https://github.com/LineageOS/android_tools_extract-utils/blob/ff6589c6b0a4a00c592e98940e9e6b6a50c76569/extract_utils/elf.py#L38-L57), +which opens each proprietary ELF to determine its architecture and dependencies. + +An actual replay of the unchanged pinned Python sources on 2026-10-22 +failed with `FileNotFoundError` for +`vendor/sony/pdx235/proprietary/vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so`. +Upstream had already written four partial vendor declaration files. Those +files are not successful or qualified preparation. The reproducible command, +source revisions, input hashes and normalized traceback are retained in +[`docs/evidence/vold-vendor-preflight-20261022.json`](evidence/vold-vendor-preflight-20261022.json). +This replay ran Python extraction code only; it did **not** compile Android. + +To avoid downloading toolchains before this known failure, the probe first +syncs exactly three pinned source projects: both Sony device trees and +`tools/extract-utils`. The [`prepare_vold_vendor_makefiles.py`](../src/ci/prepare_vold_vendor_makefiles.py) -runs that specific mode once after source sync and records a hash-only -`sony-generated-makefiles-report.json` receipt, reused on subsequent -iterations. It rejects unexpected binary blobs, symlinks and changes. +preflight verifies those revisions and writes `sony-vendor-input-report.json` +with `status: blocked` and `makefiles_generated: false`, then exits before +invoking the generator or syncing the remaining seed projects. It rejects +the former synthetic-success receipts; arbitrary existing vendor files +also remain unqualified. -Generated public makefile declarations are **not** equivalent to the -original unpublished proprietary vendor files or complete product inputs. -If `m vold` requires actual missing blobs, the build still fails closed. -An independently verified original OTA may supply some blob bytes to the -reviewed extract-utils workflow, but neither runtime compatibility nor -publisher-identity follows merely from regenerating makefiles. +**Both real build modes are currently blocked here.** A reviewed acquisition +and provenance contract for the required proprietary inputs is needed before +vendor preparation can be enabled. Authenticating the pinned original OTA +may provide a source for some of these bytes, but extraction, fixups, original +build inputs and every required file must be accounted for. The present code +does not claim that this acquisition is implemented. Do not fabricate files, +disable ELF checks or enable missing-dependency flags to bypass this gate. The on-disk dependency resolver [`src/ci/vold_source_closure.py`](../src/ci/vold_source_closure.py) verifies the complete original manifest and produces a pinned selection. -After a limited `repo sync `, it scans the available -`Android.bp` definitions for a **partial** transitive module graph. The real +After the vendor-input gate is resolved and a limited +`repo sync ` succeeds, it scans available `Android.bp` +definitions for a **partial** transitive module graph. Up to three static +scans batch missing providers from reviewed hints before compilation, +syncing only newly selected pinned projects. Unknown, ambiguous and generated +module names remain visible in diagnostic reports. The real Soong compiler, rather than the static scan, remains authoritative for conditional/generated modules. Any missing path reported in the compiler diagnostics is mapped to its **exact existing** project in the official manifest. Known module names have reviewed initial provider hints; unknown providers and unpinned vendor paths stop for manual investigation. -Within a single CI run, +Once prerequisites have passed, within a single CI run, [`src/ci/probe_original_vold.sh`](../src/ci/probe_original_vold.sh) automatically repeats up to **three** controlled passes. It caches already synchronized projects in the same working tree, fetches only newly identified @@ -103,6 +155,20 @@ unknown modules and other blockers. It never enables the exFAT timeout patch. Any unexpected source revision or modified checkout fails immediately. +The build verifier supports Repo's actual single-include `manifest.xml` +wrapper and verifies the included flattened manifest bytes. Local manifest +overrides are rejected. A resumed sync distinguishes actual Git worktrees +from parent or partially created directories. Before compiling, the helper +rejects unrecorded external source trees and inherited missing-dependency +bypasses, fixes the output directory to this workspace and removes any old +final `vold` executable. +It repeats source/workspace verification after a successful compile. +A real Repo integration against two tiny local Git projects confirmed the +include wrapper, selected-only sync, missing-only resume, and no sync when +the selection was unchanged. See +[`vold-repo-integration-20261022.json`](evidence/vold-repo-integration-20261022.json). +This exercises the real Repo tool, not an Android compilation. + You can test the graph on an ordinary GitHub-hosted runner with **Actions → Qualify original LineageOS pdx235 vold (no patch) → Run workflow**, choosing `plan` (the default). This needs **no self-hosted runner or @@ -110,14 +176,20 @@ repository variable** and produces a `vold-source-provisional-plan` artifact. It does **not** compile or qualify a binary. +Ordinary PR CI and `main` CI also run this same plan command and retain the +same `vold-source-provisional-plan` artifact. This provides a low-cost plan +on the merged commit even when the GitHub connector cannot dispatch manual +workflows. These CI steps never sync Android sources or run the compiler. + ## Runner provisioning for source compilation For `probe` or `compare`, the workflow still requires a trusted, isolated self-hosted Linux x64 runner labeled `modos-lineage-large`. Provide a dedicated scratch filesystem with **at least 100 GB free as an experimental starting allowance**, sufficient CPU/RAM, the Android 17 -build prerequisites, and `repo`, `avbroot`, `debugfs`, `unzip`, -`curl`, `openssl`, `git`, and `python3`. One iteration may require +build prerequisites, and `repo`, `git`, `python3`, and `realpath`. +The authenticated `compare` mode additionally needs `avbroot`, `debugfs`, +`unzip`, `curl`, `openssl`, and `sha256sum`. One iteration may require more storage. The 100 GB threshold is **not an empirically measured maximum or proof** that the selected dependency graph is complete. @@ -133,14 +205,59 @@ unmodified `m vold` **without downloading the official source OTA**. A successfully compiled probe is **not** binary-equality evidence; its workflow job explicitly labels it as unqualified. It is useful for discovering missing modules, vendor inputs and toolchains cheaply. +At the current reviewed revision it stops at the known vendor-input gate +before downloading the large seed or invoking `m vold`. -Use `compare` for a real proof. It first reauthenticates the exact +Only after a successful, reviewed `probe`, use `compare` for a real proof. +The compare workflow downloads the OTA before the source probe; running it +while the known vendor gate is blocked would waste that download. +It first reauthenticates the exact publisher-signed official 2026-10-16 OTA, extracts the original `/system/bin/vold`, and invokes the same bounded dependency solver. **Only** a real `byte_identical: true` report with equal original and built hashes qualifies this baseline. If the binaries differ, it emits a diagnostic comparison receipt and fails. +### Hosted-runner alternatives and observed access + +Checked on 2026-10-22: GitHub documents larger Ubuntu x64 runners with +4 CPUs / 16 GB RAM / 150 GB storage and 8 CPUs / 32 GB RAM / 300 GB storage. +Those are plausible experimental build hosts, not measured requirements +for this incomplete dependency closure. The 100 GB free-space gate does not +prove either machine has enough RAM or working space after tool installation. +See GitHub's [larger runner specifications](https://docs.github.com/en/actions/reference/runners/larger-runners). + +GitHub's managed larger runners require an organization or enterprise on +Team or Enterprise Cloud; `0cwa/ModOS` is currently owned by a user account. +They cannot be assumed available merely by changing `runs-on` to an invented +label. An actual organization runner with repository access, or a dedicated +trusted self-hosted Linux x64 machine, must be provisioned and checked first. +See [GitHub-hosted runner availability](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) +and [managing larger runners](https://docs.github.com/en/actions/how-tos/manage-runners/larger-runners/manage-larger-runners). +Standard Ubuntu runners advertise only 14 GB storage and remain suitable +for this plan/unit-test workload, not the current Android build preflight. + +The last original-baseline attempt +[#39699955476](https://github.com/0cwa/ModOS/actions/runs/39699955476) +failed its hosted prerequisites job with both readiness variables empty; +the self-hosted build job was skipped. This is evidence about that run, +not proof of the current runner inventory. The connected GitHub actions +expose repository changes, CI inspection and reruns, but no workflow dispatch, +runner inventory or variable mutation. No runner or configuration change +has been claimed on that basis. + +For an owner with the GitHub CLI configured, the explicit manual plan is: + +```sh +gh workflow run vold-baseline.yml --repo 0cwa/ModOS --ref main -f mode=plan +``` + +After resolving the authenticated vendor-input contract and verifying the +trusted runner, set the two documented repository variables and dispatch +`-f mode=probe`. Inspect its source selection and diagnostics. Dispatch +`-f mode=compare` only once that actual compile succeeds. The connector's +missing dispatch capability does not justify changing any qualification gate. + From a checked-out repository, local inspection is also possible without compiling or obtaining the proprietary vendor files: @@ -171,7 +288,7 @@ untrusted shared runner. ## After the first authentic success Review the actual run URL and download its -`original-pdx235-vold-reproducibility-report` artifact. Record the exact +`vold-probe-compare-diagnostics` artifact. Record the exact source, manifest and reference SHA-256, plus the CI commit and run ID. Verify the report contains `byte_identical: true` with equal SHA-256s. Independent assessment of the real reference extraction and Android diff --git a/locks/lineage-vold-baseline-20261016.json b/locks/lineage-vold-baseline-20261016.json index e87c2ed9..db86dfb5 100644 --- a/locks/lineage-vold-baseline-20261016.json +++ b/locks/lineage-vold-baseline-20261016.json @@ -30,6 +30,15 @@ }, "build": { "target": "lineage_pdx235-userdebug", + "product": "lineage_pdx235", + "release": "cp2a", + "variant": "userdebug", + "setup_command": "_lunch_meat lineage_pdx235 cp2a userdebug", + "release_config": { + "path": "vendor/lineage/vars/aosp_target_release", + "sha256": "258eaaeb1fc743ee5fd3a0ade38905be3bbfe3d2624d7fd477dd966a68d67eb6", + "source_url": "https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/vars/aosp_target_release" + }, "command": "m vold", "artifact": "out/target/product/pdx235/system/bin/vold", "elf_class": 64, diff --git a/locks/lineage-vold-source-probe-20261016.json b/locks/lineage-vold-source-probe-20261016.json index 77b5f6dc..03cd098c 100644 --- a/locks/lineage-vold-source-probe-20261016.json +++ b/locks/lineage-vold-source-probe-20261016.json @@ -7,7 +7,32 @@ "source_evidence": [ "https://github.com/LineageOS/android_system_vold/blob/11d8982f824d52e8122c7aa732207ba66b8f23c8/Android.bp", "https://github.com/LineageOS/android_device_sony_pdx235/blob/0559de69466e1e9af69b121985f592c4f4609283/device.mk", - "https://github.com/LineageOS/android_device_sony_sm6375-common/blob/d1d80af1ad9032cb092ff6d85db437709cc54e28/common.mk" + "https://github.com/LineageOS/android_device_sony_sm6375-common/blob/d1d80af1ad9032cb092ff6d85db437709cc54e28/common.mk", + "https://android.googlesource.com/platform/hardware/libhardware_legacy/+/2a073cd42ec822914ed4e680e7b708485d3a172b/Android.bp", + "https://android.googlesource.com/platform/system/libbase/+/fef4173e19d97cd4b4fc09a06bccbf800d0d3937/Android.bp", + "https://android.googlesource.com/platform/system/gsid/+/35a92d40f529b605fad8ff622691e76901044cc8/Android.bp", + "https://android.googlesource.com/platform/system/libhidl/+/061055fc4dad78e5d25a77d9217001794656f16b/Android.bp", + "https://android.googlesource.com/platform/system/server_configurable_flags/+/5069ba739faebf91ae72096577ae17e13061af72/libflags/Android.bp", + "https://android.googlesource.com/platform/system/libhwbinder/+/2f5282448223b97c541181a5b94a05c2ac17ff17/Android.bp", + "https://github.com/LineageOS/android_system_libfmq/blob/f727482ff0c6a26b4c64c5106c62c176415b2d81/Android.bp", + "https://android.googlesource.com/platform/system/hardware/interfaces/+/9c3469aa3d4aeec5b2bc8e8ae8fdb4437dab4edb/suspend/aidl/default/Android.bp", + "https://android.googlesource.com/platform/system/hardware/interfaces/+/9c3469aa3d4aeec5b2bc8e8ae8fdb4437dab4edb/suspend/aidl/Android.bp", + "https://android.googlesource.com/platform/system/tools/hidl/+/5d0e05e6535fea94a674a359102bdd42944714f4/Android.bp", + "https://android.googlesource.com/platform/system/tools/aidl/+/9a94664f74536bf28010207fd1804b04c6435e2c/Android.bp", + "https://github.com/LineageOS/android_build_soong/blob/183e59578606687d66096cbce86cb50c27f4e9d7/cc/gen.go", + "https://github.com/LineageOS/android_build_soong/blob/183e59578606687d66096cbce86cb50c27f4e9d7/aconfig/codegen/cc_aconfig_library.go", + "https://github.com/LineageOS/android_build/blob/73e37454cdac3892c961e554524ccc52b1bfcf00/tools/aconfig/aconfig_storage_read_api/Android.bp", + "https://android.googlesource.com/platform/external/rust/cxx/+/41972f6ebbe9c21eeed3fdf5c1b0805f2c8b824c/Android.bp", + "https://android.googlesource.com/platform/external/rust/android-crates-io/+/1f5e95cd6e996995023c9c0f7df0f15a8a43269e/crates/anyhow/Android.bp", + "https://android.googlesource.com/platform/external/rust/android-crates-io/+/1f5e95cd6e996995023c9c0f7df0f15a8a43269e/crates/memmap2/Android.bp", + "https://android.googlesource.com/platform/external/rust/android-crates-io/+/1f5e95cd6e996995023c9c0f7df0f15a8a43269e/crates/thiserror/Android.bp", + "https://github.com/LineageOS/android_system_core/blob/7c3f32fe89c52b4176fd288ba5bb4f907809c202/libcrypto_utils/Android.bp", + "https://android.googlesource.com/platform/system/libvintf/+/df89ad736e14ac375f61d50fa543ae7cd746a140/Android.bp", + "https://android.googlesource.com/platform/external/tinyxml2/+/33ba115074c80a720826efffab1937d5b355c3b8/Android.bp", + "https://android.googlesource.com/platform/system/tools/sysprop/+/deead2d06508faeca98c236feea7e59bb4c43c8c/Android.bp", + "https://github.com/LineageOS/android_device_sony_pdx235/blob/0559de69466e1e9af69b121985f592c4f4609283/extract-files.py", + "https://github.com/LineageOS/android_device_sony_sm6375-common/blob/d1d80af1ad9032cb092ff6d85db437709cc54e28/extract-files.py", + "https://github.com/LineageOS/android_tools_extract-utils/blob/ff6589c6b0a4a00c592e98940e9e6b6a50c76569/extract_utils/makefiles.py" ], "candidate_groups": { "core_build_toolchain": [ @@ -81,12 +106,28 @@ "vendor/qcom/opensource/commonsys-intf/display", "vendor/qcom/opensource/commonsys/display", "vendor/qcom/opensource/display", - "tools/extract-utils" + "tools/extract-utils", + "vendor/qcom/opensource/dataservices", + "vendor/qcom/opensource/usb" ], "module_service_api": [ "packages/modules/adb", "packages/modules/Connectivity", "packages/modules/ArtPrebuilt" + ], + "verified_native_and_generated_transitive": [ + "external/rust/android-crates-io", + "external/rust/cxx", + "hardware/libhardware_legacy", + "system/gsid", + "system/hardware/interfaces", + "system/libbase", + "system/libfmq", + "system/libhidl", + "system/libhwbinder", + "system/server_configurable_flags", + "system/tools/aidl", + "system/tools/hidl" ] }, "direct_module_to_project_hints": { @@ -105,14 +146,14 @@ "libbinder_ndk": "frameworks/native", "libcutils": "system/core", "libutils": "system/core", - "libbase": "system/core", + "libbase": "system/libbase", "libselinux": "external/selinux", "libcrypto": "external/boringssl", "liblog": "system/logging", "liblogwrap": "system/core", "libhardware": "hardware/libhardware", - "libhardware_legacy": "hardware/libhardware", - "libgsi": "system/fs/fs_mgr", + "libhardware_legacy": "hardware/libhardware_legacy", + "libgsi": "system/gsid", "libext2_uuid": "external/e2fsprogs", "libext4_utils": "system/extras", "libf2fs_sparseblock": "system/extras", @@ -128,7 +169,32 @@ "fuseMedia.bpf": "system/bpfprogs", "mke2fs": "external/e2fsprogs", "libfec_rs": "system/extras", - "libasync_safe": "bionic" + "libasync_safe": "bionic", + "libhidlbase": "system/libhidl", + "libhidlbase-combined-impl": "system/libhidl", + "server_configurable_flags": "system/server_configurable_flags", + "libhwbinder-impl-internal": "system/libhwbinder", + "libhwbinder-impl-shared-libs": "system/libhwbinder", + "libfmq-base": "system/libfmq", + "libfmq": "system/libfmq", + "hidl-module-defaults": "system/tools/hidl", + "hidl-gen": "system/tools/hidl", + "aidl": "system/tools/aidl", + "aidl-cpp": "system/tools/aidl", + "system_suspend_defaults": "system/hardware/interfaces", + "android.system.suspend-V1-ndk": "system/hardware/interfaces", + "suspend_service_flags_c_lib": "system/hardware/interfaces", + "libcrypto_utils": "system/core", + "libaconfig_storage_read_api_cc": "build/make", + "aconfig_lib_cc_static_link.defaults": "build/make", + "libcxx": "external/rust/cxx", + "cxx-bridge-header": "external/rust/cxx", + "libanyhow": "external/rust/android-crates-io", + "libmemmap2": "external/rust/android-crates-io", + "libthiserror": "external/rust/android-crates-io", + "libvintf": "system/libvintf", + "libtinyxml2": "external/tinyxml2", + "sysprop_cpp": "system/tools/sysprop" }, "known_unpinned_vendor_inputs": [ "vendor/sony/pdx235/pdx235-vendor.mk", @@ -136,5 +202,59 @@ "vendor/sony/sm6375-common/sm6375-common-vendor.mk", "vendor/sony/sm6375-common/BoardConfigVendor.mk" ], - "qualification_note": "Initial hints, not a complete Soong source closure. Repo sync and build errors must discover additional immutable manifest projects; missing proprietary vendor inputs cannot be fabricated." + "qualification_note": "Reviewed seed and provider hints, not a complete Soong source closure. Static scanning omits variables and implicit generators and may include inactive product branches. Repo sync and build errors may discover additional immutable manifest projects. Missing proprietary vendor inputs cannot be fabricated or replaced with generated declarations lacking provenance.", + "reviewed_dependency_chains": [ + { + "consumer": "vold_default_libs", + "providers": [ + "system/libbase", + "system/gsid", + "hardware/libhardware_legacy", + "system/libhidl" + ], + "finding": "Pinned Android.bp files place libbase, libgsi, libhardware_legacy and libhidlbase in distinct manifest projects; the previous hints pointed three of these at other already-selected projects." + }, + { + "consumer": "libhidlbase-combined-impl", + "providers": [ + "system/libhwbinder", + "system/libfmq", + "system/tools/hidl" + ], + "finding": "Pinned HIDL defaults require libhwbinder-impl-internal, libfmq-base and hidl-module-defaults; generated HIDL sources also require hidl-gen." + }, + { + "consumer": "libhardware_legacy -> libpower_defaults", + "providers": [ + "system/hardware/interfaces", + "system/server_configurable_flags" + ], + "finding": "Pinned legacy hardware defaults inherit system_suspend_defaults and use android.system.suspend-V1-ndk; the suspend defaults also require server_configurable_flags." + }, + { + "consumer": "libvold_binder AIDL sources", + "providers": [ + "system/tools/aidl" + ], + "finding": "Pinned Soong cc/gen.go invokes aidl-cpp for AIDL source generation; the tool is declared in system/tools/aidl." + }, + { + "consumer": "vold_flags_c_lib -> libaconfig_storage_read_api_cc", + "providers": [ + "build/make", + "external/rust/cxx", + "external/rust/android-crates-io" + ], + "finding": "Pinned Soong cc_aconfig_library adds the native aconfig reader. Its C++ bridge requires Rust libcxx, libanyhow, libmemmap2 and libthiserror; the latter three are in the single pinned android-crates-io project. This implicit generator edge is not visible in vold Android.bp alone." + }, + { + "consumer": "Sony product and vendor namespace declarations", + "providers": [ + "vendor/qcom/opensource/dataservices", + "vendor/qcom/opensource/usb" + ], + "finding": "Pinned extract-files.py namespace_imports reference dataservices; pinned common.mk adds the usb/etc namespace. These public projects do not provide the proprietary Sony vendor blobs." + } + ], + "vendor_prerequisite_note": "Pinned extract-utils --regenerate_makefiles still opens proprietary ELF files to determine machine, bitness and DT_NEEDED. Public proprietary-file lists alone cannot regenerate usable vendor declarations. A real replay fails at the missing pdx235 proprietary/vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so input; generated files from a failed replay are partial and unqualified. The workflow must stop before large source downloads until an authenticated vendor-input or exact declaration provenance plan is reviewed." } diff --git a/src/ci/build_original_vold.py b/src/ci/build_original_vold.py index bc8c669c..f5710e14 100644 --- a/src/ci/build_original_vold.py +++ b/src/ci/build_original_vold.py @@ -22,6 +22,10 @@ ROOT = Path(__file__).resolve().parents[2] LOCK = ROOT / "locks/lineage-vold-baseline-20261016.json" +MANIFEST_NAME = "modos-vold-original-20261016.xml" +VENDOR_PREREQUISITE_PROJECTS = ( + "device/sony/pdx235", "device/sony/sm6375-common", "tools/extract-utils", +) class BuildError(ValueError): @@ -47,6 +51,16 @@ def load_contract() -> tuple[dict, bytes]: raise BuildError("pinned source manifest digest differs") if lock["build"]["target"] != "lineage_pdx235-userdebug" or lock["build"]["command"] != "m vold": raise BuildError("baseline build target differs from reviewed source") + build = lock["build"] + if ( + (build.get("product"), build.get("release"), build.get("variant")) + != ("lineage_pdx235", "cp2a", "userdebug") + or build.get("setup_command") != "_lunch_meat lineage_pdx235 cp2a userdebug" + or build.get("release_config", {}).get("path") != "vendor/lineage/vars/aosp_target_release" + or build.get("release_config", {}).get("sha256") + != "258eaaeb1fc743ee5fd3a0ade38905be3bbfe3d2624d7fd477dd966a68d67eb6" + ): + raise BuildError("baseline product/release/variant differs from pinned LineageOS setup") projects = project_map(manifest) if len(projects) != 1243: raise BuildError("unexpected number of projects in source manifest") @@ -98,6 +112,54 @@ def verified_selection(selection_file: Path | None, manifest: bytes) -> list[str return validate_selection(selection, projects, seed) +def verify_active_manifest(source: Path, manifest: bytes) -> None: + """Validate Repo's selected file, including its modern include wrapper. + + Repo's Link() writes .repo/manifest.xml as a single include, not a copy + of the selected manifest. Check that include without permitting another + manifest, a local override, or a linked file outside this checkout. + """ + repo_dir = source / ".repo" + if repo_dir.is_symlink() or not repo_dir.is_dir(): + raise BuildError("repo metadata directory is missing or unsafe") + for local in (repo_dir / "local_manifest.xml", repo_dir / "local_manifests"): + if local.is_symlink() or (local.exists() and ( + not local.is_dir() or any(local.iterdir()) + )): + raise BuildError("local manifest overrides are forbidden for the pinned baseline") + active = repo_dir / "manifest.xml" + if not active.is_file(): + raise BuildError("repo selected manifest is unavailable") + if not active.is_symlink() and active.read_bytes() == manifest: + # Support an existing legacy checkout with the exact flattened file. + return + manifest_dir = repo_dir / "manifests" + pinned = manifest_dir / MANIFEST_NAME + if ( + manifest_dir.is_symlink() or not manifest_dir.is_dir() + or pinned.is_symlink() or not pinned.is_file() + or pinned.read_bytes() != manifest + ): + raise BuildError("repo selected manifest differs from the exact official snapshot") + if active.is_symlink(): + if active.resolve(strict=True) != pinned.resolve(strict=True): + raise BuildError("repo manifest link does not select the pinned snapshot") + return + try: + wrapper = ET.fromstring(active.read_bytes()) + except ET.ParseError as exc: + raise BuildError("invalid repo manifest selection wrapper") from exc + children = list(wrapper) + if ( + wrapper.tag != "manifest" or wrapper.attrib or (wrapper.text or "").strip() + or len(children) != 1 or children[0].tag != "include" + or children[0].attrib != {"name": MANIFEST_NAME} + or list(children[0]) or (children[0].text or "").strip() + or (children[0].tail or "").strip() + ): + raise BuildError("repo manifest wrapper must select only the exact pinned snapshot") + + def verify_checkout( source: Path, manifest: bytes, *, clean: bool = False, selected: list[str] | None = None, @@ -105,10 +167,9 @@ def verify_checkout( repo = shutil.which("repo") if repo is None: raise BuildError("repo is required to verify complete source revisions") + source = source.resolve(strict=True) expected = project_map(manifest) - active_manifest = source / ".repo/manifest.xml" - if active_manifest.is_dir() or not active_manifest.exists() or active_manifest.read_bytes() != manifest: - raise BuildError("repo selected manifest differs from the exact official snapshot") + verify_active_manifest(source, manifest) if selected is None: # The legacy full-checkout gate remains unchanged for fully synced trees. result = subprocess.run( @@ -136,6 +197,48 @@ def verify_checkout( raise BuildError(f"source project has unreviewed local content: {relpath}") +def verify_source_workspace(source: Path, manifest: bytes, selected: list[str] | None) -> None: + """Reject unrecorded source outside clean pinned projects before building.""" + approved = set(project_map(manifest) if selected is None else selected) + exported = {} + for project in ET.fromstring(manifest).findall("project"): + project_path = project.get("path", project.get("name", "")) + if project_path not in approved: + continue + for entry in project: + if entry.tag in ("linkfile", "copyfile"): + exported[entry.attrib["dest"]] = ( + entry.tag, source / project_path / entry.attrib["src"] + ) + parents = { + parent.as_posix() + for path in approved | set(exported) + for parent in Path(path).parents if parent != Path(".") + } + pending = [source] + while pending: + for path in pending.pop().iterdir(): + relative = path.relative_to(source).as_posix() + if relative in approved: + continue # verify_checkout checks each entire Git worktree. + if relative in exported: + kind, original = exported[relative] + if kind == "linkfile": + if not path.is_symlink() or path.resolve() != original.resolve(): + raise BuildError(f"manifest linkfile differs from pinned source: {relative}") + elif path.is_symlink() or not path.is_file() or path.read_bytes() != original.read_bytes(): + raise BuildError(f"manifest copyfile differs from pinned source: {relative}") + continue + if relative in (".repo", "out"): + if path.is_symlink() or not path.is_dir(): + raise BuildError(f"build workspace directory is unsafe: {relative}") + continue + if relative in parents and path.is_dir() and not path.is_symlink(): + pending.append(path) + continue + raise BuildError(f"unrecorded source outside selected pinned projects: {relative}") + + def sync( source: Path, lock: dict, manifest: bytes, jobs: int, selected: list[str] | None = None, resume: bool = False, @@ -143,16 +246,14 @@ def sync( if source.is_symlink() or source.exists() and not source.is_dir(): raise BuildError("source directory must be a real directory") if resume: - if not (source / ".repo/manifest.xml").is_file(): - raise BuildError("resuming a subset requires the existing pinned repo checkout") - if (source / ".repo/manifest.xml").read_bytes() != manifest: - raise BuildError("resuming checkout with a different manifest is forbidden") + verify_active_manifest(source, manifest) elif source.exists() and any(source.iterdir()): raise BuildError("source directory must be new or empty; use --resume to add pinned dependencies") repo = shutil.which("repo") if repo is None: raise BuildError("repo tool is unavailable on source build runner") source.mkdir(parents=True, exist_ok=True) + source = source.resolve(strict=True) if not resume: manifest_commit = lock["projects"]["manifest"]["revision"] command([ @@ -163,23 +264,18 @@ def sync( # recent versions regenerate it during subsequent sync/init operations. # Instead install the immutable flattened manifest in Repo's manifest # checkout and formally select it with -m, as supported by Repo. - manifest_name = "modos-vold-original-20261016.xml" manifest_dir = source / ".repo/manifests" if manifest_dir.is_symlink() or not manifest_dir.is_dir(): raise BuildError("repo manifests checkout is unavailable or unsafe") - pinned_manifest = manifest_dir / manifest_name + pinned_manifest = manifest_dir / MANIFEST_NAME if pinned_manifest.exists() or pinned_manifest.is_symlink(): raise BuildError("preexisting untrusted local pinned manifest") pinned_manifest.write_bytes(manifest) command([ repo, "init", "-u", "https://github.com/LineageOS/android.git", - "-b", manifest_commit, "-m", manifest_name + "-b", manifest_commit, "-m", MANIFEST_NAME ], source) - active_manifest = source / ".repo/manifest.xml" - if not active_manifest.exists() or active_manifest.is_dir(): - raise BuildError("repo did not activate the immutable full manifest") - if active_manifest.read_bytes() != manifest: - raise BuildError("repo's active manifest differs from the pinned source") + verify_active_manifest(source, manifest) # Explicit project-path sync prevents all 1,243 projects being fetched. # On resume, preverify existing project HEADs and cleanliness, then fetch # *only newly selected* missing projects. Preserve the original Soong out @@ -193,7 +289,10 @@ def sync( path = source / project if path.is_symlink() or (path.exists() and not path.is_dir()): raise BuildError(f"previously synced project path is unsafe: {project}") - (present if path.is_dir() else missing).append(project) + # Parent directories and interrupted checkout directories are not + # fetched projects. Repo worktrees have a .git file/link/directory. + git_marker = path / ".git" + (present if git_marker.exists() or git_marker.is_symlink() else missing).append(project) if present: verify_checkout(source, manifest, clean=True, selected=present) to_sync = missing @@ -211,7 +310,20 @@ def compile_original_vold( raise BuildError("baseline source directory is missing or unsafe") source = source.resolve(strict=True) verify_checkout(source, manifest, clean=True, selected=selected) + verify_source_workspace(source, manifest, selected) + release_config = source / lock["build"]["release_config"]["path"] + if ( + release_config.is_symlink() or not release_config.is_file() + or hashlib.sha256(release_config.read_bytes()).hexdigest() + != lock["build"]["release_config"]["sha256"] + ): + raise BuildError("pinned LineageOS release configuration is missing or changed") artifact = source / lock["build"]["artifact"] + for parent in artifact.parents: + if parent == source: + break + if parent.is_symlink() or (parent.exists() and not parent.is_dir()): + raise BuildError("Android vold output directory is unsafe") if artifact.is_symlink(): raise BuildError("Android vold build artifact cannot be a symbolic link") # An earlier trial must not be mistaken for the current compiler result. @@ -219,12 +331,34 @@ def compile_original_vold( if not artifact.is_file(): raise BuildError("unexpected prior vold output type") artifact.unlink() + # At this pinned revision, lunch requires product/release/variant and also + # invokes roomservice, which can fetch unselected sources. Its unchanged + # _lunch_meat helper performs the same configuration without roomservice. + # Kernel auto-construction is outside this individual target's scope. script = ( - "source build/envsetup.sh && " - f"lunch {lock['build']['target']} && " + "source build/envsetup.sh || exit $?; " + "function build_kernel() { echo 'Automatic kernel builds are outside the pinned vold probe' >&2; exit 1; }; " + f"check_product {lock['build']['product']} {lock['build']['release']} && " + f"{lock['build']['setup_command']} && " f"m -j{jobs} vold" ) - command(["bash", "-euc", script], source) + environment = os.environ.copy() + for variable in ("SOONG_ALLOW_MISSING_DEPENDENCIES", "ALLOW_MISSING_DEPENDENCIES"): + if environment.get(variable, "").lower() not in ("", "false", "0"): + raise BuildError(f"missing-dependency bypass must be disabled: {variable}") + environment.pop(variable, None) + # An inherited runner output override must not reuse another build tree. + for variable in ("OUT_DIR_COMMON_BASE", "ANDROID_PRODUCT_OUT", "ANDROID_BUILD_TOP", "TOP", "BASH_ENV"): + environment.pop(variable, None) + environment["OUT_DIR"] = "out" + environment["USE_LEFTOVERS"] = "0" + environment["ROOMSERVICE_DRYRUN"] = "true" + environment["PYTHONDONTWRITEBYTECODE"] = "1" + # Upstream envsetup intentionally reads unset optional variables; nounset + # would fail before configuration (for example at USE_LEFTOVERS). + command(["bash", "-ec", script], source, env=environment) + verify_checkout(source, manifest, clean=True, selected=selected) + verify_source_workspace(source, manifest, selected) if artifact.is_symlink() or not artifact.is_file() or not artifact.stat().st_mode & 0o111: raise BuildError("source build did not produce a regular executable vold") return artifact @@ -244,6 +378,9 @@ def build_baseline( artifact = compile_original_vold(source, lock, manifest, jobs, selected) built = read_vold(artifact) report = comparison(official, built, lock) + report["build_release"] = lock["build"]["release"] + report["build_setup_command"] = lock["build"]["setup_command"] + report["release_config_sha256"] = lock["build"]["release_config"]["sha256"] if selected is not None: report["selected_source_projects"] = len(selected) report["selected_source_paths_sha256"] = hashlib.sha256( @@ -260,11 +397,12 @@ def build_baseline( def main() -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="mode", required=True) - for name in ("sync", "probe", "compare"): + for name in ("sync", "sync-vendor-sources", "probe", "compare"): opt = sub.add_parser(name) opt.add_argument("--source-root", type=Path, required=True) opt.add_argument("--jobs", type=int, default=4) - opt.add_argument("--projects-file", type=Path, help="optional validated subset plan; never weakens the binary equality gate") + if name != "sync-vendor-sources": + opt.add_argument("--projects-file", type=Path, help="optional validated subset plan; never weakens the binary equality gate") if name == "sync": opt.add_argument("--resume", action="store_true", help="fetch missing approved projects within the same checkout") if name == "compare": @@ -275,8 +413,13 @@ def main() -> int: parser.error("--jobs must be between 1 and 16") try: lock, manifest = load_contract() - selected = verified_selection(args.projects_file, manifest) - if args.mode == "sync": + selected = verified_selection(getattr(args, "projects_file", None), manifest) + if args.mode == "sync-vendor-sources": + # Diagnose the known proprietary-input gate before downloading the + # large compiler/toolchain checkout. This fixed set is not a build + # selection and never authorizes compiling or qualifying a binary. + sync(args.source_root, lock, manifest, args.jobs, list(VENDOR_PREREQUISITE_PROJECTS)) + elif args.mode == "sync": if args.resume and selected is None: raise BuildError("full-manifest checkout does not support incremental resume") sync(args.source_root, lock, manifest, args.jobs, selected, args.resume) diff --git a/src/ci/prepare_vold_vendor_makefiles.py b/src/ci/prepare_vold_vendor_makefiles.py index f3e0d81b..f5bbdb24 100644 --- a/src/ci/prepare_vold_vendor_makefiles.py +++ b/src/ci/prepare_vold_vendor_makefiles.py @@ -1,23 +1,23 @@ #!/usr/bin/env python3 -"""Generate missing Sony product makefiles from pinned public extraction lists. +"""Report the proprietary inputs blocking pinned Sony makefile preparation. -This is a diagnostic build-probe convenience, not a proprietary-blob source. -LineageOS extract-utils --regenerate_makefiles explicitly skips all copying, -source image extraction, and blob fixups. The generated makefiles do NOT -establish provenance for missing proprietary binaries or runtime behavior. +At the locked extract-utils revision, --regenerate_makefiles skips extraction +but still opens proprietary ELF files to discover architecture and dependencies. +It cannot regenerate the Sony makefiles from public file lists alone. Until an +authenticated vendor-input contract exists, stop before running that generator +or writing partial vendor declarations. This diagnostic never qualifies vold. """ from __future__ import annotations import argparse import hashlib import json -import os from pathlib import Path import stat import subprocess import sys -from build_original_vold import load_contract, verify_checkout +from build_original_vold import VENDOR_PREREQUISITE_PROJECTS, load_contract, verify_checkout class VendorPreparationError(ValueError): @@ -30,92 +30,100 @@ class VendorPreparationError(ValueError): "device/sony/sm6375-common/extract-files.py", "device/sony/sm6375-common/proprietary-files.txt", "tools/extract-utils/extract_utils/main.py", + "tools/extract-utils/extract_utils/module.py", + "tools/extract-utils/extract_utils/makefiles.py", + "tools/extract-utils/extract_utils/elf.py", ) -REQUIRED = ( - "vendor/sony/pdx235/pdx235-vendor.mk", - "vendor/sony/pdx235/BoardConfigVendor.mk", - "vendor/sony/pdx235/Android.bp", - "vendor/sony/sm6375-common/sm6375-common-vendor.mk", - "vendor/sony/sm6375-common/BoardConfigVendor.mk", - "vendor/sony/sm6375-common/Android.bp", +# This is the first absent ELF opened in a replay of the unchanged, exact +# pinned scripts and public lists. It is an example, not a complete blob set. +REQUIRED_ELF_EXAMPLE = ( + "vendor/sony/pdx235/proprietary/" + "vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so" +) +EXTRACT_UTILS_REVISION = "ff6589c6b0a4a00c592e98940e9e6b6a50c76569" +EVIDENCE_BASE = ( + "https://github.com/LineageOS/android_tools_extract-utils/blob/" + + EXTRACT_UTILS_REVISION ) def safe_file(path: Path, max_bytes: int = 4_000_000) -> bytes: st = path.lstat() if not stat.S_ISREG(st.st_mode) or not 0 <= st.st_size <= max_bytes: - raise VendorPreparationError(f"generated makefile or source is unsafe: {path}") + raise VendorPreparationError(f"diagnostic receipt or source is unsafe: {path}") content = path.read_bytes() if len(content) != st.st_size: - raise VendorPreparationError(f"generated makefile changed while reading: {path}") + raise VendorPreparationError(f"diagnostic receipt or source changed while reading: {path}") return content -def report(source: Path, input_hashes: dict[str, str]) -> dict: - outputs = {} - for path in REQUIRED: - outputs[path] = hashlib.sha256(safe_file(source / path)).hexdigest() +def blocked_report(source: Path, input_hashes: dict[str, str]) -> dict: + for path in (source / "vendor", source / "vendor/sony"): + if path.is_symlink() or path.exists() and not path.is_dir(): + raise VendorPreparationError("Sony vendor output root is unsafe") sony_root = source / "vendor/sony" - for path in sony_root.rglob("*"): - if path.is_symlink(): - raise VendorPreparationError("generated Sony vendor source contains an unsafe link") - if not path.is_file(): - continue - if path.suffix.lower() in (".so", ".apk", ".jar", ".img", ".bin"): - raise VendorPreparationError("unexpected binary blob from makefiles-only mode") - safe_file(path) + preexisting = sony_root.exists() and any(sony_root.iterdir()) return { "schema_version": 1, - "kind": "synthetic-probe-vendor-makefiles-not-publisher-provenance", - "method": "pinned device extract-files.py --regenerate_makefiles", + "kind": "blocked-unqualified-proprietary-vendor-inputs", + "status": "blocked", "does_not_extract_blobs": True, "does_not_qualify_original_vold": True, + "makefiles_generated": False, + "preexisting_vendor_tree_unqualified": preexisting, "input_sha256": input_hashes, - "generated_makefile_sha256": outputs, - "warning": "Proprietary blobs remain absent. A real build may still fail; never treat these generated makefiles as verified official vendor blobs.", + "required_input_example": REQUIRED_ELF_EXAMPLE, + "required_input_example_is_complete_list": False, + "reason": ( + "The pinned --regenerate_makefiles path skips extraction, but " + "write_elfs_package still opens proprietary ELF files for machine, " + "bitness and DT_NEEDED metadata. Public source lists are insufficient." + ), + "required_next_step": ( + "Authenticate and pin the required proprietary vendor inputs and " + "their build provenance in a reviewed contract before enabling " + "vendor preparation. Preexisting files and synthetic receipts " + "are not accepted as provenance." + ), + "upstream_evidence": [ + EVIDENCE_BASE + "/extract_utils/main.py#L182-L219", + EVIDENCE_BASE + "/extract_utils/makefiles.py#L256-L271", + EVIDENCE_BASE + "/extract_utils/elf.py#L38-L57", + ], + "public_source_replay": { + "scope": "Review-time replay of pinned public source, not performed by this diagnostic", + "command": "python3 -B device/sony/pdx235/extract-files.py --regenerate_makefiles", + "exit_status": 1, + "exception": "FileNotFoundError", + "first_missing_file": REQUIRED_ELF_EXAMPLE, + "original_scripts_modified": False, + }, } -def prepare(source: Path, receipt: Path) -> dict: +def prepare(source: Path, receipt: Path) -> None: if source.is_symlink() or not source.is_dir(): raise VendorPreparationError("original source workspace is missing or linked") source = source.resolve(strict=True) - lock, manifest = load_contract() - # Probe only already-synced immutable projects; no unpinned tools/scripts. - required_projects = [ - "device/sony/pdx235", "device/sony/sm6375-common", "tools/extract-utils", - ] - verify_checkout(source, manifest, clean=True, selected=required_projects) + _lock, manifest = load_contract() + verify_checkout(source, manifest, clean=True, selected=list(VENDOR_PREREQUISITE_PROJECTS)) hashes = {path: hashlib.sha256(safe_file(source / path)).hexdigest() for path in INPUTS} - sony_root = source / "vendor/sony" - if sony_root.is_symlink() or sony_root.exists() and not sony_root.is_dir(): - raise VendorPreparationError("Sony vendor output root is unsafe") - + current = blocked_report(source, hashes) if receipt.is_symlink(): - raise VendorPreparationError("vendor makefile receipt must not be a symlink") + raise VendorPreparationError("vendor diagnostic receipt must not be a symlink") if receipt.exists(): - saved = json.loads(safe_file(receipt)) - current = report(source, hashes) - if current != saved: - raise VendorPreparationError("cached generated vendor makefiles differ from the pinned source receipt") - return current - - if sony_root.exists() and any(sony_root.iterdir()): - raise VendorPreparationError("unapproved preexisting Sony vendor workspace") - environment = os.environ.copy() - environment["PYTHONPATH"] = str(source / "tools/extract-utils") - subprocess.run( - [sys.executable, "device/sony/pdx235/extract-files.py", "--regenerate_makefiles"], - cwd=source, - env=environment, - check=True, + if json.loads(safe_file(receipt)) != current: + raise VendorPreparationError("existing vendor receipt is not the current blocked-input diagnostic") + else: + receipt.parent.mkdir(parents=True, exist_ok=True) + with receipt.open("x", encoding="utf-8") as output: + json.dump(current, output, indent=2, sort_keys=True) + output.write("\n") + raise VendorPreparationError( + "pinned Sony makefile regeneration requires authenticated proprietary " + f"ELF inputs (for example {REQUIRED_ELF_EXAMPLE}); no vendor-input " + f"provenance contract exists. Blocked-input report: {receipt}" ) - current = report(source, hashes) - receipt.parent.mkdir(parents=True, exist_ok=True) - with receipt.open("x", encoding="utf-8") as output: - json.dump(current, output, indent=2, sort_keys=True) - output.write("\n") - return current def main() -> int: @@ -124,10 +132,9 @@ def main() -> int: parser.add_argument("--report", type=Path, required=True) args = parser.parse_args() try: - result = prepare(args.source_root, args.report) - print(json.dumps({"kind": result["kind"], "output_count": len(REQUIRED)}, sort_keys=True)) + prepare(args.source_root, args.report) except (OSError, KeyError, TypeError, ValueError, subprocess.CalledProcessError) as exc: - print(f"Error: cannot produce pinned Sony makefiles: {exc}", file=sys.stderr) + print(f"Error: cannot prepare pinned Sony makefiles: {exc}", file=sys.stderr) return 1 return 0 diff --git a/src/ci/probe_original_vold.sh b/src/ci/probe_original_vold.sh index 1edf1804..245d7535 100644 --- a/src/ci/probe_original_vold.sh +++ b/src/ci/probe_original_vold.sh @@ -7,7 +7,6 @@ set -euo pipefail : "${VOLD_WORK:?isolated scratch workspace is required}" : "${VOLD_PROBE_STAGE:?probe or compare stage is required}" -: "${VOLD_SOURCE_REF:?absolute original vold path is required for compare}" [[ "${VOLD_PROBE_STAGE}" == probe || "${VOLD_PROBE_STAGE}" == compare ]] || { echo 'Error: unsupported unmodified vold dependency stage' >&2 @@ -17,7 +16,7 @@ set -euo pipefail echo 'Error: isolated scratch directory is missing or unsafe' >&2 exit 1 } -[[ "${VOLD_PROBE_STAGE}" != compare || ( -s "${VOLD_SOURCE_REF}" && ! -L "${VOLD_SOURCE_REF}" ) ]] || { +[[ "${VOLD_PROBE_STAGE}" != compare || ( "${VOLD_SOURCE_REF:-}" == /* && -s "${VOLD_SOURCE_REF}" && ! -L "${VOLD_SOURCE_REF}" ) ]] || { echo 'Error: authenticated original vold reference is required' >&2 exit 1 } @@ -29,16 +28,36 @@ jobs="${VOLD_PROBE_JOBS:-4}" selection="${VOLD_WORK}/source-selection.json" python3 src/ci/vold_source_closure.py plan --output "${selection}" -python3 src/ci/build_original_vold.py sync \ - --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" - -# The pinned Sony public device trees reference proprietary vendor-generated -# mk files absent from the source manifest. Generate ONLY makefile declarations -# using their reviewed --regenerate_makefiles option. This does not download, -# synthesize or validate vendor blobs; absent binaries remain build blockers. +# Check the three small pinned public vendor-input projects BEFORE fetching +# toolchains. At these revisions --regenerate_makefiles still reads absent +# proprietary ELF files. The preflight records that blocker without generating +# misleading partial declarations or downloading the rest of the source seed. +python3 src/ci/build_original_vold.py sync-vendor-sources \ + --source-root "${VOLD_WORK}/source" --jobs "${jobs}" python3 src/ci/prepare_vold_vendor_makefiles.py \ --source-root "${VOLD_WORK}/source" \ - --report "${VOLD_WORK}/sony-generated-makefiles-report.json" + --report "${VOLD_WORK}/sony-vendor-input-report.json" + +python3 src/ci/build_original_vold.py sync --resume \ + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" + +# Batch reviewed provider hints from the checked-out Blueprint graph before +# compiling. Unknown or conditional modules remain diagnostics for real Soong; +# they are never mapped to guessed repositories. Stop after three scans. +for scan in 1 2 3; do + next="${VOLD_WORK}/selection-static-next.json" + rm -f -- "${next}" + python3 src/ci/vold_source_closure.py index \ + --source-root "${VOLD_WORK}/source" --selection "${selection}" \ + --output "${VOLD_WORK}/source-index-static-${scan}.json" \ + --next-selection "${next}" + if [[ ! -f "${next}" ]]; then + break + fi + mv -- "${next}" "${selection}" + python3 src/ci/build_original_vold.py sync --resume \ + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" +done # Most missing dependencies are reported during Soong setup before the # expensive target compile. Resolve their exact pinned project paths once @@ -46,12 +65,6 @@ python3 src/ci/prepare_vold_vendor_makefiles.py \ # Unknown module providers must stop for review; never turn on missing # dependency flags or attempt an unverified project URL. for iteration in 1 2 3; do - python3 src/ci/vold_source_closure.py index \ - --source-root "${VOLD_WORK}/source" \ - --selection "${selection}" \ - --output "${VOLD_WORK}/source-index-${iteration}.json" || - echo "::warning::Partial Soong index could not be collected on pass ${iteration}" - log="${VOLD_WORK}/vold-build-pass-${iteration}.log" if [[ "${VOLD_PROBE_STAGE}" == probe ]]; then build_cmd=(python3 src/ci/build_original_vold.py probe diff --git a/src/ci/vold_source_closure.py b/src/ci/vold_source_closure.py index 8fbff29f..d3564944 100644 --- a/src/ci/vold_source_closure.py +++ b/src/ci/vold_source_closure.py @@ -32,9 +32,10 @@ re.I, ) QUOTED_MODULE = re.compile( - r'(?:module|dependency)\s+["\x27]([A-Za-z0-9_.+@-]+)["\x27].{0,140}' - r'(?:missing|undefined|not found)', re.I, + r'(?:module|dependency)\s+["\x27]([A-Za-z0-9_.+@-]+)["\x27]' + r'\s+(?:(?:is|was)\s+)?(?:missing(?!\s+dependenc(?:y|ies))|undefined|not found)\b', re.I, ) +MISSING_DEPENDENCIES = re.compile(r"\bmissing dependencies:\s*(.*)$", re.I) ERROR_MARKERS = ("error:", "not found", "no such file", "missing", "undefined", "cannot find") BLUEPRINT_BLOCK = re.compile(r"(?m)^\s*([A-Za-z][A-Za-z0-9_]*)\s*\{") BLUEPRINT_NAME = re.compile(r'\bname\s*:\s*"([^"]+)"') @@ -44,8 +45,13 @@ "defaults", "static_libs", "shared_libs", "whole_static_libs", "header_libs", "export_header_lib_headers", "export_shared_lib_headers", "export_static_lib_headers", "required", "vintf_fragment_modules", - "generated_headers", "tools", "tool_files", + "generated_headers", "generated_sources", "export_generated_headers", + "tools", "runtime_libs", "host_required", "target_required", + "rustlibs", "rlibs", "dylibs", "proc_macros", } +BP_MODULE_REFERENCE_FIELDS = {"srcs", "exclude_srcs", "tool_files", "data"} +BP_SINGLE_MODULE_FIELDS = {"aconfig_declarations"} +BP_STRING = re.compile(r'\b([a-z_]+)\s*:\s*"((?:[^"\\]|\\.)*)"') class ClosureError(ValueError): @@ -75,6 +81,7 @@ def verified_contract() -> tuple[dict, dict[str, str], dict]: if ( not path or not re.fullmatch(r"[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)*", path) + or any(part in (".", "..") for part in path.split("/")) or not revision or not re.fullmatch(r"[0-9a-f]{40}", revision) ): @@ -116,9 +123,7 @@ def validate_selection(selection: dict, projects: dict[str, str], seed: dict) -> paths = selection.get("selected_projects") if not isinstance(paths, list) or not paths or len(paths) > len(projects): raise ClosureError("dependency selection contains an invalid project list") - if len(paths) != len(set(paths)) or any( - not isinstance(path, str) or path not in projects for path in paths - ): + if any(not isinstance(path, str) or path not in projects for path in paths) or len(paths) != len(set(paths)): raise ClosureError("dependency selection contains unknown, unpinned or duplicate projects") if not set(initial_projects(seed)).issubset(paths): raise ClosureError("dependency selection may not omit reviewed seed projects") @@ -149,20 +154,31 @@ def missing_dependencies( unpinned: set[str] = set() unresolved_modules: set[str] = set() modules: set[str] = set() + unmapped_errors: list[str] = [] hints = seed["direct_module_to_project_hints"] for line in text.splitlines(): low = line.lower() if not any(token in low for token in ERROR_MARKERS): continue + line_modules = set() for pattern in (MISSING_MODULE, QUOTED_MODULE): - modules.update(pattern.findall(line)) + line_modules.update(pattern.findall(line)) + missing_list = MISSING_DEPENDENCIES.search(line) + if missing_list: + line_modules.update(re.findall(r"[A-Za-z0-9_.+@-]+", missing_list[1])) + modules.update(line_modules) + actionable_path = False for match in PATH_IN_LOG.finditer(line): path = match[1] project = owner(path, projects) if project is not None and project not in selected: pinned_missing.add(project) - elif project is None and path.startswith(("vendor/sony/", "device/sony/")): + actionable_path = True + elif project is None and path.startswith(("vendor/", "device/")): unpinned.add(path) + actionable_path = True + if not line_modules and not actionable_path and len(unmapped_errors) < 100: + unmapped_errors.append(line[:1000]) for module in modules: mapped = hints.get(module) if mapped is None: @@ -174,12 +190,30 @@ def missing_dependencies( "unpinned_missing_paths": sorted(unpinned), "modules_without_verified_owner": sorted(unresolved_modules), "modules_observed": sorted(modules), + "modules_with_selected_owner": sorted(module for module in modules if hints.get(module) in selected), + "unmapped_error_lines": unmapped_errors, "statically_unpinned_vendor_inputs": seed["known_unpinned_vendor_inputs"], } +def without_blueprint_comments(raw: str) -> str: + """Erase comments without interpreting comment markers inside quoted strings.""" + return re.sub( + r'"(?:[^"\\]|\\.)*"|//[^\n]*|/\*.*?\*/', + lambda match: match[0] if match[0].startswith('"') else re.sub(r"[^\n]", " ", match[0]), + raw, + flags=re.S, + ) + + +def module_reference(value: str) -> str: + """Normalize an explicit :module{.output_tag}, not an arbitrary source path.""" + return re.sub(r"\{[^{}]+\}$", "", value.removeprefix(":")) + + def iter_blueprint_blocks(raw: str): """Extract balanced module declarations, without assuming nested blocks are flat.""" + raw = without_blueprint_comments(raw) for candidate in BLUEPRINT_BLOCK.finditer(raw): start = candidate.end() depth = 1 @@ -220,6 +254,7 @@ def blueprint_module_graph(source_root: Path, selected: list[str]) -> dict: source_root = source_root.resolve(strict=True) found: dict[str, list[dict]] = {} count_files = 0 + scanned = set() for project in selected: directory = source_root / project if directory.is_symlink() or not directory.is_dir(): @@ -227,6 +262,9 @@ def blueprint_module_graph(source_root: Path, selected: list[str]) -> dict: for bp in directory.rglob("Android.bp"): if bp.is_symlink() or not bp.is_file() or bp.stat().st_size > 4_000_000: continue + if bp in scanned: + continue + scanned.add(bp) count_files += 1 if count_files > 12000: raise ClosureError("source contains unexpectedly many Blueprint files") @@ -238,9 +276,12 @@ def blueprint_module_graph(source_root: Path, selected: list[str]) -> dict: deps = set() for field, values in BP_ARRAY.findall(body): if field in BP_DEP_FIELDS: - deps.update(x.lstrip(":") for x in BP_ITEM.findall(values)) - if field in ("srcs", "generated_sources"): - deps.update(x[1:] for x in BP_ITEM.findall(values) if x.startswith(":")) + deps.update(module_reference(x) for x in BP_ITEM.findall(values)) + if field in BP_MODULE_REFERENCE_FIELDS: + deps.update(module_reference(x) for x in BP_ITEM.findall(values) if x.startswith(":")) + for field, value in BP_STRING.findall(body): + if field in BP_SINGLE_MODULE_FIELDS: + deps.add(module_reference(value)) found.setdefault(name, []).append({ "path": bp.relative_to(source_root).as_posix(), "deps": sorted(deps), @@ -269,10 +310,28 @@ def blueprint_module_graph(source_root: Path, selected: list[str]) -> dict: "reachable_module_sample": sorted(reachable)[:160], "unresolved_module_names": sorted(unresolved), "ambiguous_module_names": sorted(ambiguous), - "warning": "Over-approximation: product-selected Soong modules, generated dependencies and missing unsynced module providers still need a real Soong evaluation.", + "warning": "Partial approximation with omissions and conditional false positives: variables, namespaces, product-selected modules and generated dependencies still need a real Soong evaluation.", } +def next_projects_after_index(selected: list[str], report: dict, seed: dict) -> dict | None: + """Use reviewed provider hints only; unresolved static names are not build failures.""" + hints = seed["direct_module_to_project_hints"] + additions = sorted({ + hints[module] for module in report["unresolved_module_names"] + if module in hints and hints[module] not in selected + }) + report["proposed_project_additions"] = additions + report["modules_without_verified_owner"] = sorted( + module for module in report["unresolved_module_names"] if module not in hints + ) + if len(additions) > 30: + raise ClosureError("too many static additions: require independent review") + if not additions: + return None + return selection_record(sorted(set(selected + additions)), seed) + + def next_projects_after_diagnostic( selected: list[str], diagnostic: dict, seed: dict, ) -> dict | None: @@ -303,6 +362,7 @@ def main() -> int: cmd_index.add_argument("--selection", type=Path, required=True) cmd_index.add_argument("--source-root", type=Path, required=True) cmd_index.add_argument("--output", type=Path, required=True) + cmd_index.add_argument("--next-selection", type=Path) cmd_diag = sub.add_parser("diagnose", help="map build errors to additional pinned paths") cmd_diag.add_argument("--selection", type=Path, required=True) cmd_diag.add_argument("--log", type=Path, required=True) @@ -325,6 +385,11 @@ def main() -> int: selected = validate_selection(json.loads(args.selection.read_text()), projects, seed) if args.action == "index": report = blueprint_module_graph(args.source_root, selected) + expansion = next_projects_after_index(selected, report, seed) + if args.next_selection is not None: + if expansion is not None: + write_json(args.next_selection, expansion) + report["next_selection"] = str(args.next_selection) if expansion is not None else None else: report = missing_dependencies(args.log.read_text(errors="replace"), selected, projects, seed) if args.next_selection is not None: diff --git a/tests/vold_baseline_ci_test.py b/tests/vold_baseline_ci_test.py index 77b1ac61..99b29e6e 100644 --- a/tests/vold_baseline_ci_test.py +++ b/tests/vold_baseline_ci_test.py @@ -117,6 +117,9 @@ def test_baseline_build_removes_stale_output_and_rejects_nonidentical_bytes(self source = root / "source" (source / ".repo").mkdir(parents=True) (source / ".repo/manifest.xml").write_bytes(manifest) + release = source / lock["build"]["release_config"]["path"] + release.parent.mkdir(parents=True) + release.write_text("# Updated manually\naosp_target_release=cp2a\n") out = source / lock["build"]["artifact"] out.parent.mkdir(parents=True) out.write_bytes(b"stale build must be deleted") @@ -124,7 +127,7 @@ def test_baseline_build_removes_stale_output_and_rejects_nonidentical_bytes(self reference.write_bytes(original) def fake_compile(argv, cwd, **_kwargs): - self.assertEqual(argv[:2], ["bash", "-euc"]) + self.assertEqual(argv[:2], ["bash", "-ec"]) self.assertFalse(out.exists(), "stale binary must be removed first") self.assertIn("m -j2 vold", argv[2]) out.write_bytes(original) diff --git a/tests/vold_original_build_test.py b/tests/vold_original_build_test.py new file mode 100644 index 00000000..fdb41528 --- /dev/null +++ b/tests/vold_original_build_test.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Regression checks for the real Repo layout and fail-closed build inputs.""" +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src/ci")) +import build_original_vold as builder # noqa: E402 + + +class OriginalVoldBuildTest(unittest.TestCase): + def setUp(self): + self.lock, self.manifest = builder.load_contract() + + def release_config(self, source): + path = source / self.lock["build"]["release_config"]["path"] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# Updated manually\naosp_target_release=cp2a\n") + + @staticmethod + def selected_manifest(source, manifest): + manifests = source / ".repo/manifests" + manifests.mkdir(parents=True) + (manifests / builder.MANIFEST_NAME).write_bytes(manifest) + # Modern Repo manifest_xml.XmlManifest.Link() writes this include + # wrapper; it does not copy the selected manifest into manifest.xml. + active = source / ".repo/manifest.xml" + active.write_text( + '\n' + '\n' + '\n' + f' \n' + '\n' + ) + return active + + def test_actual_repo_include_wrapper_and_included_bytes(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + builder.verify_active_manifest(source, self.manifest) + (source / ".repo/manifests" / builder.MANIFEST_NAME).write_bytes(b"changed") + with self.assertRaisesRegex(builder.BuildError, "exact official snapshot"): + builder.verify_active_manifest(source, self.manifest) + + def test_wrapper_cannot_add_overrides_or_select_another_manifest(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + active = self.selected_manifest(source, self.manifest) + for contents in ( + '', + f'', + f'', + f'', + ): + with self.subTest(contents=contents): + active.write_text(contents) + with self.assertRaisesRegex(builder.BuildError, "wrapper"): + builder.verify_active_manifest(source, self.manifest) + + def test_local_manifests_rejected_even_with_matching_active_manifest(self): + for filename in ("local_manifest.xml", "local_manifests/extra.xml"): + with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + override = source / ".repo" / filename + override.parent.mkdir(parents=True, exist_ok=True) + override.write_text("") + with self.assertRaisesRegex(builder.BuildError, "local manifest overrides"): + builder.verify_active_manifest(source, self.manifest) + + def test_legacy_manifest_link_must_target_the_pinned_checkout_file(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "source" + active = self.selected_manifest(source, self.manifest) + active.unlink() + active.symlink_to(Path("manifests") / builder.MANIFEST_NAME) + builder.verify_active_manifest(source, self.manifest) + outside = Path(tmp) / "outside.xml" + outside.write_bytes(self.manifest) + active.unlink() + active.symlink_to(outside) + with self.assertRaisesRegex(builder.BuildError, "link does not select"): + builder.verify_active_manifest(source, self.manifest) + + def test_omitted_manifest_path_checks_real_git_revision_and_local_content(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + project = source / "device/common" + project.mkdir(parents=True) + subprocess.run(["git", "init", "--quiet", str(project)], check=True) + subprocess.run( + ["git", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", + "commit", "--quiet", "--allow-empty", "-m", "pinned fixture"], + cwd=project, check=True, + ) + revision = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=project, check=True, + capture_output=True, text=True, + ).stdout.strip() + manifest = f''.encode() + self.selected_manifest(source, manifest) + with patch.object(builder.shutil, "which", return_value="/trusted/repo"): + builder.verify_checkout(source, manifest, clean=True, selected=["device/common"]) + (project / "unexpected.bp").write_text("unreviewed source") + with self.assertRaisesRegex(builder.BuildError, "unreviewed local content"): + builder.verify_checkout(source, manifest, clean=True, selected=["device/common"]) + (project / "unexpected.bp").unlink() + bad = manifest.replace(revision.encode(), b"0" * 40) + (source / ".repo/manifests" / builder.MANIFEST_NAME).write_bytes(bad) + with self.assertRaisesRegex(builder.BuildError, "source revision changed"): + builder.verify_checkout(source, bad, clean=True, selected=["device/common"]) + + def test_resume_fetches_partial_directories_but_not_existing_worktrees(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + (source / "system/vold/.git").mkdir(parents=True) + (source / "system/update_engine").mkdir(parents=True) + calls = [] + with ( + patch.object(builder.shutil, "which", return_value="/trusted/repo"), + patch.object(builder, "verify_checkout") as verify, + patch.object(builder, "command", side_effect=lambda args, *a, **k: calls.append(args)), + ): + builder.sync(source, self.lock, self.manifest, 2, + ["system/vold", "system/update_engine"], resume=True) + self.assertEqual(calls, [["/trusted/repo", "sync", "--no-clone-bundle", "-j2", "system/update_engine"]]) + self.assertEqual(verify.call_args_list[0].kwargs["selected"], ["system/vold"]) + + def test_vendor_prerequisite_mode_only_requests_three_pinned_projects(self): + with ( + patch.object(sys, "argv", ["builder", "sync-vendor-sources", "--source-root", "/fixture", "--jobs", "2"]), + patch.object(builder, "sync") as sync, + ): + self.assertEqual(builder.main(), 0) + args = sync.call_args.args + self.assertEqual(args[0], Path("/fixture")) + self.assertEqual(args[3], 2) + self.assertEqual(args[4], ["device/sony/pdx235", "device/sony/sm6375-common", "tools/extract-utils"]) + self.assertTrue(set(args[4]).issubset(builder.project_map(self.manifest))) + + def test_unrecorded_vendor_or_unselected_source_is_not_build_input(self): + for unexpected in ("vendor/sony/pdx235/Android.bp", "system/update_engine/Android.bp"): + with self.subTest(unexpected=unexpected), tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + (source / "system/vold").mkdir(parents=True) + bad = source / unexpected + bad.parent.mkdir(parents=True, exist_ok=True) + bad.write_text("unrecorded source") + with self.assertRaisesRegex(builder.BuildError, "unrecorded source"): + builder.verify_source_workspace(source, self.manifest, ["system/vold"]) + + def test_manifest_exported_links_are_checked_without_scanning_project_contents(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + project = source / "build/make" + project.mkdir(parents=True) + (project / "envsetup.sh").write_text("pinned source") + link = source / "build/envsetup.sh" + link.symlink_to("make/envsetup.sh") + builder.verify_source_workspace(source, self.manifest, ["build/make"]) + link.unlink() + link.write_text("unreviewed replacement") + with self.assertRaisesRegex(builder.BuildError, "linkfile differs"): + builder.verify_source_workspace(source, self.manifest, ["build/make"]) + + def test_successful_noop_build_cannot_reuse_prior_vold(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + self.release_config(source) + artifact = source / self.lock["build"]["artifact"] + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"previous executable") + artifact.chmod(0o755) + with ( + patch.object(builder, "verify_checkout"), + patch.object(builder, "command"), + self.assertRaisesRegex(builder.BuildError, "did not produce"), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + self.assertFalse(artifact.exists()) + + def test_build_uses_own_output_tree_and_rejects_dependency_bypasses(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + self.release_config(source) + artifact = source / self.lock["build"]["artifact"] + artifact.parent.mkdir(parents=True) + + def compile_fixture(_args, _cwd, *, env): + self.assertEqual(_args[:2], ["bash", "-ec"]) + self.assertIn("check_product lineage_pdx235 cp2a", _args[2]) + self.assertIn("_lunch_meat lineage_pdx235 cp2a userdebug", _args[2]) + self.assertNotIn("lunch lineage_pdx235-userdebug", _args[2]) + self.assertEqual(env["USE_LEFTOVERS"], "0") + self.assertEqual(env["PYTHONDONTWRITEBYTECODE"], "1") + self.assertEqual(env["OUT_DIR"], "out") + self.assertNotIn("OUT_DIR_COMMON_BASE", env) + self.assertNotIn("ANDROID_PRODUCT_OUT", env) + self.assertNotIn("ALLOW_MISSING_DEPENDENCIES", env) + artifact.write_bytes(b"fresh executable") + artifact.chmod(0o755) + + with ( + patch.object(builder, "verify_checkout"), + patch.object(builder, "command", side_effect=compile_fixture), + patch.dict(os.environ, {"OUT_DIR": "/other-build", "OUT_DIR_COMMON_BASE": "/common", "ANDROID_PRODUCT_OUT": "/other-product", "ALLOW_MISSING_DEPENDENCIES": "false"}), + ): + self.assertEqual(builder.compile_original_vold(source, self.lock, self.manifest, 2), artifact) + for flag in ("SOONG_ALLOW_MISSING_DEPENDENCIES", "ALLOW_MISSING_DEPENDENCIES"): + with ( + self.subTest(flag=flag), + patch.object(builder, "verify_checkout"), + patch.object(builder, "command") as compile_command, + patch.dict(os.environ, {flag: "true"}), + self.assertRaisesRegex(builder.BuildError, "bypass must be disabled"), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + compile_command.assert_not_called() + + def test_failed_upstream_setup_or_kernel_autobuild_stops_before_vold(self): + for setup_failure in (True, False): + with self.subTest(setup_failure=setup_failure), tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + self.release_config(source) + make = source / "build/make" + make.mkdir(parents=True) + (source / "build/envsetup.sh").symlink_to("make/envsetup.sh") + artifact = source / self.lock["build"]["artifact"] + (make / "envsetup.sh").write_text( + "function check_product() { return 0; }\n" + "function _lunch_meat() { " + + ("return 0;" if setup_failure else "build_kernel;") + + " }\n" + "function m() { mkdir -p out/target/product/pdx235/system/bin; " + "echo stale > out/target/product/pdx235/system/bin/vold; " + "chmod +x out/target/product/pdx235/system/bin/vold; }\n" + + ("return 1\n" if setup_failure else "return 0\n") + ) + with ( + patch.object(builder, "verify_checkout"), + self.assertRaises(subprocess.CalledProcessError), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + self.assertFalse(artifact.exists()) + + def test_successful_compiler_cannot_add_an_unrecorded_source_tree(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + self.release_config(source) + artifact = source / self.lock["build"]["artifact"] + + def changed_source(_args, _cwd, **_kwargs): + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"fresh executable") + artifact.chmod(0o755) + extra = source / "vendor/unrecorded/Android.bp" + extra.parent.mkdir(parents=True) + extra.write_text("unrecorded build input") + + with ( + patch.object(builder, "verify_checkout") as verify, + patch.object(builder, "command", side_effect=changed_source), + self.assertRaisesRegex(builder.BuildError, "unrecorded source"), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + self.assertEqual(verify.call_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vold_probe_orchestration_test.py b/tests/vold_probe_orchestration_test.py new file mode 100644 index 00000000..e12b1f72 --- /dev/null +++ b/tests/vold_probe_orchestration_test.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Exercise probe ordering at process boundaries without downloading Android.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class ProbeOrchestrationTest(unittest.TestCase): + def run_probe(self, *, stage="probe", vendor_blocked=False, + static_additions=1, build_failures=0, diagnosed=True, + reference=True): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + binaries = work / "bin" + binaries.mkdir() + shim = binaries / "python3" + shim.write_text(f"#!{sys.executable}\n" + textwrap.dedent('''\ + import json + import os + from pathlib import Path + import sys + + args = sys.argv[1:] + work = Path(os.environ["VOLD_WORK"]) + history = work / "commands.jsonl" + with history.open("a") as output: + output.write(json.dumps(args) + "\\n") + previous = [json.loads(line) for line in history.read_text().splitlines()] + + def option(name): + return Path(args[args.index(name) + 1]) + + if args[0].endswith("vold_source_closure.py"): + action = args[1] + option("--output").write_text("{}") + if action == "plan": + option("--output").write_text('{"selected_projects": ["system/vold"]}') + elif action == "index": + scans = sum(x[0].endswith("vold_source_closure.py") and x[1] == "index" for x in previous) + if scans <= int(os.environ["TEST_STATIC_ADDITIONS"]): + option("--next-selection").write_text('{"selected_projects": ["system/vold", "system/libbase"]}') + elif action == "diagnose" and os.environ["TEST_DIAGNOSED"] == "1": + option("--next-selection").write_text('{"selected_projects": ["system/vold", "system/libbase", "system/gsid"]}') + elif args[0].endswith("prepare_vold_vendor_makefiles.py"): + option("--report").write_text('{"qualified": false}') + if os.environ["TEST_VENDOR_BLOCKED"] == "1": + print("missing proprietary ELF input", file=sys.stderr) + sys.exit(1) + elif args[0].endswith("build_original_vold.py"): + if args[1] in ("sync", "sync-vendor-sources"): + option("--source-root").mkdir(exist_ok=True) + else: + builds = sum(x[0].endswith("build_original_vold.py") and x[1] in ("probe", "compare") for x in previous) + if builds <= int(os.environ["TEST_BUILD_FAILURES"]): + print('error: module "vold" depends on undefined module "libgsi"') + sys.exit(1) + else: + sys.exit("unexpected command") + ''')) + shim.chmod(0o755) + environment = os.environ.copy() + environment.pop("VOLD_SOURCE_REF", None) + environment.update({ + "PATH": str(binaries) + os.pathsep + environment["PATH"], + "VOLD_WORK": str(work), + "VOLD_PROBE_STAGE": stage, + "TEST_VENDOR_BLOCKED": str(int(vendor_blocked)), + "TEST_STATIC_ADDITIONS": str(static_additions), + "TEST_BUILD_FAILURES": str(build_failures), + "TEST_DIAGNOSED": str(int(diagnosed)), + }) + if stage == "compare" and reference: + original = work / "authenticated-reference-fixture" + original.write_bytes(b"reference fixture") + environment["VOLD_SOURCE_REF"] = str(original) + result = subprocess.run( + ["bash", str(ROOT / "src/ci/probe_original_vold.sh")], + cwd=ROOT, env=environment, text=True, capture_output=True, + ) + history = work / "commands.jsonl" + calls = [json.loads(line) for line in history.read_text().splitlines()] if history.exists() else [] + return result, calls + + @staticmethod + def build_calls(calls): + return [args for args in calls if args[0].endswith("build_original_vold.py")] + + def test_vendor_blocker_precedes_toolchain_sync_and_build(self): + result, calls = self.run_probe(vendor_blocked=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("missing proprietary ELF", result.stderr) + self.assertEqual([x[1] for x in self.build_calls(calls)], ["sync-vendor-sources"]) + self.assertFalse(any(x[1] == "index" for x in calls)) + + def test_probe_needs_no_ota_and_scans_before_compiling(self): + result, calls = self.run_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("no authenticated binary equality", result.stdout) + self.assertEqual([x[1] for x in self.build_calls(calls)], + ["sync-vendor-sources", "sync", "sync", "probe"]) + resume = [x for x in self.build_calls(calls) if x[1] == "sync"] + self.assertTrue(all("--resume" in x and "--projects-file" in x for x in resume)) + actions = [x[1] for x in calls] + self.assertLess(actions.index("index"), actions.index("probe")) + + def test_static_scans_are_bounded(self): + result, calls = self.run_probe(static_additions=99) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sum(x[1] == "index" for x in calls), 3) + self.assertEqual(sum(x[1] == "probe" for x in calls), 1) + + def test_build_error_expands_once_then_reuses_checkout(self): + result, calls = self.run_probe(static_additions=0, build_failures=1) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([x[1] for x in self.build_calls(calls)], + ["sync-vendor-sources", "sync", "probe", "sync", "probe"]) + + def test_unknown_errors_and_compile_budget_stop(self): + for diagnosed, expected in ((False, 1), (True, 3)): + with self.subTest(diagnosed=diagnosed): + result, calls = self.run_probe(static_additions=0, + build_failures=99, diagnosed=diagnosed) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(sum(x[1] == "probe" for x in calls), expected) + + def test_compare_requires_reference_and_never_calls_probe(self): + result, calls = self.run_probe(stage="compare", reference=False) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(calls, []) + result, calls = self.run_probe(stage="compare") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([x[1] for x in self.build_calls(calls)][-1], "compare") + self.assertFalse(any(x[1] == "probe" for x in calls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vold_source_closure_test.py b/tests/vold_source_closure_test.py index 7ff36bdd..58a3e55d 100644 --- a/tests/vold_source_closure_test.py +++ b/tests/vold_source_closure_test.py @@ -3,6 +3,8 @@ from __future__ import annotations import json +import contextlib +import io from pathlib import Path import sys import tempfile @@ -21,18 +23,22 @@ def setUp(self): self.seed, self.manifest, self.baseline = closure.verified_contract() self.initial = closure.initial_projects(self.seed) - def test_pinned_70_project_bootstrap_is_strictly_smaller_than_full_manifest(self): + def test_pinned_84_project_bootstrap_is_strictly_smaller_than_full_manifest(self): self.assertEqual(len(self.manifest), 1243) # The official manifest omits path= for dozens of AOSP projects. self.assertIn("device/common", self.manifest) self.assertIn("toolchain/pgo-profiles", self.manifest) - self.assertEqual(len(self.initial), 70) + self.assertEqual(len(self.initial), 84) self.assertEqual(len(self.initial), len(set(self.initial))) self.assertTrue({"system/vold", "build/soong", "frameworks/base", "device/sony/pdx235"}.issubset(self.initial)) self.assertTrue(set(self.initial).issubset(self.manifest)) self.assertEqual(self.seed["manifest_sha256"], self.baseline["manifest"]["sha256"]) self.assertIn("vendor/sony/pdx235/pdx235-vendor.mk", self.seed["known_unpinned_vendor_inputs"]) self.assertNotIn("vendor/sony/pdx235", self.manifest) + self.assertEqual(self.seed["direct_module_to_project_hints"]["libbase"], "system/libbase") + self.assertEqual(self.seed["direct_module_to_project_hints"]["libgsi"], "system/gsid") + self.assertEqual(self.seed["direct_module_to_project_hints"]["libhardware_legacy"], "hardware/libhardware_legacy") + self.assertTrue({"external/rust/cxx", "external/rust/android-crates-io", "system/tools/aidl"}.issubset(self.initial)) def test_invalid_or_unpinned_expansions_fail_closed(self): correct = closure.selection_record(self.initial, self.seed) @@ -42,6 +48,7 @@ def test_invalid_or_unpinned_expansions_fail_closed(self): lambda x: x.update(selected_projects=["system/vold"]), lambda x: x.update(selected_projects=x["selected_projects"] + ["vendor/sony/pdx235"]), lambda x: x.update(selected_projects=x["selected_projects"] * 2), + lambda x: x.update(selected_projects=[[]]), ]: selection = dict(correct) mutate(selection) @@ -59,10 +66,44 @@ def test_diagnostics_identify_pinned_source_vs_unpinned_vendor(self): self.assertIn("system/update_engine", report["proposed_project_additions"]) self.assertIn("vendor/sony/pdx235/pdx235-vendor.mk", report["unpinned_missing_paths"]) self.assertIn("unsupported_vendor_blob", report["modules_without_verified_owner"]) + self.assertNotIn("vold", report["modules_observed"], "the requesting module is not the missing provider") self.assertNotIn("vendor/sony/pdx235", report["proposed_project_additions"]) self.assertEqual(closure.owner("system/fs/fs_mgr/fs_mgr/Android.bp", self.manifest), "system/fs/fs_mgr") self.assertIsNone(closure.owner("vendor/sony/pdx235/pdx235-vendor.mk", self.manifest)) + def test_real_soong_diagnostics_do_not_mistake_requesters_for_missing_modules(self): + hint_seed = {**self.seed, "direct_module_to_project_hints": {"libupdate": "system/update_engine"}} + for log in ( + 'error: module "vold" depends on undefined module "libupdate".', + 'error: module "vold" variant "android_arm64" depends on undefined module "libupdate".', + 'error: module "libupdate" is missing', + 'error: module "vold" missing dependencies: libupdate', + ): + with self.subTest(log=log): + report = closure.missing_dependencies(log, self.initial, self.manifest, hint_seed) + self.assertEqual(report["modules_observed"], ["libupdate"]) + self.assertEqual(report["modules_without_verified_owner"], []) + self.assertEqual(report["proposed_project_additions"], ["system/update_engine"]) + self.assertIsNotNone(closure.next_projects_after_diagnostic(self.initial, report, hint_seed)) + report = closure.missing_dependencies( + 'error: module "vold" missing dependencies: [libupdate, unknown_generator]', + self.initial, self.manifest, hint_seed, + ) + self.assertEqual(report["modules_without_verified_owner"], ["unknown_generator"]) + self.assertIsNone(closure.next_projects_after_diagnostic(self.initial, report, hint_seed)) + + def test_unmappable_errors_and_non_sony_vendor_inputs_remain_diagnostic(self): + report = closure.missing_dependencies( + 'error: vendor/private/required.mk missing\n' + 'error: module "vold" depends on undefined module "libbase"\n' + 'error: product configuration could not be evaluated\n', + self.initial, self.manifest, self.seed, + ) + self.assertEqual(report["unpinned_missing_paths"], ["vendor/private/required.mk"]) + self.assertEqual(report["modules_with_selected_owner"], ["libbase"]) + self.assertIn('error: product configuration could not be evaluated', report["unmapped_error_lines"]) + self.assertIsNone(closure.next_projects_after_diagnostic(self.initial, report, self.seed)) + def test_expand_only_when_all_missing_inputs_have_pinned_owners(self): good = { "proposed_project_additions": ["system/update_engine"], @@ -105,6 +146,67 @@ def test_static_soong_modules_trace_defaults_and_transitive_references(self): self.assertTrue({"libcrypto", "libbase", "libdm"}.issubset(report["unresolved_module_names"])) self.assertNotIn("vold_aidl", report["unresolved_module_names"]) + def test_static_index_preserves_generated_dependencies_and_ignores_comments_and_file_paths(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bp = root / "system/vold/Android.bp" + bp.parent.mkdir(parents=True) + bp.write_text( + '/* cc_binary { name: "vold", static_libs: ["commented"], } */\n' + 'cc_binary {\n name: "vold",\n generated_sources: ["generated_cpp"],\n' + 'srcs: [":tagged_sources{.cpp}", "literal.cpp"],\n' + 'tools: ["compiler"], tool_files: ["script.py", ":tool_sources"],\n' + 'static_libs: ["flags", "rust_bridge"], // closing brace in a comment }\n' + '}\n' + 'cc_aconfig_library { name: "flags", aconfig_declarations: "flag_declarations", }\n' + 'rust_ffi_static { name: "rust_bridge", rustlibs: ["librust"], proc_macros: ["libmacro"], }\n' + 'filegroup { name: "tagged_sources", srcs: ["source.cpp"], }\n' + 'filegroup { name: "tool_sources", srcs: ["tool.py"], }\n' + ) + report = closure.blueprint_module_graph(root, ["system", "system/vold"]) + self.assertEqual(report["scanned_android_bp_files"], 1, "overlapping projects must not duplicate modules") + self.assertEqual(report["ambiguous_module_names"], []) + self.assertEqual(report["unresolved_module_names"], ["compiler", "flag_declarations", "generated_cpp", "libmacro", "librust"]) + self.assertEqual( + closure.without_blueprint_comments('url: "https://example/path", // erased\n'), + 'url: "https://example/path", \n', + ) + + def test_static_expansion_uses_only_reviewed_hints_and_retains_unknown_names(self): + hint_seed = {**self.seed, "direct_module_to_project_hints": { + "known_generator": "system/update_engine", "libvold": "system/vold", + }} + report = {"unresolved_module_names": ["known_generator", "unknown_conditional", "libvold"]} + expansion = closure.next_projects_after_index(self.initial, report, hint_seed) + self.assertEqual(expansion["selected_projects"], sorted(self.initial + ["system/update_engine"])) + self.assertEqual(report["proposed_project_additions"], ["system/update_engine"]) + self.assertEqual(report["modules_without_verified_owner"], ["unknown_conditional"]) + self.assertIsNone(closure.next_projects_after_index(expansion["selected_projects"], report, hint_seed)) + too_many = {**hint_seed, "direct_module_to_project_hints": {f"m{i}": f"project{i}" for i in range(31)}} + with self.assertRaises(closure.ClosureError): + closure.next_projects_after_index([], {"unresolved_module_names": list(too_many["direct_module_to_project_hints"])}, too_many) + + def test_index_cli_emits_reviewed_next_selection_before_build(self): + hint_seed = {**self.seed, "direct_module_to_project_hints": {"known_generator": "system/update_engine"}} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "source" + bp = source / "system/vold/Android.bp" + bp.parent.mkdir(parents=True) + bp.write_text('cc_binary { name: "vold", generated_sources: ["known_generator", "unknown"], }\n') + selection, output, next_selection = (root / name for name in ("selection.json", "index.json", "next.json")) + selection.write_text(json.dumps(closure.selection_record(self.initial, hint_seed))) + with ( + patch.object(closure, "verified_contract", return_value=(hint_seed, self.manifest, self.baseline)), + patch.object(sys, "argv", ["closure", "index", "--source-root", str(source), + "--selection", str(selection), "--output", str(output), + "--next-selection", str(next_selection)]), + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(closure.main(), 0) + self.assertIn("system/update_engine", closure.validate_selection(json.loads(next_selection.read_text()), self.manifest, hint_seed)) + self.assertEqual(json.loads(output.read_text())["modules_without_verified_owner"], ["unknown"]) + def test_incremental_sync_reuses_checkout_and_syncs_only_pinned_projects(self): manifest_bytes = (ROOT / self.baseline["manifest"]["path"]).read_bytes() source_ref = self.baseline["projects"]["manifest"]["revision"] @@ -119,12 +221,11 @@ def fake_command(argv, cwd, **kwargs): (repo_dir / "manifests").mkdir(parents=True) (repo_dir / "manifest.xml").write_text("prior default") if argv[1] == "init" and "-m" in argv: - (repo_dir / "manifest.xml").write_bytes( - (repo_dir / "manifests" / argv[-1]).read_bytes() - ) + (repo_dir / "manifest.xml").write_text(f'') if argv[1] == "sync": for project in argv[4:]: (cwd / project).mkdir(parents=True, exist_ok=True) + (cwd / project / ".git").mkdir(exist_ok=True) with ( patch.object(original.shutil, "which", return_value="/usr/bin/repo"), @@ -185,7 +286,8 @@ def test_probe_remains_independent_of_published_vold_patch(self): for source in (script, workflow, builder): self.assertNotIn("git apply", source) self.assertNotIn("force-publish", source) - self.assertNotIn("SOONG_ALLOW_MISSING_DEPENDENCIES", source) + self.assertNotIn("SOONG_ALLOW_MISSING_DEPENDENCIES=true", source) + self.assertNotIn("ALLOW_MISSING_DEPENDENCIES=true", source) self.assertNotIn("publish: true", source) diff --git a/tests/vold_vendor_makefiles_test.py b/tests/vold_vendor_makefiles_test.py index 0fc5ae4b..76fcbf5d 100644 --- a/tests/vold_vendor_makefiles_test.py +++ b/tests/vold_vendor_makefiles_test.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 -"""Ensure Sony makefile regeneration remains a no-blob, unqualified probe.""" +"""Ensure absent or unqualified Sony inputs fail before vendor generation.""" from __future__ import annotations +from contextlib import redirect_stderr +import io import json from pathlib import Path import sys @@ -24,73 +26,133 @@ def source(self, parent: Path) -> Path: file.write_text(f"PINNED TEST CONTENT {path}\n") return root - def test_regenerate_makefiles_only_and_compare_cached_receipt(self): + def test_missing_vendor_inputs_report_blocker_without_running_generator(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) source = self.source(root) - report_path = root / "report.json" - calls = [] + receipt = root / "vendor-input-report.json" + before = sorted(p.relative_to(source).as_posix() for p in source.rglob("*")) + with ( + patch.object(prepare, "verify_checkout") as verify, + patch.object(prepare.subprocess, "run") as run, + ): + with self.assertRaisesRegex(prepare.VendorPreparationError, "authenticated proprietary"): + prepare.prepare(source, receipt) + run.assert_not_called() + self.assertEqual(verify.call_args.kwargs, { + "clean": True, + "selected": list(prepare.VENDOR_PREREQUISITE_PROJECTS), + }) + saved = json.loads(receipt.read_text()) + self.assertEqual(saved["status"], "blocked") + self.assertEqual(saved["kind"], "blocked-unqualified-proprietary-vendor-inputs") + self.assertTrue(saved["does_not_extract_blobs"]) + self.assertTrue(saved["does_not_qualify_original_vold"]) + self.assertFalse(saved["makefiles_generated"]) + self.assertFalse(saved["preexisting_vendor_tree_unqualified"]) + self.assertEqual(saved["required_input_example"], prepare.REQUIRED_ELF_EXAMPLE) + self.assertFalse(saved["required_input_example_is_complete_list"]) + self.assertEqual(set(saved["input_sha256"]), set(prepare.INPUTS)) + self.assertEqual( + before, sorted(p.relative_to(source).as_posix() for p in source.rglob("*")), + "diagnosing missing blobs must not create partial .mk/.bp files or bytecode", + ) + + def test_reused_diagnostic_still_fails_and_cannot_mask_changed_source(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + with patch.object(prepare, "verify_checkout"): + for _ in range(2): + with self.assertRaisesRegex(prepare.VendorPreparationError, "authenticated proprietary"): + prepare.prepare(source, receipt) + (source / prepare.INPUTS[0]).write_text("changed input") + with self.assertRaisesRegex(prepare.VendorPreparationError, "not the current blocked-input"): + prepare.prepare(source, receipt) - def fake_generate(argv, cwd, env, check): - calls.append(argv) - self.assertTrue(check) - self.assertEqual(cwd, source) - self.assertEqual(argv[0], sys.executable) - self.assertEqual(argv[1:], [ - "device/sony/pdx235/extract-files.py", "--regenerate_makefiles", - ]) - self.assertEqual(env["PYTHONPATH"], str(source / "tools/extract-utils")) - for path in prepare.REQUIRED: - output = source / path - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(f"generated from pinned public list: {path}\n") + def test_preexisting_vendor_elf_and_partial_makefiles_are_not_provenance(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + elf = source / prepare.REQUIRED_ELF_EXAMPLE + elf.parent.mkdir(parents=True) + elf.write_bytes(b"unqualified bytes") + (source / "vendor/sony/pdx235/Android.bp").write_text("partial upstream output") + receipt = root / "report.json" + with ( + patch.object(prepare, "verify_checkout"), + patch.object(prepare.subprocess, "run") as run, + self.assertRaisesRegex(prepare.VendorPreparationError, "provenance contract"), + ): + prepare.prepare(source, receipt) + run.assert_not_called() + self.assertTrue(json.loads(receipt.read_text())["preexisting_vendor_tree_unqualified"]) + self.assertEqual(elf.read_bytes(), b"unqualified bytes") + def test_old_synthetic_success_receipt_is_never_reused(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + receipt.write_text(json.dumps({ + "kind": "synthetic-probe-vendor-makefiles-not-publisher-provenance", + "generated_makefile_sha256": {}, + })) with ( patch.object(prepare, "verify_checkout"), - patch.object(prepare.subprocess, "run", side_effect=fake_generate), + self.assertRaisesRegex(prepare.VendorPreparationError, "not the current blocked-input"), ): - saved = prepare.prepare(source, report_path) - self.assertEqual(saved["kind"], "synthetic-probe-vendor-makefiles-not-publisher-provenance") - self.assertTrue(saved["does_not_extract_blobs"]) - self.assertTrue(saved["does_not_qualify_original_vold"]) - self.assertEqual(len(saved["generated_makefile_sha256"]), len(prepare.REQUIRED)) - self.assertEqual(len(calls), 1) - self.assertEqual(json.loads(report_path.read_text()), saved) - self.assertEqual(prepare.prepare(source, report_path), saved) - self.assertEqual(len(calls), 1, "reused makefiles must not trigger extraction again") - first = source / prepare.REQUIRED[0] - first.write_text("tampered generated files") - with self.assertRaisesRegex(prepare.VendorPreparationError, "differ"): - prepare.prepare(source, report_path) + prepare.prepare(source, receipt) - def test_proprietary_elf_or_symlink_are_rejected_even_if_generator_succeeds(self): + def test_linked_vendor_root_and_receipt_are_rejected(self): + for linked in ("vendor", "receipt"): + with self.subTest(linked=linked), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + other = root / "other" + other.mkdir() + if linked == "vendor": + (source / "vendor").symlink_to(other, target_is_directory=True) + else: + receipt.symlink_to(other / "outside.json") + with ( + patch.object(prepare, "verify_checkout"), + self.assertRaisesRegex(prepare.VendorPreparationError, "unsafe|symlink"), + ): + prepare.prepare(source, receipt) + self.assertEqual(list(other.iterdir()), []) + + def test_dirty_pinned_source_prevents_diagnostic_claim(self): with tempfile.TemporaryDirectory() as temp: root = Path(temp) source = self.source(root) - for path in prepare.REQUIRED: - dest = source / path - dest.parent.mkdir(parents=True, exist_ok=True) - dest.write_text("makefile") - (source / "vendor/sony/pdx235/libsecret.so").write_bytes(b"binary") - with self.assertRaisesRegex(prepare.VendorPreparationError, "unexpected binary blob"): - prepare.report(source, {}) - (source / "vendor/sony/pdx235/libsecret.so").unlink() - (source / "vendor/sony/pdx235/untrusted.mk").symlink_to(prepare.REQUIRED[0]) - with self.assertRaisesRegex(prepare.VendorPreparationError, "unsafe link"): - prepare.report(source, {}) + receipt = root / "report.json" + with ( + patch.object(prepare, "verify_checkout", side_effect=ValueError("source revision changed")), + self.assertRaisesRegex(ValueError, "source revision changed"), + ): + prepare.prepare(source, receipt) + self.assertFalse(receipt.exists()) - def test_workflow_is_still_unmodified_and_unqualified(self): - script = (ROOT / "src/ci/probe_original_vold.sh").read_text() - code = (ROOT / "src/ci/prepare_vold_vendor_makefiles.py").read_text() - workflow = (ROOT / ".github/workflows/vold-baseline.yml").read_text() - self.assertIn("src/ci/prepare_vold_vendor_makefiles.py", script) - self.assertIn("sony-generated-makefiles-report.json", workflow) - self.assertIn("--regenerate_makefiles", code) - self.assertIn("does_not_qualify_original_vold", code) - for x in (script, code, workflow): - self.assertNotIn("git apply", x) - self.assertNotIn("ota patch", x) - self.assertNotIn("SOONG_ALLOW_MISSING_DEPENDENCIES", x) + def test_cli_exits_nonzero_with_retained_blocker_report(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + stderr = io.StringIO() + with ( + patch.object(prepare, "verify_checkout"), + patch.object(sys, "argv", [ + "prepare_vold_vendor_makefiles.py", "--source-root", str(source), + "--report", str(receipt), + ]), + redirect_stderr(stderr), + ): + self.assertEqual(prepare.main(), 1) + self.assertIn(prepare.REQUIRED_ELF_EXAMPLE, stderr.getvalue()) + self.assertTrue(receipt.is_file()) if __name__ == "__main__":