diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97668d96..83398fa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,12 +108,18 @@ jobs: - name: Test bounded original Vold source dependency resolver run: python3 tests/vold_source_closure_test.py - - name: Test Sony vendor makefile-only regeneration guard + - name: Test immutable Sony vendor acquisition and regeneration run: python3 tests/vold_vendor_makefiles_test.py - name: Test original vold probe ordering and bounded expansion run: python3 tests/vold_probe_orchestration_test.py + - name: Test persistent local original vold build commands + run: python3 tests/vold_local_test.py + + - name: Test authenticated local build tool setup + run: python3 tests/vold_local_tools_test.py + - name: Generate the provisional original vold source plan run: python3 src/ci/vold_source_closure.py plan --output "${RUNNER_TEMP}/vold-source-plan.json" diff --git a/.github/workflows/vold-baseline.yml b/.github/workflows/vold-baseline.yml index 3f2b6871..fb525246 100644 --- a/.github/workflows/vold-baseline.yml +++ b/.github/workflows/vold-baseline.yml @@ -47,7 +47,8 @@ jobs: with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as summary: summary.write(f"### Provisional vold source graph: {count}/1243 manifest projects\n") summary.write("This is a source probe, **not** a complete Soong closure or a reproduced binary.\n") - summary.write("Sony vendor declarations cannot be regenerated from the public lists alone: the pinned generator reads proprietary ELF inputs.\n") + summary.write("Sony proprietary inputs use separately locked TheMuppets candidate trees, verified before fetching toolchains.\n") + summary.write("Local builds need no Actions runner: see docs/vold-baseline-ci.md and src/ci/vold_local.py.\n") PY - name: Upload static source dependency selection uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/README.md b/README.md index dc27fa64..b2c19b0d 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,14 @@ This repository acts as a server. ## Usage +### Build the original pdx235 vold locally + +The unmodified LineageOS `vold` reproduction workflow can run on a local Linux +x86_64 computer. It acquires pinned Sony vendor inputs from TheMuppets, retains +source downloads and incremental output, and checks the result against the +authenticated original OTA. See [local build setup and commands](docs/vold-baseline-ci.md). +This qualification workflow does not require a GitHub Actions runner. + ### Getting Started Reading the [AVBRoot docs](https://github.com/chenxiaolong/AVBRoot) is essential before proceeding with PixeneOS. diff --git a/docs/evidence/vold-local-vendor-integration-20261022.json b/docs/evidence/vold-local-vendor-integration-20261022.json new file mode 100644 index 00000000..237744cd --- /dev/null +++ b/docs/evidence/vold-local-vendor-integration-20261022.json @@ -0,0 +1,222 @@ +{ + "base_build_id": "lineage-24.0-20261016-nightly-pdx235-signed", + "environment_note": "An interrupted execution-environment file-transfer temporary artifact appeared after moving the vendor trees. The strict verifier rejected it. Its bytes were verified to be an exact prefix of the neighboring locked ELF, and only that task-created partial copy was removed before both successful runs. No verifier exception or tracked-input modification was made.", + "kind": "real-local-vold-vendor-integration", + "local_cli": { + "command": [ + "python3", + "-B", + "src/ci/vold_local.py", + "vendor", + "--work-dir", + "/workspace/scratch/83e8e6988998/local-vold-integration", + "--jobs", + "2" + ], + "fetch_metadata": { + "device/sony/pdx235": { + "mtime_ns": 1792671463480206515, + "sha256": "b7f0fe1fb0de8b13d57eb7cb936ce9d9ca464526561786f31f20012147e6ca2d", + "size": 267 + }, + "device/sony/sm6375-common": { + "mtime_ns": 1792671463451270665, + "sha256": "e3a1e94b4924ca377dc1973e3bb682535bffd0f319c59cc529ef67f1bcdc1eea", + "size": 281 + }, + "tools/extract-utils": { + "mtime_ns": 1792671463817701313, + "sha256": "896920cb620b08d65f710fbf84583780019aa82f3ed7b3ad7d61af411502d10d", + "size": 269 + }, + "vendor/sony/pdx235": { + "mtime_ns": 1792671138114364594, + "sha256": "a5a80542211fee18884d9452b0117a64838a0cf5984154013e1d359694ba282d", + "size": 149 + }, + "vendor/sony/sm6375-common": { + "mtime_ns": 1792671153566645275, + "sha256": "974a433e404d83140c82e75e53302e8309614ae95e785545e0546f91fdfc9609", + "size": 156 + } + }, + "fetch_metadata_unchanged_for_all_five_projects": true, + "receipts_identical": true, + "resume_redownloads": 0, + "successful_runs": [ + { + "elapsed_seconds": 2.085, + "exit_status": 0, + "log_sha256": "ed2ee0e49d830366f548c49c93ae3506665bdb66e9a177c0949a6ee2f028b18a", + "receipt_sha256": "eda88f294855c91d946e2806d7d8f0149ae80c5d8487a4b7d94e327150635683", + "run_directory": "20261022T122040.119895Z-vendor" + }, + { + "elapsed_seconds": 1.857, + "exit_status": 0, + "log_sha256": "1d3c33d35bc6f62deeffdd8394c76f31713e2cf7c871654b12c0a5219e20f6ae", + "receipt_sha256": "eda88f294855c91d946e2806d7d8f0149ae80c5d8487a4b7d94e327150635683", + "run_directory": "20261022T122042.217746Z-vendor" + } + ] + }, + "observed_on": "2026-10-22", + "qualification": { + "android_compilation_attempted": false, + "authenticated_ota_extraction_attempted": false, + "byte_identical_original_vold_proven": false, + "ota_published": false, + "patched_vold_authorized": false, + "source_dependency_closure_proven": false, + "vendor_candidates_available_and_verified": true + }, + "repo": { + "active_manifest_uses_supported_single_include_wrapper": true, + "execution": "Locally materialized upstream Repo implementation through the prior integration wrapper; this checks actual Repo behavior, not launcher bootstrap authentication.", + "explicit_source_projects": [ + { + "clean": true, + "path": "device/sony/pdx235", + "repository": "LineageOS/android_device_sony_pdx235", + "revision": "0559de69466e1e9af69b121985f592c4f4609283", + "shallow": true + }, + { + "clean": true, + "path": "device/sony/sm6375-common", + "repository": "LineageOS/android_device_sony_sm6375-common", + "revision": "d1d80af1ad9032cb092ff6d85db437709cc54e28", + "shallow": true + }, + { + "clean": true, + "path": "tools/extract-utils", + "repository": "LineageOS/android_tools_extract-utils", + "revision": "ff6589c6b0a4a00c592e98940e9e6b6a50c76569", + "shallow": true + } + ], + "implementation_revision": "d744eb15f38b9351463900babffd732994e3a046", + "implementation_version": "2.71", + "init_options": [ + "--depth=1" + ], + "initial_source_sync_exit_status": 0, + "repository": "https://gerrit.googlesource.com/git-repo", + "selected_manifest_is_official_flattened_snapshot": true, + "selected_manifest_sha256": "d4ded1ca84de399497b68d7ac006808059b6be9cf76725029420a3495a2e5468", + "sync_options": [ + "--current-branch", + "--no-tags", + "--no-clone-bundle", + "-j2" + ] + }, + "schema_version": 1, + "scope": "Actual local CLI with real Repo source checkout and actual pinned public vendor contents; no mocked generator or vendor files.", + "vendor_inputs": { + "acquisition": "Exact pinned public mirror commits fetched once and moved locally into this integration source root; CLI runs below verify and reuse them without another download.", + "lock_path": "locks/lineage-vold-vendor-candidate-20261016.json", + "lock_sha256": "b10dcfdf65930aed8e831df5dd17276ca24a3d66cee16dafc0989ca284bcc25d", + "pinned_common_blob_sha1_matches": 27, + "pinned_common_blob_sha1_mismatches": 0, + "projects": [ + { + "content_bytes": 380859981, + "declaration_sha256": { + "Android.bp": "04efde7c0f819ca7b096efaf3d5186c4589d8a7a1109bb36771a732b64ec5d85", + "Android.mk": "7a5e9b2a048ce3b454ca66d2be7775eaf808c0cdac69b680a5deeb072ca08eb0", + "BoardConfigVendor.mk": "31932b8ede523a06046fb6287bdaf6495bf50508dd849cb619aec7ea7d61bdec", + "pdx235-vendor.mk": "8b46c82d0d0f8876288b00840eb127a965e774512f240611fda7e193cb414912" + }, + "file_count": 232, + "inventory_sha256": "529efe19916b9dd2dd3aea6ed3d41a4bb18d4c780377299b6b5af69ef281ba94", + "path": "vendor/sony/pdx235", + "repository": "TheMuppets/proprietary_vendor_sony_pdx235", + "revision": "1e503a56c0d16de1f19f2b011581908dac29065b", + "tree": "87f37284815b03640bb3df0305356f31b43d393c", + "url": "https://github.com/TheMuppets/proprietary_vendor_sony_pdx235.git" + }, + { + "content_bytes": 304006211, + "declaration_sha256": { + "Android.bp": "0da6991ffe51153e5af2919db465b1bbe44fb39b35f9d7567e84990c690a9718", + "Android.mk": "7a5e9b2a048ce3b454ca66d2be7775eaf808c0cdac69b680a5deeb072ca08eb0", + "BoardConfigVendor.mk": "31932b8ede523a06046fb6287bdaf6495bf50508dd849cb619aec7ea7d61bdec", + "sm6375-common-vendor.mk": "fc58774d6c95c3f3ebdf7eadc9c2dcc829152cbe86fed9672703aca200d69559" + }, + "file_count": 882, + "inventory_sha256": "efb9f9e1903bb67e873125dbba5485fe6a53ab9fa370cced9b06f7cfb889469e", + "path": "vendor/sony/sm6375-common", + "repository": "TheMuppets/proprietary_vendor_sony_sm6375-common", + "revision": "2114108d5adfe787503ad0c87508d78123551991", + "tree": "7dd8d71c689778a4cdf246df63e6b76e4014d827", + "url": "https://github.com/TheMuppets/proprietary_vendor_sony_sm6375-common.git" + } + ], + "proprietary_list_destinations_checked": 1106, + "regenerated_declarations": 8, + "regeneration": { + "declarations_byte_identical": true, + "exit_status": 0, + "method": "unchanged manifest-pinned extract-files.py --regenerate_makefiles" + } + }, + "local_host_tool_setup": { + "setup_command": "/usr/bin/python3 -B src/ci/setup_vold_local_tools.py --work-dir ", + "exit_status": 0, + "repo_launcher_revision": "d744eb15f38b9351463900babffd732994e3a046", + "repo_launcher_sha256": "05f571deafd6ed6da0d20fa707dde29aa5624dbf35553d697a40dbdd4c8d178a", + "repo_normal_signed_bootstrap_used": true, + "fresh_repo_implementation_revision": "46e9866ca0f161604bd45a0c3b7c7887c9415dcb", + "fresh_repo_implementation_version": "v2.70", + "fresh_three_project_sync_exit_status": 0, + "avbroot_version_output": "avbroot 3.35.0", + "avbroot_installation": { + "schema_version": 1, + "tools": [ + { + "arch": "x86_64-unknown-linux-gnu", + "archive_sha256": "bbc7f8b4df8d9e6d2162bf354ac9f06643b2f64247a6f345d71e963f645f58c5", + "archive_size": 4250535, + "id": "avbroot", + "layout": [ + { + "mode": "0644", + "path": "LICENSE", + "sha256": "3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986", + "size": 35149, + "type": "file" + }, + { + "mode": "0644", + "path": "README.md", + "sha256": "398f4be471f467b40e33a042b13ef3a31ac47c969672521ed86cf29da335ffe6", + "size": 33214, + "type": "file" + }, + { + "mode": "0755", + "path": "avbroot", + "sha256": "2539724e0add2b8f6d855b8311897d01a4c28e851d387551f130325f006fa614", + "size": 10916976, + "type": "file" + } + ], + "signature": { + "identity": "chenxiaolong", + "namespace": "file", + "sha256": "7ab8b19e7c47bbaf7f846df5c601b3a0888d605f5a3176144b7bedf64600176b", + "signer_fingerprint": "SHA256:Ct0HoRyrFLrnF9W+A/BKEiJmwx7yWkgaW/JvghKrboA", + "type": "ssh", + "verified": true + }, + "version": "3.35.0" + } + ] + }, + "avbroot_execution": "Existing publisher-authenticated sealed executable runner through the generated absolute Python wrapper", + "runtime_note": "Ubuntu system Python supplies Linux file-sealing constants absent from the execution environment custom Python; setup gives an actionable error before acquisition for unsupported interpreters.", + "system_tools_or_profiles_modified": false + } +} diff --git a/docs/evidence/vold-vendor-candidate-20261022.json b/docs/evidence/vold-vendor-candidate-20261022.json new file mode 100644 index 00000000..969822cd --- /dev/null +++ b/docs/evidence/vold-vendor-candidate-20261022.json @@ -0,0 +1,320 @@ +{ + "base_build_id": "lineage-24.0-20261016-nightly-pdx235-signed", + "explicit_sha1_pin_verification": [ + { + "matched": "post-fixup", + "observed_sha1": "6d1f1d953c48a414e978263256e2ed3aec1580b7", + "path": "vendor/sony/sm6375-common/proprietary/system/framework/WfdCommon.jar", + "post_fixup_sha1": "6d1f1d953c48a414e978263256e2ed3aec1580b7", + "source_list_sha1": "27251f8421652af8abffb1f3fd9449fe0ec64ee5" + }, + { + "matched": "source", + "observed_sha1": "6b738758fe3d21d6d53d3fb522e1321b54592fa9", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/bin/wfdservice", + "post_fixup_sha1": null, + "source_list_sha1": "6b738758fe3d21d6d53d3fb522e1321b54592fa9" + }, + { + "matched": "source", + "observed_sha1": "907def8565d8f91f531ca7dfad880af05d540b0c", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/etc/init/wfdservice.rc", + "post_fixup_sha1": null, + "source_list_sha1": "907def8565d8f91f531ca7dfad880af05d540b0c" + }, + { + "matched": "source", + "observed_sha1": "6669d5ee462c981663b8ac573d84d334e4eed96c", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/etc/permissions/wfd-system-ext-privapp-permissions-qti.xml", + "post_fixup_sha1": null, + "source_list_sha1": "6669d5ee462c981663b8ac573d84d334e4eed96c" + }, + { + "matched": "source", + "observed_sha1": "1e331272649395423b11c575892e0bdc9433b820", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/etc/seccomp_policy/wfdservice64.policy", + "post_fixup_sha1": null, + "source_list_sha1": "1e331272649395423b11c575892e0bdc9433b820" + }, + { + "matched": "source", + "observed_sha1": "51c486b525879c1239062f0fa6b591c3e0abaaa0", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libmmrtpdecoder.so", + "post_fixup_sha1": null, + "source_list_sha1": "51c486b525879c1239062f0fa6b591c3e0abaaa0" + }, + { + "matched": "source", + "observed_sha1": "946d85b6b852ee95e17b40b00c4c85ebfa72c85c", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libmmrtpencoder.so", + "post_fixup_sha1": null, + "source_list_sha1": "946d85b6b852ee95e17b40b00c4c85ebfa72c85c" + }, + { + "matched": "source", + "observed_sha1": "e5a06c6ea07c0cd4ebb73b030b87bf5e51a553e3", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdavenhancements.so", + "post_fixup_sha1": null, + "source_list_sha1": "e5a06c6ea07c0cd4ebb73b030b87bf5e51a553e3" + }, + { + "matched": "source", + "observed_sha1": "ae443c8657e680558cce30039409cee1e6ab5505", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdclient.so", + "post_fixup_sha1": null, + "source_list_sha1": "ae443c8657e680558cce30039409cee1e6ab5505" + }, + { + "matched": "source", + "observed_sha1": "27c411978b486ccc075862fa2eff450990b9d47a", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdcommonutils.so", + "post_fixup_sha1": null, + "source_list_sha1": "27c411978b486ccc075862fa2eff450990b9d47a" + }, + { + "matched": "source", + "observed_sha1": "d05d03298caf2e233c69689cf1fed0dd19a02be1", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdconfigutils.so", + "post_fixup_sha1": null, + "source_list_sha1": "d05d03298caf2e233c69689cf1fed0dd19a02be1" + }, + { + "matched": "source", + "observed_sha1": "2717202736b678390be7518ce6f69a63079e4217", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfddisplayconfig.so", + "post_fixup_sha1": null, + "source_list_sha1": "2717202736b678390be7518ce6f69a63079e4217" + }, + { + "matched": "source", + "observed_sha1": "b560306dbc7b2bffb734c8eb6439dd3721f75c12", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdmminterface.so", + "post_fixup_sha1": null, + "source_list_sha1": "b560306dbc7b2bffb734c8eb6439dd3721f75c12" + }, + { + "matched": "source", + "observed_sha1": "a250fd4085ec04345db8067f11962f57c3b166f0", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdmmsink.so", + "post_fixup_sha1": null, + "source_list_sha1": "a250fd4085ec04345db8067f11962f57c3b166f0" + }, + { + "matched": "source", + "observed_sha1": "69e08746a9557ee5fba3e02a8170032db0fa0105", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdmmsrc_system.so", + "post_fixup_sha1": null, + "source_list_sha1": "69e08746a9557ee5fba3e02a8170032db0fa0105" + }, + { + "matched": "source", + "observed_sha1": "923acb22f26dc3aab5115cf685a161611e43c018", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdnative.so", + "post_fixup_sha1": null, + "source_list_sha1": "923acb22f26dc3aab5115cf685a161611e43c018" + }, + { + "matched": "source", + "observed_sha1": "14eec96c4fa615096c473d5a269ba315a53cfd00", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdrtsp.so", + "post_fixup_sha1": null, + "source_list_sha1": "14eec96c4fa615096c473d5a269ba315a53cfd00" + }, + { + "matched": "post-fixup", + "observed_sha1": "54624fd8c8ef2932e71c4b3ce176fb712e884f5a", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdservice.so", + "post_fixup_sha1": "54624fd8c8ef2932e71c4b3ce176fb712e884f5a", + "source_list_sha1": "429839b831a8b16fb858179241dc6183c50bdf13" + }, + { + "matched": "source", + "observed_sha1": "e714120166d6480ada0b629ddd73c499c73c55a2", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfdsinksm.so", + "post_fixup_sha1": null, + "source_list_sha1": "e714120166d6480ada0b629ddd73c499c73c55a2" + }, + { + "matched": "source", + "observed_sha1": "62a596c93efa57bdc79b078d74a3ccf4d19d1f58", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfduibcinterface.so", + "post_fixup_sha1": null, + "source_list_sha1": "62a596c93efa57bdc79b078d74a3ccf4d19d1f58" + }, + { + "matched": "source", + "observed_sha1": "a6fc5d0245e23648812a535f7338d562c9317f09", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfduibcsink.so", + "post_fixup_sha1": null, + "source_list_sha1": "a6fc5d0245e23648812a535f7338d562c9317f09" + }, + { + "matched": "source", + "observed_sha1": "2db07ec7a9a6206d2782439e247b35fbe62a550f", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfduibcsinkinterface.so", + "post_fixup_sha1": null, + "source_list_sha1": "2db07ec7a9a6206d2782439e247b35fbe62a550f" + }, + { + "matched": "source", + "observed_sha1": "ac3e0988fb5c36c6c4561f04e0e5d53d7de2ad76", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfduibcsrc.so", + "post_fixup_sha1": null, + "source_list_sha1": "ac3e0988fb5c36c6c4561f04e0e5d53d7de2ad76" + }, + { + "matched": "source", + "observed_sha1": "d19e9b5c35f61cc696a9c999dce6eb11d781bd8c", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/libwfduibcsrcinterface.so", + "post_fixup_sha1": null, + "source_list_sha1": "d19e9b5c35f61cc696a9c999dce6eb11d781bd8c" + }, + { + "matched": "source", + "observed_sha1": "7b3051498476202797fce711a6dcc42ae6a7b8ea", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/vendor.qti.hardware.wifidisplaysession@1.0.so", + "post_fixup_sha1": null, + "source_list_sha1": "7b3051498476202797fce711a6dcc42ae6a7b8ea" + }, + { + "matched": "source", + "observed_sha1": "b2f864d6ae621758bef6f4081af353e2e11e04d8", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/lib64/vendor.qti.hardware.wifidisplaysession_aidl-V2-ndk.so", + "post_fixup_sha1": null, + "source_list_sha1": "b2f864d6ae621758bef6f4081af353e2e11e04d8" + }, + { + "matched": "source", + "observed_sha1": "7d7e4ce1ec07338f1d2abf39035a8f44714a4a40", + "path": "vendor/sony/sm6375-common/proprietary/system_ext/priv-app/WfdService/WfdService.apk", + "post_fixup_sha1": null, + "source_list_sha1": "7d7e4ce1ec07338f1d2abf39035a8f44714a4a40" + } + ], + "inventory_format": "sha256(json.dumps(sorted regular files by path, objects with path/mode/size/sha256, sort_keys=True, separators=(comma,colon)))", + "kind": "reviewed-public-vendor-candidate-compatibility-evidence", + "list_coverage": [ + { + "distinct_destination_files": 228, + "entries": 228, + "explicit_sha1_pins": 0, + "missing_destination_files": [], + "source_list": "device/sony/pdx235/proprietary-files.txt", + "source_list_sha256": "93da9a7ac7957afde23ac9c59425947a29b410d01fd5b1fcde2efb5ec155855f" + }, + { + "distinct_destination_files": 878, + "entries": 878, + "explicit_sha1_pins": 27, + "missing_destination_files": [], + "source_list": "device/sony/sm6375-common/proprietary-files.txt", + "source_list_sha256": "8056cdd324881ba179fbac2553e77ddc0d1456ade7ef1e37f9630bb5f6c39a29" + } + ], + "manifest": { + "path": "locks/lineage-pdx235-20261016-build-manifest.xml", + "sha256": "d4ded1ca84de399497b68d7ac006808059b6be9cf76725029420a3495a2e5468" + }, + "observed_date_utc": "2026-10-22", + "provenance": { + "acceptance": "Authenticated original OTA extraction and a byte-identical unmodified vold comparison remain mandatory before build reproducibility is qualified.", + "android_compilation_performed": false, + "byte_identical_vold_qualified": false, + "kind": "public-git-mirror-candidate", + "original_ota_vendor_identity_proven": false, + "publisher_authenticated": false, + "scope": "Exact public mirror bytes satisfy the pinned proprietary destination lists and reproduce all eight declarations with the unchanged manifest-pinned extraction scripts." + }, + "regeneration": { + "all_files_match_pinned_inventory_after_generation": true, + "command": [ + "python3", + "-B", + "-s", + "device/sony/pdx235/extract-files.py", + "--regenerate_makefiles" + ], + "declarations_byte_identical_to_pinned_mirrors": true, + "declarations_compared": 8, + "environment": { + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": "tools/extract-utils" + }, + "evidence": "Actual local CLI vendor receipts; see docs/evidence/vold-local-vendor-integration-20261022.json for run and cache-reuse checks.", + "exit_status": 0, + "source_scripts_modified": false, + "vendor_worktrees_clean_after_generation": true, + "working_directory": "verified source root" + }, + "schema_version": 1, + "source_input_sha256": { + "device/sony/pdx235/extract-files.py": "440af4750e0496627003295606935d3a534f2e758ab42adacd3d5da6e255f0e0", + "device/sony/pdx235/proprietary-files.txt": "93da9a7ac7957afde23ac9c59425947a29b410d01fd5b1fcde2efb5ec155855f", + "device/sony/sm6375-common/extract-files.py": "266ac34e070a5755d513979ad74196b4341c363915b0cfaa24e51d4bf49d6ec8", + "device/sony/sm6375-common/proprietary-files.txt": "8056cdd324881ba179fbac2553e77ddc0d1456ade7ef1e37f9630bb5f6c39a29", + "tools/extract-utils/extract_utils/elf.py": "58d3a6db079185d94e2d283211f73fa71f32bb882618ac47b0feb6d2f5e01d31", + "tools/extract-utils/extract_utils/main.py": "3470e7a43a9788f2699c13d627750b7f5fdc448bf364a48a3a7c86f2b931562d", + "tools/extract-utils/extract_utils/makefiles.py": "2a550851c75a2a32d4d294338df3e73f7d95b4418721653224565d27ba6fa111", + "tools/extract-utils/extract_utils/module.py": "1d1f77c66e05a3f3fea1e710caca7842c38c11b76c1a7bb486724826e72f73ef" + }, + "source_projects": [ + { + "path": "device/sony/pdx235", + "repository": "LineageOS/android_device_sony_pdx235", + "revision": "0559de69466e1e9af69b121985f592c4f4609283", + "url": "https://github.com/LineageOS/android_device_sony_pdx235/tree/0559de69466e1e9af69b121985f592c4f4609283" + }, + { + "path": "device/sony/sm6375-common", + "repository": "LineageOS/android_device_sony_sm6375-common", + "revision": "d1d80af1ad9032cb092ff6d85db437709cc54e28", + "url": "https://github.com/LineageOS/android_device_sony_sm6375-common/tree/d1d80af1ad9032cb092ff6d85db437709cc54e28" + }, + { + "path": "tools/extract-utils", + "repository": "LineageOS/android_tools_extract-utils", + "revision": "ff6589c6b0a4a00c592e98940e9e6b6a50c76569", + "url": "https://github.com/LineageOS/android_tools_extract-utils/tree/ff6589c6b0a4a00c592e98940e9e6b6a50c76569" + } + ], + "total_content_bytes": 684866192, + "total_files": 1114, + "vendor_lock": { + "path": "locks/lineage-vold-vendor-candidate-20261016.json", + "sha256": "b10dcfdf65930aed8e831df5dd17276ca24a3d66cee16dafc0989ca284bcc25d" + }, + "vendor_projects": [ + { + "content_bytes": 380859981, + "declaration_sha256": { + "Android.bp": "04efde7c0f819ca7b096efaf3d5186c4589d8a7a1109bb36771a732b64ec5d85", + "Android.mk": "7a5e9b2a048ce3b454ca66d2be7775eaf808c0cdac69b680a5deeb072ca08eb0", + "BoardConfigVendor.mk": "31932b8ede523a06046fb6287bdaf6495bf50508dd849cb619aec7ea7d61bdec", + "pdx235-vendor.mk": "8b46c82d0d0f8876288b00840eb127a965e774512f240611fda7e193cb414912" + }, + "file_count": 232, + "inventory_sha256": "529efe19916b9dd2dd3aea6ed3d41a4bb18d4c780377299b6b5af69ef281ba94", + "path": "vendor/sony/pdx235", + "repository": "TheMuppets/proprietary_vendor_sony_pdx235", + "revision": "1e503a56c0d16de1f19f2b011581908dac29065b", + "tree": "87f37284815b03640bb3df0305356f31b43d393c", + "url": "https://github.com/TheMuppets/proprietary_vendor_sony_pdx235.git" + }, + { + "content_bytes": 304006211, + "declaration_sha256": { + "Android.bp": "0da6991ffe51153e5af2919db465b1bbe44fb39b35f9d7567e84990c690a9718", + "Android.mk": "7a5e9b2a048ce3b454ca66d2be7775eaf808c0cdac69b680a5deeb072ca08eb0", + "BoardConfigVendor.mk": "31932b8ede523a06046fb6287bdaf6495bf50508dd849cb619aec7ea7d61bdec", + "sm6375-common-vendor.mk": "fc58774d6c95c3f3ebdf7eadc9c2dcc829152cbe86fed9672703aca200d69559" + }, + "file_count": 882, + "inventory_sha256": "efb9f9e1903bb67e873125dbba5485fe6a53ab9fa370cced9b06f7cfb889469e", + "path": "vendor/sony/sm6375-common", + "repository": "TheMuppets/proprietary_vendor_sony_sm6375-common", + "revision": "2114108d5adfe787503ad0c87508d78123551991", + "tree": "7dd8d71c689778a4cdf246df63e6b76e4014d827", + "url": "https://github.com/TheMuppets/proprietary_vendor_sony_sm6375-common.git" + } + ] +} diff --git a/docs/vold-baseline-ci.md b/docs/vold-baseline-ci.md index 59e846ed..e70cf09b 100644 --- a/docs/vold-baseline-ci.md +++ b/docs/vold-baseline-ci.md @@ -1,4 +1,4 @@ -# First gate: reproduce LineageOS pdx235's original vold in CI +# Build and qualify LineageOS pdx235's original vold locally This is a **separate, nonpublishing, unmodified-source** workflow. It does not use or import the exFAT timeout patch, the ModOS vold injection module, @@ -9,6 +9,132 @@ or mere compatibility. First prove that a clean build of the **unmodified** source produces the identical executable found in the cryptographically authenticated original LineageOS OTA. +**A GitHub runner is optional.** The local entry point acquires the pinned +source and vendor inputs, keeps them between invocations, builds only `m vold`, +and performs the same authenticated byte comparison. The two public vendor +mirrors have been downloaded and verified with the real pinned generator; +an Android compilation and byte-identical result are still outstanding. + +## Local quick start + +Use Linux x86_64 with an SSD and a work directory without whitespace. Ubuntu +24.04 is the documented setup. On another operating system, use a Linux x86_64 +build machine or VM. The local command needs no GitHub token, repository +variables, Actions runner registration, phone, `adb`, or signing keys. + +From a checkout of this repository, install the host packages once: + +```sh +sudo apt-get update +sudo apt-get install -y \ + git git-lfs gnupg openssh-client openssl curl python3 build-essential flex bison \ + zip unzip rsync bc zlib1g-dev libc6-dev-i386 lib32z1-dev \ + libx11-dev x11proto-core-dev libgl1-mesa-dev libxml2-utils xsltproc \ + fontconfig libssl-dev libelf-dev lz4 erofs-utils e2fsprogs +``` + +This follows the [AOSP host setup](https://source.android.com/docs/setup/start/requirements) +with the extraction utilities used here. The pinned Android projects provide +their matching compiler and JDK; do not replace those with a random host JDK. + +Choose one persistent directory, install the small host tools there, then build: + +```sh +export VOLD_LOCAL_WORK="$HOME/build/modos-vold" +python3 src/ci/setup_vold_local_tools.py --work-dir "$VOLD_LOCAL_WORK" +source "$VOLD_LOCAL_WORK/host-tools/env.sh" + +# Check the host before a long build. +python3 src/ci/vold_local.py doctor --for compare --work-dir "$VOLD_LOCAL_WORK" + +# One command: vendor inputs, source sync, compile, authenticate, compare. +python3 src/ci/vold_local.py compare --work-dir "$VOLD_LOCAL_WORK" --jobs 4 +``` + +The setup command installs a SHA-256-pinned official Repo launcher and the +existing [publisher-authenticated avbroot tool](executable-tool-trust.md). +Its `avbroot` wrapper uses the existing sealed executable runner. Repo retains +its normal signed bootstrap; pinning its launcher does not claim that every +future Repo implementation version is pinned. Nothing is installed globally. +Run `source .../host-tools/env.sh` in each new terminal. If the ModOS checkout +moves, rerun setup to refresh the wrapper's absolute path. `--repo-only` skips +avbroot installation until comparison is needed. + +`compare` first performs a compile-only probe. Only after that succeeds does it +download the exact locked OTA, verify its digest and publisher signature, +extract a fresh reference, and repeat the incremental `m vold` comparison. +The source checkout and intermediate output are reused. An existing OTA can +be supplied with `--ota-file /absolute/path/lineage-24.0-20261016-nightly-pdx235-signed.zip`; +its exact filename, SHA-256 and signature are still checked. + +### Smaller stages and resuming + +All stages accept the same `--work-dir` and `--jobs` options. Repeating a command +resumes the same pinned checkout. Each invocation writes a new log directory. + +| Command | Work performed | +| --- | --- | +| `plan` | Write the provisional 84-project selection, without downloads; preserve any expanded active selection. | +| `doctor --for vendor` | Check only the host tools needed for vendor preparation. | +| `vendor` | Sync the three small public prerequisite projects, acquire/verify both pinned vendor trees, and regenerate their eight declarations. Stop before toolchains. | +| `prepare` | Run vendor preparation, sync the selected source/toolchain projects, and perform bounded static dependency scans. | +| `probe` | Prepare/resume sources and compile the unmodified `vold`, without downloading an OTA. | +| `compare` | Compile first, then authenticate the exact OTA and require byte-for-byte equality. | + +For the inexpensive first check: + +```sh +python3 src/ci/vold_local.py vendor --work-dir "$VOLD_LOCAL_WORK" --jobs 2 +``` + +The verified vendor worktree content is 684,866,192 bytes (about 685 MB), plus +Git objects and metadata. This is measured vendor content, not a full build +space estimate. The 84-project source seed is still provisional. `doctor` +reports available disk and RAM; 100 GB free is an initial planning allowance, +not a proven minimum or maximum, and it does not block a resumed workspace. +Use `--min-free-gb 100` to impose an explicit floor. Full Android's upstream +64 GB RAM / 400 GB disk guidance concerns a full platform build, not a measured +requirement for this narrower target. Reduce `--jobs` if memory is tight. + +Persistent files are arranged as follows: + +| Path under the work directory | Purpose | +| --- | --- | +| `source/` | Manifest-pinned public projects and separately pinned vendor trees. | +| `source/out/` | Incremental Android build outputs. | +| `source-selection.json` | Current approved source selection, including discovered dependencies. | +| `cache/` | Exact OTA downloads and resumable partial downloads. | +| `host-tools/` | Private Repo launcher and authenticated avbroot installation. | +| `runs/-/` | Invocation status, logs, source/vendor diagnostics and comparison receipts. | + +Failures retain the checkout, outputs and logs. Resolve the reported problem +and rerun the same command. Source revisions and inventories are rechecked; +modified or extra source files are rejected, never silently reset. Concurrent +commands against one work directory are prevented. There is no automatic +cleanup. A cached reference ELF or editable receipt is never accepted as +authentication: every comparison reextracts the reference from the verified +OTA. Large extracted partitions remain in that invocation's `reference/` +directory and may be removed after reviewing its results. + +### Interrupted initial Repo setup + +An interruption during the first `repo init` can leave `source/.repo/` before +the pinned manifest has been installed and selected. Resume reports the +specific manifest validation failure and leaves that metadata untouched. +If the failure followed an interrupted first initialization, retain only the +incomplete source directory under a new name: + +```sh +VOLD_INCOMPLETE_SOURCE="$(mktemp -d "$VOLD_LOCAL_WORK/source.incomplete.XXXXXXXX")" +mv -T -- "$VOLD_LOCAL_WORK/source" "$VOLD_INCOMPLETE_SOURCE" +``` + +Then rerun the command that failed with the same `--work-dir`. Keep +`source-selection.json`, `cache/`, `host-tools/`, and `runs/` in place; the +new initialization preserves that selection and those retained files. For +a previously complete checkout, investigate any changed pinned manifest or +local override instead of treating the validation failure as interrupted setup. + ## Exact pinned inputs - Source: `lineage-24.0-20261016-nightly-pdx235-signed.zip`. @@ -91,50 +217,69 @@ The source reasoning is grounded in three exact upstream revisions: the common product also includes proprietary `vendor/sony/sm6375-common`. Two other Sony vendor BoardConfig files are referenced. -### Verified proprietary-input blocker - -The exact public manifest does **not** contain `vendor/sony/pdx235` or -`vendor/sony/sm6375-common`. The original assumption that -`--regenerate_makefiles` can generate their declarations using only public -file lists is incorrect at the pinned revisions. This option skips -extraction, but the subsequent -[`write_elfs_package`](https://github.com/LineageOS/android_tools_extract-utils/blob/ff6589c6b0a4a00c592e98940e9e6b6a50c76569/extract_utils/makefiles.py#L256-L271) -still calls -[`get_file_machine_bits_libs`](https://github.com/LineageOS/android_tools_extract-utils/blob/ff6589c6b0a4a00c592e98940e9e6b6a50c76569/extract_utils/elf.py#L38-L57), -which opens each proprietary ELF to determine its architecture and dependencies. - -An actual replay of the unchanged pinned Python sources on 2026-10-22 -failed with `FileNotFoundError` for -`vendor/sony/pdx235/proprietary/vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so`. -Upstream had already written four partial vendor declaration files. Those -files are not successful or qualified preparation. The reproducible command, -source revisions, input hashes and normalized traceback are retained in -[`docs/evidence/vold-vendor-preflight-20261022.json`](evidence/vold-vendor-preflight-20261022.json). -This replay ran Python extraction code only; it did **not** compile Android. - -To avoid downloading toolchains before this known failure, the probe first -syncs exactly three pinned source projects: both Sony device trees and -`tools/extract-utils`. The -[`prepare_vold_vendor_makefiles.py`](../src/ci/prepare_vold_vendor_makefiles.py) -preflight verifies those revisions and writes `sony-vendor-input-report.json` -with `status: blocked` and `makefiles_generated: false`, then exits before -invoking the generator or syncing the remaining seed projects. It rejects -the former synthetic-success receipts; arbitrary existing vendor files -also remain unqualified. - -**Both real build modes are currently blocked here.** A reviewed acquisition -and provenance contract for the required proprietary inputs is needed before -vendor preparation can be enabled. Authenticating the pinned original OTA -may provide a source for some of these bytes, but extraction, fixups, original -build inputs and every required file must be accounted for. The present code -does not claim that this acquisition is implemented. Do not fabricate files, -disable ELF checks or enable missing-dependency flags to bypass this gate. +### Working vendor acquisition from pinned public mirrors + +The official public manifest omits `vendor/sony/pdx235` and +`vendor/sony/sm6375-common`, but their inputs are available from TheMuppets. +The supplementary [vendor candidate lock](../locks/lineage-vold-vendor-candidate-20261016.json) +records these exact commits, Git trees, complete SHA-256 inventories, source +input hashes, and the eight generated declaration hashes: + +| Vendor project | Immutable revision | Files / content bytes | +| --- | --- | ---: | +| [TheMuppets/proprietary_vendor_sony_pdx235](https://github.com/TheMuppets/proprietary_vendor_sony_pdx235/tree/1e503a56c0d16de1f19f2b011581908dac29065b) | `1e503a56c0d16de1f19f2b011581908dac29065b` | 232 / 380,859,981 | +| [TheMuppets/proprietary_vendor_sony_sm6375-common](https://github.com/TheMuppets/proprietary_vendor_sony_sm6375-common/tree/2114108d5adfe787503ad0c87508d78123551991) | `2114108d5adfe787503ad0c87508d78123551991` | 882 / 304,006,211 | + +These are reproducible third-party mirror candidates. Their public availability, +immutable identities and source compatibility permit a genuine build probe; +they do not prove the official OTA builder used those exact vendor commits. +The receipts say `publisher_authenticated: false` and preserve the final +byte-identical comparison requirement. + +The acquisition helper shallow-fetches those exact commits, verifies complete +file inventories, checks the three public prerequisite revisions, and runs the +unchanged pinned `extract-files.py --regenerate_makefiles`. It accepts success +only when regeneration preserves all eight locked declarations byte for byte. +Untracked/ignored files, changed revisions, unresolved LFS pointers, unsafe +links, different mirrors and changed inventories are rejected. A resumed run +verifies and reuses completed worktrees without fetching them again. The main +build verifier allows only these two independently verified supplemental roots, +and records their complete provenance in the comparison report. + +Actual verification on 2026-10-22 established: + +- All 1,106 proprietary-list destinations are present (228 device + 878 common). +- All 27 explicit common-list SHA-1 requirements match, including the two + expected post-fixup hashes. +- The exact upstream generator succeeds and all eight generated declarations + match the pinned mirrors byte for byte. +- The real local `vendor` command succeeds twice on a proper Repo checkout; + both receipts match and all five projects' fetch records are unchanged. + +The recorded evidence is +[vendor candidate compatibility](evidence/vold-vendor-candidate-20261022.json) +and [real local vendor integration](evidence/vold-local-vendor-integration-20261022.json). +These checks run real extraction Python code and Git operations, not Android +compilation. + +The earlier [source-list-only failure](evidence/vold-vendor-preflight-20261022.json) +remains historical evidence: `--regenerate_makefiles` reads proprietary ELF +architecture and dependency metadata, so empty vendor trees do fail. Acquiring +the actual blobs resolves that failure; fabricating declarations is unnecessary. + +LineageOS also officially documents +[extracting proprietary inputs directly from an installable ZIP](https://github.com/LineageOS/lineage_wiki/blob/main/pages/extracting_blobs_from_zips.md) +by passing the archive to the device's `extract-files.py`. The exact authenticated +October 16 OTA is an alternative source to investigate if a future mirror pin +is unavailable or produces a binary mismatch. This workflow currently uses the +verified mirror contract; a different extraction/fixup result needs its own +recorded inventory, rather than silently replacing these pinned inputs. The on-disk dependency resolver [`src/ci/vold_source_closure.py`](../src/ci/vold_source_closure.py) verifies the complete original manifest and produces a pinned selection. -After the vendor-input gate is resolved and a limited -`repo sync ` succeeds, it scans available `Android.bp` +After vendor acquisition and a limited +`repo sync ` succeed, it scans available `Android.bp` definitions for a **partial** transitive module graph. Up to three static scans batch missing providers from reviewed hints before compilation, syncing only newly selected pinned projects. Unknown, ambiguous and generated @@ -145,10 +290,10 @@ diagnostics is mapped to its **exact existing** project in the official manifest. Known module names have reviewed initial provider hints; unknown providers and unpinned vendor paths stop for manual investigation. -Once prerequisites have passed, within a single CI run, +Once prerequisites have passed, in each local or CI invocation, [`src/ci/probe_original_vold.sh`](../src/ci/probe_original_vold.sh) -automatically repeats up to **three** controlled passes. It caches already -synchronized projects in the same working tree, fetches only newly identified +automatically repeats up to **three** controlled passes. It retains already +synchronized projects and the expanded selection in the same working tree, fetches only newly identified pinned dependencies, and saves small diagnostic JSON documents showing unknown modules and other blockers. It never enables `SOONG_ALLOW_MISSING_DEPENDENCIES`, changes the build target, or applies @@ -181,115 +326,30 @@ same `vold-source-provisional-plan` artifact. This provides a low-cost plan on the merged commit even when the GitHub connector cannot dispatch manual workflows. These CI steps never sync Android sources or run the compiler. -## Runner provisioning for source compilation - -For `probe` or `compare`, the workflow still requires a trusted, -isolated self-hosted Linux x64 runner labeled `modos-lineage-large`. -Provide a dedicated scratch filesystem with **at least 100 GB free as an -experimental starting allowance**, sufficient CPU/RAM, the Android 17 -build prerequisites, and `repo`, `git`, `python3`, and `realpath`. -The authenticated `compare` mode additionally needs `avbroot`, `debugfs`, -`unzip`, `curl`, `openssl`, and `sha256sum`. One iteration may require -more storage. The 100 GB threshold is **not an empirically measured maximum -or proof** that the selected dependency graph is complete. - -Set repository Actions variables: - -- `MODOS_LINEAGE_LARGE_SCRATCH` — an absolute, real, isolated scratch - directory on the trusted self-hosted runner. -- `MODOS_VOLD_BASELINE_RUNNER_READY=true` — only when that runner has been - provisioned and independently checked. - -Use `probe` to attempt the smaller pinned source checkout and compile -unmodified `m vold` **without downloading the official source OTA**. -A successfully compiled probe is **not** binary-equality evidence; its -workflow job explicitly labels it as unqualified. It is useful for -discovering missing modules, vendor inputs and toolchains cheaply. -At the current reviewed revision it stops at the known vendor-input gate -before downloading the large seed or invoking `m vold`. - -Only after a successful, reviewed `probe`, use `compare` for a real proof. -The compare workflow downloads the OTA before the source probe; running it -while the known vendor gate is blocked would waste that download. -It first reauthenticates the exact -publisher-signed official 2026-10-16 OTA, extracts the original -`/system/bin/vold`, and invokes the same bounded dependency solver. -**Only** a real `byte_identical: true` report with equal original and -built hashes qualifies this baseline. If the binaries differ, it emits a -diagnostic comparison receipt and fails. - -### Hosted-runner alternatives and observed access - -Checked on 2026-10-22: GitHub documents larger Ubuntu x64 runners with -4 CPUs / 16 GB RAM / 150 GB storage and 8 CPUs / 32 GB RAM / 300 GB storage. -Those are plausible experimental build hosts, not measured requirements -for this incomplete dependency closure. The 100 GB free-space gate does not -prove either machine has enough RAM or working space after tool installation. -See GitHub's [larger runner specifications](https://docs.github.com/en/actions/reference/runners/larger-runners). - -GitHub's managed larger runners require an organization or enterprise on -Team or Enterprise Cloud; `0cwa/ModOS` is currently owned by a user account. -They cannot be assumed available merely by changing `runs-on` to an invented -label. An actual organization runner with repository access, or a dedicated -trusted self-hosted Linux x64 machine, must be provisioned and checked first. -See [GitHub-hosted runner availability](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) -and [managing larger runners](https://docs.github.com/en/actions/how-tos/manage-runners/larger-runners/manage-larger-runners). -Standard Ubuntu runners advertise only 14 GB storage and remain suitable -for this plan/unit-test workload, not the current Android build preflight. - -The last original-baseline attempt -[#39699955476](https://github.com/0cwa/ModOS/actions/runs/39699955476) -failed its hosted prerequisites job with both readiness variables empty; -the self-hosted build job was skipped. This is evidence about that run, -not proof of the current runner inventory. The connected GitHub actions -expose repository changes, CI inspection and reruns, but no workflow dispatch, -runner inventory or variable mutation. No runner or configuration change -has been claimed on that basis. - -For an owner with the GitHub CLI configured, the explicit manual plan is: - -```sh -gh workflow run vold-baseline.yml --repo 0cwa/ModOS --ref main -f mode=plan -``` +## Optional GitHub Actions execution -After resolving the authenticated vendor-input contract and verifying the -trusted runner, set the two documented repository variables and dispatch -`-f mode=probe`. Inspect its source selection and diagnostics. Dispatch -`-f mode=compare` only once that actual compile succeeds. The connector's -missing dispatch capability does not justify changing any qualification gate. - -From a checked-out repository, local inspection is also possible without -compiling or obtaining the proprietary vendor files: - -```sh -python3 src/ci/vold_source_closure.py plan --output /tmp/vold-projects.json -# After a limited repo sync on a trusted build host: -python3 src/ci/vold_source_closure.py index \ - --source-root /path/to/pinned-source \ - --selection /tmp/vold-projects.json \ - --output /tmp/vold-blueprint-graph.json -# After an unsuccessful m vold build: -python3 src/ci/vold_source_closure.py diagnose \ - --selection /tmp/vold-projects.json \ - --log /path/to/build-failure.log \ - --output /tmp/vold-diagnosis.json \ - --next-selection /tmp/vold-projects-expanded.json -``` +The local commands above require no Actions configuration. The existing manual +`.github/workflows/vold-baseline.yml` remains available for an owner who wants +to use a trusted dedicated Linux x64 runner. `plan` runs on a standard hosted +runner and needs no repository variables; ordinary PR/main CI also uploads the +same provisional plan artifact. -All selection changes must remain part of the exact original manifest. -The build helper supports `sync --projects-file ... --resume` to fetch -additional pinned projects without repeating an entire sync. +The optional `probe` and `compare` jobs remain main-only, read-only, +nonpublishing, and gated by `MODOS_VOLD_BASELINE_RUNNER_READY=true` plus an +absolute private `MODOS_LINEAGE_LARGE_SCRATCH` directory on a runner labeled +`modos-lineage-large`. The workflow's 100 GB free-space check is an initial +allowance, not measured closure evidence. It uploads small diagnostics and +cleans its temporary checkout. Local runs retain their workspace instead. -The workflow uses a read-only GitHub token, never receives release or -signing secrets, uploads diagnostics/comparison receipts only, and clears -the large checkout and original OTA after completion. Do not use an -untrusted shared runner. +No runner registration, repository setting change, or Actions dispatch is +needed for the documented local build. ## After the first authentic success -Review the actual run URL and download its -`vold-probe-compare-diagnostics` artifact. Record the exact -source, manifest and reference SHA-256, plus the CI commit and run ID. +Review the local `runs/-compare/compare/baseline-*/baseline-report.json` +and its authenticated reference receipt, or the optional Actions diagnostics +artifact. Record the exact source, manifest, vendor lock and reference SHA-256, +plus the ModOS commit and local invocation identifier or CI run ID. Verify the report contains `byte_identical: true` with equal SHA-256s. Independent assessment of the real reference extraction and Android build remains essential. Do not infer these properties from synthetic tests. diff --git a/locks/lineage-vold-vendor-candidate-20261016.json b/locks/lineage-vold-vendor-candidate-20261016.json new file mode 100644 index 00000000..c572fa0d --- /dev/null +++ b/locks/lineage-vold-vendor-candidate-20261016.json @@ -0,0 +1,82 @@ +{ + "base_build_id": "lineage-24.0-20261016-nightly-pdx235-signed", + "inventory_format": "sha256(json.dumps(sorted regular files by path, objects with path/mode/size/sha256, sort_keys=True, separators=(comma,colon)))", + "kind": "lineageos-pdx235-public-vendor-candidate", + "manifest_sha256": "d4ded1ca84de399497b68d7ac006808059b6be9cf76725029420a3495a2e5468", + "provenance": { + "acceptance": "An authenticated byte-identical unmodified vold comparison remains mandatory; mirror pinning is reproducibility evidence, not LineageOS publisher authentication.", + "compatibility_evidence": "Both pinned public source lists name Sony68.2.A.3.107; all1106 destination files exist, all27 pinned common hashes match raw or fixed-up contents, and all8 declarations regenerate identically with the manifest-pinned scripts.", + "kind": "public-git-mirror-candidate", + "original_ota_vendor_identity_proven": false, + "publisher_authenticated": false, + "selection": "Latest lineage-24.0 commits preceding 2026-10-16; identical to branch heads observed 2026-10-22." + }, + "schema_version": 1, + "source_input_sha256": { + "device/sony/pdx235/extract-files.py": "440af4750e0496627003295606935d3a534f2e758ab42adacd3d5da6e255f0e0", + "device/sony/pdx235/proprietary-files.txt": "93da9a7ac7957afde23ac9c59425947a29b410d01fd5b1fcde2efb5ec155855f", + "device/sony/sm6375-common/extract-files.py": "266ac34e070a5755d513979ad74196b4341c363915b0cfaa24e51d4bf49d6ec8", + "device/sony/sm6375-common/proprietary-files.txt": "8056cdd324881ba179fbac2553e77ddc0d1456ade7ef1e37f9630bb5f6c39a29", + "tools/extract-utils/extract_utils/elf.py": "58d3a6db079185d94e2d283211f73fa71f32bb882618ac47b0feb6d2f5e01d31", + "tools/extract-utils/extract_utils/main.py": "3470e7a43a9788f2699c13d627750b7f5fdc448bf364a48a3a7c86f2b931562d", + "tools/extract-utils/extract_utils/makefiles.py": "2a550851c75a2a32d4d294338df3e73f7d95b4418721653224565d27ba6fa111", + "tools/extract-utils/extract_utils/module.py": "1d1f77c66e05a3f3fea1e710caca7842c38c11b76c1a7bb486724826e72f73ef" + }, + "source_projects": [ + { + "path": "device/sony/pdx235", + "repository": "LineageOS/android_device_sony_pdx235", + "revision": "0559de69466e1e9af69b121985f592c4f4609283" + }, + { + "path": "device/sony/sm6375-common", + "repository": "LineageOS/android_device_sony_sm6375-common", + "revision": "d1d80af1ad9032cb092ff6d85db437709cc54e28" + }, + { + "path": "tools/extract-utils", + "repository": "LineageOS/android_tools_extract-utils", + "revision": "ff6589c6b0a4a00c592e98940e9e6b6a50c76569" + } + ], + "vendor_projects": [ + { + "commit_date": "2026-03-28T18:47:17Z", + "content_bytes": 380859981, + "declaration_sha256": { + "Android.bp": "04efde7c0f819ca7b096efaf3d5186c4589d8a7a1109bb36771a732b64ec5d85", + "Android.mk": "7a5e9b2a048ce3b454ca66d2be7775eaf808c0cdac69b680a5deeb072ca08eb0", + "BoardConfigVendor.mk": "31932b8ede523a06046fb6287bdaf6495bf50508dd849cb619aec7ea7d61bdec", + "pdx235-vendor.mk": "8b46c82d0d0f8876288b00840eb127a965e774512f240611fda7e193cb414912" + }, + "description": "Update from Sony 68.2.A.3.107", + "file_count": 232, + "inventory_sha256": "529efe19916b9dd2dd3aea6ed3d41a4bb18d4c780377299b6b5af69ef281ba94", + "observed_branch": "lineage-24.0", + "path": "vendor/sony/pdx235", + "repository": "TheMuppets/proprietary_vendor_sony_pdx235", + "revision": "1e503a56c0d16de1f19f2b011581908dac29065b", + "tree": "87f37284815b03640bb3df0305356f31b43d393c", + "url": "https://github.com/TheMuppets/proprietary_vendor_sony_pdx235.git" + }, + { + "commit_date": "2026-08-25T15:23:11Z", + "content_bytes": 304006211, + "declaration_sha256": { + "Android.bp": "0da6991ffe51153e5af2919db465b1bbe44fb39b35f9d7567e84990c690a9718", + "Android.mk": "7a5e9b2a048ce3b454ca66d2be7775eaf808c0cdac69b680a5deeb072ca08eb0", + "BoardConfigVendor.mk": "31932b8ede523a06046fb6287bdaf6495bf50508dd849cb619aec7ea7d61bdec", + "sm6375-common-vendor.mk": "fc58774d6c95c3f3ebdf7eadc9c2dcc829152cbe86fed9672703aca200d69559" + }, + "description": "Update WFD system blobs from marvel_g A171WEH.20; base Sony68.2.A.3.107", + "file_count": 882, + "inventory_sha256": "efb9f9e1903bb67e873125dbba5485fe6a53ab9fa370cced9b06f7cfb889469e", + "observed_branch": "lineage-24.0", + "path": "vendor/sony/sm6375-common", + "repository": "TheMuppets/proprietary_vendor_sony_sm6375-common", + "revision": "2114108d5adfe787503ad0c87508d78123551991", + "tree": "7dd8d71c689778a4cdf246df63e6b76e4014d827", + "url": "https://github.com/TheMuppets/proprietary_vendor_sony_sm6375-common.git" + } + ] +} diff --git a/src/ci/build_original_vold.py b/src/ci/build_original_vold.py index f5710e14..937fd2e8 100644 --- a/src/ci/build_original_vold.py +++ b/src/ci/build_original_vold.py @@ -197,9 +197,27 @@ def verify_checkout( raise BuildError(f"source project has unreviewed local content: {relpath}") -def verify_source_workspace(source: Path, manifest: bytes, selected: list[str] | None) -> None: +def verified_vendor_inputs(source: Path) -> dict: + # Vendor repositories are absent from the public release manifest. They + # have a separate immutable candidate lock, never a blanket vendor/ allowlist. + from prepare_vold_vendor_makefiles import VendorInputError, verify_vendor_checkout + try: + return verify_vendor_checkout(source) + except (OSError, ValueError, VendorInputError, subprocess.CalledProcessError) as exc: + raise BuildError(f"supplemental vendor input verification failed: {exc}") from exc + + +def verify_source_workspace( + source: Path, manifest: bytes, selected: list[str] | None, + *, require_vendor: bool = False, +) -> dict | None: """Reject unrecorded source outside clean pinned projects before building.""" approved = set(project_map(manifest) if selected is None else selected) + from prepare_vold_vendor_makefiles import VENDOR_PATHS + vendor = None + if require_vendor or any((source / path).exists() or (source / path).is_symlink() for path in VENDOR_PATHS): + vendor = verified_vendor_inputs(source) + approved.update(VENDOR_PATHS) exported = {} for project in ET.fromstring(manifest).findall("project"): project_path = project.get("path", project.get("name", "")) @@ -237,6 +255,7 @@ def verify_source_workspace(source: Path, manifest: bytes, selected: list[str] | pending.append(path) continue raise BuildError(f"unrecorded source outside selected pinned projects: {relative}") + return vendor def sync( @@ -246,19 +265,31 @@ def sync( if source.is_symlink() or source.exists() and not source.is_dir(): raise BuildError("source directory must be a real directory") if resume: - verify_active_manifest(source, manifest) + try: + verify_active_manifest(source, manifest) + except BuildError as exc: + raise BuildError( + f"{exc}. Resume never replaces unverified Repo metadata. " + "If the first Repo initialization was interrupted, rename only " + f"the incomplete source directory {source} to an unused sibling " + "path, then rerun the same local command with the same --work-dir. " + "Retain source-selection.json and the rest of the work directory. " + "For a previously completed checkout, investigate the manifest " + "mismatch instead of treating it as interrupted setup. See " + "docs/vold-baseline-ci.md#interrupted-initial-repo-setup." + ) from exc elif source.exists() and any(source.iterdir()): raise BuildError("source directory must be new or empty; use --resume to add pinned dependencies") repo = shutil.which("repo") if repo is None: - raise BuildError("repo tool is unavailable on source build runner") + raise BuildError("repo tool is unavailable; see docs/vold-baseline-ci.md for local setup") source.mkdir(parents=True, exist_ok=True) source = source.resolve(strict=True) if not resume: manifest_commit = lock["projects"]["manifest"]["revision"] command([ repo, "init", "-u", "https://github.com/LineageOS/android.git", - "-b", manifest_commit, + "-b", manifest_commit, "--depth=1", ], source) # Do not overwrite .repo/manifest.xml: Repo manages this file, and # recent versions regenerate it during subsequent sync/init operations. @@ -273,7 +304,7 @@ def sync( pinned_manifest.write_bytes(manifest) command([ repo, "init", "-u", "https://github.com/LineageOS/android.git", - "-b", manifest_commit, "-m", MANIFEST_NAME + "-b", manifest_commit, "--depth=1", "-m", MANIFEST_NAME ], source) verify_active_manifest(source, manifest) # Explicit project-path sync prevents all 1,243 projects being fetched. @@ -297,7 +328,7 @@ def sync( verify_checkout(source, manifest, clean=True, selected=present) to_sync = missing if to_sync is None or to_sync: - command([repo, "sync", "--no-clone-bundle", f"-j{jobs}", *(to_sync or [])], source) + command([repo, "sync", "--current-branch", "--no-tags", "--no-clone-bundle", f"-j{jobs}", *(to_sync or [])], source) verify_checkout(source, manifest, clean=True, selected=selected) @@ -310,7 +341,7 @@ def compile_original_vold( raise BuildError("baseline source directory is missing or unsafe") source = source.resolve(strict=True) verify_checkout(source, manifest, clean=True, selected=selected) - verify_source_workspace(source, manifest, selected) + verify_source_workspace(source, manifest, selected, require_vendor=True) release_config = source / lock["build"]["release_config"]["path"] if ( release_config.is_symlink() or not release_config.is_file() @@ -358,7 +389,7 @@ def compile_original_vold( # would fail before configuration (for example at USE_LEFTOVERS). command(["bash", "-ec", script], source, env=environment) verify_checkout(source, manifest, clean=True, selected=selected) - verify_source_workspace(source, manifest, selected) + verify_source_workspace(source, manifest, selected, require_vendor=True) if artifact.is_symlink() or not artifact.is_file() or not artifact.stat().st_mode & 0o111: raise BuildError("source build did not produce a regular executable vold") return artifact @@ -381,6 +412,7 @@ def build_baseline( report["build_release"] = lock["build"]["release"] report["build_setup_command"] = lock["build"]["setup_command"] report["release_config_sha256"] = lock["build"]["release_config"]["sha256"] + report["supplemental_vendor_inputs"] = verified_vendor_inputs(source) if selected is not None: report["selected_source_projects"] = len(selected) report["selected_source_paths_sha256"] = hashlib.sha256( @@ -403,7 +435,7 @@ def main() -> int: opt.add_argument("--jobs", type=int, default=4) if name != "sync-vendor-sources": opt.add_argument("--projects-file", type=Path, help="optional validated subset plan; never weakens the binary equality gate") - if name == "sync": + if name in ("sync", "sync-vendor-sources"): opt.add_argument("--resume", action="store_true", help="fetch missing approved projects within the same checkout") if name == "compare": opt.add_argument("--reference-vold", type=Path, required=True) @@ -415,10 +447,11 @@ def main() -> int: lock, manifest = load_contract() selected = verified_selection(getattr(args, "projects_file", None), manifest) if args.mode == "sync-vendor-sources": - # Diagnose the known proprietary-input gate before downloading the - # large compiler/toolchain checkout. This fixed set is not a build - # selection and never authorizes compiling or qualifying a binary. - sync(args.source_root, lock, manifest, args.jobs, list(VENDOR_PREREQUISITE_PROJECTS)) + # Acquire/verify the small source lists before fetching toolchains. + # Resume never resyncs existing clean worktrees or erases an + # expanded dependency selection from an earlier local invocation. + sync(args.source_root, lock, manifest, args.jobs, + list(VENDOR_PREREQUISITE_PROJECTS), args.resume) elif args.mode == "sync": if args.resume and selected is None: raise BuildError("full-manifest checkout does not support incremental resume") diff --git a/src/ci/prepare_vold_vendor_makefiles.py b/src/ci/prepare_vold_vendor_makefiles.py index f5bbdb24..4ff422b6 100644 --- a/src/ci/prepare_vold_vendor_makefiles.py +++ b/src/ci/prepare_vold_vendor_makefiles.py @@ -1,29 +1,30 @@ #!/usr/bin/env python3 -"""Report the proprietary inputs blocking pinned Sony makefile preparation. +"""Acquire and verify immutable public Sony vendor candidates for original vold. -At the locked extract-utils revision, --regenerate_makefiles skips extraction -but still opens proprietary ELF files to discover architecture and dependencies. -It cannot regenerate the Sony makefiles from public file lists alone. Until an -authenticated vendor-input contract exists, stop before running that generator -or writing partial vendor declarations. This diagnostic never qualifies vold. +The two reviewed TheMuppets commits supply real proprietary inputs and their +declarations. Git identity, complete SHA-256 inventories and pinned-source +regeneration are checked. These are reproducible mirror candidates, not proof +of LineageOS publisher identity: authentic byte-identical vold remains required. """ from __future__ import annotations import argparse import hashlib import json +import os from pathlib import Path +import re import stat import subprocess import sys +import tempfile +import xml.etree.ElementTree as ET -from build_original_vold import VENDOR_PREREQUISITE_PROJECTS, load_contract, verify_checkout - - -class VendorPreparationError(ValueError): - pass - - +ROOT = Path(__file__).resolve().parents[2] +LOCK = ROOT / "locks/lineage-vold-vendor-candidate-20261016.json" +BASELINE_LOCK = ROOT / "locks/lineage-vold-baseline-20261016.json" +VENDOR_PATHS = ("vendor/sony/pdx235", "vendor/sony/sm6375-common") +SOURCE_PATHS = ("device/sony/pdx235", "device/sony/sm6375-common", "tools/extract-utils") INPUTS = ( "device/sony/pdx235/extract-files.py", "device/sony/pdx235/proprietary-files.txt", @@ -34,96 +35,308 @@ class VendorPreparationError(ValueError): "tools/extract-utils/extract_utils/makefiles.py", "tools/extract-utils/extract_utils/elf.py", ) -# This is the first absent ELF opened in a replay of the unchanged, exact -# pinned scripts and public lists. It is an example, not a complete blob set. -REQUIRED_ELF_EXAMPLE = ( - "vendor/sony/pdx235/proprietary/" - "vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so" -) -EXTRACT_UTILS_REVISION = "ff6589c6b0a4a00c592e98940e9e6b6a50c76569" -EVIDENCE_BASE = ( - "https://github.com/LineageOS/android_tools_extract-utils/blob/" - + EXTRACT_UTILS_REVISION -) +LFS_POINTER = b"version https://git-lfs.github.com/spec/v1" +MAX_FILE_BYTES = 512 * 1024 * 1024 + + +class VendorInputError(ValueError): + pass + +# Preserve the earlier public exception name for callers. +VendorPreparationError = VendorInputError -def safe_file(path: Path, max_bytes: int = 4_000_000) -> bytes: - st = path.lstat() - if not stat.S_ISREG(st.st_mode) or not 0 <= st.st_size <= max_bytes: - raise VendorPreparationError(f"diagnostic receipt or source is unsafe: {path}") - content = path.read_bytes() - if len(content) != st.st_size: - raise VendorPreparationError(f"diagnostic receipt or source changed while reading: {path}") - return content +def _environment() -> dict[str, str]: + env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + env.update({ + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + "GIT_LFS_SKIP_SMUDGE": "1", + "GIT_NO_REPLACE_OBJECTS": "1", + }) + return env -def blocked_report(source: Path, input_hashes: dict[str, str]) -> dict: - for path in (source / "vendor", source / "vendor/sony"): - if path.is_symlink() or path.exists() and not path.is_dir(): - raise VendorPreparationError("Sony vendor output root is unsafe") - sony_root = source / "vendor/sony" - preexisting = sony_root.exists() and any(sony_root.iterdir()) + +def _git(directory: Path, *args: str) -> str: + try: + result = subprocess.run( + [ + "git", "--no-optional-locks", "-c", "core.hooksPath=" + os.devnull, + "-c", "core.fsmonitor=false", "-c", "core.autocrlf=false", + "-c", "protocol.file.allow=never", "-C", str(directory), *args, + ], + env=_environment(), check=True, text=True, capture_output=True, + ) + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or exc.stdout or "no Git diagnostic").strip()[-3000:] + raise VendorInputError(f"Git operation failed in {directory}: {detail}") from exc + return result.stdout.strip() + + +def _safe_path(root: Path, relative: str) -> Path: + parts = Path(relative).parts + if not parts or Path(relative).is_absolute() or any(p in (".", "..") for p in parts): + raise VendorInputError("unsafe path in vendor input contract") + current = root + for part in parts: + current /= part + if current.is_symlink(): + raise VendorInputError(f"linked source or vendor path is forbidden: {relative}") + return current + + +def _file_record(path: Path, relative: str) -> dict: + before = path.lstat() + if not stat.S_ISREG(before.st_mode) or not 0 <= before.st_size <= MAX_FILE_BYTES: + raise VendorInputError(f"vendor input must be a bounded regular file: {relative}") + digest = hashlib.sha256() + with path.open("rb") as stream: + first = stream.read(256) + if first.startswith(LFS_POINTER): + raise VendorInputError(f"unresolved Git LFS pointer is not a binary input: {relative}") + digest.update(first) + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + after = os.fstat(stream.fileno()) + if (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) != ( + after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns + ): + raise VendorInputError(f"vendor input changed while reading: {relative}") return { - "schema_version": 1, - "kind": "blocked-unqualified-proprietary-vendor-inputs", - "status": "blocked", - "does_not_extract_blobs": True, - "does_not_qualify_original_vold": True, - "makefiles_generated": False, - "preexisting_vendor_tree_unqualified": preexisting, - "input_sha256": input_hashes, - "required_input_example": REQUIRED_ELF_EXAMPLE, - "required_input_example_is_complete_list": False, - "reason": ( - "The pinned --regenerate_makefiles path skips extraction, but " - "write_elfs_package still opens proprietary ELF files for machine, " - "bitness and DT_NEEDED metadata. Public source lists are insufficient." - ), - "required_next_step": ( - "Authenticate and pin the required proprietary vendor inputs and " - "their build provenance in a reviewed contract before enabling " - "vendor preparation. Preexisting files and synthetic receipts " - "are not accepted as provenance." - ), - "upstream_evidence": [ - EVIDENCE_BASE + "/extract_utils/main.py#L182-L219", - EVIDENCE_BASE + "/extract_utils/makefiles.py#L256-L271", - EVIDENCE_BASE + "/extract_utils/elf.py#L38-L57", - ], - "public_source_replay": { - "scope": "Review-time replay of pinned public source, not performed by this diagnostic", - "command": "python3 -B device/sony/pdx235/extract-files.py --regenerate_makefiles", - "exit_status": 1, - "exception": "FileNotFoundError", - "first_missing_file": REQUIRED_ELF_EXAMPLE, - "original_scripts_modified": False, - }, + "path": relative, + "mode": "100755" if before.st_mode & 0o111 else "100644", + "size": before.st_size, + "sha256": digest.hexdigest(), + } + + +def inventory(path: Path) -> tuple[list[dict], str]: + """Hash every worktree file, including extra/ignored files; skip only Git metadata.""" + records = [] + for parent, directories, filenames in os.walk(path, followlinks=False): + parent = Path(parent) + if parent == path and ".git" in directories: + directories.remove(".git") + for name in directories: + candidate = parent / name + if candidate.is_symlink(): + raise VendorInputError(f"linked directory in vendor input: {candidate}") + for name in filenames: + candidate = parent / name + if parent == path and name == ".git": + raise VendorInputError("vendor repository must have its own real Git directory") + records.append(_file_record(candidate, candidate.relative_to(path).as_posix())) + records.sort(key=lambda item: item["path"]) + encoded = json.dumps(records, sort_keys=True, separators=(",", ":")).encode() + return records, hashlib.sha256(encoded).hexdigest() + + +def load_vendor_contract() -> dict: + if LOCK.is_symlink() or BASELINE_LOCK.is_symlink(): + raise VendorInputError("vendor and baseline locks must be regular checked-in files") + contract = json.loads(LOCK.read_bytes()) + baseline = json.loads(BASELINE_LOCK.read_bytes()) + if ( + contract.get("schema_version") != 1 + or contract.get("kind") != "lineageos-pdx235-public-vendor-candidate" + or contract.get("base_build_id") != baseline["base_build_id"] + or contract.get("manifest_sha256") != baseline["manifest"]["sha256"] + or contract.get("provenance", {}).get("publisher_authenticated") is not False + or contract.get("provenance", {}).get("original_ota_vendor_identity_proven") is not False + ): + raise VendorInputError("vendor candidate contract does not match the original baseline") + manifest = ROOT / baseline["manifest"]["path"] + if manifest.is_symlink(): + raise VendorInputError("source manifest must not be linked") + manifest_bytes = manifest.read_bytes() + if hashlib.sha256(manifest_bytes).hexdigest() != contract["manifest_sha256"]: + raise VendorInputError("vendor candidate source manifest digest differs") + expected = { + item.get("path", item.get("name")): (item.get("name"), item.get("revision")) + for item in ET.fromstring(manifest_bytes).findall("project") } + sources = contract.get("source_projects", []) + if [project.get("path") for project in sources] != list(SOURCE_PATHS): + raise VendorInputError("unexpected public-source prerequisites in vendor contract") + for project in sources: + if expected.get(project["path"]) != (project["repository"], project["revision"]): + raise VendorInputError("vendor generator source differs from the pinned manifest") + hashes = contract.get("source_input_sha256", {}) + if set(hashes) != set(INPUTS) or any(not re.fullmatch(r"[a-f0-9]{64}", v) for v in hashes.values()): + raise VendorInputError("vendor generator source hashes are incomplete") + projects = contract.get("vendor_projects", []) + if [project.get("path") for project in projects] != list(VENDOR_PATHS): + raise VendorInputError("unexpected supplemental vendor project paths") + for project in projects: + device = project["path"].split("/")[-1] + repository = "TheMuppets/proprietary_vendor_sony_" + device + if project.get("repository") != repository or project.get("url") != "https://github.com/" + repository + ".git": + raise VendorInputError("unreviewed vendor mirror URL") + for key in ("revision", "tree"): + if not re.fullmatch(r"[a-f0-9]{40}", project.get(key, "")): + raise VendorInputError("vendor mirror must use an immutable commit and tree") + if not re.fullmatch(r"[a-f0-9]{64}", project.get("inventory_sha256", "")): + raise VendorInputError("missing complete vendor SHA-256 inventory") + if not isinstance(project.get("file_count"), int) or not 1 <= project["file_count"] <= 5000: + raise VendorInputError("invalid vendor inventory file count") + if not isinstance(project.get("content_bytes"), int) or not 1 <= project["content_bytes"] <= 2_000_000_000: + raise VendorInputError("invalid vendor inventory size") + declarations = project.get("declaration_sha256", {}) + if set(declarations) != {"Android.bp", "Android.mk", "BoardConfigVendor.mk", device + "-vendor.mk"}: + raise VendorInputError("vendor declarations are not completely pinned") + if any(not re.fullmatch(r"[a-f0-9]{64}", v) for v in declarations.values()): + raise VendorInputError("invalid vendor declaration digest") + return contract + + +def _verify_git(path: Path, revision: str, tree: str | None = None) -> None: + if path.is_symlink() or not path.is_dir(): + raise VendorInputError(f"missing or unsafe pinned Git project: {path}") + if Path(_git(path, "rev-parse", "--show-toplevel")).resolve() != path.resolve(): + raise VendorInputError(f"project is not its own Git worktree: {path}") + if _git(path, "rev-parse", "HEAD") != revision: + raise VendorInputError(f"vendor or generator source revision changed: {path}") + if tree is not None and _git(path, "rev-parse", "HEAD^{tree}") != tree: + raise VendorInputError(f"vendor Git tree differs from contract: {path}") + # Status can otherwise hide modified imported source files in a resumed + # checkout. Reject index flags that suppress ordinary worktree checks. + tracked = _git(path, "ls-files", "-v", "-z").split("\0") + if any(entry and (entry[0].islower() or entry[0] == "S") for entry in tracked): + raise VendorInputError(f"pinned Git project has assume-unchanged or skip-worktree entries: {path}") + if _git(path, "status", "--porcelain", "--untracked-files=all", "--ignored"): + raise VendorInputError(f"vendor or generator source has unreviewed local content: {path}") + + +def verify_source_inputs(source: Path, contract: dict) -> dict[str, str]: + for project in contract["source_projects"]: + _verify_git(_safe_path(source, project["path"]), project["revision"]) + hashes = {} + for relative, expected in contract["source_input_sha256"].items(): + record = _file_record(_safe_path(source, relative), relative) + if record["sha256"] != expected: + raise VendorInputError(f"pinned vendor generator input changed: {relative}") + hashes[relative] = record["sha256"] + return hashes -def prepare(source: Path, receipt: Path) -> None: +def verify_project(path: Path, project: dict) -> dict: + if (path / ".git").is_symlink() or not (path / ".git").is_dir(): + raise VendorInputError("vendor repository must have its own real Git directory") + _verify_git(path, project["revision"], project["tree"]) + if _git(path, "remote", "get-url", "origin") != project["url"]: + raise VendorInputError(f"vendor repository origin differs from reviewed mirror: {project['path']}") + records, digest = inventory(path) + if ( + digest != project["inventory_sha256"] + or len(records) != project["file_count"] + or sum(record["size"] for record in records) != project["content_bytes"] + ): + raise VendorInputError(f"complete vendor SHA-256 inventory differs: {project['path']}") + declarations = {record["path"]: record["sha256"] for record in records if "/" not in record["path"]} + if declarations != project["declaration_sha256"]: + raise VendorInputError(f"pinned vendor declarations differ: {project['path']}") + return { + "path": project["path"], "repository": project["repository"], "url": project["url"], + "revision": project["revision"], "tree": project["tree"], + "inventory_sha256": digest, "file_count": len(records), + "content_bytes": project["content_bytes"], "declaration_sha256": declarations, + } + + +def _root(source: Path) -> Path: if source.is_symlink() or not source.is_dir(): - raise VendorPreparationError("original source workspace is missing or linked") - source = source.resolve(strict=True) - _lock, manifest = load_contract() - verify_checkout(source, manifest, clean=True, selected=list(VENDOR_PREREQUISITE_PROJECTS)) - hashes = {path: hashlib.sha256(safe_file(source / path)).hexdigest() for path in INPUTS} - current = blocked_report(source, hashes) + raise VendorInputError("original source workspace is missing or linked") + return source.resolve(strict=True) + + +def verify_vendor_checkout(source: Path) -> dict: + """Verify candidate bytes without trusting a previous receipt or fetching anything.""" + source = _root(source) + contract = load_vendor_contract() + input_hashes = verify_source_inputs(source, contract) + sony = _safe_path(source, "vendor/sony") + if not sony.is_dir() or {p.name for p in sony.iterdir()} != {"pdx235", "sm6375-common"}: + raise VendorInputError("Sony vendor tree contains missing or unreviewed project paths") + projects = [verify_project(_safe_path(source, project["path"]), project) + for project in contract["vendor_projects"]] + return { + "schema_version": 1, + "kind": "pinned-public-vendor-candidate", + "status": "ready-for-unmodified-build-probe", + "publisher_authenticated": False, + "original_ota_vendor_identity_proven": False, + "byte_identical_vold_qualified": False, + "base_build_id": contract["base_build_id"], + "source_manifest_sha256": contract["manifest_sha256"], + "vendor_lock_sha256": hashlib.sha256(LOCK.read_bytes()).hexdigest(), + "source_input_sha256": input_hashes, + "projects": projects, + "pinned_declarations_match_reviewed_regeneration": True, + "acceptance": "An authenticated byte-identical original vold comparison is still required.", + } + + +def _fetch_project(source: Path, project: dict) -> None: + destination = _safe_path(source, project["path"]) + if destination.exists(): + verify_project(destination, project) + return + destination.parent.mkdir(parents=True, exist_ok=True) + # Stage outside the source tree. Failed downloads are never accepted as a + # completed vendor checkout, and existing worktrees are never reset. + with tempfile.TemporaryDirectory(prefix="modos-vendor-fetch-", dir=source.parent) as temp: + checkout = Path(temp) / "checkout" + checkout.mkdir() + _git(checkout, "init", "--quiet") + _git(checkout, "remote", "add", "origin", project["url"]) + _git(checkout, "-c", "fetch.fsckObjects=true", "fetch", "--depth=1", + "--no-tags", "origin", project["revision"]) + _git(checkout, "checkout", "--detach", "--quiet", project["revision"]) + verify_project(checkout, project) + checkout.rename(destination) + + +def prepare(source: Path, receipt: Path) -> dict: + source = _root(source) if receipt.is_symlink(): - raise VendorPreparationError("vendor diagnostic receipt must not be a symlink") - if receipt.exists(): - if json.loads(safe_file(receipt)) != current: - raise VendorPreparationError("existing vendor receipt is not the current blocked-input diagnostic") - else: - receipt.parent.mkdir(parents=True, exist_ok=True) - with receipt.open("x", encoding="utf-8") as output: - json.dump(current, output, indent=2, sort_keys=True) - output.write("\n") - raise VendorPreparationError( - "pinned Sony makefile regeneration requires authenticated proprietary " - f"ELF inputs (for example {REQUIRED_ELF_EXAMPLE}); no vendor-input " - f"provenance contract exists. Blocked-input report: {receipt}" + raise VendorInputError("vendor input receipt must not be a symlink") + if receipt.resolve().is_relative_to(source): + raise VendorInputError("vendor input receipt must be outside the verified source tree") + contract = load_vendor_contract() + verify_source_inputs(source, contract) + sony = _safe_path(source, "vendor/sony") + if sony.exists() and (not sony.is_dir() or any(p.name not in {"pdx235", "sm6375-common"} for p in sony.iterdir())): + raise VendorInputError("Sony vendor tree has unreviewed project paths") + for project in contract["vendor_projects"]: + _fetch_project(source, project) + # The exact pinned upstream code reads actual ELF inputs. Regeneration must + # reproduce the already pinned mirror declarations without changing a byte. + environment = _environment() + environment.pop("PYTHONHOME", None) + environment["PYTHONPATH"] = str(source / "tools/extract-utils") + environment["PYTHONDONTWRITEBYTECODE"] = "1" + subprocess.run( + [sys.executable, "-B", "-s", "device/sony/pdx235/extract-files.py", "--regenerate_makefiles"], + cwd=source, env=environment, check=True, ) + result = verify_vendor_checkout(source) + result["regeneration"] = { + "method": "unchanged manifest-pinned extract-files.py --regenerate_makefiles", + "exit_status": 0, "declarations_byte_identical": True, + } + receipt.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", prefix=receipt.name + ".", + dir=receipt.parent, delete=False) as output: + temporary = Path(output.name) + json.dump(result, output, indent=2, sort_keys=True) + output.write("\n") + try: + temporary.replace(receipt) + finally: + temporary.unlink(missing_ok=True) + return result def main() -> int: @@ -132,9 +345,13 @@ def main() -> int: parser.add_argument("--report", type=Path, required=True) args = parser.parse_args() try: - prepare(args.source_root, args.report) - except (OSError, KeyError, TypeError, ValueError, subprocess.CalledProcessError) as exc: - print(f"Error: cannot prepare pinned Sony makefiles: {exc}", file=sys.stderr) + result = prepare(args.source_root, args.report) + print(json.dumps({ + "status": result["status"], "projects": len(result["projects"]), + "publisher_authenticated": False, "byte_identical_vold_qualified": False, + }, sort_keys=True)) + except (OSError, KeyError, TypeError, ValueError, ET.ParseError, subprocess.CalledProcessError) as exc: + print(f"Error: cannot prepare pinned Sony vendor inputs: {exc}", file=sys.stderr) return 1 return 0 diff --git a/src/ci/probe_original_vold.sh b/src/ci/probe_original_vold.sh index 245d7535..dafce4cc 100644 --- a/src/ci/probe_original_vold.sh +++ b/src/ci/probe_original_vold.sh @@ -5,15 +5,26 @@ # Never apply the timeout patch or publish an OTA. set -euo pipefail -: "${VOLD_WORK:?isolated scratch workspace is required}" -: "${VOLD_PROBE_STAGE:?probe or compare stage is required}" +: "${VOLD_WORK:?isolated persistent build workspace is required}" +: "${VOLD_PROBE_STAGE:?vendor, prepare, probe or compare stage is required}" -[[ "${VOLD_PROBE_STAGE}" == probe || "${VOLD_PROBE_STAGE}" == compare ]] || { +[[ "${VOLD_PROBE_STAGE}" == vendor || "${VOLD_PROBE_STAGE}" == prepare || "${VOLD_PROBE_STAGE}" == probe || "${VOLD_PROBE_STAGE}" == compare ]] || { echo 'Error: unsupported unmodified vold dependency stage' >&2 exit 1 } [[ "${VOLD_WORK}" == /* && -d "${VOLD_WORK}" && ! -L "${VOLD_WORK}" ]] || { - echo 'Error: isolated scratch directory is missing or unsafe' >&2 + echo 'Error: isolated build directory is missing or unsafe' >&2 + exit 1 +} +VOLD_WORK="$(realpath -e -- "${VOLD_WORK}")" +run_dir="${VOLD_RUN_DIR:-${VOLD_WORK}}" +[[ "${run_dir}" == /* && -d "${run_dir}" && ! -L "${run_dir}" ]] || { + echo 'Error: diagnostic directory is missing or unsafe' >&2 + exit 1 +} +run_dir="$(realpath -e -- "${run_dir}")" +[[ "${run_dir}" == "${VOLD_WORK}" || "${run_dir}" == "${VOLD_WORK}/"* ]] || { + echo 'Error: diagnostic directory must be inside the build workspace' >&2 exit 1 } [[ "${VOLD_PROBE_STAGE}" != compare || ( "${VOLD_SOURCE_REF:-}" == /* && -s "${VOLD_SOURCE_REF}" && ! -L "${VOLD_SOURCE_REF}" ) ]] || { @@ -27,16 +38,31 @@ jobs="${VOLD_PROBE_JOBS:-4}" } selection="${VOLD_WORK}/source-selection.json" -python3 src/ci/vold_source_closure.py plan --output "${selection}" -# Check the three small pinned public vendor-input projects BEFORE fetching -# toolchains. At these revisions --regenerate_makefiles still reads absent -# proprietary ELF files. The preflight records that blocker without generating -# misleading partial declarations or downloading the rest of the source seed. +if [[ -e "${selection}" || -L "${selection}" ]]; then + [[ -f "${selection}" && -s "${selection}" && ! -L "${selection}" ]] || { + echo 'Error: persistent source selection is missing or unsafe' >&2 + exit 1 + } +else + python3 src/ci/vold_source_closure.py plan --output "${selection}" +fi +# The exact public lists require real proprietary ELF inputs. Fetch/verify +# separately locked mirror trees before the toolchains; never create stubs. +# Keep earlier source selection expansions and reuse existing clean projects. +resume=() +if [[ -e "${VOLD_WORK}/source/.repo" || -L "${VOLD_WORK}/source/.repo" ]]; then + resume=(--resume) +fi python3 src/ci/build_original_vold.py sync-vendor-sources \ - --source-root "${VOLD_WORK}/source" --jobs "${jobs}" + --source-root "${VOLD_WORK}/source" --jobs "${jobs}" "${resume[@]}" python3 src/ci/prepare_vold_vendor_makefiles.py \ --source-root "${VOLD_WORK}/source" \ - --report "${VOLD_WORK}/sony-vendor-input-report.json" + --report "${run_dir}/sony-vendor-input-report.json" + +if [[ "${VOLD_PROBE_STAGE}" == vendor ]]; then + echo "Verified pinned vendor inputs; receipt: ${run_dir}/sony-vendor-input-report.json" + exit 0 +fi python3 src/ci/build_original_vold.py sync --resume \ --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" @@ -45,11 +71,11 @@ python3 src/ci/build_original_vold.py sync --resume \ # compiling. Unknown or conditional modules remain diagnostics for real Soong; # they are never mapped to guessed repositories. Stop after three scans. for scan in 1 2 3; do - next="${VOLD_WORK}/selection-static-next.json" + next="${run_dir}/selection-static-next.json" rm -f -- "${next}" python3 src/ci/vold_source_closure.py index \ --source-root "${VOLD_WORK}/source" --selection "${selection}" \ - --output "${VOLD_WORK}/source-index-static-${scan}.json" \ + --output "${run_dir}/source-index-static-${scan}.json" \ --next-selection "${next}" if [[ ! -f "${next}" ]]; then break @@ -59,13 +85,18 @@ for scan in 1 2 3; do --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" done +if [[ "${VOLD_PROBE_STAGE}" == prepare ]]; then + echo 'Prepared pinned source selection and vendor inputs; compilation has not run.' + exit 0 +fi + # Most missing dependencies are reported during Soong setup before the # expensive target compile. Resolve their exact pinned project paths once # and reuse the same checkout and Android out directory on each iteration. # Unknown module providers must stop for review; never turn on missing # dependency flags or attempt an unverified project URL. for iteration in 1 2 3; do - log="${VOLD_WORK}/vold-build-pass-${iteration}.log" + log="${run_dir}/vold-build-pass-${iteration}.log" if [[ "${VOLD_PROBE_STAGE}" == probe ]]; then build_cmd=(python3 src/ci/build_original_vold.py probe --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}") @@ -73,14 +104,14 @@ for iteration in 1 2 3; do build_cmd=(python3 src/ci/build_original_vold.py compare --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --reference-vold "${VOLD_SOURCE_REF}" - --output-dir "${VOLD_WORK}/baseline-${iteration}" --jobs "${jobs}") + --output-dir "${run_dir}/baseline-${iteration}" --jobs "${jobs}") fi if "${build_cmd[@]}" >"${log}" 2>&1; then tail -n 15 -- "${log}" if [[ "${VOLD_PROBE_STAGE}" == probe ]]; then echo '::notice::Unmodified vold compiled, but no authenticated binary equality has been established.' - echo "Compile-only probe completed; next stage MUST use authenticated compare." >"${VOLD_WORK}/probe-only.txt" + echo "Compile-only probe completed; next stage MUST use authenticated compare." >"${run_dir}/probe-only.txt" else echo '::notice::Unmodified vold is byte-identical to the authenticated reference.' fi @@ -89,11 +120,11 @@ for iteration in 1 2 3; do echo "::warning::Pinned source probe pass ${iteration} failed. Last diagnostic lines:" tail -n 55 -- "${log}" - next="${VOLD_WORK}/selection-next.json" + next="${run_dir}/selection-next.json" rm -f -- "${next}" python3 src/ci/vold_source_closure.py diagnose \ --selection "${selection}" --log "${log}" \ - --output "${VOLD_WORK}/dependency-diagnosis-${iteration}.json" \ + --output "${run_dir}/dependency-diagnosis-${iteration}.json" \ --next-selection "${next}" if [[ ! -f "${next}" || "${iteration}" == 3 ]]; then diff --git a/src/ci/setup_vold_local_tools.py b/src/ci/setup_vold_local_tools.py new file mode 100644 index 00000000..896de4f5 --- /dev/null +++ b/src/ci/setup_vold_local_tools.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Install private Repo/avbroot launchers for local vold work, without sudo. + +Repo's launcher is SHA-256 pinned; its normal signed Repo bootstrap stays on. +avbroot uses the existing publisher-authenticated executable lock and sealed +runner. No system tools, shell profiles, source trees or GitHub settings change. +""" +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import platform +import shlex +import shutil +import subprocess +import sys +import tempfile + +from vold_local import ( + DEFAULT_WORK, LocalBuildError, REPO_LAUNCHER_REV, REPO_LAUNCHER_SHA256, + ROOT, digest_file, exclusive_workspace, work_directory, +) + + +def private_directory(path: Path) -> Path: + if path.is_symlink() or (path.exists() and not path.is_dir()): + raise LocalBuildError(f"Tool directory is unsafe: {path}") + path.mkdir(mode=0o700, parents=True, exist_ok=True) + if path.stat().st_mode & 0o077: + raise LocalBuildError(f"Tool directory must be private (mode 0700): {path}") + return path + + +def write_executable(path: Path, contents: str, mode: int = 0o755) -> None: + if path.is_symlink() or (path.exists() and not path.is_file()): + raise LocalBuildError(f"Tool launcher is unsafe: {path}") + with tempfile.NamedTemporaryFile(dir=path.parent, prefix=path.name + ".", delete=False) as output: + temporary = Path(output.name) + output.write(contents.encode()) + try: + temporary.chmod(mode) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +def install_repo(binary: Path) -> None: + if binary.exists() or binary.is_symlink(): + if digest_file(binary) != REPO_LAUNCHER_SHA256: + raise LocalBuildError(f"Existing private Repo launcher does not match the reviewed SHA-256: {binary}") + binary.chmod(0o755) + return + with tempfile.TemporaryDirectory(prefix="repo-download-", dir=binary.parent) as temporary: + downloaded = Path(temporary) / "repo" + subprocess.run([ + "curl", "--fail", "--location", "--retry", "3", "--connect-timeout", "30", + "--proto", "=https", "--proto-redir", "=https", + "https://raw.githubusercontent.com/GerritCodeReview/git-repo/" + REPO_LAUNCHER_REV + "/repo", + "--output", str(downloaded), + ], check=True) + if digest_file(downloaded) != REPO_LAUNCHER_SHA256: + raise LocalBuildError("Official Repo launcher download failed its pinned SHA-256 check.") + downloaded.chmod(0o755) + downloaded.replace(binary) + + +def avbroot_wrapper(store: Path) -> str: + python = str(Path(sys.executable).resolve()) + command = [ + python, "-E", "-s", "-B", + str(ROOT / "src/bootstrap_executable_tools.py"), + "--workdir", str(store), "run", "avbroot", "--", + ] + # Bypass PATH interpreters, Python startup overrides and shell BASH_ENV. + return ( + f"#!{python} -I\nimport os\nimport sys\n" + f"os.execv({python!r}, {command!r} + sys.argv[1:])\n" + ) + + +def setup(work: Path, repo_only: bool = False) -> Path: + store = private_directory(work / "host-tools") + binaries = private_directory(store / "bin") + install_repo(binaries / "repo") + if not repo_only: + # Reuse the existing archive/member hashes, detached SSH signature, + # trusted signer and executable runner. No parallel download policy. + subprocess.run([ + sys.executable, "-E", "-s", "-B", str(ROOT / "src/bootstrap_executable_tools.py"), + "--workdir", str(store), "install", "--report", str(store / "executable-tools.json"), + "avbroot", + ], cwd=work, check=True) + write_executable(binaries / "avbroot", avbroot_wrapper(store)) + subprocess.run([str(binaries / "avbroot"), "--version"], cwd=work, check=True) + environment = store / "env.sh" + write_executable(environment, "export PATH=" + shlex.quote(str(binaries)) + ':"$PATH"\n', 0o600) + return environment + + +def sealed_runner_supported() -> bool: + import fcntl + return ( + os.execve in os.supports_fd and hasattr(os, "memfd_create") + and all(hasattr(fcntl, name) for name in ( + "F_ADD_SEALS", "F_GET_SEALS", "F_SEAL_WRITE", "F_SEAL_GROW", "F_SEAL_SHRINK", "F_SEAL_SEAL", + )) + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir", type=Path, default=DEFAULT_WORK) + parser.add_argument("--repo-only", action="store_true", help="skip avbroot until ready for authenticated comparison") + args = parser.parse_args(argv) + try: + if platform.system() != "Linux" or platform.machine().lower() not in ("x86_64", "amd64"): + raise LocalBuildError("The local build setup supports Linux x86_64.") + if not args.repo_only and not sealed_runner_supported(): + raise LocalBuildError( + "This Python lacks Linux sealed-execution support; on Ubuntu rerun this setup with /usr/bin/python3." + ) + required = ("curl", "git", "gpg") + (() if args.repo_only else ("ssh-keygen",)) + missing = [tool for tool in required if shutil.which(tool) is None] + if missing: + raise LocalBuildError("Install the documented host packages first; missing: " + ", ".join(missing)) + work = work_directory(args.work_dir) + with exclusive_workspace(work): + environment = setup(work, args.repo_only) + print("Private host tools are ready. Run this in your current shell:") + print(" source " + shlex.quote(str(environment))) + print("Repo's launcher is pinned; its normal signed tool bootstrap remains enabled.") + if not args.repo_only: + print("avbroot executes through the existing publisher-verified sealed runner.") + return 0 + except (OSError, ValueError, subprocess.SubprocessError) as exc: + print(f"Error: local tool setup failed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ci/vold_local.py b/src/ci/vold_local.py new file mode 100644 index 00000000..8ff7edfd --- /dev/null +++ b/src/ci/vold_local.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Build and qualify original pdx235 vold locally with persistent sources/cache. + +Examples: + python3 src/ci/vold_local.py doctor --work-dir /fast/modos-vold + python3 src/ci/vold_local.py vendor --work-dir /fast/modos-vold + python3 src/ci/vold_local.py compare --work-dir /fast/modos-vold --jobs 4 + +This command never patches, signs, publishes, or deploys an OTA. +""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import platform +import shlex +import shutil +import signal +import stat +import subprocess +import sys +from urllib.parse import urlparse +import uuid + +from build_original_vold import BuildError, load_contract +from vold_baseline_compare import BaselineError, read_vold +from vold_source_closure import initial_projects, selection_record, verified_contract + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_WORK = Path.home() / ".cache/modos/vold-pdx235-20261016" +STAGES = ("vendor", "prepare", "probe", "compare") +BASE_TOOLS = ("bash", "git", "gpg", "repo", "python3") +BUILD_TOOLS = ("make", "gcc", "g++", "zip", "unzip", "rsync") +COMPARE_TOOLS = ("curl", "avbroot", "unzip", "openssl", "sha256sum", "debugfs") +AUTH_FILES = ( + "locks/lineage-vold-baseline-20261016.json", + "locks/executable-tools-v1.json", + "src/bootstrap_executable_tools.py", + "src/ci/extract_original_vold.sh", + "src/source_care_map_policy.sh", + "src/ci/check_ota_certificate_member.py", + "src/ci/vold_baseline_compare.py", +) +REPO_LAUNCHER_REV = "d744eb15f38b9351463900babffd732994e3a046" +REPO_LAUNCHER_SHA256 = "05f571deafd6ed6da0d20fa707dde29aa5624dbf35553d697a40dbdd4c8d178a" + + +class LocalBuildError(ValueError): + pass + + +def timestamp() -> str: + return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + + +def digest_file(path: Path) -> str: + info = path.lstat() + if not stat.S_ISREG(info.st_mode): + raise LocalBuildError(f"Expected a regular file, not a link: {path}") + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b""): + digest.update(chunk) + after = path.stat() + if (info.st_ino, info.st_size, info.st_mtime_ns) != (after.st_ino, after.st_size, after.st_mtime_ns): + raise LocalBuildError(f"File changed while hashing: {path}") + return digest.hexdigest() + + +def real_directory(path: Path) -> Path: + if path.is_symlink() or (path.exists() and not path.is_dir()): + raise LocalBuildError(f"Workspace directory is unsafe: {path}") + path.mkdir(parents=True, exist_ok=True) + return path.resolve(strict=True) + + +def work_directory(path: Path) -> Path: + path = path.expanduser().absolute() + if any(character.isspace() for character in str(path)): + raise LocalBuildError("Use a work directory without whitespace; Android build tools require a compatible source path.") + result = real_directory(path) + if any(character.isspace() for character in str(result)): + raise LocalBuildError("The resolved Android work directory must not contain whitespace.") + return result + + +def write_json(path: Path, value: dict) -> None: + if path.is_symlink() or (path.exists() and not path.is_file()): + raise LocalBuildError(f"Refusing to replace an unsafe report: {path}") + temporary = path.with_name(path.name + ".tmp-" + uuid.uuid4().hex) + with temporary.open("x", encoding="utf-8") as output: + json.dump(value, output, indent=2, sort_keys=True) + output.write("\n") + temporary.replace(path) + + +@contextmanager +def exclusive_workspace(work: Path): + # Android compilation is Linux-only. Import lazily so plan/help still work + # on other platforms and doctor can explain the host requirement. + import fcntl + path = work / "local-workspace.lock" + descriptor = os.open(path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "a+") as handle: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as exc: + raise LocalBuildError(f"Another local vold command is using {work}; let it finish first.") from exc + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + + +def doctor(work: Path, stage: str, jobs: int, minimum_free_gb: float | None) -> dict: + required = set(BASE_TOOLS) + if stage in ("probe", "compare"): + required.update(BUILD_TOOLS) + if stage == "compare": + required.update(COMPARE_TOOLS) + paths = {tool: shutil.which(tool) for tool in sorted(required)} + missing = [tool for tool, path in paths.items() if path is None] + free = shutil.disk_usage(work).free + errors = [] + notes = [] + if platform.system() != "Linux" or platform.machine().lower() not in ("x86_64", "amd64"): + errors.append("Source builds require Linux x86_64; Ubuntu 24.04 is the documented local setup.") + if missing: + errors.append("Missing required executables: " + ", ".join(missing)) + if minimum_free_gb is not None and free < minimum_free_gb * 1_000_000_000: + errors.append(f"Free disk is below the explicit --min-free-gb {minimum_free_gb:g} requirement.") + elif stage != "vendor" and free < 100_000_000_000: + notes.append("100 GB free is an unmeasured initial planning allowance, not a resume requirement. Existing sources already occupy disk; use --min-free-gb for an explicit floor.") + memory = None + try: + for line in Path("/proc/meminfo").read_text().splitlines(): + if line.startswith("MemTotal:"): + memory = int(line.split()[1]) * 1024 + break + except (OSError, ValueError): + pass + return { + "schema_version": 1, "kind": "local-vold-host-preflight", + "stage": stage, "work_dir": str(work), "jobs": jobs, + "host": {"system": platform.system(), "machine": platform.machine()}, + "free_disk_bytes": free, "physical_memory_bytes": memory, + "minimum_free_gb": minimum_free_gb, "tool_paths": paths, + "missing_tools": missing, "errors": errors, "notes": notes, + "ready_for_requested_stage": not errors, + "does_not_establish_source_closure_or_byte_identity": True, + } + + +def print_doctor(report: dict) -> None: + print(f"Work directory: {report['work_dir']}") + print(f"Host: {report['host']['system']} {report['host']['machine']}; jobs: {report['jobs']}") + print(f"Free disk: {report['free_disk_bytes'] / 1_000_000_000:.1f} GB") + if report["physical_memory_bytes"] is not None: + print(f"Physical memory: {report['physical_memory_bytes'] / (1024 ** 3):.1f} GiB") + for note in report["notes"]: + print("Note: " + note) + for error in report["errors"]: + print("Required: " + error) + if "repo" in report["missing_tools"]: + print("Install the official Repo launcher, then put it on PATH (its normal signed bootstrap remains enabled):") + print(" mkdir -p \"$HOME/.local/bin\"") + print(" curl --fail --location --proto '=https' --proto-redir '=https' \\") + print(f" https://raw.githubusercontent.com/GerritCodeReview/git-repo/{REPO_LAUNCHER_REV}/repo \\") + print(" --output \"$HOME/.local/bin/repo.download\"") + print(f" printf '%s %s\\n' '{REPO_LAUNCHER_SHA256}' \"$HOME/.local/bin/repo.download\" | sha256sum --check - && \\") + print(" install -m 0755 \"$HOME/.local/bin/repo.download\" \"$HOME/.local/bin/repo\"") + print(" export PATH=\"$HOME/.local/bin:$PATH\"") + if report["missing_tools"]: + print("Ubuntu packages and the trusted avbroot setup are documented in docs/vold-baseline-ci.md.") + + +def run_logged(command: list[str], *, log: Path, env: dict | None = None) -> None: + print("Running: " + shlex.join(command), flush=True) + with log.open("x", encoding="utf-8") as output: + output.write("$ " + shlex.join(command) + "\n") + output.flush() + process = subprocess.Popen( + command, cwd=ROOT, env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors="replace", start_new_session=True, + ) + assert process.stdout is not None + try: + for line in process.stdout: + print(line, end="", flush=True) + output.write(line) + output.flush() + status = process.wait() + except BaseException: + # Stop the whole compile/download process group before releasing + # the workspace lock; keep its sources and logs for the next run. + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise + finally: + process.stdout.close() + if status: + raise LocalBuildError(f"Command exited with status {status}. Sources and outputs are retained; inspect {log} and rerun the same command after resolving the diagnostic.") + + +def source_stage(stage: str, work: Path, run_dir: Path, jobs: int, reference: Path | None = None) -> Path: + phase = real_directory(run_dir / stage) + environment = os.environ.copy() + environment.update(VOLD_WORK=str(work), VOLD_RUN_DIR=str(phase), + VOLD_PROBE_STAGE=stage, VOLD_PROBE_JOBS=str(jobs)) + environment.pop("VOLD_SOURCE_REF", None) + if reference is not None: + environment["VOLD_SOURCE_REF"] = str(reference) + run_logged(["bash", str(ROOT / "src/ci/probe_original_vold.sh")], + log=phase / "workflow.log", env=environment) + return phase + + +def ota_filename(contract: dict) -> str: + filename = contract["base_build_id"] + ".zip" + url = urlparse(contract["source_ota_url"]) + if url.scheme != "https" or Path(url.path).name != filename: + raise LocalBuildError("The source contract does not name the exact HTTPS OTA archive.") + return filename + + +def exact_ota(work: Path, run_dir: Path, contract: dict, supplied: Path | None = None) -> Path: + filename = ota_filename(contract) + if supplied is not None: + archive = supplied.expanduser().absolute() + if archive.name != filename or digest_file(archive) != contract["source_ota_sha256"]: + raise LocalBuildError("--ota-file must match the exact locked official filename and SHA-256; it has not been accepted as authentic.") + return archive + cache = real_directory(work / "cache") + archive = cache / filename + if archive.exists() or archive.is_symlink(): + if digest_file(archive) != contract["source_ota_sha256"]: + raise LocalBuildError(f"Cached OTA digest differs from the immutable lock. Move the corrupt file aside and rerun: {archive}") + print(f"Reusing exact cached OTA: {archive}") + return archive + partial = cache / (filename + ".part") + if partial.exists() or partial.is_symlink(): + if digest_file(partial) == contract["source_ota_sha256"]: + partial.replace(archive) + return archive + run_logged([ + "curl", "--fail", "--location", "--retry", "3", "--connect-timeout", "30", + "--proto", "=https", "--proto-redir", "=https", "--continue-at", "-", + "--output", str(partial), contract["source_ota_url"], + ], log=run_dir / "download-ota.log") + if digest_file(partial) != contract["source_ota_sha256"]: + raise LocalBuildError(f"Downloaded OTA failed the pinned SHA-256 check; reference extraction was not attempted: {partial}") + partial.replace(archive) + return archive + + +def authentication_inputs() -> dict: + inputs = {path: digest_file(ROOT / path) for path in AUTH_FILES} + executable = shutil.which("avbroot") + if executable is None: + raise LocalBuildError("avbroot is required for authentic OTA verification.") + inputs["avbroot_executable"] = digest_file(Path(executable).resolve(strict=True)) + return inputs + + +def reference_receipt(directory: Path, contract: dict, inputs: dict) -> dict: + reference = directory / "original-vold" + read_vold(reference) + return { + "schema_version": 1, "kind": "authenticated-local-original-vold-reference", + "base_build_id": contract["base_build_id"], + "source_ota_filename": ota_filename(contract), + "source_ota_sha256": contract["source_ota_sha256"], + "manifest_sha256": contract["manifest"]["sha256"], + "reference_sha256": digest_file(reference), + "certificate_sha256": digest_file(directory / "source-otacert.pem"), + "authentication_inputs": inputs, + "independent_publisher_and_ota_signature_verified": True, + } + + +def authenticated_reference(archive: Path, work: Path, run_dir: Path, contract: dict) -> Path: + # The caller first checks the entire OTA against its immutable hash. A user + # cannot supply a standalone ELF/reference path to this local entry point. + # A local JSON receipt is editable and is not an independent authenticity + # anchor. Re-extract from the verified cached OTA on every comparison; only + # the download is reused. The reference receipt remains audit evidence. + directory = run_dir / "reference" + inputs = authentication_inputs() + if directory.exists() or directory.is_symlink(): + raise LocalBuildError("Fresh reference extraction requires a new invocation directory; existing ELF files or receipts are not accepted.") + receipt_file = directory / "reference-receipt.json" + run_logged(["bash", str(ROOT / "src/ci/extract_original_vold.sh"), str(archive), str(directory)], + log=run_dir / "extract-reference.log") + write_json(receipt_file, reference_receipt(directory, contract, inputs)) + return directory / "original-vold" + + +def require_comparison_receipt(phase: Path, reference: Path, contract: dict) -> Path: + expected = digest_file(reference) + for path in sorted(phase.glob("baseline-*/baseline-report.json")): + digest_file(path) + report = json.loads(path.read_bytes()) + if ( + report.get("byte_identical") is True + and report.get("official_ota_sha256") == contract["source_ota_sha256"] + and report.get("manifest_sha256") == contract["manifest"]["sha256"] + and report.get("reference_sha256") == report.get("built_sha256") == expected + ): + return path + raise LocalBuildError("Compare returned without a matching byte-identical receipt in this invocation's diagnostics.") + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = result.add_subparsers(dest="mode", required=True) + help_text = { + "plan": "write the pinned source plan without downloads", + "doctor": "check local host, tools, memory and disk", + "vendor": "fetch only small public prerequisite projects and pinned vendor candidates", + "prepare": "prepare/resume the pinned source selection without compiling", + "probe": "compile unmodified m vold without downloading the OTA", + "compare": "compile first, authenticate the exact OTA, then require byte equality", + } + for mode, description in help_text.items(): + command = commands.add_parser(mode, help=description) + command.add_argument("--work-dir", type=Path, default=DEFAULT_WORK, + help="persistent source, cache and logs directory; no automatic cleanup") + command.add_argument("--jobs", type=int, default=min(4, os.cpu_count() or 1)) + command.add_argument("--min-free-gb", type=float, + help="optional explicit disk floor; 100 GB is only an initial recommendation") + if mode == "doctor": + command.add_argument("--for", dest="check_stage", choices=STAGES, default="probe") + if mode == "compare": + command.add_argument("--ota-file", type=Path, + help="already downloaded exact official archive; filename, digest and publisher are verified") + return result + + +def main(argv: list[str] | None = None) -> int: + arguments = parser() + args = arguments.parse_args(argv) + if not 1 <= args.jobs <= 16: + arguments.error("--jobs must be between 1 and 16") + if args.min_free_gb is not None and not 0 <= args.min_free_gb < float("inf"): + arguments.error("--min-free-gb must be a finite nonnegative number") + run_dir = None + try: + contract, _manifest = load_contract() + work = work_directory(args.work_dir) + if args.mode == "plan": + seed, projects, _baseline = verified_contract() + plan = selection_record(initial_projects(seed), seed) + output = work / "source-provisional-plan.json" + write_json(output, plan) + print(f"Pinned starting selection: {len(plan['selected_projects'])}/{len(projects)} projects") + print(f"Plan: {output}\nNo sources or OTA were downloaded.") + return 0 + report = doctor(work, getattr(args, "check_stage", args.mode), args.jobs, args.min_free_gb) + print_doctor(report) + write_json(work / "local-host-report.json", report) + if not report["ready_for_requested_stage"]: + return 1 + if args.mode == "doctor": + print("Local prerequisites passed. Source closure and binary equality still require the build stages.") + return 0 + with exclusive_workspace(work): + run_dir = real_directory(real_directory(work / "runs") / (timestamp() + "-" + args.mode)) + record = {"schema_version": 1, "mode": args.mode, "work_dir": str(work), + "jobs": args.jobs, "status": "running", "base_build_id": contract["base_build_id"]} + write_json(run_dir / "local-run.json", record) + print(f"Invocation logs: {run_dir}") + try: + if args.mode == "compare": + source_stage("probe", work, run_dir, args.jobs) + archive = exact_ota(work, run_dir, contract, args.ota_file) + reference = authenticated_reference(archive, work, run_dir, contract) + phase = source_stage("compare", work, run_dir, args.jobs, reference) + proof = require_comparison_receipt(phase, reference, contract) + record["baseline_report"] = str(proof) + print(f"Authentic byte-identical original vold confirmed: {proof}") + else: + source_stage(args.mode, work, run_dir, args.jobs) + if args.mode == "probe": + print("Original vold compiled. Byte equality is still unproved; run compare next.") + else: + print(f"{args.mode.capitalize()} completed. Reuse this work directory for the next stage.") + record["status"] = "completed" + except BaseException: + record["status"] = "interrupted-or-failed" + raise + finally: + write_json(run_dir / "local-run.json", record) + return 0 + except KeyboardInterrupt: + print("Interrupted. Downloaded sources, outputs, caches and logs are retained.", file=sys.stderr) + return 130 + except (OSError, ValueError, BuildError, BaselineError, subprocess.SubprocessError) as exc: + print(f"Error: {exc}", file=sys.stderr) + if run_dir is not None: + print(f"Invocation diagnostics: {run_dir}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/vold_baseline_ci_test.py b/tests/vold_baseline_ci_test.py index 99b29e6e..2bc28b00 100644 --- a/tests/vold_baseline_ci_test.py +++ b/tests/vold_baseline_ci_test.py @@ -135,6 +135,7 @@ def fake_compile(argv, cwd, **_kwargs): with ( patch.object(builder, "verify_checkout"), + patch.object(builder, "verified_vendor_inputs", return_value={"publisher_authenticated": False, "fixture": True}), patch.object(builder, "command", side_effect=fake_compile), ): report = builder.build_baseline( @@ -142,6 +143,7 @@ def fake_compile(argv, cwd, **_kwargs): ) self.assertTrue(report["byte_identical"]) self.assertEqual(report["reference_sha256"], report["built_sha256"]) + self.assertFalse(report["supplemental_vendor_inputs"]["publisher_authenticated"]) def fake_bad_compile(argv, cwd, **_kwargs): bad = bytearray(original) @@ -151,6 +153,7 @@ def fake_bad_compile(argv, cwd, **_kwargs): with ( patch.object(builder, "verify_checkout"), + patch.object(builder, "verified_vendor_inputs", return_value={"publisher_authenticated": False, "fixture": True}), patch.object(builder, "command", side_effect=fake_bad_compile), self.assertRaises(comparator.BaselineError), ): diff --git a/tests/vold_local_test.py b/tests/vold_local_test.py new file mode 100644 index 00000000..0db7d23f --- /dev/null +++ b/tests/vold_local_test.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Local build orchestration/cache regressions; no Android or OTA downloads.""" +from __future__ import annotations + +from contextlib import redirect_stderr, redirect_stdout +import hashlib +import io +import json +import os +from pathlib import Path +import struct +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src/ci")) +import vold_local as local # noqa: E402 + + +class LocalVoldTest(unittest.TestCase): + def setUp(self): + self.contract, _ = local.load_contract() + self.output = io.StringIO() + + @staticmethod + def elf(): + result = bytearray(256) + result[:7] = b"\x7fELF\x02\x01\x01" + struct.pack_into(" tuple[dict[str, str], Path]: + shadow = root / "path-shadow" + shadow.mkdir() + marker = root / "shadow-executed" + payload = (f"#!{Path(sys.executable).resolve()}\nfrom pathlib import Path\n" + f"Path({str(marker)!r}).write_text('untrusted startup or PATH entry')\n" + "raise SystemExit(73)\n") + for name in ("bash", "python", "python3", "avbroot"): + path = shadow / name + path.write_text(payload) + path.chmod(0o755) + (shadow / "sitecustomize.py").write_text(payload.split("\n", 1)[1]) + startup = root / "bash-startup" + startup.write_text("printf 'untrusted shell startup' > " + str(marker) + "\nexit 73\n") + environment = os.environ.copy() + environment.update({ + "PATH": str(shadow) + os.pathsep + os.defpath, + "PYTHONPATH": str(shadow), + "BASH_ENV": str(startup), + }) + return environment, marker + + def test_wrapper_preserves_arguments_and_ignores_path_and_startup_shadows(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + fixture = root / "fixture-repository" + (fixture / "src").mkdir(parents=True) + # Exercise the real sealed-runner CLI parser at the wrapper's + # execution boundary; cryptographic installation has separate tests. + (fixture / "src/bootstrap_executable_tools.py").write_text( + "import json, sys\n" + f"sys.path.insert(0, {str(ROOT / 'src')!r})\n" + "from bootstrap_executable_tools import parse_args\n" + "args = parse_args(sys.argv[1:])\n" + "print(json.dumps({'workdir': str(args.workdir), 'command': args.command,\n" + " 'tool': args.tool, 'arguments': args.arguments, 'python': sys.executable,\n" + " 'ignore_environment': sys.flags.ignore_environment,\n" + " 'no_user_site': sys.flags.no_user_site,\n" + " 'dont_write_bytecode': sys.flags.dont_write_bytecode}))\n" + ) + store = root / "private-store" + wrapper = root / "avbroot" + with patch.object(setup, "ROOT", fixture): + setup.write_executable(wrapper, setup.avbroot_wrapper(store)) + environment, marker = self.shadow_environment(root) + arguments = [ + "ota", "verify", "--", "name with spaces", "quote'and\"double", + "$(false)", "semi;colon", "line\nbreak", + ] + result = subprocess.run( + [str(wrapper), *arguments], env=environment, cwd=root, + check=True, capture_output=True, text=True, + ) + observed = json.loads(result.stdout) + self.assertEqual(observed["workdir"], str(store)) + self.assertEqual(observed["command"], "run") + self.assertEqual(observed["tool"], "avbroot") + self.assertEqual(observed["arguments"], arguments) + self.assertEqual(Path(observed["python"]).resolve(), Path(sys.executable).resolve()) + self.assertEqual(observed["ignore_environment"], 1) + self.assertEqual(observed["no_user_site"], 1) + self.assertEqual(observed["dont_write_bytecode"], 1) + self.assertFalse(marker.exists()) + + def test_real_sealed_runner_refuses_missing_install_without_path_fallback(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = root / "empty-store" + store.mkdir(mode=0o700) + wrapper = root / "avbroot" + setup.write_executable(wrapper, setup.avbroot_wrapper(store)) + environment, marker = self.shadow_environment(root) + result = subprocess.run( + [str(wrapper), "--version"], env=environment, cwd=root, + check=False, capture_output=True, text=True, + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("executable bootstrap failed:", result.stderr) + self.assertIn("cannot stat installed executable directory", result.stderr) + self.assertFalse(marker.exists()) + + def test_setup_does_not_claim_ready_if_sealed_execution_is_unavailable(self): + with tempfile.TemporaryDirectory() as tmp: + output = io.StringIO() + observed = [] + + def invoke(command, **kwargs): + observed.append(command) + if "install" in command: + self.assertEqual(command[command.index("install") - 2], "--workdir") + self.assertEqual(command[-1], "avbroot") + return subprocess.CompletedProcess(command, 0) + self.assertEqual(command, [str(Path(tmp) / "host-tools/bin/avbroot"), "--version"]) + raise subprocess.CalledProcessError(1, command) + + with ( + patch.object(setup.platform, "system", return_value="Linux"), + patch.object(setup.platform, "machine", return_value="x86_64"), + patch.object(setup, "sealed_runner_supported", return_value=True), + patch.object(setup.shutil, "which", side_effect=lambda tool: "/usr/bin/" + tool), + patch.object(setup, "install_repo"), + patch.object(setup.subprocess, "run", side_effect=invoke), + redirect_stdout(output), redirect_stderr(output), + ): + self.assertEqual(setup.main(["--work-dir", tmp]), 1) + self.assertEqual(len(observed), 2) + self.assertIn("local tool setup failed", output.getvalue()) + self.assertNotIn("Private host tools are ready", output.getvalue()) + + def test_unsupported_python_is_reported_before_any_tool_acquisition(self): + with tempfile.TemporaryDirectory() as tmp: + output = io.StringIO() + with ( + patch.object(setup.platform, "system", return_value="Linux"), + patch.object(setup.platform, "machine", return_value="x86_64"), + patch.object(setup, "sealed_runner_supported", return_value=False), + patch.object(setup, "setup", side_effect=AssertionError("unsupported host must not acquire tools")), + redirect_stdout(output), redirect_stderr(output), + ): + self.assertEqual(setup.main(["--work-dir", tmp]), 1) + self.assertIn("/usr/bin/python3", output.getvalue()) + self.assertNotIn("Private host tools are ready", output.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vold_original_build_test.py b/tests/vold_original_build_test.py index fdb41528..60a75ffb 100644 --- a/tests/vold_original_build_test.py +++ b/tests/vold_original_build_test.py @@ -132,9 +132,47 @@ def test_resume_fetches_partial_directories_but_not_existing_worktrees(self): ): builder.sync(source, self.lock, self.manifest, 2, ["system/vold", "system/update_engine"], resume=True) - self.assertEqual(calls, [["/trusted/repo", "sync", "--no-clone-bundle", "-j2", "system/update_engine"]]) + self.assertEqual(calls, [["/trusted/repo", "sync", "--current-branch", "--no-tags", "--no-clone-bundle", "-j2", "system/update_engine"]]) self.assertEqual(verify.call_args_list[0].kwargs["selected"], ["system/vold"]) + def test_interrupted_init_guidance_preserves_inputs_and_original_pin_error(self): + for state, diagnostic in ( + ("missing-active", "repo selected manifest is unavailable"), + ("initial-manifest", "repo selected manifest differs from the exact official snapshot"), + ("changed-pin", "repo selected manifest differs from the exact official snapshot"), + ): + with self.subTest(state=state), tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + source = work / "source" + manifests = source / ".repo/manifests" + manifests.mkdir(parents=True) + selection = work / "source-selection.json" + selection.write_text('{"selected_projects":["system/vold"]}\n') + if state == "initial-manifest": + (source / ".repo/manifest.xml").write_text( + '' + ) + (manifests / "default.xml").write_text("") + elif state == "changed-pin": + (source / ".repo/manifest.xml").write_text( + f'' + ) + (manifests / builder.MANIFEST_NAME).write_text("changed pinned bytes") + before = {p.relative_to(work): p.read_bytes() for p in work.rglob("*") if p.is_file()} + with ( + patch.object(builder, "command") as command, + self.assertRaises(builder.BuildError) as failure, + ): + builder.sync(source, self.lock, self.manifest, 2, ["system/vold"], resume=True) + message = str(failure.exception) + self.assertIn(diagnostic, message) + self.assertIn("If the first Repo initialization was interrupted", message) + self.assertIn(f"rename only the incomplete source directory {source}", message) + self.assertIn("For a previously completed checkout, investigate", message) + command.assert_not_called() + after = {p.relative_to(work): p.read_bytes() for p in work.rglob("*") if p.is_file()} + self.assertEqual(after, before) + def test_vendor_prerequisite_mode_only_requests_three_pinned_projects(self): with ( patch.object(sys, "argv", ["builder", "sync-vendor-sources", "--source-root", "/fixture", "--jobs", "2"]), @@ -146,6 +184,13 @@ def test_vendor_prerequisite_mode_only_requests_three_pinned_projects(self): self.assertEqual(args[3], 2) self.assertEqual(args[4], ["device/sony/pdx235", "device/sony/sm6375-common", "tools/extract-utils"]) self.assertTrue(set(args[4]).issubset(builder.project_map(self.manifest))) + self.assertFalse(args[5]) + with ( + patch.object(sys, "argv", ["builder", "sync-vendor-sources", "--source-root", "/fixture", "--resume"]), + patch.object(builder, "sync") as sync, + ): + self.assertEqual(builder.main(), 0) + self.assertTrue(sync.call_args.args[5]) def test_unrecorded_vendor_or_unselected_source_is_not_build_input(self): for unexpected in ("vendor/sony/pdx235/Android.bp", "system/update_engine/Android.bp"): @@ -155,9 +200,34 @@ def test_unrecorded_vendor_or_unselected_source_is_not_build_input(self): bad = source / unexpected bad.parent.mkdir(parents=True, exist_ok=True) bad.write_text("unrecorded source") + with self.assertRaisesRegex(builder.BuildError, "unrecorded source|supplemental vendor input verification failed"): + builder.verify_source_workspace(source, self.manifest, ["system/vold"]) + + def test_vendor_allowance_is_limited_to_independently_verified_locked_roots(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + for relpath in ("system/vold", "vendor/sony/pdx235", "vendor/sony/sm6375-common"): + (source / relpath).mkdir(parents=True) + receipt = {"publisher_authenticated": False, "fixture": True} + with patch.object(builder, "verified_vendor_inputs", return_value=receipt) as verify: + self.assertEqual(builder.verify_source_workspace(source, self.manifest, ["system/vold"]), receipt) + verify.assert_called_once_with(source) + (source / "vendor/sony/unlocked").mkdir() with self.assertRaisesRegex(builder.BuildError, "unrecorded source"): builder.verify_source_workspace(source, self.manifest, ["system/vold"]) + def test_compile_requires_real_vendor_inputs_before_running_build(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + with ( + patch.object(builder, "verify_checkout"), + patch.object(builder, "command") as command, + self.assertRaisesRegex(builder.BuildError, "supplemental vendor input verification failed"), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + command.assert_not_called() + def test_manifest_exported_links_are_checked_without_scanning_project_contents(self): with tempfile.TemporaryDirectory() as tmp: source = Path(tmp) @@ -183,6 +253,7 @@ def test_successful_noop_build_cannot_reuse_prior_vold(self): artifact.chmod(0o755) with ( patch.object(builder, "verify_checkout"), + patch.object(builder, "verified_vendor_inputs", return_value={"fixture": True}), patch.object(builder, "command"), self.assertRaisesRegex(builder.BuildError, "did not produce"), ): @@ -213,6 +284,7 @@ def compile_fixture(_args, _cwd, *, env): with ( patch.object(builder, "verify_checkout"), + patch.object(builder, "verified_vendor_inputs", return_value={"fixture": True}), patch.object(builder, "command", side_effect=compile_fixture), patch.dict(os.environ, {"OUT_DIR": "/other-build", "OUT_DIR_COMMON_BASE": "/common", "ANDROID_PRODUCT_OUT": "/other-product", "ALLOW_MISSING_DEPENDENCIES": "false"}), ): @@ -221,6 +293,7 @@ def compile_fixture(_args, _cwd, *, env): with ( self.subTest(flag=flag), patch.object(builder, "verify_checkout"), + patch.object(builder, "verified_vendor_inputs", return_value={"fixture": True}), patch.object(builder, "command") as compile_command, patch.dict(os.environ, {flag: "true"}), self.assertRaisesRegex(builder.BuildError, "bypass must be disabled"), @@ -250,6 +323,7 @@ def test_failed_upstream_setup_or_kernel_autobuild_stops_before_vold(self): ) with ( patch.object(builder, "verify_checkout"), + patch.object(builder, "verified_vendor_inputs", return_value={"fixture": True}), self.assertRaises(subprocess.CalledProcessError), ): builder.compile_original_vold(source, self.lock, self.manifest, 2) @@ -272,6 +346,7 @@ def changed_source(_args, _cwd, **_kwargs): with ( patch.object(builder, "verify_checkout") as verify, + patch.object(builder, "verified_vendor_inputs", return_value={"fixture": True}), patch.object(builder, "command", side_effect=changed_source), self.assertRaisesRegex(builder.BuildError, "unrecorded source"), ): diff --git a/tests/vold_probe_orchestration_test.py b/tests/vold_probe_orchestration_test.py index e12b1f72..bb1da152 100644 --- a/tests/vold_probe_orchestration_test.py +++ b/tests/vold_probe_orchestration_test.py @@ -17,7 +17,7 @@ class ProbeOrchestrationTest(unittest.TestCase): def run_probe(self, *, stage="probe", vendor_blocked=False, static_additions=1, build_failures=0, diagnosed=True, - reference=True): + reference=True, resume=False, run_directory=False): with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) binaries = work / "bin" @@ -58,6 +58,7 @@ def option(name): elif args[0].endswith("build_original_vold.py"): if args[1] in ("sync", "sync-vendor-sources"): option("--source-root").mkdir(exist_ok=True) + (option("--source-root") / ".repo").mkdir(exist_ok=True) else: builds = sum(x[0].endswith("build_original_vold.py") and x[1] in ("probe", "compare") for x in previous) if builds <= int(os.environ["TEST_BUILD_FAILURES"]): @@ -78,6 +79,13 @@ def option(name): "TEST_BUILD_FAILURES": str(build_failures), "TEST_DIAGNOSED": str(int(diagnosed)), }) + if resume: + (work / "source/.repo").mkdir(parents=True) + (work / "source-selection.json").write_text('{"selected_projects": ["system/vold", "system/gsid"]}') + if run_directory: + run_dir = work / "runs/fixture" + run_dir.mkdir(parents=True) + environment["VOLD_RUN_DIR"] = str(run_dir) if stage == "compare" and reference: original = work / "authenticated-reference-fixture" original.write_bytes(b"reference fixture") @@ -112,6 +120,28 @@ def test_probe_needs_no_ota_and_scans_before_compiling(self): actions = [x[1] for x in calls] self.assertLess(actions.index("index"), actions.index("probe")) + def test_vendor_stage_stops_before_toolchains_and_compile(self): + result, calls = self.run_probe(stage="vendor") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([x[1] for x in self.build_calls(calls)], ["sync-vendor-sources"]) + self.assertFalse(any(x[1] in ("index", "probe", "compare") for x in calls)) + + def test_prepare_syncs_and_scans_without_compiling(self): + result, calls = self.run_probe(stage="prepare") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([x[1] for x in self.build_calls(calls)], ["sync-vendor-sources", "sync", "sync"]) + self.assertIn("compilation has not run", result.stdout) + + def test_resume_preserves_selection_and_uses_separate_diagnostics(self): + result, calls = self.run_probe(resume=True, run_directory=True, static_additions=0) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(x[1] == "plan" for x in calls)) + self.assertIn("--resume", self.build_calls(calls)[0]) + for args in calls: + for option in ("--output", "--report"): + if option in args: + self.assertIn("/runs/fixture/", args[args.index(option) + 1]) + def test_static_scans_are_bounded(self): result, calls = self.run_probe(static_additions=99) self.assertEqual(result.returncode, 0, result.stderr) diff --git a/tests/vold_source_closure_test.py b/tests/vold_source_closure_test.py index 58a3e55d..32d53de5 100644 --- a/tests/vold_source_closure_test.py +++ b/tests/vold_source_closure_test.py @@ -223,7 +223,7 @@ def fake_command(argv, cwd, **kwargs): if argv[1] == "init" and "-m" in argv: (repo_dir / "manifest.xml").write_text(f'') if argv[1] == "sync": - for project in argv[4:]: + for project in argv[6:]: (cwd / project).mkdir(parents=True, exist_ok=True) (cwd / project / ".git").mkdir(exist_ok=True) @@ -241,7 +241,8 @@ def fake_command(argv, cwd, **kwargs): original.sync(source, self.baseline, manifest_bytes, 2, more, resume=True) self.assertEqual(len(log), 1) self.assertEqual(log[0][1], "sync") - self.assertEqual(log[0][4:], ["system/update_engine"]) + self.assertEqual(log[0][2:6], ["--current-branch", "--no-tags", "--no-clone-bundle", "-j2"]) + self.assertEqual(log[0][6:], ["system/update_engine"]) log.clear() original.sync(source, self.baseline, manifest_bytes, 2, more, resume=True) self.assertEqual(log, [], "unchanged pinned checkout should not re-run repo sync") diff --git a/tests/vold_vendor_makefiles_test.py b/tests/vold_vendor_makefiles_test.py index 76fcbf5d..8627a549 100644 --- a/tests/vold_vendor_makefiles_test.py +++ b/tests/vold_vendor_makefiles_test.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -"""Ensure absent or unqualified Sony inputs fail before vendor generation.""" +"""Exercise immutable real Git vendor candidates and their provenance gates.""" from __future__ import annotations -from contextlib import redirect_stderr -import io +import copy +import hashlib import json from pathlib import Path +import subprocess import sys import tempfile import unittest @@ -16,143 +17,229 @@ import prepare_vold_vendor_makefiles as prepare # noqa: E402 -class SonyVendorMakefileTest(unittest.TestCase): - def source(self, parent: Path) -> Path: - root = parent / "source" - root.mkdir() - for path in prepare.INPUTS: - file = root / path - file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(f"PINNED TEST CONTENT {path}\n") - return root - - def test_missing_vendor_inputs_report_blocker_without_running_generator(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - source = self.source(root) - receipt = root / "vendor-input-report.json" - before = sorted(p.relative_to(source).as_posix() for p in source.rglob("*")) - with ( - patch.object(prepare, "verify_checkout") as verify, - patch.object(prepare.subprocess, "run") as run, - ): - with self.assertRaisesRegex(prepare.VendorPreparationError, "authenticated proprietary"): - prepare.prepare(source, receipt) - run.assert_not_called() - self.assertEqual(verify.call_args.kwargs, { - "clean": True, - "selected": list(prepare.VENDOR_PREREQUISITE_PROJECTS), - }) - saved = json.loads(receipt.read_text()) - self.assertEqual(saved["status"], "blocked") - self.assertEqual(saved["kind"], "blocked-unqualified-proprietary-vendor-inputs") - self.assertTrue(saved["does_not_extract_blobs"]) - self.assertTrue(saved["does_not_qualify_original_vold"]) - self.assertFalse(saved["makefiles_generated"]) - self.assertFalse(saved["preexisting_vendor_tree_unqualified"]) - self.assertEqual(saved["required_input_example"], prepare.REQUIRED_ELF_EXAMPLE) - self.assertFalse(saved["required_input_example_is_complete_list"]) - self.assertEqual(set(saved["input_sha256"]), set(prepare.INPUTS)) - self.assertEqual( - before, sorted(p.relative_to(source).as_posix() for p in source.rglob("*")), - "diagnosing missing blobs must not create partial .mk/.bp files or bytecode", - ) - - def test_reused_diagnostic_still_fails_and_cannot_mask_changed_source(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - source = self.source(root) - receipt = root / "report.json" - with patch.object(prepare, "verify_checkout"): - for _ in range(2): - with self.assertRaisesRegex(prepare.VendorPreparationError, "authenticated proprietary"): - prepare.prepare(source, receipt) - (source / prepare.INPUTS[0]).write_text("changed input") - with self.assertRaisesRegex(prepare.VendorPreparationError, "not the current blocked-input"): - prepare.prepare(source, receipt) - - def test_preexisting_vendor_elf_and_partial_makefiles_are_not_provenance(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - source = self.source(root) - elf = source / prepare.REQUIRED_ELF_EXAMPLE - elf.parent.mkdir(parents=True) - elf.write_bytes(b"unqualified bytes") - (source / "vendor/sony/pdx235/Android.bp").write_text("partial upstream output") - receipt = root / "report.json" - with ( - patch.object(prepare, "verify_checkout"), - patch.object(prepare.subprocess, "run") as run, - self.assertRaisesRegex(prepare.VendorPreparationError, "provenance contract"), - ): - prepare.prepare(source, receipt) - run.assert_not_called() - self.assertTrue(json.loads(receipt.read_text())["preexisting_vendor_tree_unqualified"]) - self.assertEqual(elf.read_bytes(), b"unqualified bytes") - - def test_old_synthetic_success_receipt_is_never_reused(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - source = self.source(root) - receipt = root / "report.json" - receipt.write_text(json.dumps({ - "kind": "synthetic-probe-vendor-makefiles-not-publisher-provenance", - "generated_makefile_sha256": {}, - })) - with ( - patch.object(prepare, "verify_checkout"), - self.assertRaisesRegex(prepare.VendorPreparationError, "not the current blocked-input"), - ): - prepare.prepare(source, receipt) - - def test_linked_vendor_root_and_receipt_are_rejected(self): - for linked in ("vendor", "receipt"): - with self.subTest(linked=linked), tempfile.TemporaryDirectory() as temp: - root = Path(temp) - source = self.source(root) - receipt = root / "report.json" - other = root / "other" - other.mkdir() - if linked == "vendor": - (source / "vendor").symlink_to(other, target_is_directory=True) - else: - receipt.symlink_to(other / "outside.json") +def git(root, *args): + return subprocess.check_output( + ["git", "-C", str(root), *args], text=True, stderr=subprocess.DEVNULL, + ).strip() + + +def commit_files(root): + git(root, "init", "--quiet") + git(root, "config", "user.name", "Vendor Fixture") + git(root, "config", "user.email", "fixture@example.invalid") + git(root, "add", ".") + git(root, "commit", "--quiet", "-m", "immutable vendor fixture") + return git(root, "rev-parse", "HEAD") + + +class SonyVendorCandidateTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.source = self.root / "source" + self.source.mkdir() + self.receipt = self.root / "vendor-report.json" + for relative in prepare.INPUTS: + target = self.source / relative + target.parent.mkdir(parents=True, exist_ok=True) + # The actual pinned upstream generator is also exercised separately + # against the downloaded mirrors, with its evidence checked in. + target.write_text("pass\n" if relative.endswith(".py") else "# fixture list\n") + (self.source / "tools/extract-utils/extract_utils/fixture_import.py").write_text("pass\n") + source_projects = [] + for relative in prepare.SOURCE_PATHS: + target = self.source / relative + source_projects.append({ + "path": relative, "repository": "source-fixture", + "revision": commit_files(target), + }) + projects = [] + for relative in prepare.VENDOR_PATHS: + target = self.source / relative + target.mkdir(parents=True) + device = target.name + for filename in ("Android.bp", "Android.mk", "BoardConfigVendor.mk", device + "-vendor.mk"): + (target / filename).write_text("# pinned declaration\n") + library = target / "proprietary/vendor/lib64/libcandidate.so" + library.parent.mkdir(parents=True) + library.write_bytes(b"\x7fELF immutable candidate fixture") + revision = commit_files(target) + repository = "TheMuppets/proprietary_vendor_sony_" + device + url = "https://github.com/" + repository + ".git" + git(target, "remote", "add", "origin", url) + records, digest = prepare.inventory(target) + projects.append({ + "path": relative, "repository": repository, "url": url, + "revision": revision, "tree": git(target, "rev-parse", "HEAD^{tree}"), + "file_count": len(records), "content_bytes": sum(item["size"] for item in records), + "inventory_sha256": digest, + "declaration_sha256": {item["path"]: item["sha256"] for item in records + if "/" not in item["path"]}, + }) + self.contract = { + "schema_version": 1, "kind": "lineageos-pdx235-public-vendor-candidate", + "base_build_id": "fixture", "manifest_sha256": "1" * 64, + "source_projects": source_projects, + "source_input_sha256": { + path: hashlib.sha256((self.source / path).read_bytes()).hexdigest() + for path in prepare.INPUTS + }, + "vendor_projects": projects, + } + self.lock = self.root / "lock.json" + self.lock.write_text(json.dumps(self.contract)) + loader = patch.object(prepare, "load_vendor_contract", return_value=self.contract) + lock_path = patch.object(prepare, "LOCK", self.lock) + loader.start() + lock_path.start() + self.addCleanup(loader.stop) + self.addCleanup(lock_path.stop) + + def test_real_git_candidates_and_regeneration_do_not_claim_publisher_identity(self): + with patch.object(prepare, "_git", wraps=prepare._git) as calls: + first = prepare.prepare(self.source, self.receipt) + second = prepare.prepare(self.source, self.receipt) + self.assertEqual(first, second) + self.assertEqual(first["status"], "ready-for-unmodified-build-probe") + self.assertEqual(len(first["projects"]), 2) + self.assertFalse(first["publisher_authenticated"]) + self.assertFalse(first["original_ota_vendor_identity_proven"]) + self.assertFalse(first["byte_identical_vold_qualified"]) + self.assertTrue(first["regeneration"]["declarations_byte_identical"]) + self.assertEqual(json.loads(self.receipt.read_text()), first) + self.assertFalse(any("fetch" in call.args for call in calls.call_args_list), + "reusing exact vendor trees must not download them again") + self.assertEqual(list(self.source.rglob("__pycache__")), []) + + def test_receipt_cannot_authorize_changed_vendor_bytes(self): + prepare.prepare(self.source, self.receipt) + forged = json.loads(self.receipt.read_text()) + forged["publisher_authenticated"] = True + self.receipt.write_text(json.dumps(forged)) + library = self.source / prepare.VENDOR_PATHS[0] / "proprietary/vendor/lib64/libcandidate.so" + library.write_bytes(b"\x7fELF changed bytes") + with self.assertRaisesRegex(prepare.VendorInputError, "unreviewed local content"): + prepare.verify_vendor_checkout(self.source) + + def test_wrong_head_and_wrong_mirror_are_rejected(self): + target = self.source / prepare.VENDOR_PATHS[0] + git(target, "remote", "set-url", "origin", "https://example.invalid/vendor.git") + with self.assertRaisesRegex(prepare.VendorInputError, "origin differs"): + prepare.verify_vendor_checkout(self.source) + git(target, "remote", "set-url", "origin", self.contract["vendor_projects"][0]["url"]) + git(target, "commit", "--allow-empty", "--quiet", "-m", "moved head") + with self.assertRaisesRegex(prepare.VendorInputError, "revision changed"): + prepare.verify_vendor_checkout(self.source) + + def test_ignored_and_extra_files_cannot_enter_candidate_inventory(self): + target = self.source / prepare.VENDOR_PATHS[0] + (target / ".git/info/exclude").write_text("extra.bp\n") + (target / "extra.bp").write_text("unapproved module") + with self.assertRaisesRegex(prepare.VendorInputError, "unreviewed local content"): + prepare.verify_vendor_checkout(self.source) + + def test_unreviewed_vendor_project_is_rejected(self): + (self.source / "vendor/sony/other-device").mkdir() + with self.assertRaisesRegex(prepare.VendorInputError, "unreviewed project paths"): + prepare.verify_vendor_checkout(self.source) + + def test_symlinked_project_and_file_are_rejected(self): + target = self.source / prepare.VENDOR_PATHS[0] + moved = target.with_name("moved") + target.rename(moved) + target.symlink_to(moved, target_is_directory=True) + with self.assertRaisesRegex(prepare.VendorInputError, "linked source or vendor path"): + prepare._fetch_project(self.source, self.contract["vendor_projects"][0]) + target.unlink() + moved.rename(target) + library = target / "proprietary/vendor/lib64/libcandidate.so" + library.unlink() + library.symlink_to(target / "Android.bp") + with self.assertRaisesRegex(prepare.VendorInputError, "regular file"): + prepare.inventory(target) + + def test_lfs_pointer_is_not_accepted_as_real_binary(self): + target = self.source / prepare.VENDOR_PATHS[0] + library = target / "proprietary/vendor/lib64/libcandidate.so" + library.write_bytes(prepare.LFS_POINTER + b"\noid sha256:" + b"0" * 64 + b"\nsize 123\n") + with self.assertRaisesRegex(prepare.VendorInputError, "unresolved Git LFS"): + prepare.inventory(target) + + def test_complete_inventory_digest_checks_more_than_git_head(self): + bad = copy.deepcopy(self.contract["vendor_projects"][0]) + bad["inventory_sha256"] = "0" * 64 + with self.assertRaisesRegex(prepare.VendorInputError, "inventory differs"): + prepare.verify_project(self.source / bad["path"], bad) + + def test_changed_pinned_generator_fails_before_any_acquisition(self): + (self.source / prepare.INPUTS[0]).write_text("raise SystemExit('changed')\n") + with ( + patch.object(prepare, "_fetch_project") as fetch, + self.assertRaisesRegex(prepare.VendorInputError, "unreviewed local content"), + ): + prepare.prepare(self.source, self.receipt) + fetch.assert_not_called() + self.assertFalse(self.receipt.exists()) + + def test_index_flags_cannot_hide_changed_imported_source(self): + project = self.source / "tools/extract-utils" + relative = "extract_utils/fixture_import.py" + for flag in ("assume-unchanged", "skip-worktree"): + with self.subTest(flag=flag): + git(project, "update-index", "--" + flag, relative) + (project / relative).write_text("raise RuntimeError('changed imported source')\n") + self.assertEqual(git(project, "status", "--porcelain"), "") with ( - patch.object(prepare, "verify_checkout"), - self.assertRaisesRegex(prepare.VendorPreparationError, "unsafe|symlink"), + patch.object(prepare, "_fetch_project") as fetch, + self.assertRaisesRegex(prepare.VendorInputError, "assume-unchanged or skip-worktree"), ): - prepare.prepare(source, receipt) - self.assertEqual(list(other.iterdir()), []) - - def test_dirty_pinned_source_prevents_diagnostic_claim(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - source = self.source(root) - receipt = root / "report.json" - with ( - patch.object(prepare, "verify_checkout", side_effect=ValueError("source revision changed")), - self.assertRaisesRegex(ValueError, "source revision changed"), - ): - prepare.prepare(source, receipt) - self.assertFalse(receipt.exists()) - - def test_cli_exits_nonzero_with_retained_blocker_report(self): - with tempfile.TemporaryDirectory() as temp: - root = Path(temp) - source = self.source(root) - receipt = root / "report.json" - stderr = io.StringIO() - with ( - patch.object(prepare, "verify_checkout"), - patch.object(sys, "argv", [ - "prepare_vold_vendor_makefiles.py", "--source-root", str(source), - "--report", str(receipt), - ]), - redirect_stderr(stderr), - ): - self.assertEqual(prepare.main(), 1) - self.assertIn(prepare.REQUIRED_ELF_EXAMPLE, stderr.getvalue()) - self.assertTrue(receipt.is_file()) + prepare.prepare(self.source, self.receipt) + fetch.assert_not_called() + (project / relative).write_text("pass\n") + git(project, "update-index", "--no-" + flag, relative) + self.assertFalse(self.receipt.exists()) + + def test_generator_input_digest_is_independent_of_git_revision(self): + self.contract["source_input_sha256"][prepare.INPUTS[0]] = "0" * 64 + with self.assertRaisesRegex(prepare.VendorInputError, "generator input changed"): + prepare.verify_vendor_checkout(self.source) + + def test_report_cannot_modify_verified_source_or_follow_symlink(self): + with self.assertRaisesRegex(prepare.VendorInputError, "outside the verified source"): + prepare.prepare(self.source, self.source / "report.json") + self.receipt.symlink_to(self.root / "outside.json") + with self.assertRaisesRegex(prepare.VendorInputError, "must not be a symlink"): + prepare.prepare(self.source, self.receipt) + self.assertFalse((self.root / "outside.json").exists()) + + def test_failed_acquisition_never_installs_or_resets_a_vendor_tree(self): + missing = self.source / prepare.VENDOR_PATHS[0] + backup = self.root / "existing-fixture" + missing.rename(backup) + real_git = prepare._git + + def fail_fetch(directory, *args): + if "fetch" in args: + raise prepare.VendorInputError("network unavailable") + return real_git(directory, *args) + + with ( + patch.object(prepare, "_git", side_effect=fail_fetch), + self.assertRaisesRegex(prepare.VendorInputError, "network unavailable"), + ): + prepare._fetch_project(self.source, self.contract["vendor_projects"][0]) + self.assertFalse(missing.exists()) + self.assertEqual(list(self.root.glob("modos-vendor-fetch-*")), []) + self.assertEqual(git(backup, "rev-parse", "HEAD"), self.contract["vendor_projects"][0]["revision"]) + + +class ActualVendorContractTest(unittest.TestCase): + def test_checked_in_lock_is_tied_to_original_manifest_and_public_source(self): + contract = prepare.load_vendor_contract() + self.assertEqual([p["path"] for p in contract["vendor_projects"]], list(prepare.VENDOR_PATHS)) + self.assertEqual(sum(p["file_count"] for p in contract["vendor_projects"]), 1114) + self.assertFalse(contract["provenance"]["publisher_authenticated"]) if __name__ == "__main__":