diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c93ab3b..97668d96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,29 @@ jobs: - name: Test unmodified original vold CI-only baseline run: python3 tests/vold_baseline_ci_test.py + - name: Test actual Repo manifest selection and bounded build inputs + run: python3 tests/vold_original_build_test.py + + - name: Test bounded original Vold source dependency resolver + run: python3 tests/vold_source_closure_test.py + + - name: Test Sony vendor makefile-only regeneration guard + run: python3 tests/vold_vendor_makefiles_test.py + + - name: Test original vold probe ordering and bounded expansion + run: python3 tests/vold_probe_orchestration_test.py + + - name: Generate the provisional original vold source plan + run: python3 src/ci/vold_source_closure.py plan --output "${RUNNER_TEMP}/vold-source-plan.json" + + - name: Retain the provisional original vold source plan + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: vold-source-provisional-plan + path: ${{ runner.temp }}/vold-source-plan.json + if-no-files-found: error + retention-days: 14 + - name: Test pdx235 Custota metadata preparation run: bash tests/pdx235_custota_metadata_test.sh diff --git a/.github/workflows/vold-baseline.yml b/.github/workflows/vold-baseline.yml index e7899bb1..3f2b6871 100644 --- a/.github/workflows/vold-baseline.yml +++ b/.github/workflows/vold-baseline.yml @@ -4,6 +4,16 @@ name: Qualify original LineageOS pdx235 vold (no patch) # once this *baseline-only* workflow is merged to the default branch. on: workflow_dispatch: + inputs: + mode: + description: Static plan, source-only compile probe (not proof), or authenticated byte comparison + type: choice + required: true + default: plan + options: + - plan + - probe + - compare permissions: contents: read @@ -13,7 +23,42 @@ concurrency: cancel-in-progress: false jobs: + dependency_plan: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Checkout immutable manifest and source probe + uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Trace the initial pinned source selection + shell: bash + run: | + set -euo pipefail + python3 src/ci/vold_source_closure.py plan --output "${RUNNER_TEMP}/vold-source-plan.json" + python3 - <<'PY' + import json + import os + from pathlib import Path + info = json.loads((Path(os.environ["RUNNER_TEMP"]) / "vold-source-plan.json").read_text()) + assert info["schema_version"] == 1 + count = len(info["selected_projects"]) + with Path(os.environ["GITHUB_STEP_SUMMARY"]).open("a") as summary: + summary.write(f"### Provisional vold source graph: {count}/1243 manifest projects\n") + summary.write("This is a source probe, **not** a complete Soong closure or a reproduced binary.\n") + summary.write("Sony vendor declarations cannot be regenerated from the public lists alone: the pinned generator reads proprietary ELF inputs.\n") + PY + - name: Upload static source dependency selection + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: vold-source-provisional-plan + path: ${{ runner.temp }}/vold-source-plan.json + retention-days: 14 + prerequisites: + needs: dependency_plan + if: inputs.mode != 'plan' runs-on: ubuntu-24.04 permissions: contents: read @@ -37,6 +82,7 @@ jobs: } unmodified_vold: + name: ${{ inputs.mode == 'probe' && 'Source compile probe — NOT QUALIFIED' || 'Authentic byte-for-byte original vold check' }} needs: prerequisites runs-on: [self-hosted, linux, x64, modos-lineage-large] timeout-minutes: 360 @@ -53,9 +99,15 @@ jobs: - name: Validate isolated host and reserve scratch workspace shell: bash + env: + BASELINE_MODE: ${{ inputs.mode }} run: | set -euo pipefail - for binary in repo avbroot curl unzip openssl sha256sum debugfs git python3 realpath; do + binaries=(repo git python3 realpath) + if [[ "$BASELINE_MODE" == compare ]]; then + binaries+=(avbroot curl unzip openssl sha256sum debugfs) + fi + for binary in "${binaries[@]}"; do command -v "$binary" >/dev/null || { echo "::error::Missing required executable: $binary" exit 1 @@ -68,8 +120,8 @@ jobs: } path="$(realpath -e -- "$path")" free="$(df -PB1 "$path" | awk 'END {print $4}')" - (( free >= 250000000000 )) || { - echo '::error::At least 250 GB free space is required before syncing the complete 1,243-project tree' + (( free >= 100000000000 )) || { + echo '::error::The subset probe requires at least 100 GB as a conservative starting allowance (not a proven maximum)' exit 1 } workspace="$(mktemp -d "$path/modos-original-vold.XXXXXXXX")" @@ -77,6 +129,7 @@ jobs: echo "VOLD_WORK=$workspace" >> "$GITHUB_ENV" - name: Fetch only the immutable official October 16 source OTA + if: inputs.mode == 'compare' shell: bash run: | set -euo pipefail @@ -96,45 +149,35 @@ jobs: --output "$VOLD_WORK/source.zip" "$url" - name: Authenticate official publisher and extract original vold + if: inputs.mode == 'compare' shell: bash run: | set -euo pipefail bash src/ci/extract_original_vold.sh "$VOLD_WORK/source.zip" "$VOLD_WORK/reference" - - name: Synchronize the exact 1,243-project original manifest - shell: bash - run: | - set -euo pipefail - python3 src/ci/build_original_vold.py sync \ - --source-root "$VOLD_WORK/source" --jobs 4 - - name: Build unmodified vold and require byte-exact equality + - name: Probe missing project dependencies and compare original when selected shell: bash + env: + VOLD_PROBE_STAGE: ${{ inputs.mode }} + VOLD_PROBE_JOBS: '4' run: | set -euo pipefail - python3 src/ci/build_original_vold.py compare \ - --source-root "$VOLD_WORK/source" \ - --reference-vold "$VOLD_WORK/reference/original-vold" \ - --output-dir "$VOLD_WORK/baseline" --jobs 4 - python3 - <<'PY' - import json - import os - from pathlib import Path - p = Path(os.environ['VOLD_WORK']) / 'baseline' / 'baseline-report.json' - data = json.loads(p.read_text()) - assert data['byte_identical'] is True - with Path(os.environ['GITHUB_STEP_SUMMARY']).open('a') as report: - report.write('## Unmodified vold reproduced byte-for-byte\n') - report.write(f"Source OTA SHA-256: `{data['official_ota_sha256']}`\n\n") - report.write(f"Original vold SHA-256: `{data['reference_sha256']}`\n") - PY + export VOLD_SOURCE_REF="${VOLD_WORK}/reference/original-vold" + bash src/ci/probe_original_vold.sh - - name: Retain small comparison receipt only + - name: Upload only small dependency diagnostics or comparison receipt if: always() && env.VOLD_WORK != '' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: original-pdx235-vold-reproducibility-report - path: ${{ env.VOLD_WORK }}/baseline/baseline-report.json + name: vold-probe-${{ inputs.mode }}-diagnostics + path: | + ${{ env.VOLD_WORK }}/baseline-*/baseline-report.json + ${{ env.VOLD_WORK }}/source-selection.json + ${{ env.VOLD_WORK }}/sony-vendor-input-report.json + ${{ env.VOLD_WORK }}/source-index-*.json + ${{ env.VOLD_WORK }}/dependency-diagnosis-*.json + ${{ env.VOLD_WORK }}/probe-only.txt if-no-files-found: warn retention-days: 14 diff --git a/docs/evidence/vold-build-setup-review-20261022.json b/docs/evidence/vold-build-setup-review-20261022.json new file mode 100644 index 00000000..02f8c468 --- /dev/null +++ b/docs/evidence/vold-build-setup-review-20261022.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "kind": "pinned-lineage-build-setup-source-review", + "date_utc": "2026-10-22", + "source_evidence": [ + "https://github.com/LineageOS/android_build/blob/73e37454cdac3892c961e554524ccc52b1bfcf00/envsetup.sh", + "https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/build/envsetup.sh", + "https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/build/tools/roomservice.py", + "https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/vars/aosp_target_release" + ], + "findings": [ + "The pinned lunch function requires product, release and variant; its legacy syntax rejects lineage_pdx235-userdebug because the release component is absent", + "The pinned LineageOS breakfast helper reads vars/aosp_target_release; that exact file selects cp2a", + "The pinned lunch function invokes roomservice for dependency discovery; roomservice can synchronize projects not selected by the bounded probe", + "The unchanged _lunch_meat function performs product/release/variant configuration without roomservice", + "The pinned envsetup reads optional unset variables, so bash nounset is incompatible with sourcing it unchanged" + ], + "reviewed_configuration": { + "product": "lineage_pdx235", + "release": "cp2a", + "variant": "userdebug", + "precheck": "check_product lineage_pdx235 cp2a", + "setup": "_lunch_meat lineage_pdx235 cp2a userdebug", + "build_target": "m vold", + "release_file_sha256": "258eaaeb1fc743ee5fd3a0ade38905be3bbfe3d2624d7fd477dd966a68d67eb6", + "automatic_kernel_builds": "fail before kernel synchronization or compilation", + "source_files_modified": false + }, + "validation": { + "tests": "tests/vold_original_build_test.py", + "actual_bash_failure_guards_tested": ["upstream setup failure", "automatic kernel-build request"], + "source_workspace_checked_after_successful_build": true, + "android_compilation_attempted": false, + "byte_identity_established": false + } +} diff --git a/docs/evidence/vold-repo-integration-20261022.json b/docs/evidence/vold-repo-integration-20261022.json new file mode 100644 index 00000000..560373de --- /dev/null +++ b/docs/evidence/vold-repo-integration-20261022.json @@ -0,0 +1,37 @@ +{ + "schema_version": 1, + "kind": "local-repo-manifest-and-subset-sync-integration", + "date_utc": "2026-10-22", + "repo_tool": { + "source": "https://github.com/GerritCodeReview/git-repo", + "revision": "d744eb15f38b9351463900babffd732994e3a046", + "manifest_selection_source": "https://github.com/GerritCodeReview/git-repo/blob/d744eb15f38b9351463900babffd732994e3a046/manifest_xml.py" + }, + "fixture": { + "project_source": "Two tiny local Git repositories and a local manifest repository, all created for this integration check", + "project_paths": ["system/vold", "system/update_engine"], + "manifest_revisions": "Immutable commit SHAs generated by the local fixture", + "repo_launcher": "Official main.py executed against the fixture .repo directory; .repo/repo links to the recorded tool checkout", + "android_sources_downloaded": false, + "android_compilation_attempted": false + }, + "verified_steps": [ + "First actual repo init selects the fixture default.xml at the fixture manifest commit", + "The builder installs the flattened manifest into .repo/manifests/modos-vold-original-20261016.xml", + "Second actual repo init -m modos-vold-original-20261016.xml succeeds and writes the modern include wrapper", + "verify_active_manifest accepts the actual wrapper and byte-identical included manifest", + "repo sync --no-clone-bundle -j1 system/vold succeeds and system/update_engine remains absent", + "Resuming with both selected paths issues only repo sync --no-clone-bundle -j1 system/update_engine", + "Resuming the unchanged two-project selection issues no repo sync", + "Real Git HEAD and clean-worktree verification pass for each checked-out fixture project" + ], + "observed_sync_commands": [ + ["sync", "--no-clone-bundle", "-j1", "system/vold"], + ["sync", "--no-clone-bundle", "-j1", "system/update_engine"] + ], + "limitations": [ + "The first sync attempt was blocked before project checkout by the execution sandbox's local multiprocessing-socket restriction; the same fixture was resumed outside that sandbox restriction", + "The local tool wrapper emitted a repo-tool refresh warning; both recorded project sync commands completed with exit status zero", + "This verifies Repo selection and incremental synchronization behavior only. It is not Android source-closure, vendor-provenance, binary-equality, or device-qualification evidence" + ] +} diff --git a/docs/evidence/vold-vendor-preflight-20261022.json b/docs/evidence/vold-vendor-preflight-20261022.json new file mode 100644 index 00000000..8bcd81eb --- /dev/null +++ b/docs/evidence/vold-vendor-preflight-20261022.json @@ -0,0 +1,43 @@ +{ + "byte_identical_vold_qualified": false, + "command": [ + "python3", + "-B", + "device/sony/pdx235/extract-files.py", + "--regenerate_makefiles" + ], + "date": "2026-10-22", + "environment": { + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": "tools/extract-utils" + }, + "exit_status": 1, + "input_sha256": { + "device/sony/pdx235/extract-files.py": "440af4750e0496627003295606935d3a534f2e758ab42adacd3d5da6e255f0e0", + "device/sony/pdx235/proprietary-files.txt": "93da9a7ac7957afde23ac9c59425947a29b410d01fd5b1fcde2efb5ec155855f", + "device/sony/sm6375-common/extract-files.py": "266ac34e070a5755d513979ad74196b4341c363915b0cfaa24e51d4bf49d6ec8", + "device/sony/sm6375-common/proprietary-files.txt": "8056cdd324881ba179fbac2553e77ddc0d1456ade7ef1e37f9630bb5f6c39a29", + "tools/extract-utils/extract_utils/elf.py": "58d3a6db079185d94e2d283211f73fa71f32bb882618ac47b0feb6d2f5e01d31", + "tools/extract-utils/extract_utils/main.py": "3470e7a43a9788f2699c13d627750b7f5fdc448bf364a48a3a7c86f2b931562d", + "tools/extract-utils/extract_utils/makefiles.py": "2a550851c75a2a32d4d294338df3e73f7d95b4418721653224565d27ba6fa111", + "tools/extract-utils/extract_utils/module.py": "1d1f77c66e05a3f3fea1e710caca7842c38c11b76c1a7bb486724826e72f73ef" + }, + "kind": "sony-vendor-regeneration-public-source-replay", + "original_scripts_modified": false, + "partial_outputs": [ + "vendor/sony/pdx235/Android.bp", + "vendor/sony/pdx235/Android.mk", + "vendor/sony/pdx235/BoardConfigVendor.mk", + "vendor/sony/pdx235/pdx235-vendor.mk" + ], + "proprietary_binaries_present": false, + "public_source_revisions": { + "LineageOS/android_device_sony_pdx235": "0559de69466e1e9af69b121985f592c4f4609283", + "LineageOS/android_device_sony_sm6375-common": "d1d80af1ad9032cb092ff6d85db437709cc54e28", + "LineageOS/android_tools_extract-utils": "ff6589c6b0a4a00c592e98940e9e6b6a50c76569" + }, + "python_version": "Python 3.12.14", + "schema_version": 1, + "stderr": "Traceback (most recent call last):\n File \"device/sony/pdx235/extract-files.py\", line 63, in \n utils.run()\n File \"tools/extract-utils/extract_utils/main.py\", line 219, in run\n self.write_makefiles()\n File \"tools/extract-utils/extract_utils/main.py\", line 160, in write_makefiles\n module.write_makefiles(\n File \"tools/extract-utils/extract_utils/module.py\", line 738, in write_makefiles\n proprietary_file.write_makefiles(self, ctx)\n File \"tools/extract-utils/extract_utils/module.py\", line 215, in write_makefiles\n write_product_packages(\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 683, in write_product_packages\n w(write_lib_package, lib64_tree, packages_ctx)\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 626, in w\n return write_packages_group(\n ^^^^^^^^^^^^^^^^^^^^^\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 565, in write_packages_group\n package_name = fn(file, builder, *args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 382, in write_lib_package\n return write_elfs_package(\n ^^^^^^^^^^^^^^^^^^^\n File \"tools/extract-utils/extract_utils/makefiles.py\", line 266, in write_elfs_package\n machine, bits, libs = get_file_machine_bits_libs(f_path, gen_deps)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"tools/extract-utils/extract_utils/elf.py\", line 43, in get_file_machine_bits_libs\n with open(file_path, 'rb') as f:\n ^^^^^^^^^^^^^^^^^^^^^\nFileNotFoundError: [Errno 2] No such file or directory: 'vendor/sony/pdx235/proprietary/vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so'\n", + "stdout": "Parsing device/sony/pdx235/proprietary-files.txt\nParsing device/sony/sm6375-common/proprietary-files.txt\n" +} diff --git a/docs/vold-baseline-ci.md b/docs/vold-baseline-ci.md index 189361f4..59e846ed 100644 --- a/docs/vold-baseline-ci.md +++ b/docs/vold-baseline-ci.md @@ -17,8 +17,20 @@ authenticated original LineageOS OTA. - Complete 1,243-project release manifest: `locks/lineage-pdx235-20261016-build-manifest.xml`. - Source lock: `locks/lineage-vold-baseline-20261016.json`. -- Exact build: `lineage_pdx235-userdebug`, then `m vold`, without - applying any patch or introducing a replacement binary. +- Exact product/variant: `lineage_pdx235` / `userdebug`, release `cp2a`, + then `m vold`, without applying a patch or introducing a replacement binary. + +At the pinned `build/make` revision, the legacy two-component +`lunch lineage_pdx235-userdebug` syntax is rejected. The pinned +[`vendor/lineage/vars/aosp_target_release`](https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/vars/aosp_target_release) +sets `cp2a`; its exact bytes are hash-checked before build setup. The helper +uses the unchanged `_lunch_meat lineage_pdx235 cp2a userdebug` configuration +function after `check_product`, avoiding the `lunch` wrapper's roomservice +downloads. Automatic kernel builds are blocked. Upstream shell setup runs +without nounset because its optional variables are intentionally unset. +The comparison receipt records the actual release and setup invocation. +The pinned setup and roomservice findings are retained in +[`vold-build-setup-review-20261022.json`](evidence/vold-build-setup-review-20261022.json). These pins were recorded from signed ModOS source-verification runs [#39632433468](https://github.com/0cwa/ModOS/actions/runs/39632433468) @@ -29,54 +41,254 @@ match the independently pinned LineageOS official publisher key, the ZIP certificate must be unique, and `avbroot ota verify` must succeed. The original `system.img` is extracted read-only, then its -`/system/bin/vold` is recovered as the reference executable. A full, -clean, exact-revision LineageOS source checkout builds another unmodified +`/system/bin/vold` is recovered as the reference executable. A clean, +exact-revision, dependency-limited LineageOS checkout builds another unmodified `vold` for the **same** pinned product. The resulting report includes the reference and built SHA-256s, byte counts and first differing offset. Only `byte_identical: true` is success. The workflow fails on a mismatch and uploads a diagnostic JSON receipt, rather than bypassing the check. -## Runner provisioning (required before first real run) - -The GitHub Actions workflow -[`vold-baseline.yml`](../.github/workflows/vold-baseline.yml) is -manual-only (`workflow_dispatch`) and runs on the repository's `main` -branch. A regular 14 GB GitHub-hosted Ubuntu runner is too small. - -The repository owner must provision a **trusted, isolated Linux x64 -self-hosted runner**, labeled `modos-lineage-large`, with a dedicated -scratch filesystem offering at least **250 GB free** and sufficient CPU/RAM -for an Android 17 `m vold` build. It needs LineageOS/Android build -prerequisites, the `repo` tool, Git, Python 3, curl, OpenSSL, unzip, -`debugfs`, and a verified `avbroot` binary supporting `ota verify` and -`ota extract`. Install and verify these dependencies through the host's -normal trusted administration process, not from an unreviewed workflow PR. - -Configure two repository variables: - -- `MODOS_LINEAGE_LARGE_SCRATCH` — absolute path to that real, dedicated - scratch directory on the self-hosted runner. -- `MODOS_VOLD_BASELINE_RUNNER_READY=true` — set only when the owner has - confirmed the runner is isolated, sufficiently provisioned and trusted. - -The workflow's GitHub token has **read-only** repository permission and does -not receive signing keys or publishing secrets. Concurrent baseline jobs -are serialized. A failed preflight stops before allocating the source -workspace; all scratch content is deleted after uploading the small -comparison report. Do not register an untrusted shared self-hosted runner -for this purpose. - -Once the baseline-only PR has passed ordinary CI and been merged, -open **Actions → Qualify original LineageOS pdx235 vold (no patch) → -Run workflow** on `main`. Any missing runner, unavailable pinned OTA, -manifest mismatch, build failure, or byte mismatch is a failure, not an -approval. The full source tree or reference image is never uploaded. +## Dependency-closure strategy + +The original workflow fetched **all 1,243 pinned projects** before issuing +`m vold`. That is unnecessary for a single module, and it made each trial +expensive. Instead, the dependency-analysis recipe in +[`locks/lineage-vold-source-probe-20261016.json`](../locks/lineage-vold-source-probe-20261016.json) +selects **84 initial projects** after source tracing corrected the original +70-project seed. The plan artifact records the exact +current selection and manifest digest. This is a conservative dependency +seed, NOT a complete or proven Soong build graph. + +The additional 14 projects cover verified native/HIDL/AIDL providers, +suspend interfaces, Rust generated flag-library dependencies, and Sony USB +and dataservices namespaces. Three former hints pointed to the wrong project: + +| Module | Verified pinned project | +| --- | --- | +| `libbase` | `system/libbase` | +| `libgsi` | `system/gsid` | +| `libhardware_legacy` | `hardware/libhardware_legacy` | + +The less obvious chain is `vold_flags_c_lib` through Soong's +`cc_aconfig_library` generator to `libaconfig_storage_read_api_cc` and its +Rust bridge/crates. This requires `external/rust/cxx` and +`external/rust/android-crates-io`. Exact-revision evidence for these choices +and the remaining additions is retained in the seed's `source_evidence` +and `reviewed_dependency_chains` fields. This static research saves repeated +failed builds; it does not make the selection complete. + +The source reasoning is grounded in three exact upstream revisions: + +- [`system/vold/Android.bp`](https://github.com/LineageOS/android_system_vold/blob/11d8982f824d52e8122c7aa732207ba66b8f23c8/Android.bp): + `vold` links `libvold`, and its defaults include Binder, cryptography, + AVB, fs_mgr, native libc, and system service libraries. The module also + directly requires `mke2fs`, `vold_prepare_subdirs`, and `fuseMedia.bpf`. +- [`device/sony/pdx235/device.mk`](https://github.com/LineageOS/android_device_sony_pdx235/blob/0559de69466e1e9af69b121985f592c4f4609283/device.mk): + the pdx235 build inherits chipset definitions and a generated proprietary + Sony vendor makefile. +- [`device/sony/sm6375-common/common.mk`](https://github.com/LineageOS/android_device_sony_sm6375-common/blob/d1d80af1ad9032cb092ff6d85db437709cc54e28/common.mk): + the common product also includes proprietary `vendor/sony/sm6375-common`. + Two other Sony vendor BoardConfig files are referenced. + +### Verified proprietary-input blocker + +The exact public manifest does **not** contain `vendor/sony/pdx235` or +`vendor/sony/sm6375-common`. The original assumption that +`--regenerate_makefiles` can generate their declarations using only public +file lists is incorrect at the pinned revisions. This option skips +extraction, but the subsequent +[`write_elfs_package`](https://github.com/LineageOS/android_tools_extract-utils/blob/ff6589c6b0a4a00c592e98940e9e6b6a50c76569/extract_utils/makefiles.py#L256-L271) +still calls +[`get_file_machine_bits_libs`](https://github.com/LineageOS/android_tools_extract-utils/blob/ff6589c6b0a4a00c592e98940e9e6b6a50c76569/extract_utils/elf.py#L38-L57), +which opens each proprietary ELF to determine its architecture and dependencies. + +An actual replay of the unchanged pinned Python sources on 2026-10-22 +failed with `FileNotFoundError` for +`vendor/sony/pdx235/proprietary/vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so`. +Upstream had already written four partial vendor declaration files. Those +files are not successful or qualified preparation. The reproducible command, +source revisions, input hashes and normalized traceback are retained in +[`docs/evidence/vold-vendor-preflight-20261022.json`](evidence/vold-vendor-preflight-20261022.json). +This replay ran Python extraction code only; it did **not** compile Android. + +To avoid downloading toolchains before this known failure, the probe first +syncs exactly three pinned source projects: both Sony device trees and +`tools/extract-utils`. The +[`prepare_vold_vendor_makefiles.py`](../src/ci/prepare_vold_vendor_makefiles.py) +preflight verifies those revisions and writes `sony-vendor-input-report.json` +with `status: blocked` and `makefiles_generated: false`, then exits before +invoking the generator or syncing the remaining seed projects. It rejects +the former synthetic-success receipts; arbitrary existing vendor files +also remain unqualified. + +**Both real build modes are currently blocked here.** A reviewed acquisition +and provenance contract for the required proprietary inputs is needed before +vendor preparation can be enabled. Authenticating the pinned original OTA +may provide a source for some of these bytes, but extraction, fixups, original +build inputs and every required file must be accounted for. The present code +does not claim that this acquisition is implemented. Do not fabricate files, +disable ELF checks or enable missing-dependency flags to bypass this gate. + +The on-disk dependency resolver +[`src/ci/vold_source_closure.py`](../src/ci/vold_source_closure.py) +verifies the complete original manifest and produces a pinned selection. +After the vendor-input gate is resolved and a limited +`repo sync ` succeeds, it scans available `Android.bp` +definitions for a **partial** transitive module graph. Up to three static +scans batch missing providers from reviewed hints before compilation, +syncing only newly selected pinned projects. Unknown, ambiguous and generated +module names remain visible in diagnostic reports. The real +Soong compiler, rather than the static scan, remains authoritative for +conditional/generated modules. Any missing path reported in the compiler +diagnostics is mapped to its **exact existing** project in the official +manifest. Known module names have reviewed initial provider hints; unknown +providers and unpinned vendor paths stop for manual investigation. + +Once prerequisites have passed, within a single CI run, +[`src/ci/probe_original_vold.sh`](../src/ci/probe_original_vold.sh) +automatically repeats up to **three** controlled passes. It caches already +synchronized projects in the same working tree, fetches only newly identified +pinned dependencies, and saves small diagnostic JSON documents showing +unknown modules and other blockers. It never enables +`SOONG_ALLOW_MISSING_DEPENDENCIES`, changes the build target, or applies +the exFAT timeout patch. Any unexpected source revision or modified +checkout fails immediately. + +The build verifier supports Repo's actual single-include `manifest.xml` +wrapper and verifies the included flattened manifest bytes. Local manifest +overrides are rejected. A resumed sync distinguishes actual Git worktrees +from parent or partially created directories. Before compiling, the helper +rejects unrecorded external source trees and inherited missing-dependency +bypasses, fixes the output directory to this workspace and removes any old +final `vold` executable. +It repeats source/workspace verification after a successful compile. +A real Repo integration against two tiny local Git projects confirmed the +include wrapper, selected-only sync, missing-only resume, and no sync when +the selection was unchanged. See +[`vold-repo-integration-20261022.json`](evidence/vold-repo-integration-20261022.json). +This exercises the real Repo tool, not an Android compilation. + +You can test the graph on an ordinary GitHub-hosted runner with **Actions → +Qualify original LineageOS pdx235 vold (no patch) → Run workflow**, +choosing `plan` (the default). This needs **no self-hosted runner or +repository variable** and produces a +`vold-source-provisional-plan` artifact. It does **not** compile or +qualify a binary. + +Ordinary PR CI and `main` CI also run this same plan command and retain the +same `vold-source-provisional-plan` artifact. This provides a low-cost plan +on the merged commit even when the GitHub connector cannot dispatch manual +workflows. These CI steps never sync Android sources or run the compiler. + +## Runner provisioning for source compilation + +For `probe` or `compare`, the workflow still requires a trusted, +isolated self-hosted Linux x64 runner labeled `modos-lineage-large`. +Provide a dedicated scratch filesystem with **at least 100 GB free as an +experimental starting allowance**, sufficient CPU/RAM, the Android 17 +build prerequisites, and `repo`, `git`, `python3`, and `realpath`. +The authenticated `compare` mode additionally needs `avbroot`, `debugfs`, +`unzip`, `curl`, `openssl`, and `sha256sum`. One iteration may require +more storage. The 100 GB threshold is **not an empirically measured maximum +or proof** that the selected dependency graph is complete. + +Set repository Actions variables: + +- `MODOS_LINEAGE_LARGE_SCRATCH` — an absolute, real, isolated scratch + directory on the trusted self-hosted runner. +- `MODOS_VOLD_BASELINE_RUNNER_READY=true` — only when that runner has been + provisioned and independently checked. + +Use `probe` to attempt the smaller pinned source checkout and compile +unmodified `m vold` **without downloading the official source OTA**. +A successfully compiled probe is **not** binary-equality evidence; its +workflow job explicitly labels it as unqualified. It is useful for +discovering missing modules, vendor inputs and toolchains cheaply. +At the current reviewed revision it stops at the known vendor-input gate +before downloading the large seed or invoking `m vold`. + +Only after a successful, reviewed `probe`, use `compare` for a real proof. +The compare workflow downloads the OTA before the source probe; running it +while the known vendor gate is blocked would waste that download. +It first reauthenticates the exact +publisher-signed official 2026-10-16 OTA, extracts the original +`/system/bin/vold`, and invokes the same bounded dependency solver. +**Only** a real `byte_identical: true` report with equal original and +built hashes qualifies this baseline. If the binaries differ, it emits a +diagnostic comparison receipt and fails. + +### Hosted-runner alternatives and observed access + +Checked on 2026-10-22: GitHub documents larger Ubuntu x64 runners with +4 CPUs / 16 GB RAM / 150 GB storage and 8 CPUs / 32 GB RAM / 300 GB storage. +Those are plausible experimental build hosts, not measured requirements +for this incomplete dependency closure. The 100 GB free-space gate does not +prove either machine has enough RAM or working space after tool installation. +See GitHub's [larger runner specifications](https://docs.github.com/en/actions/reference/runners/larger-runners). + +GitHub's managed larger runners require an organization or enterprise on +Team or Enterprise Cloud; `0cwa/ModOS` is currently owned by a user account. +They cannot be assumed available merely by changing `runs-on` to an invented +label. An actual organization runner with repository access, or a dedicated +trusted self-hosted Linux x64 machine, must be provisioned and checked first. +See [GitHub-hosted runner availability](https://docs.github.com/en/actions/reference/runners/github-hosted-runners) +and [managing larger runners](https://docs.github.com/en/actions/how-tos/manage-runners/larger-runners/manage-larger-runners). +Standard Ubuntu runners advertise only 14 GB storage and remain suitable +for this plan/unit-test workload, not the current Android build preflight. + +The last original-baseline attempt +[#39699955476](https://github.com/0cwa/ModOS/actions/runs/39699955476) +failed its hosted prerequisites job with both readiness variables empty; +the self-hosted build job was skipped. This is evidence about that run, +not proof of the current runner inventory. The connected GitHub actions +expose repository changes, CI inspection and reruns, but no workflow dispatch, +runner inventory or variable mutation. No runner or configuration change +has been claimed on that basis. + +For an owner with the GitHub CLI configured, the explicit manual plan is: + +```sh +gh workflow run vold-baseline.yml --repo 0cwa/ModOS --ref main -f mode=plan +``` + +After resolving the authenticated vendor-input contract and verifying the +trusted runner, set the two documented repository variables and dispatch +`-f mode=probe`. Inspect its source selection and diagnostics. Dispatch +`-f mode=compare` only once that actual compile succeeds. The connector's +missing dispatch capability does not justify changing any qualification gate. + +From a checked-out repository, local inspection is also possible without +compiling or obtaining the proprietary vendor files: + +```sh +python3 src/ci/vold_source_closure.py plan --output /tmp/vold-projects.json +# After a limited repo sync on a trusted build host: +python3 src/ci/vold_source_closure.py index \ + --source-root /path/to/pinned-source \ + --selection /tmp/vold-projects.json \ + --output /tmp/vold-blueprint-graph.json +# After an unsuccessful m vold build: +python3 src/ci/vold_source_closure.py diagnose \ + --selection /tmp/vold-projects.json \ + --log /path/to/build-failure.log \ + --output /tmp/vold-diagnosis.json \ + --next-selection /tmp/vold-projects-expanded.json +``` + +All selection changes must remain part of the exact original manifest. +The build helper supports `sync --projects-file ... --resume` to fetch +additional pinned projects without repeating an entire sync. + +The workflow uses a read-only GitHub token, never receives release or +signing secrets, uploads diagnostics/comparison receipts only, and clears +the large checkout and original OTA after completion. Do not use an +untrusted shared runner. ## After the first authentic success Review the actual run URL and download its -`original-pdx235-vold-reproducibility-report` artifact. Record the exact +`vold-probe-compare-diagnostics` artifact. Record the exact source, manifest and reference SHA-256, plus the CI commit and run ID. Verify the report contains `byte_identical: true` with equal SHA-256s. Independent assessment of the real reference extraction and Android diff --git a/locks/lineage-vold-baseline-20261016.json b/locks/lineage-vold-baseline-20261016.json index e87c2ed9..db86dfb5 100644 --- a/locks/lineage-vold-baseline-20261016.json +++ b/locks/lineage-vold-baseline-20261016.json @@ -30,6 +30,15 @@ }, "build": { "target": "lineage_pdx235-userdebug", + "product": "lineage_pdx235", + "release": "cp2a", + "variant": "userdebug", + "setup_command": "_lunch_meat lineage_pdx235 cp2a userdebug", + "release_config": { + "path": "vendor/lineage/vars/aosp_target_release", + "sha256": "258eaaeb1fc743ee5fd3a0ade38905be3bbfe3d2624d7fd477dd966a68d67eb6", + "source_url": "https://github.com/LineageOS/android_vendor_lineage/blob/87daf527b0288b60c9cc07c05a5f5aad1660c234/vars/aosp_target_release" + }, "command": "m vold", "artifact": "out/target/product/pdx235/system/bin/vold", "elf_class": 64, diff --git a/locks/lineage-vold-source-probe-20261016.json b/locks/lineage-vold-source-probe-20261016.json new file mode 100644 index 00000000..03cd098c --- /dev/null +++ b/locks/lineage-vold-source-probe-20261016.json @@ -0,0 +1,260 @@ +{ + "schema_version": 1, + "kind": "unmodified-vold-provisional-source-probe", + "manifest_sha256": "d4ded1ca84de399497b68d7ac006808059b6be9cf76725029420a3495a2e5468", + "base_build_id": "lineage-24.0-20261016-nightly-pdx235-signed", + "source_revision": "11d8982f824d52e8122c7aa732207ba66b8f23c8", + "source_evidence": [ + "https://github.com/LineageOS/android_system_vold/blob/11d8982f824d52e8122c7aa732207ba66b8f23c8/Android.bp", + "https://github.com/LineageOS/android_device_sony_pdx235/blob/0559de69466e1e9af69b121985f592c4f4609283/device.mk", + "https://github.com/LineageOS/android_device_sony_sm6375-common/blob/d1d80af1ad9032cb092ff6d85db437709cc54e28/common.mk", + "https://android.googlesource.com/platform/hardware/libhardware_legacy/+/2a073cd42ec822914ed4e680e7b708485d3a172b/Android.bp", + "https://android.googlesource.com/platform/system/libbase/+/fef4173e19d97cd4b4fc09a06bccbf800d0d3937/Android.bp", + "https://android.googlesource.com/platform/system/gsid/+/35a92d40f529b605fad8ff622691e76901044cc8/Android.bp", + "https://android.googlesource.com/platform/system/libhidl/+/061055fc4dad78e5d25a77d9217001794656f16b/Android.bp", + "https://android.googlesource.com/platform/system/server_configurable_flags/+/5069ba739faebf91ae72096577ae17e13061af72/libflags/Android.bp", + "https://android.googlesource.com/platform/system/libhwbinder/+/2f5282448223b97c541181a5b94a05c2ac17ff17/Android.bp", + "https://github.com/LineageOS/android_system_libfmq/blob/f727482ff0c6a26b4c64c5106c62c176415b2d81/Android.bp", + "https://android.googlesource.com/platform/system/hardware/interfaces/+/9c3469aa3d4aeec5b2bc8e8ae8fdb4437dab4edb/suspend/aidl/default/Android.bp", + "https://android.googlesource.com/platform/system/hardware/interfaces/+/9c3469aa3d4aeec5b2bc8e8ae8fdb4437dab4edb/suspend/aidl/Android.bp", + "https://android.googlesource.com/platform/system/tools/hidl/+/5d0e05e6535fea94a674a359102bdd42944714f4/Android.bp", + "https://android.googlesource.com/platform/system/tools/aidl/+/9a94664f74536bf28010207fd1804b04c6435e2c/Android.bp", + "https://github.com/LineageOS/android_build_soong/blob/183e59578606687d66096cbce86cb50c27f4e9d7/cc/gen.go", + "https://github.com/LineageOS/android_build_soong/blob/183e59578606687d66096cbce86cb50c27f4e9d7/aconfig/codegen/cc_aconfig_library.go", + "https://github.com/LineageOS/android_build/blob/73e37454cdac3892c961e554524ccc52b1bfcf00/tools/aconfig/aconfig_storage_read_api/Android.bp", + "https://android.googlesource.com/platform/external/rust/cxx/+/41972f6ebbe9c21eeed3fdf5c1b0805f2c8b824c/Android.bp", + "https://android.googlesource.com/platform/external/rust/android-crates-io/+/1f5e95cd6e996995023c9c0f7df0f15a8a43269e/crates/anyhow/Android.bp", + "https://android.googlesource.com/platform/external/rust/android-crates-io/+/1f5e95cd6e996995023c9c0f7df0f15a8a43269e/crates/memmap2/Android.bp", + "https://android.googlesource.com/platform/external/rust/android-crates-io/+/1f5e95cd6e996995023c9c0f7df0f15a8a43269e/crates/thiserror/Android.bp", + "https://github.com/LineageOS/android_system_core/blob/7c3f32fe89c52b4176fd288ba5bb4f907809c202/libcrypto_utils/Android.bp", + "https://android.googlesource.com/platform/system/libvintf/+/df89ad736e14ac375f61d50fa543ae7cd746a140/Android.bp", + "https://android.googlesource.com/platform/external/tinyxml2/+/33ba115074c80a720826efffab1937d5b355c3b8/Android.bp", + "https://android.googlesource.com/platform/system/tools/sysprop/+/deead2d06508faeca98c236feea7e59bb4c43c8c/Android.bp", + "https://github.com/LineageOS/android_device_sony_pdx235/blob/0559de69466e1e9af69b121985f592c4f4609283/extract-files.py", + "https://github.com/LineageOS/android_device_sony_sm6375-common/blob/d1d80af1ad9032cb092ff6d85db437709cc54e28/extract-files.py", + "https://github.com/LineageOS/android_tools_extract-utils/blob/ff6589c6b0a4a00c592e98940e9e6b6a50c76569/extract_utils/makefiles.py" + ], + "candidate_groups": { + "core_build_toolchain": [ + "android", + "art", + "bionic", + "build/make", + "build/soong", + "build/blueprint", + "build/release", + "build/pesto", + "libcore", + "prebuilts/clang/host/linux-x86", + "prebuilts/build-tools", + "prebuilts/go/linux-x86", + "prebuilts/rust", + "prebuilts/jdk/jdk21", + "prebuilts/bazel/common", + "prebuilts/ndk" + ], + "vold_direct_and_native": [ + "bootable/libbootloader", + "bootable/recovery", + "external/avb", + "external/boringssl", + "external/dtc", + "external/e2fsprogs", + "external/expat", + "external/f2fs-tools", + "external/fmtlib", + "external/googletest", + "external/icu", + "external/libcap", + "external/libcxx", + "external/protobuf", + "external/selinux", + "external/sqlite", + "external/zlib", + "frameworks/av", + "frameworks/base", + "frameworks/hardware/interfaces", + "frameworks/native", + "hardware/interfaces", + "hardware/libhardware", + "system/bpf", + "system/bpfprogs", + "system/core", + "system/extras", + "system/fs/casefolding_remover", + "system/fs/fs_mgr", + "system/incremental_delivery", + "system/logging", + "system/memory/libion", + "system/memory/libmeminfo", + "system/memory/libwrapfd", + "system/security", + "system/sepolicy", + "system/vold" + ], + "lineage_sony_product": [ + "device/sony/pdx235", + "device/sony/sm6375-common", + "device/lineage/sepolicy", + "device/qcom/sepolicy_vndr/legacy-um", + "hardware/lineage/interfaces", + "hardware/sony", + "hardware/qcom-caf/common", + "hardware/qcom-caf/sm8350/audio", + "hardware/qcom-caf/wlan", + "vendor/lineage", + "vendor/qcom/opensource/commonsys-intf/display", + "vendor/qcom/opensource/commonsys/display", + "vendor/qcom/opensource/display", + "tools/extract-utils", + "vendor/qcom/opensource/dataservices", + "vendor/qcom/opensource/usb" + ], + "module_service_api": [ + "packages/modules/adb", + "packages/modules/Connectivity", + "packages/modules/ArtPrebuilt" + ], + "verified_native_and_generated_transitive": [ + "external/rust/android-crates-io", + "external/rust/cxx", + "hardware/libhardware_legacy", + "system/gsid", + "system/hardware/interfaces", + "system/libbase", + "system/libfmq", + "system/libhidl", + "system/libhwbinder", + "system/server_configurable_flags", + "system/tools/aidl", + "system/tools/hidl" + ] + }, + "direct_module_to_project_hints": { + "android.security.maintenance-ndk": "system/security", + "libavb": "external/avb", + "libfs_mgr": "system/fs/fs_mgr", + "libdm": "system/fs/fs_mgr", + "libcasefoldingremover": "system/fs/casefolding_remover", + "libbootloader_message": "bootable/libbootloader", + "libvold_binder": "system/vold", + "vold_flags_c_lib": "system/vold", + "libvold": "system/vold", + "android.system.vold-V1-cpp": "system/vold", + "libincremental_aidl-cpp": "frameworks/base", + "libbinder": "frameworks/native", + "libbinder_ndk": "frameworks/native", + "libcutils": "system/core", + "libutils": "system/core", + "libbase": "system/libbase", + "libselinux": "external/selinux", + "libcrypto": "external/boringssl", + "liblog": "system/logging", + "liblogwrap": "system/core", + "libhardware": "hardware/libhardware", + "libhardware_legacy": "hardware/libhardware_legacy", + "libgsi": "system/gsid", + "libext2_uuid": "external/e2fsprogs", + "libext4_utils": "system/extras", + "libf2fs_sparseblock": "system/extras", + "libfec": "system/fs/fs_mgr", + "libfs_avb": "system/fs/fs_mgr", + "libmeminfo_headers": "system/memory/libmeminfo", + "libboot_control_client": "hardware/interfaces", + "android.hardware.boot@1.0": "hardware/interfaces", + "android.hardware.boot-V1-ndk": "hardware/interfaces", + "libkeymint_support": "hardware/interfaces", + "libsysutils": "system/core", + "libincfs": "system/incremental_delivery", + "fuseMedia.bpf": "system/bpfprogs", + "mke2fs": "external/e2fsprogs", + "libfec_rs": "system/extras", + "libasync_safe": "bionic", + "libhidlbase": "system/libhidl", + "libhidlbase-combined-impl": "system/libhidl", + "server_configurable_flags": "system/server_configurable_flags", + "libhwbinder-impl-internal": "system/libhwbinder", + "libhwbinder-impl-shared-libs": "system/libhwbinder", + "libfmq-base": "system/libfmq", + "libfmq": "system/libfmq", + "hidl-module-defaults": "system/tools/hidl", + "hidl-gen": "system/tools/hidl", + "aidl": "system/tools/aidl", + "aidl-cpp": "system/tools/aidl", + "system_suspend_defaults": "system/hardware/interfaces", + "android.system.suspend-V1-ndk": "system/hardware/interfaces", + "suspend_service_flags_c_lib": "system/hardware/interfaces", + "libcrypto_utils": "system/core", + "libaconfig_storage_read_api_cc": "build/make", + "aconfig_lib_cc_static_link.defaults": "build/make", + "libcxx": "external/rust/cxx", + "cxx-bridge-header": "external/rust/cxx", + "libanyhow": "external/rust/android-crates-io", + "libmemmap2": "external/rust/android-crates-io", + "libthiserror": "external/rust/android-crates-io", + "libvintf": "system/libvintf", + "libtinyxml2": "external/tinyxml2", + "sysprop_cpp": "system/tools/sysprop" + }, + "known_unpinned_vendor_inputs": [ + "vendor/sony/pdx235/pdx235-vendor.mk", + "vendor/sony/pdx235/BoardConfigVendor.mk", + "vendor/sony/sm6375-common/sm6375-common-vendor.mk", + "vendor/sony/sm6375-common/BoardConfigVendor.mk" + ], + "qualification_note": "Reviewed seed and provider hints, not a complete Soong source closure. Static scanning omits variables and implicit generators and may include inactive product branches. Repo sync and build errors may discover additional immutable manifest projects. Missing proprietary vendor inputs cannot be fabricated or replaced with generated declarations lacking provenance.", + "reviewed_dependency_chains": [ + { + "consumer": "vold_default_libs", + "providers": [ + "system/libbase", + "system/gsid", + "hardware/libhardware_legacy", + "system/libhidl" + ], + "finding": "Pinned Android.bp files place libbase, libgsi, libhardware_legacy and libhidlbase in distinct manifest projects; the previous hints pointed three of these at other already-selected projects." + }, + { + "consumer": "libhidlbase-combined-impl", + "providers": [ + "system/libhwbinder", + "system/libfmq", + "system/tools/hidl" + ], + "finding": "Pinned HIDL defaults require libhwbinder-impl-internal, libfmq-base and hidl-module-defaults; generated HIDL sources also require hidl-gen." + }, + { + "consumer": "libhardware_legacy -> libpower_defaults", + "providers": [ + "system/hardware/interfaces", + "system/server_configurable_flags" + ], + "finding": "Pinned legacy hardware defaults inherit system_suspend_defaults and use android.system.suspend-V1-ndk; the suspend defaults also require server_configurable_flags." + }, + { + "consumer": "libvold_binder AIDL sources", + "providers": [ + "system/tools/aidl" + ], + "finding": "Pinned Soong cc/gen.go invokes aidl-cpp for AIDL source generation; the tool is declared in system/tools/aidl." + }, + { + "consumer": "vold_flags_c_lib -> libaconfig_storage_read_api_cc", + "providers": [ + "build/make", + "external/rust/cxx", + "external/rust/android-crates-io" + ], + "finding": "Pinned Soong cc_aconfig_library adds the native aconfig reader. Its C++ bridge requires Rust libcxx, libanyhow, libmemmap2 and libthiserror; the latter three are in the single pinned android-crates-io project. This implicit generator edge is not visible in vold Android.bp alone." + }, + { + "consumer": "Sony product and vendor namespace declarations", + "providers": [ + "vendor/qcom/opensource/dataservices", + "vendor/qcom/opensource/usb" + ], + "finding": "Pinned extract-files.py namespace_imports reference dataservices; pinned common.mk adds the usb/etc namespace. These public projects do not provide the proprietary Sony vendor blobs." + } + ], + "vendor_prerequisite_note": "Pinned extract-utils --regenerate_makefiles still opens proprietary ELF files to determine machine, bitness and DT_NEEDED. Public proprietary-file lists alone cannot regenerate usable vendor declarations. A real replay fails at the missing pdx235 proprietary/vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so input; generated files from a failed replay are partial and unqualified. The workflow must stop before large source downloads until an authenticated vendor-input or exact declaration provenance plan is reviewed." +} diff --git a/src/ci/build_original_vold.py b/src/ci/build_original_vold.py index c12e5982..f5710e14 100644 --- a/src/ci/build_original_vold.py +++ b/src/ci/build_original_vold.py @@ -22,6 +22,10 @@ ROOT = Path(__file__).resolve().parents[2] LOCK = ROOT / "locks/lineage-vold-baseline-20261016.json" +MANIFEST_NAME = "modos-vold-original-20261016.xml" +VENDOR_PREREQUISITE_PROJECTS = ( + "device/sony/pdx235", "device/sony/sm6375-common", "tools/extract-utils", +) class BuildError(ValueError): @@ -47,6 +51,16 @@ def load_contract() -> tuple[dict, bytes]: raise BuildError("pinned source manifest digest differs") if lock["build"]["target"] != "lineage_pdx235-userdebug" or lock["build"]["command"] != "m vold": raise BuildError("baseline build target differs from reviewed source") + build = lock["build"] + if ( + (build.get("product"), build.get("release"), build.get("variant")) + != ("lineage_pdx235", "cp2a", "userdebug") + or build.get("setup_command") != "_lunch_meat lineage_pdx235 cp2a userdebug" + or build.get("release_config", {}).get("path") != "vendor/lineage/vars/aosp_target_release" + or build.get("release_config", {}).get("sha256") + != "258eaaeb1fc743ee5fd3a0ade38905be3bbfe3d2624d7fd477dd966a68d67eb6" + ): + raise BuildError("baseline product/release/variant differs from pinned LineageOS setup") projects = project_map(manifest) if len(projects) != 1243: raise BuildError("unexpected number of projects in source manifest") @@ -82,18 +96,93 @@ def command(args: list[str], root: Path, **kwargs: object) -> subprocess.Complet return subprocess.run(args, cwd=root, check=True, **kwargs) -def verify_checkout(source: Path, manifest: bytes, *, clean: bool = False) -> None: +def verified_selection(selection_file: Path | None, manifest: bytes) -> list[str] | None: + if selection_file is None: + return None + if selection_file.is_symlink() or not selection_file.is_file(): + raise BuildError("dependency selection must be a real checked-in or generated JSON file") + from vold_source_closure import verified_contract, validate_selection + seed, projects, _baseline = verified_contract() + if hashlib.sha256(manifest).hexdigest() != seed["manifest_sha256"]: + raise BuildError("dependency selection does not match the authenticated manifest") + try: + selection = json.loads(selection_file.read_bytes()) + except (ValueError, UnicodeError) as exc: + raise BuildError("dependency selection is not valid JSON") from exc + return validate_selection(selection, projects, seed) + + +def verify_active_manifest(source: Path, manifest: bytes) -> None: + """Validate Repo's selected file, including its modern include wrapper. + + Repo's Link() writes .repo/manifest.xml as a single include, not a copy + of the selected manifest. Check that include without permitting another + manifest, a local override, or a linked file outside this checkout. + """ + repo_dir = source / ".repo" + if repo_dir.is_symlink() or not repo_dir.is_dir(): + raise BuildError("repo metadata directory is missing or unsafe") + for local in (repo_dir / "local_manifest.xml", repo_dir / "local_manifests"): + if local.is_symlink() or (local.exists() and ( + not local.is_dir() or any(local.iterdir()) + )): + raise BuildError("local manifest overrides are forbidden for the pinned baseline") + active = repo_dir / "manifest.xml" + if not active.is_file(): + raise BuildError("repo selected manifest is unavailable") + if not active.is_symlink() and active.read_bytes() == manifest: + # Support an existing legacy checkout with the exact flattened file. + return + manifest_dir = repo_dir / "manifests" + pinned = manifest_dir / MANIFEST_NAME + if ( + manifest_dir.is_symlink() or not manifest_dir.is_dir() + or pinned.is_symlink() or not pinned.is_file() + or pinned.read_bytes() != manifest + ): + raise BuildError("repo selected manifest differs from the exact official snapshot") + if active.is_symlink(): + if active.resolve(strict=True) != pinned.resolve(strict=True): + raise BuildError("repo manifest link does not select the pinned snapshot") + return + try: + wrapper = ET.fromstring(active.read_bytes()) + except ET.ParseError as exc: + raise BuildError("invalid repo manifest selection wrapper") from exc + children = list(wrapper) + if ( + wrapper.tag != "manifest" or wrapper.attrib or (wrapper.text or "").strip() + or len(children) != 1 or children[0].tag != "include" + or children[0].attrib != {"name": MANIFEST_NAME} + or list(children[0]) or (children[0].text or "").strip() + or (children[0].tail or "").strip() + ): + raise BuildError("repo manifest wrapper must select only the exact pinned snapshot") + + +def verify_checkout( + source: Path, manifest: bytes, *, clean: bool = False, + selected: list[str] | None = None, +) -> None: repo = shutil.which("repo") if repo is None: raise BuildError("repo is required to verify complete source revisions") - result = subprocess.run( - [repo, "manifest", "-r"], cwd=source, check=True, capture_output=True - ) + source = source.resolve(strict=True) expected = project_map(manifest) - if project_map(result.stdout) != expected: - raise BuildError("source checkout does not match the 1,243 pinned revisions") + verify_active_manifest(source, manifest) + if selected is None: + # The legacy full-checkout gate remains unchanged for fully synced trees. + result = subprocess.run( + [repo, "manifest", "-r"], cwd=source, check=True, capture_output=True + ) + if project_map(result.stdout) != expected: + raise BuildError("complete source checkout does not match pinned revisions") + else: + if not selected or any(relpath not in expected for relpath in selected): + raise BuildError("unreviewed project outside the immutable manifest") if clean: - for relpath, (_name, revision) in sorted(expected.items()): + to_verify = expected if selected is None else {p: expected[p] for p in selected} + for relpath, (_name, revision) in sorted(to_verify.items()): path = source / relpath if path.is_symlink() or not path.is_dir() or not path.resolve().is_relative_to(source): raise BuildError(f"source project is missing or unsafe: {relpath}") @@ -108,77 +197,198 @@ def verify_checkout(source: Path, manifest: bytes, *, clean: bool = False) -> No raise BuildError(f"source project has unreviewed local content: {relpath}") -def sync(source: Path, lock: dict, manifest: bytes, jobs: int) -> None: - if source.is_symlink() or (source.exists() and ( - not source.is_dir() or any(source.iterdir()) - )): - raise BuildError("source directory must be new or empty and not a symlink") +def verify_source_workspace(source: Path, manifest: bytes, selected: list[str] | None) -> None: + """Reject unrecorded source outside clean pinned projects before building.""" + approved = set(project_map(manifest) if selected is None else selected) + exported = {} + for project in ET.fromstring(manifest).findall("project"): + project_path = project.get("path", project.get("name", "")) + if project_path not in approved: + continue + for entry in project: + if entry.tag in ("linkfile", "copyfile"): + exported[entry.attrib["dest"]] = ( + entry.tag, source / project_path / entry.attrib["src"] + ) + parents = { + parent.as_posix() + for path in approved | set(exported) + for parent in Path(path).parents if parent != Path(".") + } + pending = [source] + while pending: + for path in pending.pop().iterdir(): + relative = path.relative_to(source).as_posix() + if relative in approved: + continue # verify_checkout checks each entire Git worktree. + if relative in exported: + kind, original = exported[relative] + if kind == "linkfile": + if not path.is_symlink() or path.resolve() != original.resolve(): + raise BuildError(f"manifest linkfile differs from pinned source: {relative}") + elif path.is_symlink() or not path.is_file() or path.read_bytes() != original.read_bytes(): + raise BuildError(f"manifest copyfile differs from pinned source: {relative}") + continue + if relative in (".repo", "out"): + if path.is_symlink() or not path.is_dir(): + raise BuildError(f"build workspace directory is unsafe: {relative}") + continue + if relative in parents and path.is_dir() and not path.is_symlink(): + pending.append(path) + continue + raise BuildError(f"unrecorded source outside selected pinned projects: {relative}") + + +def sync( + source: Path, lock: dict, manifest: bytes, jobs: int, + selected: list[str] | None = None, resume: bool = False, +) -> None: + if source.is_symlink() or source.exists() and not source.is_dir(): + raise BuildError("source directory must be a real directory") + if resume: + verify_active_manifest(source, manifest) + elif source.exists() and any(source.iterdir()): + raise BuildError("source directory must be new or empty; use --resume to add pinned dependencies") repo = shutil.which("repo") if repo is None: raise BuildError("repo tool is unavailable on source build runner") source.mkdir(parents=True, exist_ok=True) - manifest_commit = lock["projects"]["manifest"]["revision"] - command([ - repo, "init", "-u", "https://github.com/LineageOS/android.git", - "-b", manifest_commit - ], source) - # Do not overwrite .repo/manifest.xml: Repo manages this file, and - # recent versions regenerate it during subsequent sync/init operations. - # Instead install the immutable flattened manifest in Repo's manifest - # checkout and formally select it with -m, as supported by Repo. - manifest_name = "modos-vold-original-20261016.xml" - manifest_dir = source / ".repo/manifests" - if manifest_dir.is_symlink() or not manifest_dir.is_dir(): - raise BuildError("repo manifests checkout is unavailable or unsafe") - pinned_manifest = manifest_dir / manifest_name - if pinned_manifest.exists() or pinned_manifest.is_symlink(): - raise BuildError("preexisting untrusted local pinned manifest") - pinned_manifest.write_bytes(manifest) - command([ - repo, "init", "-u", "https://github.com/LineageOS/android.git", - "-b", manifest_commit, "-m", manifest_name - ], source) - active_manifest = source / ".repo/manifest.xml" - if not active_manifest.exists() or active_manifest.is_dir(): - raise BuildError("repo did not activate the immutable full manifest") - if active_manifest.read_bytes() != manifest: - raise BuildError("repo's active manifest differs from the pinned source") - command([repo, "sync", "--no-clone-bundle", f"-j{jobs}"], source) - verify_checkout(source, manifest, clean=True) + source = source.resolve(strict=True) + if not resume: + manifest_commit = lock["projects"]["manifest"]["revision"] + command([ + repo, "init", "-u", "https://github.com/LineageOS/android.git", + "-b", manifest_commit, + ], source) + # Do not overwrite .repo/manifest.xml: Repo manages this file, and + # recent versions regenerate it during subsequent sync/init operations. + # Instead install the immutable flattened manifest in Repo's manifest + # checkout and formally select it with -m, as supported by Repo. + manifest_dir = source / ".repo/manifests" + if manifest_dir.is_symlink() or not manifest_dir.is_dir(): + raise BuildError("repo manifests checkout is unavailable or unsafe") + pinned_manifest = manifest_dir / MANIFEST_NAME + if pinned_manifest.exists() or pinned_manifest.is_symlink(): + raise BuildError("preexisting untrusted local pinned manifest") + pinned_manifest.write_bytes(manifest) + command([ + repo, "init", "-u", "https://github.com/LineageOS/android.git", + "-b", manifest_commit, "-m", MANIFEST_NAME + ], source) + verify_active_manifest(source, manifest) + # Explicit project-path sync prevents all 1,243 projects being fetched. + # On resume, preverify existing project HEADs and cleanliness, then fetch + # *only newly selected* missing projects. Preserve the original Soong out + # tree and all previously fetched Git objects across source expansions. + to_sync = selected + if resume: + assert selected is not None + present = [] + missing = [] + for project in selected: + path = source / project + if path.is_symlink() or (path.exists() and not path.is_dir()): + raise BuildError(f"previously synced project path is unsafe: {project}") + # Parent directories and interrupted checkout directories are not + # fetched projects. Repo worktrees have a .git file/link/directory. + git_marker = path / ".git" + (present if git_marker.exists() or git_marker.is_symlink() else missing).append(project) + if present: + verify_checkout(source, manifest, clean=True, selected=present) + to_sync = missing + if to_sync is None or to_sync: + command([repo, "sync", "--no-clone-bundle", f"-j{jobs}", *(to_sync or [])], source) + verify_checkout(source, manifest, clean=True, selected=selected) -def build_baseline( - source: Path, dest: Path, reference: Path, lock: dict, manifest: bytes, - jobs: int -) -> dict: +def compile_original_vold( + source: Path, lock: dict, manifest: bytes, jobs: int, + selected: list[str] | None = None, +) -> Path: + """Do not apply or import the patch. A successful compile is NOT proof.""" if source.is_symlink() or not (source / ".repo/manifest.xml").exists(): raise BuildError("baseline source directory is missing or unsafe") source = source.resolve(strict=True) - verify_checkout(source, manifest, clean=True) - official = read_vold(reference) - if dest.is_symlink() or (dest.exists() and ( - not dest.is_dir() or any(dest.iterdir()) - )): - raise BuildError("baseline output directory must be new or empty") - dest.mkdir(parents=True, exist_ok=True) + verify_checkout(source, manifest, clean=True, selected=selected) + verify_source_workspace(source, manifest, selected) + release_config = source / lock["build"]["release_config"]["path"] + if ( + release_config.is_symlink() or not release_config.is_file() + or hashlib.sha256(release_config.read_bytes()).hexdigest() + != lock["build"]["release_config"]["sha256"] + ): + raise BuildError("pinned LineageOS release configuration is missing or changed") artifact = source / lock["build"]["artifact"] + for parent in artifact.parents: + if parent == source: + break + if parent.is_symlink() or (parent.exists() and not parent.is_dir()): + raise BuildError("Android vold output directory is unsafe") if artifact.is_symlink(): raise BuildError("Android vold build artifact cannot be a symbolic link") - # Never mistake an existing build's vold for a result of this build. + # An earlier trial must not be mistaken for the current compiler result. if artifact.exists(): if not artifact.is_file(): raise BuildError("unexpected prior vold output type") artifact.unlink() + # At this pinned revision, lunch requires product/release/variant and also + # invokes roomservice, which can fetch unselected sources. Its unchanged + # _lunch_meat helper performs the same configuration without roomservice. + # Kernel auto-construction is outside this individual target's scope. script = ( - "source build/envsetup.sh && " - f"lunch {lock['build']['target']} && " + "source build/envsetup.sh || exit $?; " + "function build_kernel() { echo 'Automatic kernel builds are outside the pinned vold probe' >&2; exit 1; }; " + f"check_product {lock['build']['product']} {lock['build']['release']} && " + f"{lock['build']['setup_command']} && " f"m -j{jobs} vold" ) - command(["bash", "-euc", script], source) - if not artifact.exists() or not artifact.stat().st_mode & 0o111: - raise BuildError("source build did not produce an executable vold") + environment = os.environ.copy() + for variable in ("SOONG_ALLOW_MISSING_DEPENDENCIES", "ALLOW_MISSING_DEPENDENCIES"): + if environment.get(variable, "").lower() not in ("", "false", "0"): + raise BuildError(f"missing-dependency bypass must be disabled: {variable}") + environment.pop(variable, None) + # An inherited runner output override must not reuse another build tree. + for variable in ("OUT_DIR_COMMON_BASE", "ANDROID_PRODUCT_OUT", "ANDROID_BUILD_TOP", "TOP", "BASH_ENV"): + environment.pop(variable, None) + environment["OUT_DIR"] = "out" + environment["USE_LEFTOVERS"] = "0" + environment["ROOMSERVICE_DRYRUN"] = "true" + environment["PYTHONDONTWRITEBYTECODE"] = "1" + # Upstream envsetup intentionally reads unset optional variables; nounset + # would fail before configuration (for example at USE_LEFTOVERS). + command(["bash", "-ec", script], source, env=environment) + verify_checkout(source, manifest, clean=True, selected=selected) + verify_source_workspace(source, manifest, selected) + if artifact.is_symlink() or not artifact.is_file() or not artifact.stat().st_mode & 0o111: + raise BuildError("source build did not produce a regular executable vold") + return artifact + + +def build_baseline( + source: Path, dest: Path, reference: Path, lock: dict, manifest: bytes, + jobs: int, selected: list[str] | None = None, +) -> dict: + """A build is only *qualified* if every byte matches the official OTA.""" + official = read_vold(reference) + if dest.is_symlink() or (dest.exists() and ( + not dest.is_dir() or any(dest.iterdir()) + )): + raise BuildError("baseline output directory must be new or empty") + dest.mkdir(parents=True, exist_ok=True) + artifact = compile_original_vold(source, lock, manifest, jobs, selected) built = read_vold(artifact) report = comparison(official, built, lock) + report["build_release"] = lock["build"]["release"] + report["build_setup_command"] = lock["build"]["setup_command"] + report["release_config_sha256"] = lock["build"]["release_config"]["sha256"] + if selected is not None: + report["selected_source_projects"] = len(selected) + report["selected_source_paths_sha256"] = hashlib.sha256( + ("\n".join(sorted(selected)) + "\n").encode() + ).hexdigest() + report["source_checkout_scope"] = "manifest-pinned-subset" + else: + report["source_checkout_scope"] = "complete-manifest" write_receipt(dest / "baseline-report.json", report) require_identical(report) return report @@ -187,10 +397,14 @@ def build_baseline( def main() -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="mode", required=True) - for name in ("sync", "compare"): + for name in ("sync", "sync-vendor-sources", "probe", "compare"): opt = sub.add_parser(name) opt.add_argument("--source-root", type=Path, required=True) opt.add_argument("--jobs", type=int, default=4) + if name != "sync-vendor-sources": + opt.add_argument("--projects-file", type=Path, help="optional validated subset plan; never weakens the binary equality gate") + if name == "sync": + opt.add_argument("--resume", action="store_true", help="fetch missing approved projects within the same checkout") if name == "compare": opt.add_argument("--reference-vold", type=Path, required=True) opt.add_argument("--output-dir", type=Path, required=True) @@ -199,12 +413,25 @@ def main() -> int: parser.error("--jobs must be between 1 and 16") try: lock, manifest = load_contract() - if args.mode == "sync": - sync(args.source_root, lock, manifest, args.jobs) + selected = verified_selection(getattr(args, "projects_file", None), manifest) + if args.mode == "sync-vendor-sources": + # Diagnose the known proprietary-input gate before downloading the + # large compiler/toolchain checkout. This fixed set is not a build + # selection and never authorizes compiling or qualifying a binary. + sync(args.source_root, lock, manifest, args.jobs, list(VENDOR_PREREQUISITE_PROJECTS)) + elif args.mode == "sync": + if args.resume and selected is None: + raise BuildError("full-manifest checkout does not support incremental resume") + sync(args.source_root, lock, manifest, args.jobs, selected, args.resume) + elif args.mode == "probe": + if selected is None: + raise BuildError("compilation probe requires an explicit pinned subset selection") + compile_original_vold(args.source_root, lock, manifest, args.jobs, selected) + print("Dependency probe compiled; this is NOT byte-identical proof or release approval.") else: report = build_baseline( args.source_root, args.output_dir, args.reference_vold, - lock, manifest, args.jobs, + lock, manifest, args.jobs, selected, ) print(json.dumps(report, sort_keys=True)) except (OSError, BuildError, BaselineError, subprocess.CalledProcessError) as exc: diff --git a/src/ci/prepare_vold_vendor_makefiles.py b/src/ci/prepare_vold_vendor_makefiles.py new file mode 100644 index 00000000..f5bbdb24 --- /dev/null +++ b/src/ci/prepare_vold_vendor_makefiles.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Report the proprietary inputs blocking pinned Sony makefile preparation. + +At the locked extract-utils revision, --regenerate_makefiles skips extraction +but still opens proprietary ELF files to discover architecture and dependencies. +It cannot regenerate the Sony makefiles from public file lists alone. Until an +authenticated vendor-input contract exists, stop before running that generator +or writing partial vendor declarations. This diagnostic never qualifies vold. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import stat +import subprocess +import sys + +from build_original_vold import VENDOR_PREREQUISITE_PROJECTS, load_contract, verify_checkout + + +class VendorPreparationError(ValueError): + pass + + +INPUTS = ( + "device/sony/pdx235/extract-files.py", + "device/sony/pdx235/proprietary-files.txt", + "device/sony/sm6375-common/extract-files.py", + "device/sony/sm6375-common/proprietary-files.txt", + "tools/extract-utils/extract_utils/main.py", + "tools/extract-utils/extract_utils/module.py", + "tools/extract-utils/extract_utils/makefiles.py", + "tools/extract-utils/extract_utils/elf.py", +) +# This is the first absent ELF opened in a replay of the unchanged, exact +# pinned scripts and public lists. It is an example, not a complete blob set. +REQUIRED_ELF_EXAMPLE = ( + "vendor/sony/pdx235/proprietary/" + "vendor/lib64/camera/com.qti.eeprom.hi846_back_eeprom.so" +) +EXTRACT_UTILS_REVISION = "ff6589c6b0a4a00c592e98940e9e6b6a50c76569" +EVIDENCE_BASE = ( + "https://github.com/LineageOS/android_tools_extract-utils/blob/" + + EXTRACT_UTILS_REVISION +) + + +def safe_file(path: Path, max_bytes: int = 4_000_000) -> bytes: + st = path.lstat() + if not stat.S_ISREG(st.st_mode) or not 0 <= st.st_size <= max_bytes: + raise VendorPreparationError(f"diagnostic receipt or source is unsafe: {path}") + content = path.read_bytes() + if len(content) != st.st_size: + raise VendorPreparationError(f"diagnostic receipt or source changed while reading: {path}") + return content + + +def blocked_report(source: Path, input_hashes: dict[str, str]) -> dict: + for path in (source / "vendor", source / "vendor/sony"): + if path.is_symlink() or path.exists() and not path.is_dir(): + raise VendorPreparationError("Sony vendor output root is unsafe") + sony_root = source / "vendor/sony" + preexisting = sony_root.exists() and any(sony_root.iterdir()) + return { + "schema_version": 1, + "kind": "blocked-unqualified-proprietary-vendor-inputs", + "status": "blocked", + "does_not_extract_blobs": True, + "does_not_qualify_original_vold": True, + "makefiles_generated": False, + "preexisting_vendor_tree_unqualified": preexisting, + "input_sha256": input_hashes, + "required_input_example": REQUIRED_ELF_EXAMPLE, + "required_input_example_is_complete_list": False, + "reason": ( + "The pinned --regenerate_makefiles path skips extraction, but " + "write_elfs_package still opens proprietary ELF files for machine, " + "bitness and DT_NEEDED metadata. Public source lists are insufficient." + ), + "required_next_step": ( + "Authenticate and pin the required proprietary vendor inputs and " + "their build provenance in a reviewed contract before enabling " + "vendor preparation. Preexisting files and synthetic receipts " + "are not accepted as provenance." + ), + "upstream_evidence": [ + EVIDENCE_BASE + "/extract_utils/main.py#L182-L219", + EVIDENCE_BASE + "/extract_utils/makefiles.py#L256-L271", + EVIDENCE_BASE + "/extract_utils/elf.py#L38-L57", + ], + "public_source_replay": { + "scope": "Review-time replay of pinned public source, not performed by this diagnostic", + "command": "python3 -B device/sony/pdx235/extract-files.py --regenerate_makefiles", + "exit_status": 1, + "exception": "FileNotFoundError", + "first_missing_file": REQUIRED_ELF_EXAMPLE, + "original_scripts_modified": False, + }, + } + + +def prepare(source: Path, receipt: Path) -> None: + if source.is_symlink() or not source.is_dir(): + raise VendorPreparationError("original source workspace is missing or linked") + source = source.resolve(strict=True) + _lock, manifest = load_contract() + verify_checkout(source, manifest, clean=True, selected=list(VENDOR_PREREQUISITE_PROJECTS)) + hashes = {path: hashlib.sha256(safe_file(source / path)).hexdigest() for path in INPUTS} + current = blocked_report(source, hashes) + if receipt.is_symlink(): + raise VendorPreparationError("vendor diagnostic receipt must not be a symlink") + if receipt.exists(): + if json.loads(safe_file(receipt)) != current: + raise VendorPreparationError("existing vendor receipt is not the current blocked-input diagnostic") + else: + receipt.parent.mkdir(parents=True, exist_ok=True) + with receipt.open("x", encoding="utf-8") as output: + json.dump(current, output, indent=2, sort_keys=True) + output.write("\n") + raise VendorPreparationError( + "pinned Sony makefile regeneration requires authenticated proprietary " + f"ELF inputs (for example {REQUIRED_ELF_EXAMPLE}); no vendor-input " + f"provenance contract exists. Blocked-input report: {receipt}" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + args = parser.parse_args() + try: + prepare(args.source_root, args.report) + except (OSError, KeyError, TypeError, ValueError, subprocess.CalledProcessError) as exc: + print(f"Error: cannot prepare pinned Sony makefiles: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ci/probe_original_vold.sh b/src/ci/probe_original_vold.sh new file mode 100644 index 00000000..245d7535 --- /dev/null +++ b/src/ci/probe_original_vold.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-or-later +# Same-workspace, immutable-revision probing: sync only selected projects, +# reuse downloaded Git objects, map errors to the pinned manifest, expand. +# Never apply the timeout patch or publish an OTA. +set -euo pipefail + +: "${VOLD_WORK:?isolated scratch workspace is required}" +: "${VOLD_PROBE_STAGE:?probe or compare stage is required}" + +[[ "${VOLD_PROBE_STAGE}" == probe || "${VOLD_PROBE_STAGE}" == compare ]] || { + echo 'Error: unsupported unmodified vold dependency stage' >&2 + exit 1 +} +[[ "${VOLD_WORK}" == /* && -d "${VOLD_WORK}" && ! -L "${VOLD_WORK}" ]] || { + echo 'Error: isolated scratch directory is missing or unsafe' >&2 + exit 1 +} +[[ "${VOLD_PROBE_STAGE}" != compare || ( "${VOLD_SOURCE_REF:-}" == /* && -s "${VOLD_SOURCE_REF}" && ! -L "${VOLD_SOURCE_REF}" ) ]] || { + echo 'Error: authenticated original vold reference is required' >&2 + exit 1 +} +jobs="${VOLD_PROBE_JOBS:-4}" +[[ "${jobs}" =~ ^[1-9][0-9]?$ ]] && (( jobs <= 16 )) || { + echo 'Error: probe concurrency must be between 1 and 16' >&2 + exit 1 +} + +selection="${VOLD_WORK}/source-selection.json" +python3 src/ci/vold_source_closure.py plan --output "${selection}" +# Check the three small pinned public vendor-input projects BEFORE fetching +# toolchains. At these revisions --regenerate_makefiles still reads absent +# proprietary ELF files. The preflight records that blocker without generating +# misleading partial declarations or downloading the rest of the source seed. +python3 src/ci/build_original_vold.py sync-vendor-sources \ + --source-root "${VOLD_WORK}/source" --jobs "${jobs}" +python3 src/ci/prepare_vold_vendor_makefiles.py \ + --source-root "${VOLD_WORK}/source" \ + --report "${VOLD_WORK}/sony-vendor-input-report.json" + +python3 src/ci/build_original_vold.py sync --resume \ + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" + +# Batch reviewed provider hints from the checked-out Blueprint graph before +# compiling. Unknown or conditional modules remain diagnostics for real Soong; +# they are never mapped to guessed repositories. Stop after three scans. +for scan in 1 2 3; do + next="${VOLD_WORK}/selection-static-next.json" + rm -f -- "${next}" + python3 src/ci/vold_source_closure.py index \ + --source-root "${VOLD_WORK}/source" --selection "${selection}" \ + --output "${VOLD_WORK}/source-index-static-${scan}.json" \ + --next-selection "${next}" + if [[ ! -f "${next}" ]]; then + break + fi + mv -- "${next}" "${selection}" + python3 src/ci/build_original_vold.py sync --resume \ + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}" +done + +# Most missing dependencies are reported during Soong setup before the +# expensive target compile. Resolve their exact pinned project paths once +# and reuse the same checkout and Android out directory on each iteration. +# Unknown module providers must stop for review; never turn on missing +# dependency flags or attempt an unverified project URL. +for iteration in 1 2 3; do + log="${VOLD_WORK}/vold-build-pass-${iteration}.log" + if [[ "${VOLD_PROBE_STAGE}" == probe ]]; then + build_cmd=(python3 src/ci/build_original_vold.py probe + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" --jobs "${jobs}") + else + build_cmd=(python3 src/ci/build_original_vold.py compare + --source-root "${VOLD_WORK}/source" --projects-file "${selection}" + --reference-vold "${VOLD_SOURCE_REF}" + --output-dir "${VOLD_WORK}/baseline-${iteration}" --jobs "${jobs}") + fi + + if "${build_cmd[@]}" >"${log}" 2>&1; then + tail -n 15 -- "${log}" + if [[ "${VOLD_PROBE_STAGE}" == probe ]]; then + echo '::notice::Unmodified vold compiled, but no authenticated binary equality has been established.' + echo "Compile-only probe completed; next stage MUST use authenticated compare." >"${VOLD_WORK}/probe-only.txt" + else + echo '::notice::Unmodified vold is byte-identical to the authenticated reference.' + fi + exit 0 + fi + + echo "::warning::Pinned source probe pass ${iteration} failed. Last diagnostic lines:" + tail -n 55 -- "${log}" + next="${VOLD_WORK}/selection-next.json" + rm -f -- "${next}" + python3 src/ci/vold_source_closure.py diagnose \ + --selection "${selection}" --log "${log}" \ + --output "${VOLD_WORK}/dependency-diagnosis-${iteration}.json" \ + --next-selection "${next}" + + if [[ ! -f "${next}" || "${iteration}" == 3 ]]; then + echo '::error::No automatically identifiable missing pinned projects remain, or expansion budget exhausted.' + echo '::error::Review diagnostic JSON for unknown Soong modules, missing vendor inputs or real build errors.' + exit 1 + fi + mv -- "${next}" "${selection}" + echo "::notice::Expanding the same pinned checkout on pass $((iteration + 1))." + python3 src/ci/build_original_vold.py sync --resume \ + --source-root "${VOLD_WORK}/source" \ + --projects-file "${selection}" --jobs "${jobs}" +done + +echo '::error::Source probing exceeded the bounded three-pass limit' >&2 +exit 1 diff --git a/src/ci/vold_source_closure.py b/src/ci/vold_source_closure.py new file mode 100644 index 00000000..d3564944 --- /dev/null +++ b/src/ci/vold_source_closure.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +"""Conservative, auditable source-dependency probing for unmodified pdx235 vold. + +A seed is NOT an authoritative Soong closure. Derive candidate projects from +locked source and build diagnostics, then verify exact Git HEADs. Missing +proprietary inputs and unknown module providers are reported, not fabricated. +Never loosen the byte-for-byte comparison or call the timeout patch. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re +import sys +import xml.etree.ElementTree as ET + + +ROOT = Path(__file__).resolve().parents[2] +SEED = ROOT / "locks/lineage-vold-source-probe-20261016.json" +BASELINE = ROOT / "locks/lineage-vold-baseline-20261016.json" +PATH_IN_LOG = re.compile( + r"(? tuple[dict, dict[str, str], dict]: + seed = json.loads(SEED.read_text()) + baseline = json.loads(BASELINE.read_text()) + raw = (ROOT / baseline["manifest"]["path"]).read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != baseline["manifest"]["sha256"] or seed.get("manifest_sha256") != digest: + raise ClosureError("source subset plan does not match the immutable 1,243-project manifest") + if seed.get("base_build_id") != baseline.get("base_build_id") or seed.get("schema_version") != 1: + raise ClosureError("source subset plan has an unexpected target or schema") + try: + xml = ET.fromstring(raw) + except ET.ParseError as exc: + raise ClosureError("pinned project manifest cannot be parsed") from exc + projects: dict[str, str] = {} + for element in xml.findall("project"): + # Repo defaults omitted project paths to the project name. + # The official pinned manifest uses that form for 44 projects. + path, revision = element.get("path", element.get("name")), element.get("revision") + # One-segment roots (eg. art, bionic) are valid. Any path escaping + # source root or any branch/tag-based revision is not. + if ( + not path + or not re.fullmatch(r"[A-Za-z0-9_.+-]+(?:/[A-Za-z0-9_.+-]+)*", path) + or any(part in (".", "..") for part in path.split("/")) + or not revision + or not re.fullmatch(r"[0-9a-f]{40}", revision) + ): + raise ClosureError("manifest contains an invalid project identity") + if path in projects: + raise ClosureError("source manifest contains duplicate project paths") + projects[path] = revision + if len(projects) != 1243: + raise ClosureError("pinned source manifest does not contain exactly 1,243 projects") + groups = seed.get("candidate_groups") + if not isinstance(groups, dict) or not groups: + raise ClosureError("source seed has no reviewed dependency groups") + for title, selected in groups.items(): + if not isinstance(title, str) or not isinstance(selected, list) or not selected: + raise ClosureError("seed group is empty or malformed") + if any(not isinstance(path, str) or path not in projects for path in selected): + raise ClosureError(f"seed group {title} references non-pinned projects") + if seed.get("direct_module_to_project_hints") is None or any( + path not in projects for path in seed["direct_module_to_project_hints"].values() + ): + raise ClosureError("module provider hints are not bound to immutable manifest projects") + for vendor in seed["known_unpinned_vendor_inputs"]: + if not vendor.startswith("vendor/sony/"): + raise ClosureError("unexpected unreviewed vendor-input exception") + if any(vendor == p or vendor.startswith(p + "/") for p in projects): + raise ClosureError("vendor-input exception unexpectedly became manifest-pinned") + return seed, projects, baseline + + +def initial_projects(seed: dict) -> list[str]: + return sorted({path for group in seed["candidate_groups"].values() for path in group}) + + +def validate_selection(selection: dict, projects: dict[str, str], seed: dict) -> list[str]: + if not isinstance(selection, dict) or selection.get("schema_version") != 1: + raise ClosureError("invalid dependency probe selection") + if selection.get("manifest_sha256") != seed["manifest_sha256"]: + raise ClosureError("dependency selection has the wrong source revision") + paths = selection.get("selected_projects") + if not isinstance(paths, list) or not paths or len(paths) > len(projects): + raise ClosureError("dependency selection contains an invalid project list") + if any(not isinstance(path, str) or path not in projects for path in paths) or len(paths) != len(set(paths)): + raise ClosureError("dependency selection contains unknown, unpinned or duplicate projects") + if not set(initial_projects(seed)).issubset(paths): + raise ClosureError("dependency selection may not omit reviewed seed projects") + return sorted(paths) + + +def selection_record(paths: list[str], seed: dict) -> dict: + return { + "schema_version": 1, + "kind": "unmodified-vold-provisional-source-selection", + "manifest_sha256": seed["manifest_sha256"], + "selected_projects": sorted(paths), + "warning": "Not complete or qualified. Must run the real unmodified build and pass a byte-exact reference check.", + } + + +def owner(path: str, projects: dict[str, str]) -> str | None: + clean = path.strip(".,:;()[]{}'\"") + # A file in system/fs/fs_mgr belongs to that project, not 'system'. + matches = [root for root in projects if clean == root or clean.startswith(root + "/")] + return max(matches, key=len) if matches else None + + +def missing_dependencies( + text: str, selected: list[str], projects: dict[str, str], seed: dict +) -> dict: + pinned_missing: set[str] = set() + unpinned: set[str] = set() + unresolved_modules: set[str] = set() + modules: set[str] = set() + unmapped_errors: list[str] = [] + hints = seed["direct_module_to_project_hints"] + for line in text.splitlines(): + low = line.lower() + if not any(token in low for token in ERROR_MARKERS): + continue + line_modules = set() + for pattern in (MISSING_MODULE, QUOTED_MODULE): + line_modules.update(pattern.findall(line)) + missing_list = MISSING_DEPENDENCIES.search(line) + if missing_list: + line_modules.update(re.findall(r"[A-Za-z0-9_.+@-]+", missing_list[1])) + modules.update(line_modules) + actionable_path = False + for match in PATH_IN_LOG.finditer(line): + path = match[1] + project = owner(path, projects) + if project is not None and project not in selected: + pinned_missing.add(project) + actionable_path = True + elif project is None and path.startswith(("vendor/", "device/")): + unpinned.add(path) + actionable_path = True + if not line_modules and not actionable_path and len(unmapped_errors) < 100: + unmapped_errors.append(line[:1000]) + for module in modules: + mapped = hints.get(module) + if mapped is None: + unresolved_modules.add(module) + elif mapped not in selected: + pinned_missing.add(mapped) + return { + "proposed_project_additions": sorted(pinned_missing), + "unpinned_missing_paths": sorted(unpinned), + "modules_without_verified_owner": sorted(unresolved_modules), + "modules_observed": sorted(modules), + "modules_with_selected_owner": sorted(module for module in modules if hints.get(module) in selected), + "unmapped_error_lines": unmapped_errors, + "statically_unpinned_vendor_inputs": seed["known_unpinned_vendor_inputs"], + } + + +def without_blueprint_comments(raw: str) -> str: + """Erase comments without interpreting comment markers inside quoted strings.""" + return re.sub( + r'"(?:[^"\\]|\\.)*"|//[^\n]*|/\*.*?\*/', + lambda match: match[0] if match[0].startswith('"') else re.sub(r"[^\n]", " ", match[0]), + raw, + flags=re.S, + ) + + +def module_reference(value: str) -> str: + """Normalize an explicit :module{.output_tag}, not an arbitrary source path.""" + return re.sub(r"\{[^{}]+\}$", "", value.removeprefix(":")) + + +def iter_blueprint_blocks(raw: str): + """Extract balanced module declarations, without assuming nested blocks are flat.""" + raw = without_blueprint_comments(raw) + for candidate in BLUEPRINT_BLOCK.finditer(raw): + start = candidate.end() + depth = 1 + quote = False + escape = False + index = start + while index < len(raw) and depth: + char = raw[index] + if quote: + if escape: + escape = False + elif char == "\\": + escape = True + elif char == '"': + quote = False + else: + if char == '"': + quote = True + elif char == "{": + depth += 1 + elif char == "}": + depth -= 1 + index += 1 + if depth == 0: + body = raw[start:index - 1] + name_match = BLUEPRINT_NAME.search(body) + if name_match: + yield name_match[1], body + + +def blueprint_module_graph(source_root: Path, selected: list[str]) -> dict: + """Best-effort static Soong graph of synced projects only. + + Does NOT evaluate Soong product conditionals or generated modules. + """ + if source_root.is_symlink(): + raise ClosureError("unsafe symbolic-link source checkout") + source_root = source_root.resolve(strict=True) + found: dict[str, list[dict]] = {} + count_files = 0 + scanned = set() + for project in selected: + directory = source_root / project + if directory.is_symlink() or not directory.is_dir(): + continue + for bp in directory.rglob("Android.bp"): + if bp.is_symlink() or not bp.is_file() or bp.stat().st_size > 4_000_000: + continue + if bp in scanned: + continue + scanned.add(bp) + count_files += 1 + if count_files > 12000: + raise ClosureError("source contains unexpectedly many Blueprint files") + try: + raw = bp.read_text(errors="replace") + except OSError as exc: + raise ClosureError("unable to read checked-out module metadata") from exc + for name, body in iter_blueprint_blocks(raw): + deps = set() + for field, values in BP_ARRAY.findall(body): + if field in BP_DEP_FIELDS: + deps.update(module_reference(x) for x in BP_ITEM.findall(values)) + if field in BP_MODULE_REFERENCE_FIELDS: + deps.update(module_reference(x) for x in BP_ITEM.findall(values) if x.startswith(":")) + for field, value in BP_STRING.findall(body): + if field in BP_SINGLE_MODULE_FIELDS: + deps.add(module_reference(value)) + found.setdefault(name, []).append({ + "path": bp.relative_to(source_root).as_posix(), + "deps": sorted(deps), + }) + reachable = set() + stack = ["vold"] + unresolved = set() + ambiguous = set() + while stack: + current = stack.pop() + if current in reachable: + continue + reachable.add(current) + matches = found.get(current, []) + if not matches: + unresolved.add(current) + if len(matches) > 1: + ambiguous.add(current) + for match in matches: + stack.extend(d for d in match["deps"] if d not in reachable) + return { + "kind": "partial-static-soong-graph", + "scanned_android_bp_files": count_files, + "indexed_module_names": len(found), + "reachable_module_names": len(reachable), + "reachable_module_sample": sorted(reachable)[:160], + "unresolved_module_names": sorted(unresolved), + "ambiguous_module_names": sorted(ambiguous), + "warning": "Partial approximation with omissions and conditional false positives: variables, namespaces, product-selected modules and generated dependencies still need a real Soong evaluation.", + } + + +def next_projects_after_index(selected: list[str], report: dict, seed: dict) -> dict | None: + """Use reviewed provider hints only; unresolved static names are not build failures.""" + hints = seed["direct_module_to_project_hints"] + additions = sorted({ + hints[module] for module in report["unresolved_module_names"] + if module in hints and hints[module] not in selected + }) + report["proposed_project_additions"] = additions + report["modules_without_verified_owner"] = sorted( + module for module in report["unresolved_module_names"] if module not in hints + ) + if len(additions) > 30: + raise ClosureError("too many static additions: require independent review") + if not additions: + return None + return selection_record(sorted(set(selected + additions)), seed) + + +def next_projects_after_diagnostic( + selected: list[str], diagnostic: dict, seed: dict, +) -> dict | None: + """Stop expensive source iteration when no safe bounded next step exists.""" + if diagnostic["unpinned_missing_paths"] or diagnostic["modules_without_verified_owner"]: + return None + additions = diagnostic["proposed_project_additions"] + if len(additions) > 30: + raise ClosureError("too many automatic additions: require independent review") + if not additions: + return None + return selection_record(sorted(set(selected + additions)), seed) + + +def write_json(path: Path, report: dict) -> None: + if path.is_symlink(): + raise ClosureError("refusing to replace a symbolic-link source report") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, sort_keys=True, indent=2) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + cmd_plan = sub.add_parser("plan", help="generate a small immutable-manifest selection without downloading") + cmd_plan.add_argument("--output", type=Path, required=True) + cmd_index = sub.add_parser("index", help="index checked-out Android.bp module references") + cmd_index.add_argument("--selection", type=Path, required=True) + cmd_index.add_argument("--source-root", type=Path, required=True) + cmd_index.add_argument("--output", type=Path, required=True) + cmd_index.add_argument("--next-selection", type=Path) + cmd_diag = sub.add_parser("diagnose", help="map build errors to additional pinned paths") + cmd_diag.add_argument("--selection", type=Path, required=True) + cmd_diag.add_argument("--log", type=Path, required=True) + cmd_diag.add_argument("--output", type=Path, required=True) + cmd_diag.add_argument("--next-selection", type=Path) + args = parser.parse_args() + try: + seed, projects, baseline = verified_contract() + if args.action == "plan": + selected = initial_projects(seed) + write_json(args.output, selection_record(selected, seed)) + print(json.dumps({ + "selected_projects": len(selected), + "total_manifest_projects": len(projects), + "source_ota_sha256": baseline["source_ota_sha256"], + "known_external_vendor_inputs": seed["known_unpinned_vendor_inputs"], + "selection_path": str(args.output), + }, sort_keys=True)) + else: + selected = validate_selection(json.loads(args.selection.read_text()), projects, seed) + if args.action == "index": + report = blueprint_module_graph(args.source_root, selected) + expansion = next_projects_after_index(selected, report, seed) + if args.next_selection is not None: + if expansion is not None: + write_json(args.next_selection, expansion) + report["next_selection"] = str(args.next_selection) if expansion is not None else None + else: + report = missing_dependencies(args.log.read_text(errors="replace"), selected, projects, seed) + if args.next_selection is not None: + expansion = next_projects_after_diagnostic(selected, report, seed) + if expansion is not None: + write_json(args.next_selection, expansion) + report["next_selection"] = str(args.next_selection) + else: + report["next_selection"] = None + report["blocked_on_unresolved_inputs"] = bool( + report["unpinned_missing_paths"] + or report["modules_without_verified_owner"] + ) + write_json(args.output, report) + print(json.dumps(report, sort_keys=True)) + except (OSError, ValueError, KeyError, TypeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/vold_baseline_ci_test.py b/tests/vold_baseline_ci_test.py index 9f22afc3..99b29e6e 100644 --- a/tests/vold_baseline_ci_test.py +++ b/tests/vold_baseline_ci_test.py @@ -117,6 +117,9 @@ def test_baseline_build_removes_stale_output_and_rejects_nonidentical_bytes(self source = root / "source" (source / ".repo").mkdir(parents=True) (source / ".repo/manifest.xml").write_bytes(manifest) + release = source / lock["build"]["release_config"]["path"] + release.parent.mkdir(parents=True) + release.write_text("# Updated manually\naosp_target_release=cp2a\n") out = source / lock["build"]["artifact"] out.parent.mkdir(parents=True) out.write_bytes(b"stale build must be deleted") @@ -124,7 +127,7 @@ def test_baseline_build_removes_stale_output_and_rejects_nonidentical_bytes(self reference.write_bytes(original) def fake_compile(argv, cwd, **_kwargs): - self.assertEqual(argv[:2], ["bash", "-euc"]) + self.assertEqual(argv[:2], ["bash", "-ec"]) self.assertFalse(out.exists(), "stale binary must be removed first") self.assertIn("m -j2 vold", argv[2]) out.write_bytes(original) @@ -166,17 +169,20 @@ def test_workflow_is_opt_in_readonly_original_only(self): self.assertNotIn(" schedule:", flow) self.assertIn("runs-on: [self-hosted, linux, x64, modos-lineage-large]", flow) self.assertIn("MODOS_VOLD_BASELINE_RUNNER_READY", flow) - self.assertIn("free >= 250000000000", flow) + self.assertIn("free >= 100000000000", flow) self.assertIn("contents: read", flow) self.assertIn("baseline-report.json", flow) self.assertIn("rm -rf -- \"$VOLD_WORK\"", flow) - self.assertIn("src/ci/build_original_vold.py compare", flow) + self.assertIn("src/ci/probe_original_vold.sh", flow) + probe = (ROOT / "src/ci/probe_original_vold.sh").read_text() + self.assertIn("src/ci/build_original_vold.py compare", probe) + self.assertIn("src/ci/build_original_vold.py probe", probe) self.assertIn("source_care_map_require_lineageos_pdx235_publisher", auth) self.assertIn("avbroot ota verify", auth) self.assertIn("avbroot ota extract", auth) self.assertIn("comparison(official, built, lock)", original) - self.assertIn("verify_checkout(source, manifest, clean=True)", original) - for text in (flow, original, auth): + self.assertIn("verify_checkout(source, manifest, clean=True, selected=selected)", original) + for text in (flow, original, auth, probe): self.assertNotIn("git apply", text) self.assertNotIn("ota patch", text) self.assertNotIn("force-publish", text) diff --git a/tests/vold_original_build_test.py b/tests/vold_original_build_test.py new file mode 100644 index 00000000..fdb41528 --- /dev/null +++ b/tests/vold_original_build_test.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Regression checks for the real Repo layout and fail-closed build inputs.""" +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src/ci")) +import build_original_vold as builder # noqa: E402 + + +class OriginalVoldBuildTest(unittest.TestCase): + def setUp(self): + self.lock, self.manifest = builder.load_contract() + + def release_config(self, source): + path = source / self.lock["build"]["release_config"]["path"] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# Updated manually\naosp_target_release=cp2a\n") + + @staticmethod + def selected_manifest(source, manifest): + manifests = source / ".repo/manifests" + manifests.mkdir(parents=True) + (manifests / builder.MANIFEST_NAME).write_bytes(manifest) + # Modern Repo manifest_xml.XmlManifest.Link() writes this include + # wrapper; it does not copy the selected manifest into manifest.xml. + active = source / ".repo/manifest.xml" + active.write_text( + '\n' + '\n' + '\n' + f' \n' + '\n' + ) + return active + + def test_actual_repo_include_wrapper_and_included_bytes(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + builder.verify_active_manifest(source, self.manifest) + (source / ".repo/manifests" / builder.MANIFEST_NAME).write_bytes(b"changed") + with self.assertRaisesRegex(builder.BuildError, "exact official snapshot"): + builder.verify_active_manifest(source, self.manifest) + + def test_wrapper_cannot_add_overrides_or_select_another_manifest(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + active = self.selected_manifest(source, self.manifest) + for contents in ( + '', + f'', + f'', + f'', + ): + with self.subTest(contents=contents): + active.write_text(contents) + with self.assertRaisesRegex(builder.BuildError, "wrapper"): + builder.verify_active_manifest(source, self.manifest) + + def test_local_manifests_rejected_even_with_matching_active_manifest(self): + for filename in ("local_manifest.xml", "local_manifests/extra.xml"): + with self.subTest(filename=filename), tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + override = source / ".repo" / filename + override.parent.mkdir(parents=True, exist_ok=True) + override.write_text("") + with self.assertRaisesRegex(builder.BuildError, "local manifest overrides"): + builder.verify_active_manifest(source, self.manifest) + + def test_legacy_manifest_link_must_target_the_pinned_checkout_file(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "source" + active = self.selected_manifest(source, self.manifest) + active.unlink() + active.symlink_to(Path("manifests") / builder.MANIFEST_NAME) + builder.verify_active_manifest(source, self.manifest) + outside = Path(tmp) / "outside.xml" + outside.write_bytes(self.manifest) + active.unlink() + active.symlink_to(outside) + with self.assertRaisesRegex(builder.BuildError, "link does not select"): + builder.verify_active_manifest(source, self.manifest) + + def test_omitted_manifest_path_checks_real_git_revision_and_local_content(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + project = source / "device/common" + project.mkdir(parents=True) + subprocess.run(["git", "init", "--quiet", str(project)], check=True) + subprocess.run( + ["git", "-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", + "commit", "--quiet", "--allow-empty", "-m", "pinned fixture"], + cwd=project, check=True, + ) + revision = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=project, check=True, + capture_output=True, text=True, + ).stdout.strip() + manifest = f''.encode() + self.selected_manifest(source, manifest) + with patch.object(builder.shutil, "which", return_value="/trusted/repo"): + builder.verify_checkout(source, manifest, clean=True, selected=["device/common"]) + (project / "unexpected.bp").write_text("unreviewed source") + with self.assertRaisesRegex(builder.BuildError, "unreviewed local content"): + builder.verify_checkout(source, manifest, clean=True, selected=["device/common"]) + (project / "unexpected.bp").unlink() + bad = manifest.replace(revision.encode(), b"0" * 40) + (source / ".repo/manifests" / builder.MANIFEST_NAME).write_bytes(bad) + with self.assertRaisesRegex(builder.BuildError, "source revision changed"): + builder.verify_checkout(source, bad, clean=True, selected=["device/common"]) + + def test_resume_fetches_partial_directories_but_not_existing_worktrees(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + (source / "system/vold/.git").mkdir(parents=True) + (source / "system/update_engine").mkdir(parents=True) + calls = [] + with ( + patch.object(builder.shutil, "which", return_value="/trusted/repo"), + patch.object(builder, "verify_checkout") as verify, + patch.object(builder, "command", side_effect=lambda args, *a, **k: calls.append(args)), + ): + builder.sync(source, self.lock, self.manifest, 2, + ["system/vold", "system/update_engine"], resume=True) + self.assertEqual(calls, [["/trusted/repo", "sync", "--no-clone-bundle", "-j2", "system/update_engine"]]) + self.assertEqual(verify.call_args_list[0].kwargs["selected"], ["system/vold"]) + + def test_vendor_prerequisite_mode_only_requests_three_pinned_projects(self): + with ( + patch.object(sys, "argv", ["builder", "sync-vendor-sources", "--source-root", "/fixture", "--jobs", "2"]), + patch.object(builder, "sync") as sync, + ): + self.assertEqual(builder.main(), 0) + args = sync.call_args.args + self.assertEqual(args[0], Path("/fixture")) + self.assertEqual(args[3], 2) + self.assertEqual(args[4], ["device/sony/pdx235", "device/sony/sm6375-common", "tools/extract-utils"]) + self.assertTrue(set(args[4]).issubset(builder.project_map(self.manifest))) + + def test_unrecorded_vendor_or_unselected_source_is_not_build_input(self): + for unexpected in ("vendor/sony/pdx235/Android.bp", "system/update_engine/Android.bp"): + with self.subTest(unexpected=unexpected), tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + (source / "system/vold").mkdir(parents=True) + bad = source / unexpected + bad.parent.mkdir(parents=True, exist_ok=True) + bad.write_text("unrecorded source") + with self.assertRaisesRegex(builder.BuildError, "unrecorded source"): + builder.verify_source_workspace(source, self.manifest, ["system/vold"]) + + def test_manifest_exported_links_are_checked_without_scanning_project_contents(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + project = source / "build/make" + project.mkdir(parents=True) + (project / "envsetup.sh").write_text("pinned source") + link = source / "build/envsetup.sh" + link.symlink_to("make/envsetup.sh") + builder.verify_source_workspace(source, self.manifest, ["build/make"]) + link.unlink() + link.write_text("unreviewed replacement") + with self.assertRaisesRegex(builder.BuildError, "linkfile differs"): + builder.verify_source_workspace(source, self.manifest, ["build/make"]) + + def test_successful_noop_build_cannot_reuse_prior_vold(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + self.release_config(source) + artifact = source / self.lock["build"]["artifact"] + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"previous executable") + artifact.chmod(0o755) + with ( + patch.object(builder, "verify_checkout"), + patch.object(builder, "command"), + self.assertRaisesRegex(builder.BuildError, "did not produce"), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + self.assertFalse(artifact.exists()) + + def test_build_uses_own_output_tree_and_rejects_dependency_bypasses(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + self.release_config(source) + artifact = source / self.lock["build"]["artifact"] + artifact.parent.mkdir(parents=True) + + def compile_fixture(_args, _cwd, *, env): + self.assertEqual(_args[:2], ["bash", "-ec"]) + self.assertIn("check_product lineage_pdx235 cp2a", _args[2]) + self.assertIn("_lunch_meat lineage_pdx235 cp2a userdebug", _args[2]) + self.assertNotIn("lunch lineage_pdx235-userdebug", _args[2]) + self.assertEqual(env["USE_LEFTOVERS"], "0") + self.assertEqual(env["PYTHONDONTWRITEBYTECODE"], "1") + self.assertEqual(env["OUT_DIR"], "out") + self.assertNotIn("OUT_DIR_COMMON_BASE", env) + self.assertNotIn("ANDROID_PRODUCT_OUT", env) + self.assertNotIn("ALLOW_MISSING_DEPENDENCIES", env) + artifact.write_bytes(b"fresh executable") + artifact.chmod(0o755) + + with ( + patch.object(builder, "verify_checkout"), + patch.object(builder, "command", side_effect=compile_fixture), + patch.dict(os.environ, {"OUT_DIR": "/other-build", "OUT_DIR_COMMON_BASE": "/common", "ANDROID_PRODUCT_OUT": "/other-product", "ALLOW_MISSING_DEPENDENCIES": "false"}), + ): + self.assertEqual(builder.compile_original_vold(source, self.lock, self.manifest, 2), artifact) + for flag in ("SOONG_ALLOW_MISSING_DEPENDENCIES", "ALLOW_MISSING_DEPENDENCIES"): + with ( + self.subTest(flag=flag), + patch.object(builder, "verify_checkout"), + patch.object(builder, "command") as compile_command, + patch.dict(os.environ, {flag: "true"}), + self.assertRaisesRegex(builder.BuildError, "bypass must be disabled"), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + compile_command.assert_not_called() + + def test_failed_upstream_setup_or_kernel_autobuild_stops_before_vold(self): + for setup_failure in (True, False): + with self.subTest(setup_failure=setup_failure), tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + self.release_config(source) + make = source / "build/make" + make.mkdir(parents=True) + (source / "build/envsetup.sh").symlink_to("make/envsetup.sh") + artifact = source / self.lock["build"]["artifact"] + (make / "envsetup.sh").write_text( + "function check_product() { return 0; }\n" + "function _lunch_meat() { " + + ("return 0;" if setup_failure else "build_kernel;") + + " }\n" + "function m() { mkdir -p out/target/product/pdx235/system/bin; " + "echo stale > out/target/product/pdx235/system/bin/vold; " + "chmod +x out/target/product/pdx235/system/bin/vold; }\n" + + ("return 1\n" if setup_failure else "return 0\n") + ) + with ( + patch.object(builder, "verify_checkout"), + self.assertRaises(subprocess.CalledProcessError), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + self.assertFalse(artifact.exists()) + + def test_successful_compiler_cannot_add_an_unrecorded_source_tree(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + self.selected_manifest(source, self.manifest) + self.release_config(source) + artifact = source / self.lock["build"]["artifact"] + + def changed_source(_args, _cwd, **_kwargs): + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"fresh executable") + artifact.chmod(0o755) + extra = source / "vendor/unrecorded/Android.bp" + extra.parent.mkdir(parents=True) + extra.write_text("unrecorded build input") + + with ( + patch.object(builder, "verify_checkout") as verify, + patch.object(builder, "command", side_effect=changed_source), + self.assertRaisesRegex(builder.BuildError, "unrecorded source"), + ): + builder.compile_original_vold(source, self.lock, self.manifest, 2) + self.assertEqual(verify.call_count, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vold_probe_orchestration_test.py b/tests/vold_probe_orchestration_test.py new file mode 100644 index 00000000..e12b1f72 --- /dev/null +++ b/tests/vold_probe_orchestration_test.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Exercise probe ordering at process boundaries without downloading Android.""" +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +class ProbeOrchestrationTest(unittest.TestCase): + def run_probe(self, *, stage="probe", vendor_blocked=False, + static_additions=1, build_failures=0, diagnosed=True, + reference=True): + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + binaries = work / "bin" + binaries.mkdir() + shim = binaries / "python3" + shim.write_text(f"#!{sys.executable}\n" + textwrap.dedent('''\ + import json + import os + from pathlib import Path + import sys + + args = sys.argv[1:] + work = Path(os.environ["VOLD_WORK"]) + history = work / "commands.jsonl" + with history.open("a") as output: + output.write(json.dumps(args) + "\\n") + previous = [json.loads(line) for line in history.read_text().splitlines()] + + def option(name): + return Path(args[args.index(name) + 1]) + + if args[0].endswith("vold_source_closure.py"): + action = args[1] + option("--output").write_text("{}") + if action == "plan": + option("--output").write_text('{"selected_projects": ["system/vold"]}') + elif action == "index": + scans = sum(x[0].endswith("vold_source_closure.py") and x[1] == "index" for x in previous) + if scans <= int(os.environ["TEST_STATIC_ADDITIONS"]): + option("--next-selection").write_text('{"selected_projects": ["system/vold", "system/libbase"]}') + elif action == "diagnose" and os.environ["TEST_DIAGNOSED"] == "1": + option("--next-selection").write_text('{"selected_projects": ["system/vold", "system/libbase", "system/gsid"]}') + elif args[0].endswith("prepare_vold_vendor_makefiles.py"): + option("--report").write_text('{"qualified": false}') + if os.environ["TEST_VENDOR_BLOCKED"] == "1": + print("missing proprietary ELF input", file=sys.stderr) + sys.exit(1) + elif args[0].endswith("build_original_vold.py"): + if args[1] in ("sync", "sync-vendor-sources"): + option("--source-root").mkdir(exist_ok=True) + else: + builds = sum(x[0].endswith("build_original_vold.py") and x[1] in ("probe", "compare") for x in previous) + if builds <= int(os.environ["TEST_BUILD_FAILURES"]): + print('error: module "vold" depends on undefined module "libgsi"') + sys.exit(1) + else: + sys.exit("unexpected command") + ''')) + shim.chmod(0o755) + environment = os.environ.copy() + environment.pop("VOLD_SOURCE_REF", None) + environment.update({ + "PATH": str(binaries) + os.pathsep + environment["PATH"], + "VOLD_WORK": str(work), + "VOLD_PROBE_STAGE": stage, + "TEST_VENDOR_BLOCKED": str(int(vendor_blocked)), + "TEST_STATIC_ADDITIONS": str(static_additions), + "TEST_BUILD_FAILURES": str(build_failures), + "TEST_DIAGNOSED": str(int(diagnosed)), + }) + if stage == "compare" and reference: + original = work / "authenticated-reference-fixture" + original.write_bytes(b"reference fixture") + environment["VOLD_SOURCE_REF"] = str(original) + result = subprocess.run( + ["bash", str(ROOT / "src/ci/probe_original_vold.sh")], + cwd=ROOT, env=environment, text=True, capture_output=True, + ) + history = work / "commands.jsonl" + calls = [json.loads(line) for line in history.read_text().splitlines()] if history.exists() else [] + return result, calls + + @staticmethod + def build_calls(calls): + return [args for args in calls if args[0].endswith("build_original_vold.py")] + + def test_vendor_blocker_precedes_toolchain_sync_and_build(self): + result, calls = self.run_probe(vendor_blocked=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("missing proprietary ELF", result.stderr) + self.assertEqual([x[1] for x in self.build_calls(calls)], ["sync-vendor-sources"]) + self.assertFalse(any(x[1] == "index" for x in calls)) + + def test_probe_needs_no_ota_and_scans_before_compiling(self): + result, calls = self.run_probe() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("no authenticated binary equality", result.stdout) + self.assertEqual([x[1] for x in self.build_calls(calls)], + ["sync-vendor-sources", "sync", "sync", "probe"]) + resume = [x for x in self.build_calls(calls) if x[1] == "sync"] + self.assertTrue(all("--resume" in x and "--projects-file" in x for x in resume)) + actions = [x[1] for x in calls] + self.assertLess(actions.index("index"), actions.index("probe")) + + def test_static_scans_are_bounded(self): + result, calls = self.run_probe(static_additions=99) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(sum(x[1] == "index" for x in calls), 3) + self.assertEqual(sum(x[1] == "probe" for x in calls), 1) + + def test_build_error_expands_once_then_reuses_checkout(self): + result, calls = self.run_probe(static_additions=0, build_failures=1) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([x[1] for x in self.build_calls(calls)], + ["sync-vendor-sources", "sync", "probe", "sync", "probe"]) + + def test_unknown_errors_and_compile_budget_stop(self): + for diagnosed, expected in ((False, 1), (True, 3)): + with self.subTest(diagnosed=diagnosed): + result, calls = self.run_probe(static_additions=0, + build_failures=99, diagnosed=diagnosed) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(sum(x[1] == "probe" for x in calls), expected) + + def test_compare_requires_reference_and_never_calls_probe(self): + result, calls = self.run_probe(stage="compare", reference=False) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(calls, []) + result, calls = self.run_probe(stage="compare") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual([x[1] for x in self.build_calls(calls)][-1], "compare") + self.assertFalse(any(x[1] == "probe" for x in calls)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vold_source_closure_test.py b/tests/vold_source_closure_test.py new file mode 100644 index 00000000..58a3e55d --- /dev/null +++ b/tests/vold_source_closure_test.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Regression tests for pin-bound, iterative unmodified vold dependency scans.""" +from __future__ import annotations + +import json +import contextlib +import io +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src/ci")) + +import build_original_vold as original # noqa: E402 +import vold_source_closure as closure # noqa: E402 + + +class VoldSourceClosureTest(unittest.TestCase): + def setUp(self): + self.seed, self.manifest, self.baseline = closure.verified_contract() + self.initial = closure.initial_projects(self.seed) + + def test_pinned_84_project_bootstrap_is_strictly_smaller_than_full_manifest(self): + self.assertEqual(len(self.manifest), 1243) + # The official manifest omits path= for dozens of AOSP projects. + self.assertIn("device/common", self.manifest) + self.assertIn("toolchain/pgo-profiles", self.manifest) + self.assertEqual(len(self.initial), 84) + self.assertEqual(len(self.initial), len(set(self.initial))) + self.assertTrue({"system/vold", "build/soong", "frameworks/base", "device/sony/pdx235"}.issubset(self.initial)) + self.assertTrue(set(self.initial).issubset(self.manifest)) + self.assertEqual(self.seed["manifest_sha256"], self.baseline["manifest"]["sha256"]) + self.assertIn("vendor/sony/pdx235/pdx235-vendor.mk", self.seed["known_unpinned_vendor_inputs"]) + self.assertNotIn("vendor/sony/pdx235", self.manifest) + self.assertEqual(self.seed["direct_module_to_project_hints"]["libbase"], "system/libbase") + self.assertEqual(self.seed["direct_module_to_project_hints"]["libgsi"], "system/gsid") + self.assertEqual(self.seed["direct_module_to_project_hints"]["libhardware_legacy"], "hardware/libhardware_legacy") + self.assertTrue({"external/rust/cxx", "external/rust/android-crates-io", "system/tools/aidl"}.issubset(self.initial)) + + def test_invalid_or_unpinned_expansions_fail_closed(self): + correct = closure.selection_record(self.initial, self.seed) + self.assertEqual(closure.validate_selection(correct, self.manifest, self.seed), self.initial) + for mutate in [ + lambda x: x.update(manifest_sha256="0" * 64), + lambda x: x.update(selected_projects=["system/vold"]), + lambda x: x.update(selected_projects=x["selected_projects"] + ["vendor/sony/pdx235"]), + lambda x: x.update(selected_projects=x["selected_projects"] * 2), + lambda x: x.update(selected_projects=[[]]), + ]: + selection = dict(correct) + mutate(selection) + with self.assertRaises(closure.ClosureError): + closure.validate_selection(selection, self.manifest, self.seed) + + def test_diagnostics_identify_pinned_source_vs_unpinned_vendor(self): + log = ( + 'error: module "vold" depends on undefined module "fuseMedia.bpf"\n' + 'error: source file system/update_engine/Android.bp: No such file or directory\n' + 'error: vendor/sony/pdx235/pdx235-vendor.mk missing\n' + 'error: module "vold" depends on undefined module "unsupported_vendor_blob"\n' + ) + report = closure.missing_dependencies(log, self.initial, self.manifest, self.seed) + self.assertIn("system/update_engine", report["proposed_project_additions"]) + self.assertIn("vendor/sony/pdx235/pdx235-vendor.mk", report["unpinned_missing_paths"]) + self.assertIn("unsupported_vendor_blob", report["modules_without_verified_owner"]) + self.assertNotIn("vold", report["modules_observed"], "the requesting module is not the missing provider") + self.assertNotIn("vendor/sony/pdx235", report["proposed_project_additions"]) + self.assertEqual(closure.owner("system/fs/fs_mgr/fs_mgr/Android.bp", self.manifest), "system/fs/fs_mgr") + self.assertIsNone(closure.owner("vendor/sony/pdx235/pdx235-vendor.mk", self.manifest)) + + def test_real_soong_diagnostics_do_not_mistake_requesters_for_missing_modules(self): + hint_seed = {**self.seed, "direct_module_to_project_hints": {"libupdate": "system/update_engine"}} + for log in ( + 'error: module "vold" depends on undefined module "libupdate".', + 'error: module "vold" variant "android_arm64" depends on undefined module "libupdate".', + 'error: module "libupdate" is missing', + 'error: module "vold" missing dependencies: libupdate', + ): + with self.subTest(log=log): + report = closure.missing_dependencies(log, self.initial, self.manifest, hint_seed) + self.assertEqual(report["modules_observed"], ["libupdate"]) + self.assertEqual(report["modules_without_verified_owner"], []) + self.assertEqual(report["proposed_project_additions"], ["system/update_engine"]) + self.assertIsNotNone(closure.next_projects_after_diagnostic(self.initial, report, hint_seed)) + report = closure.missing_dependencies( + 'error: module "vold" missing dependencies: [libupdate, unknown_generator]', + self.initial, self.manifest, hint_seed, + ) + self.assertEqual(report["modules_without_verified_owner"], ["unknown_generator"]) + self.assertIsNone(closure.next_projects_after_diagnostic(self.initial, report, hint_seed)) + + def test_unmappable_errors_and_non_sony_vendor_inputs_remain_diagnostic(self): + report = closure.missing_dependencies( + 'error: vendor/private/required.mk missing\n' + 'error: module "vold" depends on undefined module "libbase"\n' + 'error: product configuration could not be evaluated\n', + self.initial, self.manifest, self.seed, + ) + self.assertEqual(report["unpinned_missing_paths"], ["vendor/private/required.mk"]) + self.assertEqual(report["modules_with_selected_owner"], ["libbase"]) + self.assertIn('error: product configuration could not be evaluated', report["unmapped_error_lines"]) + self.assertIsNone(closure.next_projects_after_diagnostic(self.initial, report, self.seed)) + + def test_expand_only_when_all_missing_inputs_have_pinned_owners(self): + good = { + "proposed_project_additions": ["system/update_engine"], + "unpinned_missing_paths": [], + "modules_without_verified_owner": [], + } + next_selection = closure.next_projects_after_diagnostic(self.initial, good, self.seed) + self.assertEqual( + next_selection["selected_projects"], + sorted(set(self.initial) | {"system/update_engine"}), + ) + for name, value in ( + ("unpinned_missing_paths", ["vendor/sony/pdx235/BoardConfigVendor.mk"]), + ("modules_without_verified_owner", ["unknown_genrule"]), + ): + bad = {**good, name: value} + self.assertIsNone( + closure.next_projects_after_diagnostic(self.initial, bad, self.seed) + ) + oversized = {**good, "proposed_project_additions": ["p" + str(i) for i in range(31)]} + with self.assertRaises(closure.ClosureError): + closure.next_projects_after_diagnostic(self.initial, oversized, self.seed) + + def test_static_soong_modules_trace_defaults_and_transitive_references(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bp = root / "system/vold/Android.bp" + bp.parent.mkdir(parents=True) + bp.write_text( + 'cc_defaults {\n name: "vold_default",\n shared_libs: ["libbase"],\n}\n' + 'cc_binary {\n name: "vold",\n defaults: ["vold_default"],\n' + 'static_libs: ["libvold"],\n srcs: [":vold_aidl"],\n' + 'arch: { arm64: { shared_libs: ["libcrypto"], }, },\n}\n' + 'cc_library_static {\n name: "libvold",\n static_libs: ["libdm"],\n}\n' + 'filegroup {\n name: "vold_aidl",\n srcs: ["binder/IVold.aidl"],\n}\n' + ) + report = closure.blueprint_module_graph(root, self.initial) + self.assertEqual(report["scanned_android_bp_files"], 1) + self.assertGreaterEqual(report["reachable_module_names"], 6) + self.assertTrue({"libcrypto", "libbase", "libdm"}.issubset(report["unresolved_module_names"])) + self.assertNotIn("vold_aidl", report["unresolved_module_names"]) + + def test_static_index_preserves_generated_dependencies_and_ignores_comments_and_file_paths(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + bp = root / "system/vold/Android.bp" + bp.parent.mkdir(parents=True) + bp.write_text( + '/* cc_binary { name: "vold", static_libs: ["commented"], } */\n' + 'cc_binary {\n name: "vold",\n generated_sources: ["generated_cpp"],\n' + 'srcs: [":tagged_sources{.cpp}", "literal.cpp"],\n' + 'tools: ["compiler"], tool_files: ["script.py", ":tool_sources"],\n' + 'static_libs: ["flags", "rust_bridge"], // closing brace in a comment }\n' + '}\n' + 'cc_aconfig_library { name: "flags", aconfig_declarations: "flag_declarations", }\n' + 'rust_ffi_static { name: "rust_bridge", rustlibs: ["librust"], proc_macros: ["libmacro"], }\n' + 'filegroup { name: "tagged_sources", srcs: ["source.cpp"], }\n' + 'filegroup { name: "tool_sources", srcs: ["tool.py"], }\n' + ) + report = closure.blueprint_module_graph(root, ["system", "system/vold"]) + self.assertEqual(report["scanned_android_bp_files"], 1, "overlapping projects must not duplicate modules") + self.assertEqual(report["ambiguous_module_names"], []) + self.assertEqual(report["unresolved_module_names"], ["compiler", "flag_declarations", "generated_cpp", "libmacro", "librust"]) + self.assertEqual( + closure.without_blueprint_comments('url: "https://example/path", // erased\n'), + 'url: "https://example/path", \n', + ) + + def test_static_expansion_uses_only_reviewed_hints_and_retains_unknown_names(self): + hint_seed = {**self.seed, "direct_module_to_project_hints": { + "known_generator": "system/update_engine", "libvold": "system/vold", + }} + report = {"unresolved_module_names": ["known_generator", "unknown_conditional", "libvold"]} + expansion = closure.next_projects_after_index(self.initial, report, hint_seed) + self.assertEqual(expansion["selected_projects"], sorted(self.initial + ["system/update_engine"])) + self.assertEqual(report["proposed_project_additions"], ["system/update_engine"]) + self.assertEqual(report["modules_without_verified_owner"], ["unknown_conditional"]) + self.assertIsNone(closure.next_projects_after_index(expansion["selected_projects"], report, hint_seed)) + too_many = {**hint_seed, "direct_module_to_project_hints": {f"m{i}": f"project{i}" for i in range(31)}} + with self.assertRaises(closure.ClosureError): + closure.next_projects_after_index([], {"unresolved_module_names": list(too_many["direct_module_to_project_hints"])}, too_many) + + def test_index_cli_emits_reviewed_next_selection_before_build(self): + hint_seed = {**self.seed, "direct_module_to_project_hints": {"known_generator": "system/update_engine"}} + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "source" + bp = source / "system/vold/Android.bp" + bp.parent.mkdir(parents=True) + bp.write_text('cc_binary { name: "vold", generated_sources: ["known_generator", "unknown"], }\n') + selection, output, next_selection = (root / name for name in ("selection.json", "index.json", "next.json")) + selection.write_text(json.dumps(closure.selection_record(self.initial, hint_seed))) + with ( + patch.object(closure, "verified_contract", return_value=(hint_seed, self.manifest, self.baseline)), + patch.object(sys, "argv", ["closure", "index", "--source-root", str(source), + "--selection", str(selection), "--output", str(output), + "--next-selection", str(next_selection)]), + contextlib.redirect_stdout(io.StringIO()), + ): + self.assertEqual(closure.main(), 0) + self.assertIn("system/update_engine", closure.validate_selection(json.loads(next_selection.read_text()), self.manifest, hint_seed)) + self.assertEqual(json.loads(output.read_text())["modules_without_verified_owner"], ["unknown"]) + + def test_incremental_sync_reuses_checkout_and_syncs_only_pinned_projects(self): + manifest_bytes = (ROOT / self.baseline["manifest"]["path"]).read_bytes() + source_ref = self.baseline["projects"]["manifest"]["revision"] + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "source" + log = [] + + def fake_command(argv, cwd, **kwargs): + log.append(argv) + repo_dir = cwd / ".repo" + if argv[1] == "init" and "-m" not in argv: + (repo_dir / "manifests").mkdir(parents=True) + (repo_dir / "manifest.xml").write_text("prior default") + if argv[1] == "init" and "-m" in argv: + (repo_dir / "manifest.xml").write_text(f'') + if argv[1] == "sync": + for project in argv[4:]: + (cwd / project).mkdir(parents=True, exist_ok=True) + (cwd / project / ".git").mkdir(exist_ok=True) + + with ( + patch.object(original.shutil, "which", return_value="/usr/bin/repo"), + patch.object(original, "command", side_effect=fake_command), + patch.object(original, "verify_checkout"), + ): + original.sync(source, self.baseline, manifest_bytes, 2, self.initial) + self.assertEqual([a[1] for a in log], ["init", "init", "sync"]) + self.assertEqual(log[-1][-len(self.initial):], self.initial) + self.assertIn(source_ref, log[0]) + log.clear() + more = sorted(set(self.initial) | {"system/update_engine"}) + original.sync(source, self.baseline, manifest_bytes, 2, more, resume=True) + self.assertEqual(len(log), 1) + self.assertEqual(log[0][1], "sync") + self.assertEqual(log[0][4:], ["system/update_engine"]) + log.clear() + original.sync(source, self.baseline, manifest_bytes, 2, more, resume=True) + self.assertEqual(log, [], "unchanged pinned checkout should not re-run repo sync") + + def test_selections_are_checked_against_unchanged_manifest(self): + raw = (ROOT / self.baseline["manifest"]["path"]).read_bytes() + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) + (source / ".repo").mkdir() + (source / ".repo/manifest.xml").write_bytes(raw) + first = self.initial[:2] + revisions = original.project_map(raw) + + def mocked_command(argv, cwd, **kwargs): + result = revisions[cwd.relative_to(source).as_posix()][1] if argv[1] == "rev-parse" else "" + return type("Result", (), {"stdout": result})() + + for project in first: + (source / project).mkdir(parents=True) + with ( + patch.object(original.shutil, "which", return_value="/usr/bin/repo"), + patch.object(original, "command", side_effect=mocked_command), + patch.object(original.subprocess, "run", + side_effect=AssertionError("subset should never request a full repo manifest -r")), + ): + original.verify_checkout(source, raw, clean=True, selected=first) + (source / ".repo/manifest.xml").write_text("tampered") + with self.assertRaises(original.BuildError): + original.verify_checkout(source, raw, clean=True, selected=first) + + def test_probe_remains_independent_of_published_vold_patch(self): + script = (ROOT / "src/ci/probe_original_vold.sh").read_text() + workflow = (ROOT / ".github/workflows/vold-baseline.yml").read_text() + builder = (ROOT / "src/ci/build_original_vold.py").read_text() + self.assertIn("sync --resume", script) + self.assertIn("for iteration in 1 2 3", script) + self.assertIn("vold_source_closure.py diagnose", script) + self.assertIn("dependency_plan:", workflow) + self.assertIn("default: plan", workflow) + self.assertIn("Source compile probe — NOT QUALIFIED", workflow) + self.assertIn("probe", builder) + for source in (script, workflow, builder): + self.assertNotIn("git apply", source) + self.assertNotIn("force-publish", source) + self.assertNotIn("SOONG_ALLOW_MISSING_DEPENDENCIES=true", source) + self.assertNotIn("ALLOW_MISSING_DEPENDENCIES=true", source) + self.assertNotIn("publish: true", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/vold_vendor_makefiles_test.py b/tests/vold_vendor_makefiles_test.py new file mode 100644 index 00000000..76fcbf5d --- /dev/null +++ b/tests/vold_vendor_makefiles_test.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Ensure absent or unqualified Sony inputs fail before vendor generation.""" +from __future__ import annotations + +from contextlib import redirect_stderr +import io +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src/ci")) +import prepare_vold_vendor_makefiles as prepare # noqa: E402 + + +class SonyVendorMakefileTest(unittest.TestCase): + def source(self, parent: Path) -> Path: + root = parent / "source" + root.mkdir() + for path in prepare.INPUTS: + file = root / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(f"PINNED TEST CONTENT {path}\n") + return root + + def test_missing_vendor_inputs_report_blocker_without_running_generator(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "vendor-input-report.json" + before = sorted(p.relative_to(source).as_posix() for p in source.rglob("*")) + with ( + patch.object(prepare, "verify_checkout") as verify, + patch.object(prepare.subprocess, "run") as run, + ): + with self.assertRaisesRegex(prepare.VendorPreparationError, "authenticated proprietary"): + prepare.prepare(source, receipt) + run.assert_not_called() + self.assertEqual(verify.call_args.kwargs, { + "clean": True, + "selected": list(prepare.VENDOR_PREREQUISITE_PROJECTS), + }) + saved = json.loads(receipt.read_text()) + self.assertEqual(saved["status"], "blocked") + self.assertEqual(saved["kind"], "blocked-unqualified-proprietary-vendor-inputs") + self.assertTrue(saved["does_not_extract_blobs"]) + self.assertTrue(saved["does_not_qualify_original_vold"]) + self.assertFalse(saved["makefiles_generated"]) + self.assertFalse(saved["preexisting_vendor_tree_unqualified"]) + self.assertEqual(saved["required_input_example"], prepare.REQUIRED_ELF_EXAMPLE) + self.assertFalse(saved["required_input_example_is_complete_list"]) + self.assertEqual(set(saved["input_sha256"]), set(prepare.INPUTS)) + self.assertEqual( + before, sorted(p.relative_to(source).as_posix() for p in source.rglob("*")), + "diagnosing missing blobs must not create partial .mk/.bp files or bytecode", + ) + + def test_reused_diagnostic_still_fails_and_cannot_mask_changed_source(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + with patch.object(prepare, "verify_checkout"): + for _ in range(2): + with self.assertRaisesRegex(prepare.VendorPreparationError, "authenticated proprietary"): + prepare.prepare(source, receipt) + (source / prepare.INPUTS[0]).write_text("changed input") + with self.assertRaisesRegex(prepare.VendorPreparationError, "not the current blocked-input"): + prepare.prepare(source, receipt) + + def test_preexisting_vendor_elf_and_partial_makefiles_are_not_provenance(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + elf = source / prepare.REQUIRED_ELF_EXAMPLE + elf.parent.mkdir(parents=True) + elf.write_bytes(b"unqualified bytes") + (source / "vendor/sony/pdx235/Android.bp").write_text("partial upstream output") + receipt = root / "report.json" + with ( + patch.object(prepare, "verify_checkout"), + patch.object(prepare.subprocess, "run") as run, + self.assertRaisesRegex(prepare.VendorPreparationError, "provenance contract"), + ): + prepare.prepare(source, receipt) + run.assert_not_called() + self.assertTrue(json.loads(receipt.read_text())["preexisting_vendor_tree_unqualified"]) + self.assertEqual(elf.read_bytes(), b"unqualified bytes") + + def test_old_synthetic_success_receipt_is_never_reused(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + receipt.write_text(json.dumps({ + "kind": "synthetic-probe-vendor-makefiles-not-publisher-provenance", + "generated_makefile_sha256": {}, + })) + with ( + patch.object(prepare, "verify_checkout"), + self.assertRaisesRegex(prepare.VendorPreparationError, "not the current blocked-input"), + ): + prepare.prepare(source, receipt) + + def test_linked_vendor_root_and_receipt_are_rejected(self): + for linked in ("vendor", "receipt"): + with self.subTest(linked=linked), tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + other = root / "other" + other.mkdir() + if linked == "vendor": + (source / "vendor").symlink_to(other, target_is_directory=True) + else: + receipt.symlink_to(other / "outside.json") + with ( + patch.object(prepare, "verify_checkout"), + self.assertRaisesRegex(prepare.VendorPreparationError, "unsafe|symlink"), + ): + prepare.prepare(source, receipt) + self.assertEqual(list(other.iterdir()), []) + + def test_dirty_pinned_source_prevents_diagnostic_claim(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + with ( + patch.object(prepare, "verify_checkout", side_effect=ValueError("source revision changed")), + self.assertRaisesRegex(ValueError, "source revision changed"), + ): + prepare.prepare(source, receipt) + self.assertFalse(receipt.exists()) + + def test_cli_exits_nonzero_with_retained_blocker_report(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = self.source(root) + receipt = root / "report.json" + stderr = io.StringIO() + with ( + patch.object(prepare, "verify_checkout"), + patch.object(sys, "argv", [ + "prepare_vold_vendor_makefiles.py", "--source-root", str(source), + "--report", str(receipt), + ]), + redirect_stderr(stderr), + ): + self.assertEqual(prepare.main(), 1) + self.assertIn(prepare.REQUIRED_ELF_EXAMPLE, stderr.getvalue()) + self.assertTrue(receipt.is_file()) + + +if __name__ == "__main__": + unittest.main()