diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e8ed4b307..249896f4e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -54,6 +54,8 @@ jobs: PYTHONPYCACHEPREFIX: ${{ runner.temp }}/d4d-tests-pycache run: >- poetry run python -m pytest tests -v -p no:cacheprovider + -p utils.native_ci_evidence + --native-ci-evidence-root=${{ runner.temp }}/d4d-native-cases-${{ github.run_id }}-${{ github.run_attempt }}-shard${{ matrix.shard }} -p utils.pytest_shard --ci-shard=${{ matrix.shard }}/${{ github.event_name == 'pull_request' && 6 || 4 }} --ci-shard-timings=utils/ci_test_durations.json --durations=25 -n logical --maxprocesses=4 --dist load --maxschedchunk=1 @@ -76,6 +78,16 @@ jobs: path: test-results/*.xml retention-days: 14 + - name: Preserve the two synthetic native cases when their shard runs them + if: always() + uses: actions/upload-artifact@v4 + with: + name: native-synthetic-cases-${{ github.sha }}-${{ github.run_id }}-${{ github.run_attempt }}-shard${{ matrix.shard }} + path: ${{ runner.temp }}/d4d-native-cases-${{ github.run_id }}-${{ github.run_attempt }}-shard${{ matrix.shard }}/ + if-no-files-found: warn + compression-level: 0 + retention-days: 14 + # Run these controls once with their own existing 45-minute job budget. # They must not share the remainder of a full shard's budget (#4466). offline-canary: diff --git a/notes/native_ci_case_evidence_2026-10-08/README.md b/notes/native_ci_case_evidence_2026-10-08/README.md new file mode 100644 index 000000000..c48b0c9f9 --- /dev/null +++ b/notes/native_ci_case_evidence_2026-10-08/README.md @@ -0,0 +1,104 @@ +# Ordinary CI retention for two synthetic native cases (#4675) + +At PR #4659 head `cfe61d29afc6edbf79cf6970e014fd951bee6d15`, CI run +37752376080 reported the existing native-attempt deadline as the first stop in +both full synthetic acceptance tests. Neutral failed after 996.515 seconds of +test time and omission after 990.305 seconds, each with phase 1 and core sealed +but no completed assembly. These durations include test setup/finalization. +Neither result identifies the expensive operation. The two shard artifacts +contained only JUnit; their 11.4 MB captured transcripts, control logs and case +files were unavailable for independent diagnosis. + +The ordinary shard command now explicitly enables `utils.native_ci_evidence`. +Without `--native-ci-evidence-root`, the plugin performs no source reads, +capture or output writes. It does not select, filter, reorder or rerun tests. +Only these exact existing nodes can trigger case retention: + +- `tests/test_native_shared_execution.py::test_actual_public_correction_three_workers_and_saved_completion` +- `tests/test_native_shared_execution_variant.py::test_actual_public_merged_omissions_and_all_chunk_statuses` + +After their teardown report, the plugin uses their fixture-owned `tmp_path` +and the fixed `native26` or `native26-omissions` child. It creates a new output +under an external run/attempt/shard directory, then a worker directory and exact +node digest. Existing destinations refuse; no source or earlier evidence is +overwritten. The upload runs with `always()` and a 14-day retention period. +Missing upload files produce a workflow warning, not an invented passing case. + +Each case manifest retains setup/call/teardown outcomes, selected source +head/tree and working-file hashes, the unchanged declared 900/180/120-second +limits, and before/archive/after inventories of file bytes and modes. Source +identity covers the named working files; it is not complete imported-code or +environment attestation. The source checkout and referenced schema/dependency +files outside the case are not copied by following paths in registration JSON. +Their available declarations remain in the captured case. + +`retained` means that the snapshot bytes were stable during capture. It says +nothing about case completion or successful runtime/scientific acceptance. +`case_completeness` remains `not_assessed`. A fixed artifact availability roster +distinguishes regular files, links, missing files and uninspected/unavailable +cases, so an early partial directory cannot masquerade as complete evidence. +The inventories retain every case entry, including additional request/response +files. Cases exceeding 512 MiB of regular-file content or 10,000 entries refuse +with an explicit capture error; content is not silently truncated. + +Directory traversal and regular-file reads use held descriptors and +`O_NOFOLLOW`; links are inventoried and archived as links. The collector does +not follow external links, JSON authority references or unrelated temporary +directories, and does not export process environments or invoke providers. +Nonregular files refuse. Before/archive/after disagreement remains an explicit +`capture_changed` outcome. Existing test failures and pytest exit codes remain +unchanged when capture succeeds or fails. Per-worker summaries distinguish +observed nodes from nodes not observed in that worker; another shard or worker +may have run them. A worker crash may leave only partial diagnostic output. + +The two acceptance test bodies, production modules, helper commands, +registered deadlines, assertions and shard scheduling remain unchanged. +The dedicated `review/4354-offline-acceptance` workflow keeps its separate +serialized acceptance behavior. This diagnostic successor depends on the +unmerged native implementation in PR #4659 and is not a standalone main-based +adapter delivery. #4537 and #4354 remain open until their actual acceptance +obligations pass; all human/scientific/provider holds remain in place. + +## Validation status + +The first focused coordinator run at +`0893589bc8153a4416bf9c6f1370dd5771d01da2` completed with 34 passed and one +failed test (6.26 seconds reported by the coordinator; no warnings). The +failure was a test assumption that a symlink always has mode `0777`; this macOS +fixture's actual `lstat` mode was `0755`, which the implementation correctly +preserved. Under #4677, the assertion now uses the actual link mode and also +checks that mode in the archive. External-byte and non-dereference controls are +unchanged. No production behavior changed. + +The fresh run passed **35 tests**, with no failures, errors or skips, at commit +`b90ef8a142cdc5770674059b08e81b648e963f5e`, tree +`6324c0d1a479be31f6d3a64a1a301befaf8de73b`. The coordinator reported 6.16 seconds +and zero warnings; JUnit records 6.130 seconds. Module counts are 16 retention +controls, six independent adversarial controls, and 13 existing acceptance-runner +controls. The independent controls include small invented pytest children that +prove capture follows fixture teardown and preserves original exit outcomes. + +[validation.json](validation.json) records both JUnit artifact hashes, the exact +coordinator-reported invocation, and 54 source/test/workflow/dependency file +pins verified against the tested Git blobs and current bytes. Of those, 46 +native production, acceptance, fixture and dedicated-workflow files also match +the `cfe61d29afc6edbf79cf6970e014fd951bee6d15` base. All files under +`src/data_sheets_schema` are unchanged from that base. JUnit itself does not +attest the source commit or command environment. + +The same validation record binds the externally retained logs, JUnit, extracted +results and archive hashes from the preceding hosted failure: shard 1 had +2,237 passed, one omission-case failure and two skips; shard 2 had 2,854 passed, +one neutral-case failure and two skips. Both returned the native deadline stop +reason, and both downloaded archives contained only `pytest.xml`. The local +35-test result establishes retention behavior only. It is not a hosted case +capture or native completion result; no new 900-second native test was run +locally. Fresh exact-head hosted acceptance remains pending. + +The focused controls cover failed and partial cases, missing fixtures, collisions, +mode/byte drift, bounded refusal, unrelated-node exclusion, symlink isolation, +separate workers and preservation of original test outcomes. No native test, +provider call, diagnostic replay or CI rerun has been performed for this change. +Review issue #4677 adds an explicit archive root-directory member with its +captured mode, including for empty or early partial cases; the inventory must +describe actual archived entries. diff --git a/notes/native_ci_case_evidence_2026-10-08/validation.json b/notes/native_ci_case_evidence_2026-10-08/validation.json new file mode 100644 index 000000000..3820d062d --- /dev/null +++ b/notes/native_ci_case_evidence_2026-10-08/validation.json @@ -0,0 +1,501 @@ +{ + "format": "native_ci_case_evidence_validation_v1", + "issues": [ + 4675, + 4677 + ], + "pr": 4659, + "open_parent_issues": [ + 4537, + 4354 + ], + "base_commit": "cfe61d29afc6edbf79cf6970e014fd951bee6d15", + "tested_commit": "b90ef8a142cdc5770674059b08e81b648e963f5e", + "tested_tree": "6324c0d1a479be31f6d3a64a1a301befaf8de73b", + "verification": { + "coordinator_provenance": "Coordinator reports the invocation at this clean commit. JUnit does not attest source or environment. Console duration/warning count are coordinator reports.", + "independent_checks": [ + "Parsed both JUnit files and counted every testcase/failure/error/skip.", + "Compared every pinned working file byte with its tested-commit blob.", + "Compared unchanged native/acceptance/fixture paths with the cfe base.", + "Verified retained cfe CI archive, log, JUnit and extracted-result digests." + ], + "dependency_scope": "Dependency declarations and selected source bytes; not installed environment or complete imported closure." + }, + "tests": { + "artifact": { + "name": "native-ci-evidence-tests-02.xml", + "bytes": 5506, + "sha256": "44df51e2b64c874fd57be5c356485c954decc391353d23241e4efb3e18a5dd2c" + }, + "cases": 35, + "passed": 35, + "failed": 0, + "errors": 0, + "skipped": 0, + "junit_seconds": 6.13, + "modules": { + "tests/test_native_ci_evidence.py": 16, + "tests/test_native_ci_evidence_adversarial.py": 6, + "tests/test_native_shared_acceptance_runner.py": 13 + }, + "exit_code": 0, + "console_seconds": 6.16, + "console_warnings": 0 + }, + "initial_run": { + "artifact": { + "name": "native-ci-evidence-tests-01.xml", + "bytes": 6928, + "sha256": "275669ce0d230733a695f9783d8ad1bc76497ad2f9b4acf6b8ff790e965088fb" + }, + "cases": 35, + "passed": 34, + "failed": 1, + "errors": 0, + "skipped": 0, + "junit_seconds": 6.225, + "modules": { + "tests/test_native_ci_evidence.py": 16, + "tests/test_native_ci_evidence_adversarial.py": 6, + "tests/test_native_shared_acceptance_runner.py": 13 + }, + "tested_commit": "0893589bc8153a4416bf9c6f1370dd5771d01da2", + "tested_tree": "cae3a5dcc9121a592e81672d962cee44587d675d", + "console_seconds": 6.26, + "console_warnings": 0, + "failure": "test_external_symlinks_and_json_authority_paths_are_never_followed assumed symlink mode0777; observed macOS0755 was correctly retained. Fixed only the test to compare actual lstat and archive modes." + }, + "invocation": { + "cwd": "$CHECKOUT", + "environment": { + "PYTHONDONTWRITEBYTECODE": "1", + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + "PYTHONPATH": "src:." + }, + "command": [ + "$PYTHON", + "-B", + "-m", + "pytest", + "tests/test_native_ci_evidence.py", + "tests/test_native_ci_evidence_adversarial.py", + "tests/test_native_shared_acceptance_runner.py", + "-q", + "-p", + "no:cacheprovider", + "--basetemp=$EVIDENCE/native-ci-evidence-pytest-02", + "--junitxml=$EVIDENCE/native-ci-evidence-tests-02.xml" + ] + }, + "files": { + ".github/workflows/main.yaml": { + "bytes": 13867, + "sha256": "7e56e902e3177a19ff36de56394510fba3a37b35872c6fdc74f77d9ef951a496", + "git_blob": "249896f4e5bd5a7f26ee4c3081d877eb52862ac8" + }, + ".github/workflows/native-shared-acceptance.yaml": { + "bytes": 2016, + "sha256": "f76d6aedc0e7a3ceef5538ab3ba303dea9cd1c237992f83cd92174398ff07a5e", + "git_blob": "cbee64510cf23bc6652fc5a9226dceb7118cf181", + "unchanged_from_base": true + }, + "poetry.lock": { + "bytes": 592155, + "sha256": "923ad962d2646e1a4be044a76ff6a66abf49c3e10ea87fc85ed3cd523a907e21", + "git_blob": "cfe519359b0be36a52544377584f0217c6763fed" + }, + "pyproject.toml": { + "bytes": 4446, + "sha256": "bac863ac39cc39d6767627e0a7d8fd085be693555e5220015afbe6aec45c0c8b", + "git_blob": "87a84c564d243c134fead266600e96561562a75e" + }, + "src/data_sheets_schema/native_attempt_supervisor.py": { + "bytes": 26002, + "sha256": "3ac7a91f6ce5ebb60b284ccf3610ad4883a13d34b42570a8f530d6c813358d34", + "git_blob": "37c74c0b3789ec571f63e7fc1bf39b49e21aa75a", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_attribution_controller.py": { + "bytes": 21181, + "sha256": "1213d4704f0754efee6a3b1b09db0d3210eaf4fd64195bf733890ac3b1106967", + "git_blob": "1c9dcd77992671b5f8b54a56e24f56ee1b3a1fb6", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_attribution_registration.py": { + "bytes": 24342, + "sha256": "135be1958e93c682a2083b53d338c44c3acf66efe74cd5da94f7c66cf72e0ab6", + "git_blob": "b8c1a3943792d67ec2eabe858994b182f263b5e2", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_attribution_results.py": { + "bytes": 54114, + "sha256": "303d26d22d3003d6e02298311cef1c43879af50ebd0ee3b111beb41afab3ce39", + "git_blob": "d570e02c2bdb8094b3783bc7198645997d11635f", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_catalog_reuse.py": { + "bytes": 20455, + "sha256": "62658bceb86474ed651351557d9bea1be5ac7ac65f7779a92ce03e00d1ffe3c1", + "git_blob": "0a03786814d43992fbeadc2c0f17c93419f2db75", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_execution.py": { + "bytes": 10183, + "sha256": "8075fcbaacab60a1e73583b9d9b9eed8281ca147a0106eb7532d505f514067fe", + "git_blob": "a694300070cca99529aec6ce019de8cc8c9085d8", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_execution_authority.py": { + "bytes": 4317, + "sha256": "bbb10b0d678ad68a1bd71207f9d5fb3d09086674fb7da739e24c6e440861025a", + "git_blob": "9308a9030ab56d73158c1df52b0e0fc13eaec716", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_execution_engine.py": { + "bytes": 32195, + "sha256": "d87f036382c007dae2a6ce396a8106293ba0d70da570a70d71240d11a16f3870", + "git_blob": "b3b992171de71d3385dab6f1f97fe0bf57c6c95d", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_execution_gates.py": { + "bytes": 13665, + "sha256": "7b75a3d2ee302616628e379765449c00d298df565dd2dcdfeca6a6c2e6290bb6", + "git_blob": "37c41698b7d8401659e3a5f14d45858da866a0d5", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_execution_permissions.py": { + "bytes": 24358, + "sha256": "ea6d866aca57643f647907a00f3d6a7c124f2602faace20907a9c818bd771e21", + "git_blob": "a455f0110a2b17dd0c140748d4c86f36bb2253c1", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_execution_registration.py": { + "bytes": 20356, + "sha256": "7e4002de19c36b3379b1da3702c28ea0c27f6d42fd27ebaf9fbed6b0de82e689", + "git_blob": "ffd60a25507e919f96cdc12151c4b96f655bd082", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_gate_engine.py": { + "bytes": 11399, + "sha256": "dcc939a17f522656830f591f9a135cad45febd6edac5448445619fca75dd243b", + "git_blob": "64e7a13aeeb3964512ddb4b5bb7a58d1d0675033", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_neutral_runtime_v1.py": { + "bytes": 5370, + "sha256": "ae77c08defcc8b9eef3aa2967f884b503d2643c511a587c125e09530e8881826", + "git_blob": "54a41a241483fa326feea831d2a7bc5850bcd72a", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_receipt_origin.py": { + "bytes": 7225, + "sha256": "bfc46c6f7d9ddc287169633e185661dd0a09fd4b47709981e330d446b69df741", + "git_blob": "2511acaf5dc19e2ed512717932652f9baa322fe9", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_attribution.py": { + "bytes": 11549, + "sha256": "f2fff9aa2d953f0122a8c05cbc4f624c182d257ddde15928c07004ea31ba3dda", + "git_blob": "129c963ebbfa6e2b56c5650b62bb12279211ad15", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_capture.py": { + "bytes": 77215, + "sha256": "0f41fa2db916f66d45082f680e9bcad30704273f02fe1b90972067c0fab2aa05", + "git_blob": "0c4ec08608b2b4e3d6c75c5d4117172e53afbae3", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_contract.py": { + "bytes": 38808, + "sha256": "76664d4d9023a9a3a1975ca66f82bee0a4ce4b789c99c19606987ab2e50e65b9", + "git_blob": "de25b43beceba21ffc73b5f5fac09ee7d6454f07", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_controller.py": { + "bytes": 23927, + "sha256": "f04676ae57a4cc65b65351ecbda155ca62269fb21b5ecb327890e9396712ad5c", + "git_blob": "24d1fb73e1d4763b78c670a265ef29459b9d7f44", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_current.py": { + "bytes": 1628, + "sha256": "a4c66c191119060e626873570ad65e4a4626ee331471b6d65f3347d3fe5951bc", + "git_blob": "5b5b658f26dfa9a2cab6d5e22a0d8983c4dccd47", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_effects.py": { + "bytes": 8651, + "sha256": "e4c8fee4621abe29cc484a291216680da01f4f26a28cd746273b687445ef864c", + "git_blob": "3a36f9cde14a97e8185abf4304d4eac7beb8a9f7", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_evidence.py": { + "bytes": 12908, + "sha256": "a50b042d1825456d70571dbf5e314ea5204f9d1451319e1c8353462852e294e5", + "git_blob": "c6c7fb5b0b3521721684b28e9498d3cf1b356074", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_execution.py": { + "bytes": 3297, + "sha256": "fd7f635d44ac877f6b57f3f1c58990ed1da134b281ad54a7779428ef3ee456a9", + "git_blob": "fa91798213076c72da43a2e540d5c7b231642e92", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_gates.py": { + "bytes": 9978, + "sha256": "386d83172e9b065a71e85205c5f1d61873327bd9a482eb6abc283b3d748aa94d", + "git_blob": "4e613124ae6083a9d94795ec896bb608fab91966", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_observations.py": { + "bytes": 18263, + "sha256": "4dc91259ecb079e8efe3067b0eeb01e584b575bdda813fcc6e7494bf39bcbba4", + "git_blob": "c3cee904f06a9c5a9dad6697d91d8a964255c378", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_permissions.py": { + "bytes": 61966, + "sha256": "64d058d685b308da9da08efc61ea73d28a9d2e4901c0d6f86b7d816854a43883", + "git_blob": "f19d4bbd6642bae95ac1095a1e5c70104095bd4b", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_phase.py": { + "bytes": 25228, + "sha256": "fd702cd2ef62991643b81e33579ce45d70a7ef634d7d32eea144be020e598124", + "git_blob": "931f4cdac4f4f866b44b6a12c41d479f61230057", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_policy.py": { + "bytes": 9926, + "sha256": "c92d8f15ba658a6d4c63e560a79d7f02a896ca451f59a60e907ca0e8eda06b02", + "git_blob": "2cc7ab2437e152eaf05044c15b867706beea4770", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_publication.py": { + "bytes": 9077, + "sha256": "bbb8026187644f9d536c743238a9321cdfe41fa16a772398a857e244fbd3660a", + "git_blob": "60d8c34e84666751ac06e31de0e99f8a28170f22", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_receipts.py": { + "bytes": 23106, + "sha256": "c2a4725c84e893563605779ec8476658767635b5b065d399f59be0b58112d4ce", + "git_blob": "95a93a026feaca05bf50ba946d4c3be7f1b3088e", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_registration.py": { + "bytes": 11654, + "sha256": "8abbf4b1a5fefb1e221e69e1fec09d4a6841b42422aa94948d7350a9ad34711d", + "git_blob": "615ee27d533d1803fc60be62e027a73c9213111b", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_render.py": { + "bytes": 24063, + "sha256": "8b09502eeac5488b74334365914d8a1470d84c4a67f6f057b81e0cd61a94c2f7", + "git_blob": "2cb9e6a5d9c61ca3c11e293cc8a99af9cbdc3896", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_results.py": { + "bytes": 32198, + "sha256": "2047abb3ced650fcb526abf0dc22f23b68b26241f284039ac1c58d42333f3297", + "git_blob": "173c189a3318890b84ebfb80a186e1e188eca5d4", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_schema_gates.py": { + "bytes": 2601, + "sha256": "23e0222ce3ad36604d8326d3e2a84b3d01d6a69a1f09cc156f61b0ffee524534", + "git_blob": "6ec86248286b83dbd89f4a3c2f576de019e276c0", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_selection.py": { + "bytes": 23850, + "sha256": "ab398ec47c2c5828b77a0a1a7cd3840052f51e1781656d28482c9569031f0af1", + "git_blob": "150bbd2a99298fcf8d3719b158106dc49fa587b2", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_stage.py": { + "bytes": 38928, + "sha256": "4c85b31383a1e367a180d9952e33be27b4a35577fdc41038812c2a7cf7a8096e", + "git_blob": "64ad2a1d85e527c923b2c233aa2ecdf641dccda9", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_shared_streams.py": { + "bytes": 6276, + "sha256": "d499822bb9ab2f147a60bced938dd2eeebf71cb924130690de105e76d328dc8e", + "git_blob": "9b7d23d436138b662aa6b7dbb4339cdb393e6f8a", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_source_attribution.py": { + "bytes": 3068, + "sha256": "8d2322b4b0c366a00eab5fda9dba3073acf640825d0a2f6fae353eb2ebaa6d0d", + "git_blob": "16da04d20d0b33049cd076964aba914f904a35a3", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_supervisor_authority.py": { + "bytes": 3536, + "sha256": "dd48bee05889e9c9c3bc9dd933999ece38e9edba383e117ed7ec6923171f6247", + "git_blob": "162a4fd8c958f0a827b19b11ceed85c6e7e65e67", + "unchanged_from_base": true + }, + "src/data_sheets_schema/native_supervisor_gates.py": { + "bytes": 22677, + "sha256": "7174047c001ec8fb3eb710928545f642f22a7b9f4cd2e9d3fa3b27caa57fe885", + "git_blob": "76d5e11993d8562c3d537d2f3797e064aa443c65", + "unchanged_from_base": true + }, + "tests/fixtures/native_shared_execution/fake_cli.py": { + "bytes": 9434, + "sha256": "82622dc176fff332fc2d44dcb5de2920a3e70fdc00a0605ef8b87550d47bb646", + "git_blob": "e9ea332dd223f7a5d2a039d64352532381184b14", + "unchanged_from_base": true + }, + "tests/native_shared_fixture.py": { + "bytes": 14883, + "sha256": "305ccabdcdf1a511f562d842d22a1c26ca5b0857acff0b57abf6e0e94acfe582", + "git_blob": "1b8800074cf4942aff8c4e54639d076c804bb151", + "unchanged_from_base": true + }, + "tests/native_shared_omission_fixture.py": { + "bytes": 17977, + "sha256": "35229bcbe1be3867ad5b2a7b85023c94eac0f948ee9ed749735f36d0731c610d", + "git_blob": "75840d54c17e3b2640e6badce294174254339fae", + "unchanged_from_base": true + }, + "tests/test_native_ci_evidence.py": { + "bytes": 12185, + "sha256": "48f086181b5592e2269e2e28a8471730c832e9f12983e39f02c88f4b074d0d0f", + "git_blob": "ae52b4d06e7477d0a490ead3169a5d5a67e2b00f" + }, + "tests/test_native_ci_evidence_adversarial.py": { + "bytes": 7782, + "sha256": "68142307d4b962522372a6473b7d9612d8d1290a7abd4b12f5a47d2eab9577be", + "git_blob": "78c23c417dc0a3ab96bc0acdc5085e0115441cb2" + }, + "tests/test_native_shared_acceptance_runner.py": { + "bytes": 9269, + "sha256": "aee7a39f2050bbad1d037c6b4b1a15be4f20dff5d28a09b4f92bd0117cb523f7", + "git_blob": "22d6578eb707a0d72eccf864c660d00c89af0b73" + }, + "tests/test_native_shared_execution.py": { + "bytes": 8137, + "sha256": "31ac17b1d66f60570ab0e26b5f96dc9170a8707a56f0798643897822383212c1", + "git_blob": "132a176192a6b4eb63fc07050bec21016c699758", + "unchanged_from_base": true + }, + "tests/test_native_shared_execution_failure.py": { + "bytes": 2113, + "sha256": "dd661ca9ec7ea6eaa3099a03baea11971791a6c26c7bba7ccdadb9983f8f1c07", + "git_blob": "bb9c5d7e7548805c9296b422f793ed87629931c1", + "unchanged_from_base": true + }, + "tests/test_native_shared_execution_variant.py": { + "bytes": 7957, + "sha256": "54e31f2b3740c96bc1dfefe9c377aa7525a2657fc115ef961f55cf0894a4feb5", + "git_blob": "b9f98c24a438cca623e539ef4c8f27364fdf5d76", + "unchanged_from_base": true + }, + "utils/native_ci_evidence.py": { + "bytes": 12788, + "sha256": "7931517c28fcdc90d2b3ad572095c738ecad5db5c69be4fa61822cf329d62f30", + "git_blob": "eb128688163f1063f6fa5d15ce89613242f8e914" + }, + "utils/native_shared_acceptance.py": { + "bytes": 10131, + "sha256": "9e793fb0c9ce9b6d9bb5c9df985a653ec94a8d50692503d936c6b5061acfd9d3", + "git_blob": "4144a25835a2821827909cf28d979134ce70d793" + } + }, + "preservation": { + "all_src_data_sheets_schema_unchanged": true, + "original_test_nodes_and_fixtures_unchanged": true, + "dedicated_acceptance_workflow_unchanged": true, + "ordinary_workflow_delta": "Add explicit retention plugin/option and separate always-upload only; test selection, xdist scheduling and limits unchanged." + }, + "preceding_hosted_ci": { + "run_id": 37752376080, + "pr_head": "cfe61d29afc6edbf79cf6970e014fd951bee6d15", + "tested_merge": "6b677db3c23f4aa69f5bc9c288d15014bfa86f96", + "declared_attempt_seconds": 900, + "shards": { + "1": { + "job_id": 113228480772, + "tests": 2240, + "passed": 2237, + "failures": 1, + "errors": 0, + "skipped": 2, + "failed_class": "tests.test_native_shared_execution_variant", + "failed_test": "test_actual_public_merged_omissions_and_all_chunk_statuses", + "test_seconds": "990.305", + "first_stop": "native attempt deadline elapsed; retain all incomplete charge reservations", + "phase1_sealed": true, + "core_sealed": true, + "assembly_complete": false, + "pending_tool_ids": [ + "shared-test-23" + ] + }, + "2": { + "job_id": 113228480731, + "tests": 2857, + "passed": 2854, + "failures": 1, + "errors": 0, + "skipped": 2, + "failed_class": "tests.test_native_shared_execution", + "failed_test": "test_actual_public_correction_three_workers_and_saved_completion", + "test_seconds": "996.515", + "first_stop": "native attempt deadline elapsed; retain all incomplete charge reservations", + "phase1_sealed": true, + "core_sealed": true, + "assembly_complete": false, + "pending_tool_ids": [ + "shared-test-22" + ] + } + }, + "artifact_scope": "Both downloaded shard archives contain only pytest.xml. Captured case bytes were not available.", + "external_artifacts": { + "4659-cfe-shard1-artifact.zip": { + "bytes": 47112, + "sha256": "a09108280b6982bdf52d21811063d83ebd04407f042fef8df44c26d3b814dd5b" + }, + "4659-cfe-shard1-job.log": { + "bytes": 826080, + "sha256": "d6329f42bdc4a113a68ecfdc976377d7d9aadddc861177e52c0cf76d73874a0c" + }, + "4659-cfe-shard1-pytest.xml": { + "bytes": 409547, + "sha256": "ed7fb16b2ed2691f5079fabf400cf7130898ff403d48f222698b58336f0408b2" + }, + "4659-cfe-shard1-result.json": { + "bytes": 20415, + "sha256": "65189caeb09006720b4deff11dff424b719cbc15a12421aa8566faed2baae6c4" + }, + "4659-cfe-shard2-artifact.zip": { + "bytes": 63521, + "sha256": "73ec9d5868b3ad1d9981b913068fafce74c868c2d0cb1c89810cf87f79525494" + }, + "4659-cfe-shard2-job.log": { + "bytes": 1400674, + "sha256": "5f6cbffbe95895baae590bd9b06fae514649d3799488358d17be044768ed702d" + }, + "4659-cfe-shard2-pytest.xml": { + "bytes": 693729, + "sha256": "1724834a33b7b2e5f9057da7f9cf98d1199a2067b6606f6a23ba13bb3a3707b2" + }, + "4659-cfe-shard2-result.json": { + "bytes": 17755, + "sha256": "f4d755f296f2bc1f00a3c49ea99dd48ed4a5a3c10fe676aa50e3a8d40ef1440e" + } + } + }, + "limitations": [ + "35 passing tests prove only offline retention and small invented pytest lifecycles, not native acceptance completion.", + "No new local900-second native run, provider call, hosted case capture, diagnostic replay or CI rerun.", + "No source/case content is authenticated by these manifests.", + "Partial/unavailable cases and capture failures remain explicit; retained does not mean completed.", + "Existing PR4659 dependency only; no standalone main integration or closure of4537/4354." + ] +} diff --git a/notes/native_stream_diagnostics_2026-10-08/README.md b/notes/native_stream_diagnostics_2026-10-08/README.md new file mode 100644 index 000000000..0cb2b4338 --- /dev/null +++ b/notes/native_stream_diagnostics_2026-10-08/README.md @@ -0,0 +1,49 @@ +# First live-stream refusal diagnostics + +Issue [#4682](https://github.com/bridge2ai/data-sheets-schema/issues/4682) adds diagnostic context to the existing native stream refusal. It does not repair or establish the cause of [#4680](https://github.com/bridge2ai/data-sheets-schema/issues/4680). The candidate lineage contains the native implementation; this change is not an integration of that implementation into main. + +`read_live_stream` retains its existing physical identity, single-link, extent, append, two-pass byte equality and prior-prefix guards. Every metadata acquisition and byte-read call remains in its original position. A refusal from `_progress` may now carry a private `ValueError` subclass whose arguments are the original error's arguments. The adapter continues to report the original `ValueError: ...` first-stop text. + +The optional `stream_diagnostic` object contains only observations already available when that guard failed: + +- `kind: native_live_stream_refusal`, `version: 1`; +- stream `role`, exact selected `path`, and `max_bytes`; +- `stage` (`opened`, `before_read`, `after_read`, `final_fd`, or `final_path`), `pass_number` (1 or 2 for read stages, otherwise null), and byte `offset`; +- accepted `opened_extent`, or null if the opening metadata comparison itself refused; +- complete `earlier` and `later` metadata dictionaries already sampled by the reader; +- `previous_prefix` containing its byte count and SHA256, or null when none was supplied. + +The exception and adapter retain canonical bytes bounded to 16 KiB. Each report decodes a detached object. There are no raw stream contents, extra observations, retries, sleeps, new writes, altered admission, or changed deadlines. Small context dictionaries are assembled at existing samples; diagnostic serialization runs only after a guard refuses. The exact selected path is retained in the local failure report and should be considered when sharing that report. + +Diagnostic serialization or decoding failure falls back to the original refusal or the existing report without this optional field. Only the first failure can supply a diagnostic; a later callback still refuses immediately. Cursor behavior is unchanged: endpoint-acquisition failure advances neither cursor, while a later live recapture failure preserves the cursor updates already completed by endpoint acquisition. Success and unrelated failures do not acquire an invented diagnostic. + +The deterministic tests inject metadata schedules into real stream reads, including an equal-size/changed-mtime pair at every sampled stage and a split-field ambiguity. These are synthetic observation controls, not reproductions or explanations of the hosted failure. Existing stream tests retain append success, same-size rewriting, truncation, replacement, hardlink/symlink, prior-prefix and byte-bound controls. Adapter tests exercise the real failure latch, endpoint cursor updates and report path without a native process, controller helper, or provider. + +The independently reviewed implementation passed all **105 focused tests**, with no failures, errors or skips, at commit `d01ef23c37e22137be367a73f252cc74d4a58d73` (tree `b072f25cbcec36306ab8de421beecb3c3f6af21a`). JUnit records 141.224 seconds; the coordinator recorded 141.32 seconds at the console and one dateutil deprecation warning. The recorded interpreter was the existing primary-checkout virtual environment using CPython 3.13.12. These invocation and console observations are coordinator provenance; JUnit does not independently attest the interpreter or tested commit. + +| Test module | Passed | +| --- | ---: | +| `test_native_shared_stream_diagnostics.py` | 26 | +| `test_native_shared_stream_diagnostics_adversarial.py` | 15 | +| `test_native_shared_streams.py` | 31 | +| `test_native_shared_stream_bindings.py` | 7 | +| `test_native_shared_controller.py` | 26 | + +[validation.json](validation.json) records the complete tested change list, eight selected source pins, five test-module pins, dependency declarations, and the unchanged frozen controller manifest plus all 12 dependency pins. Each selected working file was checked against its tested Git blob. The manifest and its dependencies also match their declared hashes and parent bytes. This is selected source evidence, not a complete installed-environment or import-cache snapshot. + +The external JUnit artifact, `native-stream-diagnostic-tests-01.xml`, is 17,180 bytes with SHA256 `7d36cd9da496dad8b3cc60b43ca9f2679a346d13e34f211cb9af69012da1c394`. The coordinator executed the following command from the candidate checkout; `$PYTHON` and `$EVIDENCE` stand for the selected interpreter and external evidence directory: + +```bash +PYTHONDONTWRITEBYTECODE=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 PYTHONPATH=src:. \ + "$PYTHON" -B -m pytest \ + tests/test_native_shared_stream_diagnostics.py \ + tests/test_native_shared_stream_diagnostics_adversarial.py \ + tests/test_native_shared_streams.py \ + tests/test_native_shared_stream_bindings.py \ + tests/test_native_shared_controller.py \ + -q -p no:cacheprovider \ + --basetemp="$EVIDENCE/native-stream-diagnostic-pytest-01" \ + --junitxml="$EVIDENCE/native-stream-diagnostic-tests-01.xml" +``` + +The original retained CI evidence and reviewed verifier remain separately preserved in [PR #4683](https://github.com/bridge2ai/data-sheets-schema/pull/4683). These diagnostics do not alter or reinterpret those historical artifacts. No native workload, hosted-race reproduction or performance comparison is included. Issues #4680, #4537 and #4354 remain open for their separate cause, performance and acceptance obligations; native execution acceptance and scientific eligibility are not established here. diff --git a/notes/native_stream_diagnostics_2026-10-08/validation.json b/notes/native_stream_diagnostics_2026-10-08/validation.json new file mode 100644 index 000000000..6531b5258 --- /dev/null +++ b/notes/native_stream_diagnostics_2026-10-08/validation.json @@ -0,0 +1,340 @@ +{ + "format": "native_live_stream_diagnostic_validation_v1", + "implementation_issue": 4682, + "unresolved_parent_issues": [ + 4680, + 4537, + 4354 + ], + "tested_commit": "d01ef23c37e22137be367a73f252cc74d4a58d73", + "tested_tree": "b072f25cbcec36306ab8de421beecb3c3f6af21a", + "parent_commit": "aa845f2e083185707feab0ee8b090d6d9c9f52ef", + "test_binding_provenance": "Coordinator reported this clean committed head/tree immediately before and after the serialized run. The author independently checked current selected bytes against tested Git blobs. JUnit itself does not attest a commit or invocation.", + "tested_commit_changed_paths": [ + "notes/native_stream_diagnostics_2026-10-08/README.md", + "src/data_sheets_schema/native_shared_controller.py", + "src/data_sheets_schema/native_shared_streams.py", + "tests/test_native_shared_stream_diagnostics.py", + "tests/test_native_shared_stream_diagnostics_adversarial.py" + ], + "validation": { + "status": "focused_tests_passed", + "cases": 105, + "failures": 0, + "errors": 0, + "skipped": 0, + "junit": { + "artifact": "native-stream-diagnostic-tests-01.xml", + "bytes": 17180, + "sha256": "7d36cd9da496dad8b3cc60b43ca9f2679a346d13e34f211cb9af69012da1c394", + "suite_seconds": 141.224, + "module_counts": [ + { + "path": "tests/test_native_shared_stream_diagnostics.py", + "testcases": 26 + }, + { + "path": "tests/test_native_shared_stream_diagnostics_adversarial.py", + "testcases": 15 + }, + { + "path": "tests/test_native_shared_streams.py", + "testcases": 31 + }, + { + "path": "tests/test_native_shared_stream_bindings.py", + "testcases": 7 + }, + { + "path": "tests/test_native_shared_controller.py", + "testcases": 26 + } + ] + }, + "coordinator_reported": { + "exit_code": 0, + "console_seconds": 141.32, + "warning_count": 1, + "warning_description": "dateutil deprecation warning", + "python_version": "CPython 3.13.12", + "interpreter_selection": "existing primary checkout virtual environment", + "source_clean_unchanged_after_run": true + }, + "invocation": { + "provenance": "coordinator-supplied invocation; public paths are symbolic", + "cwd": "candidate checkout", + "environment": { + "PYTHONDONTWRITEBYTECODE": "1", + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + "PYTHONPATH": "src:." + }, + "argv": [ + "$PYTHON", + "-B", + "-m", + "pytest", + "tests/test_native_shared_stream_diagnostics.py", + "tests/test_native_shared_stream_diagnostics_adversarial.py", + "tests/test_native_shared_streams.py", + "tests/test_native_shared_stream_bindings.py", + "tests/test_native_shared_controller.py", + "-q", + "-p", + "no:cacheprovider", + "--basetemp=$EVIDENCE/native-stream-diagnostic-pytest-01", + "--junitxml=$EVIDENCE/native-stream-diagnostic-tests-01.xml" + ] + }, + "independent_author_checks": [ + "Parsed saved JUnit and checked exact artifact size/SHA256, suite duration and all five module counts.", + "Checked all selected source, test and declared dependency bytes against tested Git blobs.", + "Checked frozen controller manifest hash and all 12 dependencies against the manifest and unchanged parent bytes." + ] + }, + "selected_sources": [ + { + "path": "src/data_sheets_schema/native_shared_streams.py", + "bytes": 8575, + "sha256": "9600cac865c50f2e863bb5fecb206bbeafaa721b49dda9ae2206044c88d15f7a", + "working_matches_tested_git_blob": true + }, + { + "path": "src/data_sheets_schema/native_shared_controller.py", + "bytes": 25063, + "sha256": "a08b957584f7a253c872806350f28e511562a71136a06d14e6804e6ebb51068c", + "working_matches_tested_git_blob": true + }, + { + "path": "src/data_sheets_schema/native_shared_contract.py", + "bytes": 38808, + "sha256": "76664d4d9023a9a3a1975ca66f82bee0a4ce4b789c99c19606987ab2e50e65b9", + "working_matches_tested_git_blob": true + }, + { + "path": "src/data_sheets_schema/native_shared_evidence.py", + "bytes": 12908, + "sha256": "a50b042d1825456d70571dbf5e314ea5204f9d1451319e1c8353462852e294e5", + "working_matches_tested_git_blob": true + }, + { + "path": "src/data_sheets_schema/native_shared_capture.py", + "bytes": 77215, + "sha256": "0f41fa2db916f66d45082f680e9bcad30704273f02fe1b90972067c0fab2aa05", + "working_matches_tested_git_blob": true + }, + { + "path": "src/data_sheets_schema/native_shared_results.py", + "bytes": 32198, + "sha256": "2047abb3ced650fcb526abf0dc22f23b68b26241f284039ac1c58d42333f3297", + "working_matches_tested_git_blob": true + }, + { + "path": "src/data_sheets_schema/native_shared_receipts.py", + "bytes": 23106, + "sha256": "c2a4725c84e893563605779ec8476658767635b5b065d399f59be0b58112d4ce", + "working_matches_tested_git_blob": true + }, + { + "path": "src/data_sheets_schema/native_attribution_controller.py", + "bytes": 21181, + "sha256": "1213d4704f0754efee6a3b1b09db0d3210eaf4fd64195bf733890ac3b1106967", + "working_matches_tested_git_blob": true + } + ], + "test_modules": [ + { + "path": "tests/test_native_shared_stream_diagnostics.py", + "bytes": 7400, + "sha256": "458be79ba4bb67fd05591b08763b23bdaed5d2ac0fba5a0ee407e2cb1e246b8f", + "working_matches_tested_git_blob": true + }, + { + "path": "tests/test_native_shared_stream_diagnostics_adversarial.py", + "bytes": 11275, + "sha256": "162376229298f5cc8944cae12ce69a1935babaa2a8b296927aeed40be688d1dc", + "working_matches_tested_git_blob": true + }, + { + "path": "tests/test_native_shared_streams.py", + "bytes": 12721, + "sha256": "95703cd932f11a851bf7f262a3f7973aa63edd2952899e6f04c9a39294a6aad5", + "working_matches_tested_git_blob": true + }, + { + "path": "tests/test_native_shared_stream_bindings.py", + "bytes": 5082, + "sha256": "9f76ab60c848dc112614661699a5ef63f92454ab19436b9b329c09d645ce1a76", + "working_matches_tested_git_blob": true + }, + { + "path": "tests/test_native_shared_controller.py", + "bytes": 5227, + "sha256": "b24ab493c539036037fb3ec2240f18f99dea9ebdbe276a4910f28fe8484e0849", + "working_matches_tested_git_blob": true + } + ], + "dependency_declarations": [ + { + "path": "pyproject.toml", + "bytes": 4446, + "sha256": "bac863ac39cc39d6767627e0a7d8fd085be693555e5220015afbe6aec45c0c8b", + "working_matches_tested_git_blob": true + }, + { + "path": "poetry.lock", + "bytes": 592155, + "sha256": "923ad962d2646e1a4be044a76ff6a66abf49c3e10ea87fc85ed3cd523a907e21", + "working_matches_tested_git_blob": true + } + ], + "frozen_controller": { + "manifest": { + "path": "src/data_sheets_schema/native_controller_sources_v1.json", + "bytes": 2459, + "sha256": "d1f13def58c0832aea3644fb68806d829eb4ab6e087045f14d86f439aed557c1", + "working_matches_tested_git_blob": true + }, + "unchanged_from_parent": true, + "declared_source_commit": "4ee87580c5472aee2d30cabec4dff1b230de8436", + "dependencies": [ + { + "module": "budget_amendment", + "path": "notes/matched_cborg_2026-09-13/budget_amendment.py", + "bytes": 31267, + "sha256": "7347b407611871349a50cdb2e11ade7ace1f23dd703451dba5abd9f627cbd3f9", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "budgeted_cborg", + "path": "notes/matched_cborg_2026-09-13/budgeted_cborg.py", + "bytes": 32252, + "sha256": "0aef1b59d05fd9a1f7f6e2f83b863800fb48e6c35e22030cbcbcfc373c5375e2", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "native_command_policy", + "path": "notes/matched_cborg_2026-09-13/native_controls/native_command_policy.py", + "bytes": 34944, + "sha256": "99db1879752ea73fea0002ea5a3a2d8333365b4e3843abdab3e1a1f0008709cc", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "native_control", + "path": "notes/matched_cborg_2026-09-13/native_controls/native_control.py", + "bytes": 45573, + "sha256": "e53f9627e629f06f40046460dbe43bb7a62c26d3f28bedf41408531bc5374a36", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "native_file_policy", + "path": "notes/matched_cborg_2026-09-13/native_controls/native_file_policy.py", + "bytes": 9211, + "sha256": "c6fc332b237fe06200437e4e899b0e6dbb2830b9a40d537b4e8db36bd5e5b33e", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "native_phase_history", + "path": "notes/matched_cborg_2026-09-13/native_controls/native_phase_history.py", + "bytes": 17636, + "sha256": "0a318e7bb3eea8d99067da168de09985d92f7f28c6ed0e0b01d7c01dcf85a1d6", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "native_proxy", + "path": "notes/matched_cborg_2026-09-13/native_controls/native_proxy.py", + "bytes": 49654, + "sha256": "21f2d16e6c9b93f033664c7b95521076b27786a2b1679c008713f2e40bf599b7", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "native_readonly", + "path": "notes/matched_cborg_2026-09-13/native_controls/native_readonly.py", + "bytes": 8254, + "sha256": "a3b40463f1748f5d109b9d018a7d074a6e8521e9bd413151293f37857e3006b8", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "prepare_overlay_roster", + "path": "notes/matched_cborg_2026-09-13/native_controls/prepare_overlay_roster.py", + "bytes": 916, + "sha256": "36575c39ae3e291320513072337e1b22c890f4498746fa24289e67bf4ce45819", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "prepare_registration", + "path": "notes/matched_cborg_2026-09-13/prepare_registration.py", + "bytes": 19949, + "sha256": "06a31323dfd893e7e16b8d6778c19ed4489f24b8bc3d74a43391296f881c49ae", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "run_api_canary", + "path": "notes/matched_cborg_2026-09-13/run_api_canary.py", + "bytes": 13596, + "sha256": "8b5ce15695b5808060edfb3d2fb85fd441244c4ee23ea8d8cf556352b8bbbbc4", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + }, + { + "module": "run_native_canary", + "path": "notes/matched_cborg_2026-09-13/native_controls/run_native_canary.py", + "bytes": 46837, + "sha256": "fbb98d66b36b392f5291866ede6acff4b162ad626d98ca7d104c0a45ae96ac5d", + "working_matches_tested_git_blob": true, + "matches_frozen_declared_sha256": true, + "unchanged_from_parent": true + } + ] + }, + "diagnostic_contract": { + "optional_report_key": "stream_diagnostic", + "max_canonical_bytes": 16384, + "exception_base": "ValueError", + "first_stop_type_text": "ValueError", + "raw_stream_contents_retained": false, + "additional_reader_io": false, + "guard_or_admission_changes": false, + "deadline_changes": false, + "retries_added": false, + "cursor_update_timing_changed": false, + "diagnostic_failures_preserve_original_refusal": true + }, + "historical_evidence": { + "pull_request": 4683, + "url": "https://github.com/bridge2ai/data-sheets-schema/pull/4683", + "scope": "Original retained CI artifacts and reviewed evidence verifier; not regenerated or reclassified by this diagnostic change.", + "cause_issue": 4680, + "cause_established": false + }, + "execution_acceptance_established": false, + "scientific_eligibility_established": false, + "limitations": [ + "Deterministic metadata schedules are synthetic observation controls, not a reproduction or cause determination of the hosted failure.", + "Focused tests verify reader and adapter behavior without a native workload, controller helper or provider campaign.", + "No performance comparison or native completion claim is made.", + "Selected byte pins and dependency declarations are not a complete installed-environment or import-cache snapshot.", + "Console warnings, interpreter version, invocation and commit-to-run binding are coordinator provenance, separate from what JUnit directly records.", + "Historical failure artifacts remain preserved separately in PR #4683. #4680, #4537 and #4354 remain open." + ] +} diff --git a/src/data_sheets_schema/cache_dependencies.py b/src/data_sheets_schema/cache_dependencies.py index ed0686c4c..159bfa91c 100644 --- a/src/data_sheets_schema/cache_dependencies.py +++ b/src/data_sheets_schema/cache_dependencies.py @@ -92,6 +92,20 @@ def setting_key(value: Any) -> tuple: raise TypeError('unsupported mutable cache dependency') +from yaml._yaml import CParser as _CParser + +_CALLABLE_TYPES = ( + types.FunctionType, types.BuiltinFunctionType, + types.MethodDescriptorType, types.WrapperDescriptorType, + types.ClassMethodDescriptorType, type(_CParser.check_event), +) +_IMMUTABLE_DESCRIPTOR_TYPES = ( + types.BuiltinFunctionType, types.MethodDescriptorType, + types.WrapperDescriptorType, types.ClassMethodDescriptorType, + type(_CParser.check_event), +) + + class FunctionBindings: """A closed set of effective functions; changes bypass the consumer cache.""" @@ -105,6 +119,8 @@ def __init__(self, *, additional_modules: frozenset[str] = frozenset(), self._callable_defaults = callable_defaults self._modules = _SHIPPED_MODULES | additional_modules self._functions: dict[Any, tuple[tuple, int]] = {} + self._fast_functions: dict[Any, tuple[Any, tuple[int]]] = {} + self._fast_descriptors: dict[Any, tuple[int]] = {} self._binding = False self._dataclass_reprs: set[Any] = set() self._bound = False @@ -118,8 +134,18 @@ def bind(self, capture: Callable[[], Any]) -> None: capture() except Exception: self._functions.clear() + self._fast_functions.clear() + self._fast_descriptors.clear() self._dataclass_reprs.clear() else: + for fn, (state, token) in self._functions.items(): + if type(fn) in _IMMUTABLE_DESCRIPTOR_TYPES: + self._fast_descriptors[fn] = (token,) + elif (type(fn) is types.FunctionType and fn not in self._dataclass_reprs + and state[2] == (type(None), None) and state[3] == (type(None), None) + and state[4] == () and state[5] == () and state[6] == () + and not fn.__dict__): + self._fast_functions[fn] = (state[1], (token,)) self.ready = True finally: self._binding = False @@ -166,14 +192,23 @@ def _default_key(self, value: Any, seen: frozenset[int]) -> tuple: return setting_key(value) def key(self, function: Any, seen: frozenset[int] = frozenset()) -> tuple: - from yaml._yaml import CParser if function is None: return () - if not self._binding and (not self.ready or function not in self._functions): - raise TypeError('unfamiliar cache dependency callable') - if not isinstance(function, (types.FunctionType, types.BuiltinFunctionType, - types.MethodDescriptorType, types.WrapperDescriptorType, - types.ClassMethodDescriptorType, type(CParser.check_event))): + if not self._binding: + if not self.ready: + raise TypeError('unfamiliar cache dependency callable') + if not seen: + desc_token = self._fast_descriptors.get(function) + if desc_token is not None: + return desc_token + fast = self._fast_functions.get(function) + if (fast is not None and function.__code__ is fast[0] + and function.__defaults__ is None and function.__kwdefaults__ is None + and function.__closure__ is None and not function.__dict__): + return fast[1] + if function not in self._functions: + raise TypeError('unfamiliar cache dependency callable') + if not isinstance(function, _CALLABLE_TYPES): raise TypeError('unsupported cache dependency callable state') if (self._binding and isinstance(function, types.FunctionType) and function.__module__ not in self._modules): diff --git a/src/data_sheets_schema/native_catalog_reuse.py b/src/data_sheets_schema/native_catalog_reuse.py index 0a0378681..75dc8b9b9 100644 --- a/src/data_sheets_schema/native_catalog_reuse.py +++ b/src/data_sheets_schema/native_catalog_reuse.py @@ -347,10 +347,13 @@ def supported_snapshot(snapshot): """The initial optimization excludes all ambient import/prefix-map state.""" try: if (type(snapshot) is not snapshots.SchemaSnapshot or type(snapshot.sources) is not tuple - or len(snapshot.sources) != 1): + or not snapshot.sources): return False - _name, _imports, _prefixes, maps = snapshots._metadata(snapshot.sources[0][2]) - return not _imports and not maps + if len(snapshot.sources) == 1: + _name, _imports, _prefixes, maps = snapshots._metadata(snapshot.sources[0][2]) + return not _imports and not maps + return all(all(imp == 'linkml:types' for imp in snapshots._metadata(raw)[1]) + for _name, _path, raw in snapshot.sources) except Exception: return False diff --git a/src/data_sheets_schema/native_shared_capture.py b/src/data_sheets_schema/native_shared_capture.py index 0c4ec0860..9da858156 100644 --- a/src/data_sheets_schema/native_shared_capture.py +++ b/src/data_sheets_schema/native_shared_capture.py @@ -261,11 +261,27 @@ def _prefix(member, endpoint=None): return c.EvidencePrefix(item.pin.role, item.pin.path, raw, len(raw), c.sha(raw), raw.count(b'\n')) +_THROUGH_CACHE = {} + + def _through(prefix, line): if type(line) is not int or not 0 <= line <= prefix.lines: raise ValueError('observation line is outside the captured prefix') - raw = b''.join(part + b'\n' for part in prefix.raw.split(b'\n')[:line]) - return c.EvidencePrefix(prefix.stream, prefix.path, raw, len(raw), c.sha(raw), line) + if line == prefix.lines: + return prefix + key = (prefix.sha256, prefix.stream, prefix.path, line) + cached = _THROUGH_CACHE.get(key) + if cached is not None and cached[0] is prefix.raw: + return cached[1] + pos = 0 + for _ in range(line): + pos = prefix.raw.index(b'\n', pos) + 1 + raw = prefix.raw[:pos] + result = c.EvidencePrefix(prefix.stream, prefix.path, raw, len(raw), c.sha(raw), line) + if len(_THROUGH_CACHE) > 4096: + _THROUGH_CACHE.clear() + _THROUGH_CACHE[key] = (prefix.raw, result) + return result def _observation_paths(selection): @@ -362,6 +378,135 @@ def activate(selection, execution_raw, started_raw): execution_sha256=e.pin.sha256, attempt_id=value['attempt_id'], records=(record.pin,))) +def _extend_trace(prev, transcript, control, runtime): + if (prev is None or transcript.stream != 'transcript' or control.stream != 'control' + or transcript.bytes < prev.transcript.bytes or control.bytes < prev.control.bytes + or not transcript.raw.startswith(prev.transcript.raw) + or not control.raw.startswith(prev.control.raw)): + return None + if transcript == prev.transcript and control == prev.control: + return prev + t_tail = transcript.raw[prev.transcript.bytes:] + c_tail = control.raw[prev.control.bytes:] + native_tail = [] + if t_tail: + for offset_line, part in enumerate(t_tail.split(b'\n')[:-1], prev.transcript.lines + 1): + raw = part + b'\n' + if not raw.endswith(b'\n') or not raw.strip(): + return None + value = c.strict_json(raw, 'native observed row', 16 * 1024 * 1024) + if type(value) is not dict or value.get('type') == 'control_response' or ( + value.get('type') == 'system' and value.get('subtype') == 'init'): + return None + native_tail.append((offset_line, raw, value)) + parent_tail = [] + if c_tail: + for offset_line, part in enumerate(c_tail.split(b'\n')[:-1], prev.control.lines + 1): + raw = part + b'\n' + if not raw.endswith(b'\n') or not raw.strip(): + return None + value = c.strict_json(raw, 'native observed row', 16 * 1024 * 1024) + if type(value) is not dict or value.get('kind') in ('initialize_sent', 'initialize_ack'): + return None + parent_tail.append((offset_line, raw, value)) + sub = object.__new__(observed.Trace) + sub.transcript, sub.control = transcript, control + sub.session, sub.policy = prev.session, prev.policy + sub.calls = dict(prev.calls) + sub.callbacks = dict(prev.callbacks) + sub.results = dict(prev.results) + sub.decisions = dict(prev.decisions) + sub.frames = dict(prev.frames) + sub.terminal = prev.terminal + sub._req_cache = dict(getattr(prev, '_req_cache', {})) + sub._adm_cache = dict(getattr(prev, '_adm_cache', {})) + sub._set_cache = dict(getattr(prev, '_set_cache', {})) + callback_ids = {v.get('request_id') for _, v in sub.callbacks.values()} + policy = sub.policy + session_id = sub.session + for number, raw, value in native_tail: + sub.frames[number] = value + kind = value.get('type') + if value.get('parent_tool_use_id') is not None: + return None + if kind in ('assistant', 'user', 'result'): + message = value.get('message') + if value.get('model', runtime['model']) != runtime['model'] or ( + type(message) is dict and message.get('model', runtime['model']) != runtime['model']): + return None + if sub.terminal is not None or kind == 'control_cancel_request': + return None + if kind == 'result': + if value.get('session_id') != session_id or value.get('parent_tool_use_id') is not None: + return None + sub.terminal = number + if kind in ('assistant', 'user'): + message = value.get('message') + if (value.get('session_id') != session_id or value.get('parent_tool_use_id') is not None + or type(message) is not dict or message.get('role', kind) != kind): + return None + content = message.get('content') + if type(content) is not list: + return None + for offset, item in enumerate(content): + if type(item) is not dict: + return None + identity = item.get('id') if item.get('type') == 'tool_use' else item.get('tool_use_id') + if item.get('type') == 'tool_use': + if (kind != 'assistant' or type(identity) is not str or not identity or identity in sub.calls + or item.get('name') not in ('Read', 'Write', 'Bash') or type(item.get('input')) is not dict): + return None + sub.calls[identity] = (number, offset, item) + elif item.get('type') == 'tool_result': + if (kind != 'user' or identity not in sub.calls or identity in sub.results + or sub.calls[identity][0] >= number): + return None + sub.results[identity] = (number, offset, item) + if kind == 'control_request': + request = value.get('request') + data = request.get('input') if type(request) is dict else None + if type(data) is not dict: + return None + identity = data.get('tool_use_id') + callback = value.get('request_id') + if (identity not in sub.calls or identity in sub.callbacks or identity in sub.results + or type(callback) is not str or not callback or callback in callback_ids + or request.get('subtype') != 'hook_callback' + or request.get('callback_id') != policy['pretool_control']['callback_id'] + or request.get('tool_use_id') not in (None, identity) + or data.get('hook_event_name') != policy['pretool_control']['event'] + or data.get('cwd') != policy['readonly_lookups']['repository'] + or data.get('session_id', session_id) != session_id): + return None + call = sub.calls[identity] + if (call[0] >= number or data.get('tool_name') != call[2]['name'] + or not observed._same(data.get('tool_input'), call[2]['input'])): + return None + sub.callbacks[identity] = (number, value) + callback_ids.add(callback) + for number, raw, value in parent_tail: + if value.get('kind') != 'decision': + continue + request = value.get('request') + data = request.get('request', {}).get('input', {}) if type(request) is dict else {} + identity = data.get('tool_use_id') + if (identity not in sub.callbacks or identity in sub.decisions + or not observed._same(request, sub.callbacks[identity][1]) + or value.get('classification') not in ('prescribed', 'not_prescribed') + or type(value.get('basis')) is not str): + return None + classification, basis = value['classification'], value['basis'] + output = {} if classification == 'prescribed' else {'hookSpecificOutput': { + 'hookEventName': 'PreToolUse', 'permissionDecision': 'deny', + 'permissionDecisionReason': 'Outside the registered tool policy: ' + basis}} + expected = {'type': 'control_response', 'response': {'subtype': 'success', + 'request_id': request['request_id'], 'response': output}} + if not observed._same(value.get('response'), expected): + return None + sub.decisions[identity] = (number, value) + return sub + + @dataclass class _CapturedRun: selection: c.NativeSelectionCapture @@ -378,15 +523,39 @@ class _CapturedRun: phase1: object _catalogs: object = field(default_factory=receipt_api._ReceiptCatalogContext, init=False, compare=False, repr=False) + _trace: object = field(default=None, init=False, compare=False, repr=False) + _prefixes: dict = field(default_factory=dict, init=False, compare=False, repr=False) + _chronology_cache: object = field(default=None, init=False, compare=False, repr=False) + _callback_state_cache: dict = field(default_factory=dict, init=False, compare=False, repr=False) + + def __post_init__(self): + self._prefixes = self._catalogs._prefixes def trace(self, transcript=None, control=None): + if (transcript is None or transcript == self.transcript) and (control is None or control == self.control): + if self._trace is None: + token = receipt_api.catalog_reuse.runtime_token() + meta_key = None if (token is None or self._catalogs is None) else ( + token, self.binding.session_id, + c.canonical(self.composition['policy']), c.canonical(self.value['runtime']), + ) + last = getattr(self._catalogs, '_last_trace', None) if meta_key is not None else None + extended = (_extend_trace(last[1], self.transcript, self.control, self.value['runtime']) + if last is not None and last[0] == meta_key else None) + if extended is not None: + self._trace = extended + else: + self._trace = observed.Trace(self.transcript, self.control, + session_id=self.binding.session_id, policy=self.composition['policy'], runtime=self.value['runtime']) + if meta_key is not None and receipt_api.catalog_reuse.runtime_token() is token: + self._catalogs._last_trace = (meta_key, self._trace) + return self._trace return observed.Trace(transcript or self.transcript, control or self.control, session_id=self.binding.session_id, policy=self.composition['policy'], runtime=self.value['runtime']) def decision(self): - if self.phase1 is None: - return None - return stage._prepare_next(self.selection, self.binding, self.phase1, self.history, self._catalogs) + return None if self.phase1 is None else stage._prepare_next( + self.selection, self.binding, self.phase1, self.history, self._catalogs) def _binding_document(selection, value, composition, execution, started, observation_item, session): @@ -426,8 +595,8 @@ def _binding(selection, value, composition, reader, pool): return result -def _phase1(selection, execution, history, reader): - rows, _ = stage._history(selection, execution, history) +def _phase1(selection, execution, history, reader, *, _rows=None): + rows = _rows if _rows is not None else stage._history(selection, execution, history)[0] full = [row['payload'] for row in rows if row['record_type'] == 'phase1_sealed'] core = [row['payload'] for row in rows if row['record_type'] == 'core_sealed'] if not full: @@ -497,14 +666,52 @@ def _declared_stream_files(selection, reader): return live_streams.identities(doc['payload']['stream_files']) -def _base(selection_path): +def _base(selection_path, *, _catalogs=None): + from copy import deepcopy + import sys from . import native_shared_selection as selected from . import native_shared_registration as registration + from . import native_execution_authority as authority selection = selected.capture(selection_path) reader = _Reader(selection) e = reader.read('execution_capture', selection.role('execution_capture')) - value = registration.verified(e.raw) - composition, spec, doc, _ = registration._selection(value['composition_raw_json'].encode('utf-8')) + ctx = _catalogs if type(_catalogs) is receipt_api._ReceiptCatalogContext else None + token = receipt_api.catalog_reuse.runtime_token() if ctx is not None else None + base_key = None if token is None else ( + token, selection_path, selection.registration.raw, e.raw, + registration.verified, registration.registration, registration._selection, authority.dependency_identity, + ) + hit = False + if base_key is not None and base_key in ctx._base: + saved_inputs, cached_val, cached_comp, cached_spec, cached_doc = ctx._base[base_key] + try: + current_inputs = ( + registration._read(cached_val['composition'], 'composition', c.HARD_LIMITS['request_bytes']), + registration._read(cached_val['system_path'], 'system', c.HARD_LIMITS['input_bytes']), + registration._read(cached_val['permission_probe'], 'saved permission', c.HARD_LIMITS['permission_wire_bytes']), + registration._read(cached_comp['runtime_path'], 'runtime_declaration', c.HARD_LIMITS['input_bytes']), + registration._read(cached_comp['instruction_path'], 'instruction', c.HARD_LIMITS['request_bytes']), + str(Path.cwd().resolve()), sys.executable, sys.version, + ) + if current_inputs == saved_inputs: + value, composition, spec, doc = deepcopy(cached_val), deepcopy(cached_comp), cached_spec, deepcopy(cached_doc) + registration._paths(value, composition, fresh=False) + hit = True + except Exception: + hit = False + if not hit: + value = registration.verified(e.raw) + composition, spec, doc, _ = registration._selection(value['composition_raw_json'].encode('utf-8')) + if base_key is not None and receipt_api.catalog_reuse.runtime_token() is token: + saved_inputs = ( + registration._read(value['composition'], 'composition', c.HARD_LIMITS['request_bytes']), + registration._read(value['system_path'], 'system', c.HARD_LIMITS['input_bytes']), + registration._read(value['permission_probe'], 'saved permission', c.HARD_LIMITS['permission_wire_bytes']), + registration._read(composition['runtime_path'], 'runtime_declaration', c.HARD_LIMITS['input_bytes']), + registration._read(composition['instruction_path'], 'instruction', c.HARD_LIMITS['request_bytes']), + str(Path.cwd().resolve()), sys.executable, sys.version, + ) + ctx._base[base_key] = (saved_inputs, deepcopy(value), deepcopy(composition), spec, deepcopy(doc)) if (doc['registration_path'] != selection_path or composition['selection_raw_json'].encode('utf-8') != selection.registration.raw or value['selection_sha256'] != selection.registration.pin.sha256): raise ValueError('actual E/C/S binding differs from the caller-selected registration') @@ -559,14 +766,28 @@ def initialize(selection_path, *, transcript_bytes, control_bytes, stream_files, {'binding': c.pin_dict(binding.binding_artifact.pin), 'init_observation': c.pin_dict(item.pin)}) +def _journal_prefix_digest(run, end): + pins = tuple(item.pin for item in run.history.records[:end]) + key = (run.selection.registration.pin.sha256, run.binding.execution.pin.sha256, + run.binding.attempt_id, pins) + prefixes = getattr(run, '_prefixes', None) + if prefixes is not None and key in prefixes: + return prefixes[key] + raw = stage.journal_bytes(selection_sha256=run.selection.registration.pin.sha256, + execution_sha256=run.binding.execution.pin.sha256, attempt_id=run.binding.attempt_id, + records=pins) + pair = (c.sha(raw), raw) + if prefixes is not None: + prefixes[key] = pair + return pair + + def _history_prefix(run, digest): """Resolve an exact recorded predecessor; never substitute current history.""" found = [] for end in range(1, len(run.history.records) + 1): - raw = stage.journal_bytes(selection_sha256=run.selection.registration.pin.sha256, - execution_sha256=run.binding.execution.pin.sha256, attempt_id=run.binding.attempt_id, - records=tuple(item.pin for item in run.history.records[:end])) - if c.sha(raw) == digest: + prefix_digest, raw = _journal_prefix_digest(run, end) + if prefix_digest == digest: found.append(replace(run.history, journal=_captured('journal', run.selection.role('journal'), raw), records=run.history.records[:end])) if len(found) != 1: @@ -589,12 +810,84 @@ def _require_completed_read_before_write(request, read): raise ValueError('response Write precedes its complete current request Read') +def _bind_trace_cache(trace, settled_cache, base=None): + if settled_cache is None or trace is None: + return trace + owner = base if base is not None else trace + + def request(identity, _trace=trace, _owner=owner): + if identity not in _trace.calls or identity not in _trace.callbacks: + return observed.Trace.request(_trace, identity) + key = ('request', identity) + if key not in settled_cache: + settled_cache[key] = observed.Trace.request(_owner, identity) + return settled_cache[key] + + def admitted(identity, _trace=trace, _owner=owner): + if (identity not in _trace.calls or identity not in _trace.callbacks + or identity not in _trace.decisions + or _trace.decisions[identity][1]['classification'] != 'prescribed'): + return observed.Trace.admitted(_trace, identity) + key = ('admitted', identity) + if key not in settled_cache: + settled_cache[key] = observed.Trace.admitted(_owner, identity) + return settled_cache[key] + + def settled(identity, _trace=trace, _owner=owner): + if (identity not in _trace.calls or identity not in _trace.callbacks + or identity not in _trace.decisions or identity not in _trace.results + or _trace.decisions[identity][1]['classification'] != 'prescribed' + or _trace.results[identity][0] <= _trace.callbacks[identity][0]): + return observed.Trace.settled(_trace, identity) + key = ('settled', identity) + if key not in settled_cache: + settled_cache[key] = observed.Trace.settled(_owner, identity) + return settled_cache[key] + + trace.request = request + trace.admitted = admitted + trace.settled = settled + return trace + + +def _prefix_trace(base, transcript, control, init_lines=None, settled_cache=None): + """Project an already-verified Trace onto an exact earlier stream prefix.""" + if (base is None or transcript.stream != 'transcript' or control.stream != 'control' + or not base.transcript.raw.startswith(transcript.raw) + or not base.control.raw.startswith(control.raw) + or init_lines is None or transcript.lines < init_lines[0] or control.lines < init_lines[1]): + return None + t_lines, c_lines = transcript.lines, control.lines + for identity, (d_line, _) in base.decisions.items(): + if d_line <= c_lines and base.callbacks[identity][0] > t_lines: + return None + if transcript == base.transcript and control == base.control and settled_cache is None: + return base + sub = object.__new__(observed.Trace) + sub.transcript, sub.control = transcript, control + sub.session, sub.policy = base.session, base.policy + sub.calls = {k: v for k, v in base.calls.items() if v[0] <= t_lines} + sub.callbacks = {k: v for k, v in base.callbacks.items() if v[0] <= t_lines} + sub.results = {k: v for k, v in base.results.items() if v[0] <= t_lines} + sub.decisions = {k: v for k, v in base.decisions.items() if v[0] <= c_lines} + sub.frames = {k: v for k, v in base.frames.items() if k <= t_lines} + sub.terminal = base.terminal if (base.terminal is not None and base.terminal <= t_lines) else None + return _bind_trace_cache(sub, settled_cache, base=base) + + def _checked_observations(run): """Validate raw joins and current-request chronology, not saved success flags.""" previous = {'transcript': 0, 'control': 0} seen_reads, seen_intents, seen_writes, seen_advances = {}, {}, set(), set() checked = [] previous_decision = None + base_trace = None + base_attempted = False + init_lines = None + settled_cache = {} + token = receipt_api.catalog_reuse.runtime_token() + ctx = getattr(run, '_catalogs', None) if token is not None else None + verified_obs = getattr(ctx, '_verified_obs', None) if ctx is not None else None def decision_at(digest): # This invocation owns one immutable captured run. Reuse only the @@ -609,6 +902,28 @@ def decision_at(digest): previous_decision = (digest, value) return value + def trace_at(transcript, control): + nonlocal base_trace, base_attempted, settled_cache + if not base_attempted and init_lines is not None: + base_attempted = True + try: + base_trace = run.trace() + if ctx is not None and hasattr(ctx, '_settled'): + meta = (token, base_trace.session, c.canonical(base_trace.policy)) + saved = ctx._settled + if (saved is not None and saved[0] == meta + and base_trace.transcript.raw.startswith(saved[1]) + and base_trace.control.raw.startswith(saved[2])): + settled_cache = saved[3] + ctx._settled = (meta, base_trace.transcript.raw, base_trace.control.raw, settled_cache) + _bind_trace_cache(base_trace, settled_cache) + except ValueError: + base_trace = None + projected = _prefix_trace(base_trace, transcript, control, init_lines, settled_cache) + if projected is not None: + return projected + return run.trace(transcript, control) + for index, artifact in enumerate(run.observations): doc, transcript, control = observation(artifact, run.selection, run.binding, run.pool) for prefix in (transcript, control): @@ -618,10 +933,15 @@ def decision_at(digest): kind, payload = doc['observation_type'], doc['payload'] if (index == 0) != (kind == 'initialized'): raise ValueError('durable observations lack one first actual initialization') - trace = run.trace(transcript, control) if kind == 'initialized': + trace = run.trace(transcript, control) + init_lines = (transcript.lines, control.lines) if artifact != run.binding.init_observation: raise ValueError('initial observation differs from bound session') + else: + trace = trace_at(transcript, control) + if kind == 'initialized': + pass elif kind in ('request_read', 'response_written', 'advance_settled'): call = _call(payload['call']) actual = trace.settled(call.tool_use_id) @@ -633,10 +953,17 @@ def decision_at(digest): current = decision_at(payload['history_sha256']) if current.state != 'awaiting_response' or current.request != request: raise ValueError('Read observation names a stale or unpublished request') - observed.complete_request_read(trace, actual, request) + obs_key = None if verified_obs is None else ( + 'request_read', token, artifact.pin.sha256, request.pin.sha256, actual, + ) + obs_val = None if obs_key is None else (artifact.raw, request.raw) + if obs_key is None or verified_obs.get(obs_key) != obs_val: + observed.complete_request_read(trace, actual, request) + if obs_key is not None and receipt_api.catalog_reuse.runtime_token() is token: + verified_obs[obs_key] = obs_val if call.tool_use_id in seen_reads or request.pin.path in {row[1].pin.path for row in seen_reads.values()}: raise ValueError('current request has duplicate delivered-read authority') - seen_reads[call.tool_use_id] = (artifact, request, current) + seen_reads[call.tool_use_id] = (artifact, request, current, call) elif kind == 'response_written': intent = seen_intents.get(payload['intent_observation']['sha256']) if intent is None or c.pin_dict(intent[0].pin) != payload['intent_observation']: @@ -647,13 +974,27 @@ def decision_at(digest): or payload['history_sha256'] != intent_doc['history_sha256'] or response.pin.path != intent_doc['response']['path'] or response.pin.path in seen_writes): raise ValueError('response was replaced, retried or moved to another stage') - observed.first_response_write(trace, actual, response, intent_input_json=call.input_json) + obs_key = None if verified_obs is None else ( + 'response_written', token, artifact.pin.sha256, response.pin.sha256, actual, + ) + obs_val = None if obs_key is None else (artifact.raw, response.raw, call.input_json) + if obs_key is None or verified_obs.get(obs_key) != obs_val: + observed.first_response_write(trace, actual, response, intent_input_json=call.input_json) + if obs_key is not None and receipt_api.catalog_reuse.runtime_token() is token: + verified_obs[obs_key] = obs_val seen_writes.add(response.pin.path) else: result = payload['advance_result'] if result.get('advance_tool_use_id') != call.tool_use_id or call.tool_use_id not in seen_advances: raise ValueError('advance settlement lacks actual earlier selected admission') - observed.helper_result(trace, actual, result) + obs_key = None if verified_obs is None else ( + 'advance_settled', token, artifact.pin.sha256, actual, + ) + obs_val = None if obs_key is None else (artifact.raw, result) + if obs_key is None or verified_obs.get(obs_key) != obs_val: + observed.helper_result(trace, actual, result) + if obs_key is not None and receipt_api.catalog_reuse.runtime_token() is token: + verified_obs[obs_key] = obs_val _history_prefix(run, result['before_history_sha256']) after = decision_at(result['after_history_sha256']) if after.state != result['state']: @@ -676,12 +1017,19 @@ def decision_at(digest): reads = [row for row in seen_reads.values() if c.pin_dict(row[0].pin) == payload['read_observation']] if len(reads) != 1 or reads[0][1] != current.request or current.response.path in {v[1]['response']['path'] for v in seen_intents.values()}: raise ValueError('Write intent reuses a stale Read or spent response destination') - inputs = _document(request.input_json, 'Write intent') - if (set(inputs) != {'file_path', 'content'} or inputs['file_path'] != current.response.path - or type(inputs['content']) is not str or not inputs['content'] - or len(inputs['content'].encode('utf-8')) > current.response.max_bytes): - raise ValueError('Write intent exceeds the exact response contract') - read_call = _call(c.strict_json(reads[0][0].raw)['payload']['call']) + obs_key = None if verified_obs is None else ( + 'response_intent', token, artifact.pin.sha256, current.response.path, current.response.max_bytes, + ) + obs_val = None if obs_key is None else (artifact.raw, request.input_json) + if obs_key is None or verified_obs.get(obs_key) != obs_val: + inputs = _document(request.input_json, 'Write intent') + if (set(inputs) != {'file_path', 'content'} or inputs['file_path'] != current.response.path + or type(inputs['content']) is not str or not inputs['content'] + or len(inputs['content'].encode('utf-8')) > current.response.max_bytes): + raise ValueError('Write intent exceeds the exact response contract') + if obs_key is not None and receipt_api.catalog_reuse.runtime_token() is token: + verified_obs[obs_key] = obs_val + read_call = reads[0][3] _require_completed_read_before_write(request, trace.settled(read_call.tool_use_id)) seen_intents[artifact.pin.sha256] = (artifact, payload) elif kind == 'advance_admitted': @@ -705,16 +1053,19 @@ def decision_at(digest): return tuple(checked) -def _load(selection_path, *, transcript_bytes=None, control_bytes=None): +_LAST_VERIFIED_CATALOGS = None + + +def _load(selection_path, *, transcript_bytes=None, control_bytes=None, _catalogs=None): """Strict complete-file route for final and recorded consumers.""" - base = _base(selection_path) + base = _base(selection_path, _catalogs=_catalogs) streams, prefixes = _streams(base[2], transcript_bytes=transcript_bytes, control_bytes=control_bytes) - return _loaded(base, streams, prefixes) + return _loaded(base, streams, prefixes, _catalogs=_catalogs) def _load_live(selection_path, *, transcript_bytes=None, control_bytes=None, stream_files=None, _catalogs=None): """Fresh live authority, optionally with owner-reused pure catalog storage.""" - base = _base(selection_path) + base = _base(selection_path, _catalogs=_catalogs) anchors = _declared_stream_files(base[0], base[1]) if stream_files is not None and live_streams.identities(stream_files) != anchors: raise ValueError('observed stream identities differ from initialized authority') @@ -726,12 +1077,10 @@ def _load_live(selection_path, *, transcript_bytes=None, control_bytes=None, str def _loaded(base, streams, prefixes, *, _catalogs=None): selection, reader, value, composition, spec = base transcript, control = prefixes - for path in _observation_paths(selection): - reader.read('observation', path) + observations = tuple(reader.read('observation', path) for path in _observation_paths(selection)) pool = _pool(reader, streams) binding = _binding(selection, value, composition, reader, pool) history = reader.history(binding) - observations = tuple(reader.read('observation', path) for path in _observation_paths(selection)) # The complete closed observation catalog includes pre-grant intents that # intentionally cannot yet be referenced by response-consumed records. history = replace(history, observations=observations) @@ -740,6 +1089,7 @@ def _loaded(base, streams, prefixes, *, _catalogs=None): transcript, control, binding, history, observations, phase1) if _catalogs is not None: run._catalogs = receipt_api._catalog_context(_catalogs) + run._prefixes = run._catalogs._prefixes _checked_observations(run) return run @@ -750,9 +1100,9 @@ def _persist_observation(run, kind, payload): run.transcript, run.control, payload), ordinal=len(run.observations)) -def capture_stage(selection_registration_path): +def capture_stage(selection_registration_path, *, _catalogs=None): """Capture the sole pending, actually admitted fixed advance invocation.""" - run = _load_live(str(selection_registration_path)) + run = _load_live(str(selection_registration_path), _catalogs=_catalogs) if run.phase1 is None: raise ValueError('advance precedes actual sealed originals') phase, _ = phase_replay(run) @@ -767,10 +1117,11 @@ def capture_stage(selection_registration_path): # A helper cannot use bytes prefetched after its own actual callback. transcript = _through(run.transcript, call.callback.line) control = _through(run.control, call.admission.line) - candidates = [item for item in run.observations if c.strict_json(item.raw)['observation_type'] == 'advance_admitted' - and c.strict_json(item.raw)['payload']['call']['tool_use_id'] == call.tool_use_id] + obs_docs = [(item, c.strict_json(item.raw)) for item in run.observations] + candidates = [item for item, doc in obs_docs if doc['observation_type'] == 'advance_admitted' + and doc['payload']['call']['tool_use_id'] == call.tool_use_id] if not candidates: - if sum(c.strict_json(item.raw)['observation_type'] == 'advance_admitted' for item in run.observations) >= c.HARD_LIMITS['advance_invocations']: + if sum(doc['observation_type'] == 'advance_admitted' for _, doc in obs_docs) >= c.HARD_LIMITS['advance_invocations']: raise ValueError('stage helper invocation count exceeds its complete bound') admission = _persist_observation(replace(run, transcript=transcript, control=control), 'advance_admitted', {'call': call_document(call), 'before_history_sha256': run.history.journal.pin.sha256}) @@ -789,23 +1140,27 @@ def capture_stage(selection_registration_path): working_directory=run.binding.working_directory)) -def publish_stage(invocation, proposal): +def publish_stage(invocation, proposal, *, _catalogs=None): """Recheck the same current invocation, then perform only derived effects.""" from .native_shared_publication import publish_derived if type(invocation) is not c.StageInvocationCapture: raise ValueError('publication requires the actual selected captured invocation') - fresh = capture_stage(invocation.selection.registration.pin.path) + fresh = capture_stage(invocation.selection.registration.pin.path, _catalogs=_catalogs) # A newly persisted admission is an external observation, not part of the # journal predecessor; include it consistently in the freshly read history. old = replace(invocation, history=replace(invocation.history, observations=fresh.history.observations)) if old != fresh: raise ValueError('selected capture changed before stage publication') - result, _ = publish_derived(fresh, proposal) + result, _ = publish_derived(fresh, proposal, _catalogs=_catalogs) return result def _chronology(run): + key = (tuple(item.pin for item in run.observations), tuple(item.pin for item in run.history.records)) + cached = getattr(run, '_chronology_cache', None) + if cached is not None and cached[0] == key: + return cached[1] catalog = {} advance_before, advance_after = {}, {} visible = {run.history.records[0].pin.sha256: 0} @@ -835,7 +1190,9 @@ def _chronology(run): if pin['sha256'] not in catalog: raise ValueError('stage record lacks its actual observation visibility') visible[item.pin.sha256] = catalog[pin['sha256']][1] - return catalog, advance_before, advance_after, visible, seal_observations + result = (catalog, advance_before, advance_after, visible, seal_observations) + run._chronology_cache = (key, result) + return result def phase_replay(run, *, complete=False): @@ -861,27 +1218,29 @@ def decision_at(digest): previous = (digest, value) return value - for line, _, frame in observed.rows(run.transcript): + cached_trace = getattr(run, '_trace', None) + rows = (((line, None, frame) for line, frame in cached_trace.frames.items()) + if cached_trace is not None and cached_trace.transcript == run.transcript + else observed.rows(run.transcript)) + for line, _, frame in rows: count = 0 for item in run.history.records: if item.pin.sha256 not in visible or visible[item.pin.sha256] >= line: break count += 1 - decision = None + digest = None if count >= 3: - raw = stage.journal_bytes(selection_sha256=run.selection.registration.pin.sha256, - execution_sha256=run.binding.execution.pin.sha256, attempt_id=run.binding.attempt_id, - records=tuple(item.pin for item in run.history.records[:count])) - decision = decision_at(c.sha(raw)) + digest = _journal_prefix_digest(run, count)[0] blocks = frame.get('message', {}).get('content', []) for block in blocks if type(blocks) is list else (): if type(block) is not dict: continue identity = block.get('id') if block.get('type') == 'tool_use' else block.get('tool_use_id') if block.get('type') == 'tool_use' and identity in advance_before: - decision = decision_at(advance_before[identity]) + digest = advance_before[identity] if block.get('type') == 'tool_result' and identity in advance_after: - decision = decision_at(advance_after[identity]) + digest = advance_after[identity] + decision = None if digest is None else decision_at(digest) emitted = phase.observe(frame, stage_decision=decision) if emitted: actions[line] = (emitted, phase.report()) @@ -992,8 +1351,9 @@ def _effect_view_from_decision(run, decision, *, correction_window=False, exclud run.composition['policy']['readonly_lookups']['inputs']) if path not in {role.path for role in protected}] if run.phase1 is not None: protected.append(c.RolePath('sealed_original_receipt_output', run.spec._agentic_artifact_paths['receipt'])) - pending = [identity for identity in run.trace().pending() if identity != exclude_pending - and run.trace().calls[identity][2].get('input', {}).get('command') == run.composition['policy']['native_shared_helpers']['advance']] + trace = run.trace() + pending = [identity for identity in trace.pending() if identity != exclude_pending + and trace.calls[identity][2].get('input', {}).get('command') == run.composition['policy']['native_shared_helpers']['advance']] if len(pending) > 1: raise ValueError('overlapping selected advance effects') return c.NativeEffectView(correction_window=correction_window, cursor=None if decision is None else decision.cursor, @@ -1050,9 +1410,9 @@ def observe_response_written(run, identity): current = run.decision(); trace = run.trace(); actual = trace.settled(identity) if current is None or current.state != 'awaiting_response' or current.response is None: raise ValueError('response settlement has no current selected destination') - matches = [(item, _document(item.raw, 'response intent')['payload']) for item in run.observations - if _document(item.raw, 'response intent')['observation_type'] == 'response_intent' - and _document(item.raw, 'response intent')['payload']['tool_use_id'] == identity] + matches = [(item, doc['payload']) for item, doc in ( + (item, _document(item.raw, 'response intent')) for item in run.observations + ) if doc['observation_type'] == 'response_intent' and doc['payload']['tool_use_id'] == identity] if len(matches) != 1: raise ValueError('response settlement lacks one preceding exact pre-grant intent') intent, payload = matches[0] @@ -1130,14 +1490,27 @@ def _receipt_assessment(run, *, full_path, receipt_path): if fresh_selection != run.selection: raise ValueError('final receipt selection differs from independent reconstruction') # Final reconstruction must not reuse the live run's catalog context. - decision = stage.prepare_next(fresh_selection, run.binding, run.phase1, run.history) - if decision.state != 'assembly_complete' or decision.completion is None: - raise ValueError('final native receipts require the freshly checked complete stage history') - completion = stage.check_assembly(fresh_selection, run.binding, run.phase1, run.history, - decision.completion.assembly.raw) - assessed = completion.effective_receipt - block = receipts.check_final(fresh_selection, run.binding, run.phase1, completion, - final_full=full.raw, final_receipt=assessed.raw) + if (stage.prepare_next is getattr(stage, '_DEFAULT_PREPARE_NEXT', None) + and stage.check_assembly is getattr(stage, '_DEFAULT_CHECK_ASSEMBLY', None) + and receipts.check_final is getattr(receipts, '_DEFAULT_CHECK_FINAL', None)): + fresh_catalogs = getattr(run, '_catalogs', None) or receipts._catalog_context() + decision = stage._prepare_next(fresh_selection, run.binding, run.phase1, run.history, fresh_catalogs) + if decision.state != 'assembly_complete' or decision.completion is None: + raise ValueError('final native receipts require the freshly checked complete stage history') + completion = stage._check_assembly(fresh_selection, run.binding, run.phase1, run.history, + decision.completion.assembly.raw, fresh_catalogs) + assessed = completion.effective_receipt + block = receipts._check_final(fresh_selection, run.binding, run.phase1, completion, + final_full=full.raw, final_receipt=assessed.raw, _catalogs=fresh_catalogs) + else: + decision = stage.prepare_next(fresh_selection, run.binding, run.phase1, run.history) + if decision.state != 'assembly_complete' or decision.completion is None: + raise ValueError('final native receipts require the freshly checked complete stage history') + completion = stage.check_assembly(fresh_selection, run.binding, run.phase1, run.history, + decision.completion.assembly.raw) + assessed = completion.effective_receipt + block = receipts.check_final(fresh_selection, run.binding, run.phase1, completion, + final_full=full.raw, final_receipt=assessed.raw) block['native_receipt_inputs'] = {'selected_original': c.pin_dict(original.pin), 'sealed_original': c.pin_dict(run.phase1.original_receipt.pin), 'assessed_effective': c.pin_dict(assessed.pin), 'final_full': c.pin_dict(full.pin), 'basis': 'Original caller-selected receipt preserved; reconstructed effective receipt assessed.'} @@ -1145,16 +1518,21 @@ def _receipt_assessment(run, *, full_path, receipt_path): def live_receipt_block(selection_registration_path, *, full_path, receipt_path): + global _LAST_VERIFIED_CATALOGS run = _load_live(str(selection_registration_path)) phase_replay(run) - return _receipt_assessment(run, full_path=full_path, receipt_path=receipt_path)[0] + block = _receipt_assessment(run, full_path=full_path, receipt_path=receipt_path)[0] + if getattr(run, '_catalogs', None) is not None: + _LAST_VERIFIED_CATALOGS = (str(selection_registration_path), run._catalogs) + return block def _provenance_block(run): decision = run.decision() if decision is None or decision.state != 'assembly_complete' or decision.completion is None: raise ValueError('native provenance requires complete freshly reconstructed selected stages') - completion = stage.check_assembly(run.selection, run.binding, run.phase1, run.history, decision.completion.assembly.raw) + completion = stage._check_assembly(run.selection, run.binding, run.phase1, run.history, + decision.completion.assembly.raw, run._catalogs) artifact = run.reader.read('completion', run.selection.role('completion')) return {'protocol': c.NAME, 'selection_sha256': run.selection.registration.pin.sha256, 'execution_sha256': run.binding.execution.pin.sha256, 'attempt_id': run.binding.attempt_id, @@ -1165,6 +1543,7 @@ def _provenance_block(run): def live_provenance_block(selection_registration_path): + global _LAST_VERIFIED_CATALOGS run = _load_live(str(selection_registration_path)) phase, _ = phase_replay(run) required = {'draft', 'final_evidence', 'final_source_inventory', 'phase1_receipts', 'final_scope', @@ -1180,10 +1559,14 @@ def live_provenance_block(selection_registration_path): from .canary import receipt_floors if any(receipt_floors(block).values()): raise ValueError('recorder current selected receipt floors did not pass') - return _provenance_block(run) + result = _provenance_block(run) + if getattr(run, '_catalogs', None) is not None: + _LAST_VERIFIED_CATALOGS = (str(selection_registration_path), run._catalogs) + return result def _recorded_run(spec, record): + global _LAST_VERIFIED_CATALOGS from .native_shared_render import validate_metadata if type(record) is not dict: raise ValueError('native recorded reader requires one explicit provenance mapping') @@ -1192,7 +1575,11 @@ def _recorded_run(spec, record): document = validate_metadata(render_spec) except (KeyError, TypeError) as exc: raise ValueError('native record lacks its complete selected render authority') from exc - run = _load(document['registration_path']) + reg_path = str(document['registration_path']) + saved_catalogs = (_LAST_VERIFIED_CATALOGS[1] + if _LAST_VERIFIED_CATALOGS is not None and _LAST_VERIFIED_CATALOGS[0] == reg_path + else None) + run = _load(document['registration_path'], _catalogs=saved_catalogs) if c.canonical(render_spec) != c.canonical(run.composition['render_spec']): raise ValueError('native record rendering differs from its observed execution composition') if spec is not None and c.canonical(spec.render_spec()) != c.canonical(render_spec): @@ -1202,6 +1589,8 @@ def _recorded_run(spec, record): phase, _ = phase_replay(run) if not phase['passed']: raise ValueError('native recorded stage chronology did not pass') + if getattr(run, '_catalogs', None) is not None: + _LAST_VERIFIED_CATALOGS = (reg_path, run._catalogs) return run @@ -1220,7 +1609,7 @@ def recorded_receipt_claims(spec, *, record, full_path, receipt_path, expected_b if (block.get('native_receipt_stage') != 'final' or block.get('final_stage_complete') is not True or c.canonical(expected_block) != c.canonical(block)): raise ValueError('claim sidecar expected block differs from the same freshly captured final receipt assessment') - _, _, _, _, _, bases, _, _ = receipts._inputs(run.selection, run.phase1) + _, _, _, _, _, bases, _, _ = receipts._inputs(run.selection, run.phase1, _catalogs=run._catalogs) return rc.claim_receipts(_mapping(effective.raw, 'native effective receipt'), _mapping(full.raw, 'native final full'), _mapping(run.phase1.full.raw, 'native original full'), identifier_bases=bases) @@ -1228,7 +1617,7 @@ def recorded_receipt_claims(spec, *, record, full_path, receipt_path, expected_b def at_callback(run, line): """Pure current authority at one real callback; no future observation credit.""" - _, _, _, visible, _ = _chronology(run) + catalog, _, _, visible, _ = _chronology(run) count = 0 for item in run.history.records: if item.pin.sha256 not in visible or visible[item.pin.sha256] >= line: @@ -1236,20 +1625,42 @@ def at_callback(run, line): count += 1 if count < 2: raise ValueError('callback precedes actual session publication') - raw = stage.journal_bytes(selection_sha256=run.selection.registration.pin.sha256, - execution_sha256=run.binding.execution.pin.sha256, attempt_id=run.binding.attempt_id, - records=tuple(item.pin for item in run.history.records[:count])) - history = _history_prefix(run, c.sha(raw)) observations = tuple(item for item in run.observations - if observation(item, run.selection, run.binding, run.pool)[1].lines < line) - history = replace(history, observations=observations) + if catalog[item.pin.sha256][1] < line) + state_key = (count, tuple(item.pin.sha256 for item in observations)) + cached_state = getattr(run, '_callback_state_cache', None) + if cached_state is not None and state_key in cached_state: + history, phase1 = cached_state[state_key] + else: + digest, _ = _journal_prefix_digest(run, count) + history = replace(_history_prefix(run, digest), observations=observations) + phase1 = _phase1(run.selection, run.binding, history, run.reader) + if cached_state is not None: + cached_state[state_key] = (history, phase1) transcript = _through(run.transcript, line) - frames = {c.canonical(frame) for _, _, frame in observed.rows(transcript)} - control_lines = 0 - for number, _, row in observed.rows(run.control): - if row.get('kind') == 'decision' and c.canonical(row.get('request')) not in frames: - break - control_lines = number + base_trace = None + try: + base_trace = run.trace() + except ValueError: + pass + if base_trace is not None and 0 < line <= base_trace.transcript.lines: + later = [d_line for identity, (d_line, _) in base_trace.decisions.items() + if base_trace.callbacks[identity][0] > line] + control_lines = (min(later) - 1) if later else run.control.lines + else: + frames = {c.canonical(frame) for _, _, frame in observed.rows(transcript)} + control_lines = 0 + for number, _, row in observed.rows(run.control): + if row.get('kind') == 'decision' and c.canonical(row.get('request')) not in frames: + break + control_lines = number control = _through(run.control, control_lines) - return replace(run, history=history, observations=observations, transcript=transcript, control=control, - phase1=_phase1(run.selection, run.binding, history, run.reader)) + result = replace(run, history=history, observations=observations, transcript=transcript, control=control, + phase1=phase1) + result._catalogs = run._catalogs + result._prefixes = run._prefixes + result._chronology_cache = run._chronology_cache + result._callback_state_cache = run._callback_state_cache + if base_trace is not None and 0 < line <= base_trace.transcript.lines and 0 < control_lines <= base_trace.control.lines: + result._trace = _prefix_trace(base_trace, transcript, control, (1, 1)) + return result diff --git a/src/data_sheets_schema/native_shared_contract.py b/src/data_sheets_schema/native_shared_contract.py index de25b43be..0dff4cbde 100644 --- a/src/data_sheets_schema/native_shared_contract.py +++ b/src/data_sheets_schema/native_shared_contract.py @@ -528,7 +528,7 @@ def _lexical_budget(text): class _JSONValidationCache: """Bound successful pure byte validation, never mutable values or authority.""" - def __init__(self, max_entries=128, max_bytes=16 * 1024 * 1024): + def __init__(self, max_entries=2048, max_bytes=128 * 1024 * 1024): if type(max_entries) is not int or max_entries < 1 or type(max_bytes) is not int or max_bytes < 1: raise ValueError("JSON validation cache requires positive integer bounds") self._max_entries, self._max_bytes = max_entries, max_bytes @@ -574,7 +574,7 @@ def strict_json(raw: bytes, label="native shared JSON", max_bytes=2_000_000): try: text = raw.decode("utf-8") if _JSON_VALIDATION.contains(key): - return json.loads(text, object_pairs_hook=_pairs, parse_constant=_reject_constant) + return json.loads(text) _lexical_budget(text) value = json.loads(text, object_pairs_hook=_pairs, parse_constant=_reject_constant) _json_values(value) diff --git a/src/data_sheets_schema/native_shared_controller.py b/src/data_sheets_schema/native_shared_controller.py index 24d1fb73e..8ad64b236 100644 --- a/src/data_sheets_schema/native_shared_controller.py +++ b/src/data_sheets_schema/native_shared_controller.py @@ -127,11 +127,14 @@ def __init__(self, composition_raw, *, execution, registration_raw): report_path=paths['report'], receipt_path=paths['receipt'], working_directory=self.value['working_directory']) self.state = attribution.live(self.selection, execution_raw=registration_raw, spec=self.spec, composition_raw=composition_raw) self.failure = self.final_evidence = self.recorder_identity = None + self._stream_diagnostic_json = None self.recorder_call = None self.recorder_done = False self._activated = self._initialized = False self._stream_prefixes = {} self._stream_files = {} + self._stream_members = () + self._phase_cache = None self._receipt_catalogs = _ReceiptCatalogContext() def require_phase_authority(self): @@ -152,6 +155,20 @@ def _stop(self, reason): self.failure = str(reason) raise self.controls['budgeted_cborg'].BudgetStop(self.failure) + def _failure_detail(self, exc): + # Preserve the historical first-stop text, not the private subclass + # name. The first failure alone may contribute optional diagnostics. + is_stream = isinstance(exc, live_streams._LiveStreamRefusal) + if self.failure is None and is_stream: + try: + raw = exc.diagnostic_json + live_streams.diagnostic_document(raw) + if getattr(self, '_stream_diagnostic_json', None) is None: + self._stream_diagnostic_json = raw + except Exception: + pass # Diagnostic failure must never replace the refusal. + return ('ValueError' if is_stream else type(exc).__name__) + ': ' + str(exc) + def _observed_endpoint(self, event): members, prefixes = [], {} for role in live_streams.ROLES: @@ -175,6 +192,7 @@ def _observed_endpoint(self, event): # Neither cursor advances if either acquisition or the event fails. anchors = live_streams.stream_files(tuple(members)) self._stream_prefixes, self._stream_files = prefixes, anchors + self._stream_members = tuple(members) return prefixes['transcript'].bytes, prefixes['control'].bytes def _run(self, endpoints=None): @@ -188,6 +206,27 @@ def _run(self, endpoints=None): raise ValueError('recaptured streams differ from exact parent-observed prefixes') return run + def _phase_state(self, endpoints, *, refresh=False): + from dataclasses import replace + from . import native_shared_capture as capture + if refresh or getattr(self, '_phase_cache', None) is None: + run = self._run(endpoints) + decision = run.decision() + self._phase_cache = (run, decision) + return run, decision + cached_run, decision = self._phase_cache + members = getattr(self, '_stream_members', ()) + pool = capture._pool(cached_run.reader, members) if members else cached_run.pool + run = replace(cached_run, pool=pool, + transcript=self._stream_prefixes['transcript'], + control=self._stream_prefixes['control']) + run._catalogs = cached_run._catalogs + run._prefixes = cached_run._prefixes + run._chronology_cache = cached_run._chronology_cache + run._callback_state_cache = cached_run._callback_state_cache + self._phase_cache = (run, decision) + return run, decision + def _current_final(self, run): from .native_shared_gates import current_evidence result = current_evidence(run) @@ -240,14 +279,14 @@ def observe(self, event): transcript_bytes=endpoints[0], control_bytes=endpoints[1], stream_files=self._stream_files, observed_prefixes=self._stream_prefixes) self._initialized = True + self._phase_cache = None if not self._initialized: if event.get('type') != 'control_response': raise ValueError('native effects precede actual initialization binding') self.phase.observe(event) self.state.observe(event) return - run = self._run(endpoints) - decision = run.decision() + run, decision = self._phase_state(endpoints) content = event.get('message', {}).get('content', []) content = content if type(content) is list else () # Validate actual advance effects before the phase checker can @@ -257,8 +296,9 @@ def observe(self, event): continue call = run.trace().calls.get(block.get('tool_use_id')) if call and call[2].get('input', {}).get('command') == self.policy['native_shared_helpers']['advance']: + run = self._run(endpoints) capture.observe_advance_settled(run, block['tool_use_id']) - run = self._run(endpoints); decision = run.decision() + run, decision = self._phase_state(endpoints, refresh=True) if event.get('type') == 'control_request': data = event.get('request', {}).get('input', {}) name, inputs = data.get('tool_name'), data.get('tool_input') @@ -274,6 +314,7 @@ def observe(self, event): raise ValueError('native stage callback refused: ' + basis) if name == 'Write': capture.observe_response_intent(run, data['tool_use_id']) + self._phase_cache = None if inputs.get('command') == self.policy['native_shared_helpers']['recorder']: if self.final_evidence is None or self.state.accepted is None or self.recorder_call != data['tool_use_id']: raise ValueError('recorder callback lacks its settled fresh draft/final evidence') @@ -313,9 +354,12 @@ def observe(self, event): inputs = call['input'] if call['name'] == 'Read' and decision is not None and decision.request is not None and inputs.get('file_path') == decision.request.pin.path: capture.observe_request_read(run, identity) + self._phase_cache = None elif call['name'] == 'Write' and decision is not None and decision.response is not None and inputs.get('file_path') == decision.response.path: capture.observe_response_written(run, identity) + self._phase_cache = None elif call['name'] == 'Bash' and inputs.get('command') == self.policy['native_shared_helpers']['final_evidence']: + run, decision = self._phase_state(endpoints, refresh=True) current = self._current_final(run) actual = trace.settled(identity) observed_result = evidence.event(run.transcript, actual.result) @@ -325,6 +369,7 @@ def observe(self, event): raise ValueError('final helper stdout differs from actual captured final evidence') self.final_evidence = current elif identity == self.recorder_call: + run, decision = self._phase_state(endpoints, refresh=True) metadata = event.get('tool_use_result', {}) codes = [metadata[k] for k in ('exitCode', 'exit_code') if k in metadata] if not codes or any(type(code) is not int or code != 0 for code in codes) or block.get('is_error') is not False: @@ -340,13 +385,14 @@ def observe(self, event): # These fixed side effects independently replay the settled # phase boundary before creating any original seal. capture.seal_originals(self._run(endpoints), action) + self._phase_cache = None self.state.observe(event, preserved_shell_commands=preserved) problems = self.state.report()['problems'] if problems: raise ValueError('native attribution: ' + '; '.join(problems)) self._final_unchanged(run) except Exception as exc: - self._stop('native shared observation failed: ' + type(exc).__name__ + ': ' + str(exc)) + self._stop('native shared observation failed: ' + self._failure_detail(exc)) def classify(self, command, python, programs, selected_policy): if self.failure is not None: @@ -364,13 +410,19 @@ def report(self, *, complete=False): raise ValueError('; '.join(phase['problems'])) self._final_unchanged(run) except Exception as exc: - self.failure = 'completed native shared history cannot be verified: ' + type(exc).__name__ + ': ' + str(exc) + self.failure = 'completed native shared history cannot be verified: ' + self._failure_detail(exc) out = self.state.report(complete=complete) + diagnostic = {} + if self.failure is not None and getattr(self, '_stream_diagnostic_json', None) is not None: + try: + diagnostic['stream_diagnostic'] = live_streams.diagnostic_document(self._stream_diagnostic_json) + except Exception: + pass # Existing failed reports remain available without it. return {**out, 'execution': EXECUTION, 'composition_sha256': c.sha(self.raw), 'controller_stop': self.failure, 'final_evidence': deepcopy(self.final_evidence), 'recorder_completed': self.recorder_done, 'recorder_identity': deepcopy(self.recorder_identity), 'phase_history': self.phase.report(complete=complete), - 'draft_gate_passed': out['draft_gate_passed'] and self.failure is None} + 'draft_gate_passed': out['draft_gate_passed'] and self.failure is None, **diagnostic} def capture_phase(run, *, complete): diff --git a/src/data_sheets_schema/native_shared_evidence.py b/src/data_sheets_schema/native_shared_evidence.py index c6c7fb5b0..851d5c784 100644 --- a/src/data_sheets_schema/native_shared_evidence.py +++ b/src/data_sheets_schema/native_shared_evidence.py @@ -155,6 +155,8 @@ def encode(self): raise ValueError('capture pool exceeds its complete encoded byte bound') return raw + _PREFIX_CACHE={} + def prefix(self,reference,*,execution_sha256): exact(reference,{'stream_id','stream','path','through_bytes','lines','sha256'},'stream prefix reference') selected=stream_id(execution_sha256,reference['stream'],reference['path']) @@ -168,8 +170,15 @@ def prefix(self,reference,*,execution_sha256): end=reference['through_bytes'] if type(end) is not int or end<0 or end>len(member.raw): raise ValueError('prefix endpoint exceeds the captured stream') - return EvidencePrefix(stream=reference['stream'],path=reference['path'],raw=member.raw[:end], + key=(member.pin.sha256,reference['stream'],reference['path'],end,reference['lines'],reference['sha256']) + cached=CapturePool._PREFIX_CACHE.get(key) + if cached is not None and cached[0] is member.raw: + return cached[1] + result=EvidencePrefix(stream=reference['stream'],path=reference['path'],raw=member.raw[:end], bytes=end,lines=reference['lines'],sha256=reference['sha256']) + if len(CapturePool._PREFIX_CACHE)>4096:CapturePool._PREFIX_CACHE.clear() + CapturePool._PREFIX_CACHE[key]=(member.raw,result) + return result def decode(raw): @@ -216,25 +225,77 @@ def prefix_reference(prefix,*,execution_sha256): 'path':prefix.path,'through_bytes':prefix.bytes,'lines':prefix.lines,'sha256':prefix.sha256} +_PREFIX_LINES_CACHE: dict = {} +_EVENT_HASH_CACHE: dict = {} +_LAST_PARSED_LINE: tuple | None = None + + +def _prefix_lines(prefix): + key = (prefix.stream, prefix.sha256, prefix.bytes) + entry = _PREFIX_LINES_CACHE.get(key) + if entry is not None and entry[0] == prefix.raw: + return entry[1] + raw_bytes = prefix.raw + last = _PREFIX_LINES_CACHE.get((prefix.stream, 'last')) + if last is not None and last[0].endswith(b'\n') and raw_bytes.startswith(last[0]): + tail = raw_bytes[len(last[0]):] + lines = last[1] + tuple(part + b'\n' for part in tail.split(b'\n')[:-1]) if tail else last[1] + else: + lines = tuple(part + b'\n' for part in raw_bytes.split(b'\n')[:-1]) + if len(_PREFIX_LINES_CACHE) > 32: + _PREFIX_LINES_CACHE.clear() + _PREFIX_LINES_CACHE[key] = (raw_bytes, lines) + if raw_bytes.endswith(b'\n'): + _PREFIX_LINES_CACHE[(prefix.stream, 'last')] = (raw_bytes, lines) + return lines + + +def _line_raw(prefix, line, error_message='event lies beyond its observed prefix'): + lines = _prefix_lines(prefix) + if type(line) is not int or not 1 <= line <= len(lines): + raise ValueError(error_message) + return lines[line - 1] + + +def _limits_token(): + return ( + HARD_LIMITS['stream_bytes'], + HARD_LIMITS['json_yaml_depth'], + HARD_LIMITS['json_yaml_nodes_per_document'], + strict_json, + canonical, + sha, + ) + + def event(prefix,reference): """Resolve only an exact event inside the actual observed prefix.""" + global _LAST_PARSED_LINE if type(prefix) is not EvidencePrefix or type(reference) is not EventRef or reference.stream!=prefix.stream: raise ValueError('event reference differs from its observed stream prefix') - lines=[part+b'\n' for part in prefix.raw.split(b'\n')[:-1]] - if reference.line>len(lines):raise ValueError('event lies beyond its observed prefix') - raw=lines[reference.line-1] - if sha(raw)!=reference.raw_line_sha256:raise ValueError('event physical line hash differs') + raw=_line_raw(prefix,reference.line,'event lies beyond its observed prefix') + ltok=_limits_token() + hkey=(raw,reference.block,ltok) if type(reference.block) in (int,type(None)) else None + cached=_EVENT_HASH_CACHE.get(hkey) if hkey is not None else None + raw_sha=cached[0] if cached is not None else sha(raw) + if raw_sha!=reference.raw_line_sha256:raise ValueError('event physical line hash differs') value=strict_json(raw,'observed event',HARD_LIMITS['stream_bytes']) if reference.block is not None: message=value.get('message') if type(value) is dict else None content=message.get('content') if type(message) is dict else None - if type(content) is not list or reference.block>=len(content): + if type(content) is not list or type(reference.block) is not int or not 0<=reference.block512: + _EVENT_HASH_CACHE.clear() + _EVENT_HASH_CACHE[hkey]=(raw_sha,val_sha) return value + def require_append_only(earlier,later): if (type(earlier) is not EvidencePrefix or type(later) is not EvidencePrefix or earlier.stream!=later.stream or earlier.path!=later.path diff --git a/src/data_sheets_schema/native_shared_gates.py b/src/data_sheets_schema/native_shared_gates.py index 4e613124a..3582f2f17 100644 --- a/src/data_sheets_schema/native_shared_gates.py +++ b/src/data_sheets_schema/native_shared_gates.py @@ -58,6 +58,7 @@ def current_evidence(run): No declared success flag or ambient original/audit discovery is consulted. """ from . import evidence_assertions as evidence + from . import native_catalog_reuse as catalog_reuse from .api_runner import _audit_shape_problem from .native_shared_capture import _CapturedRun if type(run) is not _CapturedRun or type(run.selection) is not c.NativeSelectionCapture: @@ -76,8 +77,24 @@ def current_evidence(run): raise ValueError('native final evidence requires its selected source manifest') source_raw = selected.raw('source_manifest') authority = {'source_manifest_raw': source_raw, 'project': document['run']['project']} - chunks, pins = evidence.source_chunks_from_bytes(selected.raw('bundle'), selected.raw('chunk_manifest')) + bundle_raw = selected.raw('bundle') + manifest_raw = selected.raw('chunk_manifest') audit_raw = run.reader.read('audit', selected.role('audit')).raw + token = catalog_reuse.runtime_token() + evidence_cache = getattr(getattr(run, '_catalogs', None), '_evidence', None) if token is not None else None + pre_key = None if evidence_cache is None else ( + token, document['run']['project'], + raws['original_full'], raws['original_core'], raws['final_full'], raws['final_core'], + source_raw, bundle_raw, manifest_raw, audit_raw, + evidence.source_chunks_from_bytes, evidence.load_json, _audit_shape_problem, + evidence.check_audit, evidence.check_relationship_removals, + evidence.load_record, evidence.check_report, evidence._problem, + ) + if pre_key is not None and pre_key in evidence_cache: + cached_report_raw, cached_out = evidence_cache[pre_key] + if _current(run, 'final_report').raw == cached_report_raw: + return deepcopy(cached_out) + chunks, pins = evidence.source_chunks_from_bytes(bundle_raw, manifest_raw) audit = evidence.load_json(audit_raw) shape = _audit_shape_problem(audit) if isinstance(audit, dict) else 'audit must be an object' if shape: @@ -102,6 +119,8 @@ def current_evidence(run): out['findings'].append(evidence._problem('evidence_contract', str(exc))) out['artifact_sha256'] = pins out['scope'] = 'Declared evidence only; semantic support and omitted claims require independent review.' + if pre_key is not None and catalog_reuse.runtime_token() is token: + evidence_cache[pre_key] = (report_raw, deepcopy(out)) return out @@ -158,12 +177,17 @@ def receipt_gate(prepared): if prepared['snapshot'].read(original.pin.path, original.pin.role) != original.raw: raise ValueError('ordinary native receipt differs between captured views') assembly = run.reader.read('typed_assembly', run.selection.role('typed_assembly')) - completion = stage.check_assembly(run.selection, run.binding, run.phase1, run.history, assembly.raw) + completion = (stage._check_assembly(run.selection, run.binding, run.phase1, run.history, assembly.raw, run._catalogs) + if stage.check_assembly is getattr(stage, '_DEFAULT_CHECK_ASSEMBLY', None) + else stage.check_assembly(run.selection, run.binding, run.phase1, run.history, assembly.raw)) effective = run.reader.read('effective_receipt', run.selection.role('effective_receipt')) if effective != completion.effective_receipt: raise ValueError('effective native receipt differs from reconstructed stage completion') - block = receipts.check_final(run.selection, run.binding, run.phase1, completion, - final_full=_final(prepared, 'final_full'), final_receipt=effective.raw) + block = (receipts._check_final(run.selection, run.binding, run.phase1, completion, + final_full=_final(prepared, 'final_full'), final_receipt=effective.raw, _catalogs=run._catalogs) + if receipts.check_final is getattr(receipts, '_DEFAULT_CHECK_FINAL', None) + else receipts.check_final(run.selection, run.binding, run.phase1, completion, + final_full=_final(prepared, 'final_full'), final_receipt=effective.raw)) if block.get('checked') is not True or block.get('final_stage_complete') is not True: raise ValueError('native final receipt instrument did not complete') floors = canary.receipt_floors(block) diff --git a/src/data_sheets_schema/native_shared_observations.py b/src/data_sheets_schema/native_shared_observations.py index 32b4d2a0e..36dffa53a 100644 --- a/src/data_sheets_schema/native_shared_observations.py +++ b/src/data_sheets_schema/native_shared_observations.py @@ -9,7 +9,25 @@ import json from data_sheets_schema import native_shared_contract as c -from data_sheets_schema.native_shared_evidence import event +from data_sheets_schema.native_shared_evidence import ( + _EVENT_HASH_CACHE, + _line_raw, + _prefix_lines, + event, +) + +_LAST_REF_FRAME: tuple | None = None + + +def _ref_limits_token(): + return ( + 16 * 1024 * 1024, + c.HARD_LIMITS['json_yaml_depth'], + c.HARD_LIMITS['json_yaml_nodes_per_document'], + c.strict_json, + c.canonical, + c.sha, + ) def _same(left,right): @@ -20,8 +38,7 @@ def rows(prefix): if type(prefix) is not c.EvidencePrefix: raise ValueError('observation requires an exact captured prefix') result=[] - for number,part in enumerate(prefix.raw.split(b'\n')[:-1],1): - raw=part+b'\n' + for number,raw in enumerate(_prefix_lines(prefix),1): # Split only physical LF, not embedded Unicode line separators. if not raw.endswith(b'\n') or not raw.strip(): raise ValueError('native prefix contains an incomplete or blank physical row') @@ -33,17 +50,33 @@ def rows(prefix): def reference(prefix,line,block=None): - lines=prefix.raw.split(b'\n') - if type(line) is not int or not 1<=line512: + _EVENT_HASH_CACHE.clear() + _EVENT_HASH_CACHE[hkey]=(raw_sha,val_sha) + return c.EventRef(prefix.stream,line,block,raw_sha,val_sha) def _frame(value,session,role=None): @@ -74,15 +107,19 @@ def __init__(self,transcript,control,*,session_id,policy,runtime): raise ValueError('observation streams have exchanged roles') self.transcript,self.control=transcript,control self.session,self.policy=session_id,policy - initialized,*_=initialization(transcript,control,policy=policy,runtime=runtime) + # Decode in initialization's original order, then reuse only this + # constructor's rows for indexing. No parsed state survives in a cache. + native=list(rows(transcript));parent=list(rows(control)) + initialized,*_=_initialization_from_rows(transcript,control,native,parent,policy=policy,runtime=runtime) if initialized!=session_id: raise ValueError('supplied observation session differs from actual init') if type(session_id) is not str or not session_id: raise ValueError('observation requires actual initialized session') self.calls={};self.callbacks={};self.results={};self.decisions={} self.frames={};self.terminal=None + self._req_cache={};self._adm_cache={};self._set_cache={} callback_ids=set() - for number,raw,value in rows(transcript): + for number,raw,value in native: self.frames[number]=value kind=value.get('type') if value.get('parent_tool_use_id') is not None: @@ -137,7 +174,7 @@ def __init__(self,transcript,control,*,session_id,policy,runtime): or not _same(data.get('tool_input'),call[2]['input'])): raise ValueError('callback input differs in spelling, value or type from its observed call') self.callbacks[identity]=(number,value);callback_ids.add(callback) - for number,raw,value in rows(control): + for number,raw,value in parent: if value.get('kind')!='decision':continue request=value.get('request');data=request.get('request',{}).get('input',{}) if type(request) is dict else {} identity=data.get('tool_use_id') @@ -159,19 +196,31 @@ def __init__(self,transcript,control,*,session_id,policy,runtime): def request(self,identity): if identity not in self.calls or identity not in self.callbacks: raise ValueError('tool lacks an observed call and callback') + rcache=getattr(self,'_req_cache',None) + if rcache is not None and identity in rcache: + return rcache[identity] line,offset,call=self.calls[identity] - return c.ObservedToolRequest(call=reference(self.transcript,line,offset), + res=c.ObservedToolRequest(call=reference(self.transcript,line,offset), callback=reference(self.transcript,self.callbacks[identity][0]), input_json=c.canonical(call['input']),session_id=self.session, tool_name=call['name'],tool_use_id=identity) + if rcache is not None: + rcache[identity]=res + return res def admitted(self,identity): request=self.request(identity) if identity not in self.decisions or self.decisions[identity][1]['classification']!='prescribed': raise ValueError('tool has no exact parent admission') - return c.ObservedCall(call=request.call,callback=request.callback, + acache=getattr(self,'_adm_cache',None) + if acache is not None and identity in acache: + return acache[identity] + res=c.ObservedCall(call=request.call,callback=request.callback, admission=reference(self.control,self.decisions[identity][0]),input_json=request.input_json, session_id=request.session_id,tool_name=request.tool_name,tool_use_id=identity) + if acache is not None: + acache[identity]=res + return res def settled(self,identity): admitted=self.admitted(identity) @@ -180,7 +229,13 @@ def settled(self,identity): line,offset,_=self.results[identity] if line<=self.callbacks[identity][0]: raise ValueError('tool result precedes its callback') - return ToolEvidence(admitted,reference(self.transcript,line,offset),reference(self.transcript,line)) + scache=getattr(self,'_set_cache',None) + if scache is not None and identity in scache: + return scache[identity] + res=ToolEvidence(admitted,reference(self.transcript,line,offset),reference(self.transcript,line)) + if scache is not None: + scache[identity]=res + return res def pending(self): return tuple(identity for identity in self.calls if identity not in self.results) @@ -262,6 +317,11 @@ def helper_result(trace,evidence,expected): def initialization(transcript,control,*,policy,runtime): """Bind actual ordered initialization to the selected runtime and policy.""" native=list(rows(transcript));parent=list(rows(control)) + return _initialization_from_rows(transcript,control,native,parent,policy=policy,runtime=runtime) + + +def _initialization_from_rows(transcript,control,native,parent,*,policy,runtime): + """Validate already decoded rows owned by this one caller.""" sent=[row for row in parent if row[2].get('kind')=='initialize_sent'] acknowledgements=[row for row in parent if row[2].get('kind')=='initialize_ack'] replies=[row for row in native if row[2].get('type')=='control_response'] diff --git a/src/data_sheets_schema/native_shared_permissions.py b/src/data_sheets_schema/native_shared_permissions.py index f19d4bbd6..d4f6af00a 100644 --- a/src/data_sheets_schema/native_shared_permissions.py +++ b/src/data_sheets_schema/native_shared_permissions.py @@ -334,9 +334,11 @@ def mapped(value): return str(target_root / relative) dirs = lookup['output_directories'] _need(type(dirs) is list and dirs.count(str(source_root)) == 1 and len(set(dirs)) == len(dirs), 'ambiguous stage envelope') - lookup['output_directories'] = [effect_root if value == str(source_root) else str(_path(value)) for value in dirs] + dir_paths = [_path(value) for value in dirs] + lookup['output_directories'] = [effect_root if value == str(source_root) else str(path) for value, path in zip(dirs, dir_paths)] inputs = lookup['inputs'] - _need(type(inputs) is list and len(inputs) == len(set(inputs)), 'ambiguous input paths') + input_set = set(inputs) if type(inputs) is list else set() + _need(type(inputs) is list and len(inputs) == len(input_set), 'ambiguous input paths') lookup['inputs'] = [mapped(value) for value in inputs] _need(len(lookup['inputs']) == len(set(lookup['inputs'])), 'projected input collision') for field in ('bounded_reads', 'write_paths'): @@ -350,9 +352,10 @@ def mapped(value): path = mapped(value) _need(path not in new, 'projected file-map collision') if field == 'bounded_reads': - _need(value in inputs, 'bounded Read is not an input') + _need(value in input_set, 'bounded Read is not an input') else: - _need(any(_path(folder) in _path(value).parents for folder in dirs), 'Write outside output envelope') + value_parents = _path(value).parents + _need(any(folder in value_parents for folder in dir_paths), 'Write outside output envelope') new[path] = cap lookup[field] = new if lookup.get('write_paths') is not None: @@ -367,9 +370,9 @@ def mapped(value): required = [str(target_root / 'responses' / ('%06d.bin' % number)) for number in range(1, selection['bounds']['max_submissions'] + 1)] for value in ordinary: - _path(value) _need(source_root not in _path(value).parents, 'ordinary Write domain contains stage paths') - _need(all(path in lookup['write_paths'] for path in required + ordinary), 'incomplete global Write map') + write_set = set(lookup['write_paths']) + _need(all(path in write_set for path in required + ordinary), 'incomplete global Write map') return policy @@ -403,9 +406,10 @@ def _artifact(role, path, raw): return {'role': role, 'path': path, 'sha256': _sha(raw), 'bytes': len(raw)} -def _recipe_data(expected, selection, launch, session): +def _recipe_data(expected, selection, launch, session, *, roots_checked=False): contract, _ = _dependencies() - _roots(expected, selection, launch) + if not roots_checked: + _roots(expected, selection, launch) root = _path(launch['effect_root']) basis = {'domain': 'native_shared_permission_projection_v1', 'selection_sha256': expected['selection']['sha256'], @@ -424,10 +428,12 @@ def _recipe_data(expected, selection, launch, session): return basis, fixed, requests, responses -def _view(expected, selection, launch, session, context, *, state='awaiting_response', read=False): +def _view(expected, selection, launch, session, context, *, state='awaiting_response', read=False, policy=None, recipe=None): contract, _ = _dependencies() - basis, fixed, requests, responses = _recipe_data(expected, selection, launch, session) - root = _path(launch['effect_root']); policy = projected_policy(expected, selection, str(root)) + basis, fixed, requests, responses = _recipe_data(expected, selection, launch, session) if recipe is None else recipe + root = _path(launch['effect_root']) + if policy is None: + policy = projected_policy(expected, selection, str(root)) active = state == 'awaiting_response' and context in CONTEXTS ordinal = CONTEXTS.index(context) + 1 if context in CONTEXTS else 1 cursor = contract.StageCursor(ordinal=ordinal, kind=context, @@ -458,17 +464,21 @@ def _view(expected, selection, launch, session, context, *, state='awaiting_resp response_intent_observation_sha256=None) -def projection(expected, selection, launch, session_id): +def projection(expected, selection, launch, session_id, *, policy=None, recipe=None): """Deterministic software projection; never evidence of a real stage.""" - policy = projected_policy(expected, selection, launch['effect_root']) + if policy is None: + policy = projected_policy(expected, selection, launch['effect_root']) + if recipe is None: + recipe = _recipe_data(expected, selection, launch, session_id) states = [] for context in CONTEXTS: for read in (False, True): states.append({'name': context + ('-read' if read else '-unread'), - 'view': asdict(_view(expected, selection, launch, session_id, context, read=read))}) + 'view': asdict(_view(expected, selection, launch, session_id, context, read=read, + policy=policy, recipe=recipe))}) for state in NONRESPONSE: states.append({'name': state, 'view': asdict(_view(expected, selection, launch, session_id, - 'worker', state=state))}) + 'worker', state=state, policy=policy, recipe=recipe))}) return {'kind': 'native_shared_permission_projection', 'version': VERSION, 'effect_root': launch['effect_root'], 'session_id': session_id, 'states': states, 'production_policy_sha256': _sha(_encoded(expected['policy'])), 'probe_policy': policy, @@ -476,9 +486,9 @@ def projection(expected, selection, launch, session_id): 'role_recipe_sha256': expected['probe_recipe']['descriptor_sha256']} -def probe_cases(expected, selection, launch, session_id): +def probe_cases(expected, selection, launch, session_id, *, policy=None, recipe=None): """Exact twenty-five observed cases and no caller-selected case subset.""" - _, _, requests, responses = _recipe_data(expected, selection, launch, session_id) + _, _, requests, responses = _recipe_data(expected, selection, launch, session_id) if recipe is None else recipe root = _path(launch['effect_root']) def row(identity, tool, payload, context, mode): return {'id': identity, 'tool': tool, 'input': payload, 'context': context, 'mode': mode} @@ -513,7 +523,8 @@ def row(identity, tool, payload, context, mode): 'content': 'D4D_SOURCE_MUST_NOT_CHANGE\n'}, 'baseline', 'observer_stopped'), row('helper_owned_write', 'Write', {'file_path': str(root / 'journal.json'), 'content': 'D4D_FORBIDDEN\n'}, 'integration', 'observer_stopped')] - policy = projected_policy(expected, selection, launch['effect_root']) + if policy is None: + policy = projected_policy(expected, selection, launch['effect_root']) for case in cases: path = case['input'].get('file_path') if case['tool'] == 'Read' and path in policy['readonly_lookups'].get('bounded_reads', {}): @@ -522,8 +533,8 @@ def row(identity, tool, payload, context, mode): return cases -def _inventory(expected, selection, launch, session, *, completed): - _, fixed, requests, responses = _recipe_data(expected, selection, launch, session) +def _inventory(expected, selection, launch, session, *, completed, recipe=None): + _, fixed, requests, responses = _recipe_data(expected, selection, launch, session) if recipe is None else recipe root = _path(launch['effect_root']); files = dict(fixed) for ordinal, context in enumerate(CONTEXTS, 1): files[str(root / 'requests' / ('%06d.json' % ordinal))] = requests[context] @@ -563,11 +574,13 @@ def _file_classification(policy, case): return 'not_prescribed', 'a path outside the registered inputs and outputs' -def _classification(expected, selection, launch, session, case, controls): +def _classification(expected, selection, launch, session, case, controls, *, policy=None, recipe=None): _, effects = _dependencies() context = case['context'] if case['context'] in CONTEXTS else 'worker' - view = _view(expected, selection, launch, session, context, read=case['tool'] == 'Write') - policy = projected_policy(expected, selection, launch['effect_root']) + if policy is None: + policy = projected_policy(expected, selection, launch['effect_root']) + view = _view(expected, selection, launch, session, context, read=case['tool'] == 'Write', + policy=policy, recipe=recipe) governed = command_policy.stage_overlay_governs(view, tool_name=case['tool'], tool_input=case['input'], policy=policy) _need(type(governed) is bool, 'effect routing is not a strict boolean') @@ -654,14 +667,16 @@ def _settled_text(events, offset, session, model): return offset -def _read_content(case, expected, selection, launch, session, source_raw, *, observed_lines=None): +def _read_content(case, expected, selection, launch, session, source_raw, *, observed_lines=None, policy=None, recipe=None): if case['id'] == 'source_read': raw = source_raw else: - raw = _recipe_data(expected, selection, launch, session)[2][case['context']] + raw = (_recipe_data(expected, selection, launch, session) if recipe is None else recipe)[2][case['context']] text = raw.decode(); lines = text.split('\n') if 'limit' in case['input']: - count = projected_policy(expected, selection, launch['effect_root'])['readonly_lookups']['bounded_reads'][case['input']['file_path']] + if policy is None: + policy = projected_policy(expected, selection, launch['effect_root']) + count = policy['readonly_lookups']['bounded_reads'][case['input']['file_path']] _need(count == len(lines), 'bounded Read count contradicts captured raw file') if case['id'] != 'source_read': _need(case['input']['limit'] == len(lines), 'current request cannot be completely read in this range') @@ -676,7 +691,7 @@ def _read_content(case, expected, selection, launch, session, source_raw, *, obs return text, '\n'.join(str(i+1) + '\t' + line for i,line in enumerate(selected)), len(selected), len(lines) -def _tool_result(event, case, expected, selection, launch, session, source_raw): +def _tool_result(event, case, expected, selection, launch, session, source_raw, *, policy=None, recipe=None): content = event.get('message', {}).get('content') _need(type(content) is list and len(content) == 1 and type(content[0]) is dict, 'one unique tool result required') item = content[0] @@ -713,7 +728,8 @@ def _tool_result(event, case, expected, selection, launch, session, source_raw): file = metadata.get('file') _need(type(file) is dict and metadata.get('type') == 'text', 'Read lacks native file metadata') raw_text, rendered, count, total = _read_content(case, expected, selection, launch, session, source_raw, - observed_lines=file.get('numLines') if case['id'] == 'source_read' else None) + observed_lines=file.get('numLines') if case['id'] == 'source_read' else None, + policy=policy, recipe=recipe) wanted = {'filePath': case['input']['file_path'], 'content': raw_text, 'numLines': count, 'startLine': 1, 'totalLines': total} _need(_same(file, wanted) and text == rendered, 'Read does not contain exact complete captured range') @@ -784,13 +800,16 @@ def _trace(expected, selection, trace, raw, controls): _need(type(session) is str and re.fullmatch(r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', session), 'invalid parent session') _native_scope(init, session, model) - projected = projection(expected, selection, launch, session) + recipe = _recipe_data(expected, selection, launch, session, roots_checked=True) + projected = projection(expected, selection, launch, session, policy=policy, recipe=recipe) _need(_same(_json(raw[prefix + 'projection.json']), projected), 'projection is not independently derived') - _need(_same(_json(raw[prefix + 'neutral.before.json']), _inventory(expected, selection, launch, session, completed=False)), + _need(_same(_json(raw[prefix + 'neutral.before.json']), + _inventory(expected, selection, launch, session, completed=False, recipe=recipe)), 'neutral before inventory differs') - _need(_same(_json(raw[prefix + 'neutral.after.json']), _inventory(expected, selection, launch, session, completed=trace == 'settled')), + _need(_same(_json(raw[prefix + 'neutral.after.json']), + _inventory(expected, selection, launch, session, completed=trace == 'settled', recipe=recipe)), 'neutral after inventory shows missing or forbidden effects') - all_cases = probe_cases(expected, selection, launch, session) + all_cases = probe_cases(expected, selection, launch, session, policy=policy, recipe=recipe) cases = [c for c in all_cases if c['mode'] != 'observer_stopped'] if trace == 'settled' else [ c for c in all_cases if c['id'] == trace[5:]] _need(len(clean) == (len(cases) + 2 if trace == 'settled' else 2), 'control has missing/extra grants') @@ -812,7 +831,8 @@ def _trace(expected, selection, trace, raw, controls): if 'effort' in (callback.get('request') or {}).get('input', {}): wanted['request']['input']['effort'] = expected['runtime']['effort'] _need(_same(callback, wanted), 'callback differs from observed call/selected authority') - classification, view = _classification(expected, selection, launch, session, case, controls) + classification, view = _classification(expected, selection, launch, session, case, controls, + policy=policy, recipe=recipe) stop_sha, result_sha = None, None if case['mode'] == 'observer_stopped': _need(offset == len(events), 'stopped file case contains a result or continuation') @@ -830,7 +850,8 @@ def _trace(expected, selection, trace, raw, controls): _need(_same(clean[number+2], decision), 'parent decision contradicts recomputed classification') _need(result.get('type') == 'user', 'missing settled result') _native_scope(result, session, model, 'user') - _tool_result(result, case, expected, selection, launch, session, raw['source.before']) + _tool_result(result, case, expected, selection, launch, session, raw['source.before'], + policy=policy, recipe=recipe) result_sha = _sha(_encoded(result)) outcomes.append({'id': case['id'], 'trace_id': trace, 'state': view.state, 'tool': case['tool'], 'input_sha256': _sha(_encoded(case['input'])), 'classification': classification[0], 'basis': classification[1], @@ -871,11 +892,15 @@ def _trace(expected, selection, trace, raw, controls): 'outcome_sha256': _sha(raw[prefix + 'outcome.json'])}, outcomes, launch -def _derived_controls(expected, selection, launch, session): +def _derived_controls(expected, selection, launch, session, *, policy=None, recipe=None): _, effects = _dependencies() + if policy is None: + policy = projected_policy(expected, selection, launch['effect_root']) + if recipe is None: + recipe = _recipe_data(expected, selection, launch, session) checks = [] for state in NONRESPONSE: - view = _view(expected, selection, launch, session, 'worker', state=state) + view = _view(expected, selection, launch, session, 'worker', state=state, policy=policy, recipe=recipe) command = {'command': expected['commands']['stage']} _need(effects.governs_stage_effect(view, tool_name='Bash', tool_input=command) is True, 'fixed stage helper is outside the selected effect router') @@ -893,7 +918,7 @@ def _derived_controls(expected, selection, launch, session): checks.append({'state': state, 'stage_helper_classification': helper[0], 'response_is_none': True, 'denied_response_paths': paths}) for context in CONTEXTS: - view = _view(expected, selection, launch, session, context, read=True) + view = _view(expected, selection, launch, session, context, read=True, policy=policy, recipe=recipe) for ordinal in range(1, 6): path = str(_path(launch['effect_root']) / 'responses' / ('%06d.bin' % ordinal)) if path == view.response.path: @@ -953,10 +978,14 @@ def _verify(raw_manifest, expected): for previous in all_roots), 'probe traces reuse or overlap isolation roots') all_roots.extend(roots) if name == 'settled': + policy = projected_policy(expected, selection, launch['effect_root']) + recipe = _recipe_data(expected, selection, launch, summary['session_id'], roots_checked=True) roster = [{k:v for k,v in c.items() if k != 'input'} - for c in probe_cases(expected, selection, launch, summary['session_id'])] + for c in probe_cases(expected, selection, launch, summary['session_id'], + policy=policy, recipe=recipe)] _need(_same(_json(raw['cases.json']), roster), 'closed case roster differs') - derived = _derived_controls(expected, selection, launch, summary['session_id']) + derived = _derived_controls(expected, selection, launch, summary['session_id'], + policy=policy, recipe=recipe) traces.append(summary); cases.extend(outcomes) _need(len(cases) == 25 and len({case['id'] for case in cases}) == 25, 'not every required case was checked once') # These two digests bind ordered per-trace raw-member hashes; the complete diff --git a/src/data_sheets_schema/native_shared_policy.py b/src/data_sheets_schema/native_shared_policy.py index 2cc7ab243..75515a503 100644 --- a/src/data_sheets_schema/native_shared_policy.py +++ b/src/data_sheets_schema/native_shared_policy.py @@ -77,6 +77,9 @@ def command_policy(spec, instruction_path, controls=None): return policy +_VERIFIED_HELPER_TABLES: set[tuple[str, tuple[tuple[str, str], ...]]] = set() + + def helper_commands(policy): if (type(policy) is not dict or type(policy.get('native_shared_generation_version')) is not int or policy['native_shared_generation_version'] != 1): @@ -84,14 +87,19 @@ def helper_commands(policy): commands=policy.get('native_shared_helpers') c.exact(commands,HELPERS,'native shared helper command table') python=c.canonical_path(policy.get('python'),'registered Python') - for name,command in commands.items(): - if type(command) is not str or not command or len(command.encode('utf-8'))>1_000_000: - raise ValueError('native helper spelling is absent or oversized') - try:words=shlex.split(command) - except ValueError as exc:raise ValueError('native helper spelling is malformed') from exc - if (len(words)<3 or words[0]!=python or words[1] not in ('-m','-c') - or shlex.join(words)!=command): - raise ValueError('native helper must retain exact selected Python argv spelling') + table_key = ((python, tuple(sorted(commands.items()))) + if all(type(v) is str for v in commands.values()) else None) + if table_key is None or table_key not in _VERIFIED_HELPER_TABLES: + for name,command in commands.items(): + if type(command) is not str or not command or len(command.encode('utf-8'))>1_000_000: + raise ValueError('native helper spelling is absent or oversized') + try:words=shlex.split(command) + except ValueError as exc:raise ValueError('native helper spelling is malformed') from exc + if (len(words)<3 or words[0]!=python or words[1] not in ('-m','-c') + or shlex.join(words)!=command): + raise ValueError('native helper must retain exact selected Python argv spelling') + if table_key is not None and len(_VERIFIED_HELPER_TABLES) < 64: + _VERIFIED_HELPER_TABLES.add(table_key) if (policy.get('attribution_command')!=commands['draft'] or policy.get('post_final_recorder',{}).get('command')!=commands['recorder']): raise ValueError('native helper roles disagree with draft/recorder authority') diff --git a/src/data_sheets_schema/native_shared_publication.py b/src/data_sheets_schema/native_shared_publication.py index 60d8c34e8..7efbf92fe 100644 --- a/src/data_sheets_schema/native_shared_publication.py +++ b/src/data_sheets_schema/native_shared_publication.py @@ -18,10 +18,11 @@ from data_sheets_schema.native_shared_evidence import read_regular -def proposal_publications(invocation,proposal): +def proposal_publications(invocation,proposal,*,_catalogs=None): """Independently derive all effects before opening any output file.""" if type(invocation) is not c.StageInvocationCapture: raise ValueError('publication requires an exact captured stage invocation') + catalogs=stage.nr._catalog_context(_catalogs) selected=(invocation.selection,invocation.execution,invocation.phase1,invocation.history) advance=invocation.current_advance argv=(invocation.execution.registered_python,'-m','data_sheets_schema.native_shared_stage', @@ -33,15 +34,15 @@ def proposal_publications(invocation,proposal): or advance.predecessor_history_sha256!=invocation.history.journal.pin.sha256): raise ValueError('publisher invocation differs from current selected advance/history') if type(proposal) is c.StageDecision: - expected=stage.prepare_next(*selected) + expected=stage._prepare_next(*selected,catalogs) if proposal!=expected: raise ValueError('publication decision differs from fresh pure reconstruction') publications=proposal.publications elif type(proposal) is c.StageTransition: - current=stage.prepare_next(*selected) + current=stage._prepare_next(*selected,catalogs) if current.state!='awaiting_response' or current.request is None: raise ValueError('response publication has no current consumed request') - expected=stage.check_response(*selected,current.request.raw,proposal.first_response.raw) + expected=stage._check_response(*selected,current.request.raw,proposal.first_response.raw,catalogs) if proposal!=expected: raise ValueError('publication transition differs from first captured response') publications=proposal.publications+tuple(c.HelperPublication(action='create_once',artifact=item, @@ -101,14 +102,15 @@ def _replace_journal(destination,before,after): return observed -def publish_derived(invocation,proposal): +def publish_derived(invocation,proposal,*,_catalogs=None): """Publish only reconstructed effects after all destinations pass preflight. Used solely by native_shared_capture.publish_stage after its current invocation/context and unchanged-authority checks. No caller paths or alternate command, cursor, journal, session or worker can be selected here. """ - publications=proposal_publications(invocation,proposal) + catalogs=stage.nr._catalog_context(_catalogs) + publications=proposal_publications(invocation,proposal,_catalogs=catalogs) root=invocation.selection.role('stage_root') for pub in publications: destination=_physical_destination(pub.artifact.pin.path,root) @@ -140,7 +142,7 @@ def publish_derived(invocation,proposal): if item.captured.pin.role not in ('record','journal'))) # Report the actual reconstructed state after publication, never the # earlier proposal's state or an unobserved next helper's effects. - state=stage.prepare_next(invocation.selection,invocation.execution,invocation.phase1,updated).state + state=stage._prepare_next(invocation.selection,invocation.execution,invocation.phase1,updated,catalogs).state result={'kind':c.KINDS['advance_result'],'version':c.VERSION, 'selection_sha256':invocation.selection.registration.pin.sha256, 'execution_sha256':invocation.execution.execution.pin.sha256, diff --git a/src/data_sheets_schema/native_shared_receipts.py b/src/data_sheets_schema/native_shared_receipts.py index 95a93a026..041b9c663 100644 --- a/src/data_sheets_schema/native_shared_receipts.py +++ b/src/data_sheets_schema/native_shared_receipts.py @@ -83,9 +83,35 @@ class _ReceiptCatalogContext: def __init__(self): self._entry = None self._typed_marker = None + self._derivations = None + self._derivations_marker = None + self._prefixes = {} + self._raw_inputs = {} + self._prepared = {} + self._completed = {} + self._requests = {} + self._responses = {} + self._completions = {} + self._packets = {} + self._base = {} + self._last_trace = None + self._verified_obs = {} + self._warm_key_cache = None + self._prep_key_cache = None + self._evidence = {} + self._settled = None self._bytes = 0 self._lock = Lock() + def derivations(self): + from . import audit_omissions as omissions, typed_audit as typed + token = (typed._derive, typed._derive_uncached, typed._seal, omissions._prepare) + with self._lock: + if self._derivations is None or self._derivations_marker != token: + self._derivations = typed.DerivationCache() + self._derivations_marker = token + return self._derivations + def catalog(self, selection, snapshot): return self.schema_data(selection, snapshot)[1] @@ -215,6 +241,7 @@ def _inputs(selection, phase1, *, _catalogs=None): def _raw_inputs(selection, full_raw, receipt_raw, *, _catalogs=None): + from copy import deepcopy chosen = selection.document()['inputs'] bounds = selection.bounds() for raw in (full_raw, receipt_raw): @@ -222,10 +249,35 @@ def _raw_inputs(selection, full_raw, receipt_raw, *, _catalogs=None): raise ValueError('native initial inputs require captured bytes') if len(raw) > min(bounds['max_input_bytes'], contract.HARD_LIMITS['original_full_bytes']): raise ValueError('phase-1 input exceeds the supplied native byte bound') + bundle = selection.raw(chosen['bundle']['path']) + manifest_raw = selection.raw(chosen['chunk_manifest']['path']) + context_raw = selection.raw(chosen['context']['path']) + snapshot = schema_snapshot(selection) + ctx = _catalog_context(_catalogs) + cat_key = _catalog_key(selection, snapshot) + token = catalog_reuse.runtime_token() if _catalogs is not None else None + raw_key = None if token is None else ( + token, omissions._owners, receipts.populated_leaves, chunk_texts, validate_manifest_mapping, + selection.registration.pin.sha256, selection.registration.raw, + bundle, manifest_raw, context_raw, full_raw, receipt_raw, + cat_key, + ) + if (raw_key is not None and type(ctx).schema_data is _ReceiptCatalogContext.schema_data + and 'schema_data' not in vars(ctx) and ctx._entry is not None + and ctx._bytes <= _CATALOG_PAYLOAD_BYTES and ctx._entry[0] == cat_key): + if ctx._prep_key_cache is not None and ctx._prep_key_cache[0] == 'raw' and ctx._prep_key_cache[1] == raw_key: + raw_key = ctx._prep_key_cache[1] + if raw_key in ctx._raw_inputs: + ctx._prep_key_cache = ('raw', raw_key) + full, receipt, manifest, texts, md5, cached_bases, cached_catalog, owners = ctx._raw_inputs[raw_key] + return full, receipt, manifest, dict(texts), md5, cached_bases, cached_catalog, dict(owners) + bases, catalog = ctx.schema_data(selection, snapshot) + if raw_key is not None and raw_key in ctx._raw_inputs: + full, receipt, manifest, texts, md5, cached_bases, _cached_catalog, owners = ctx._raw_inputs[raw_key] + return full, receipt, manifest, dict(texts), md5, cached_bases, catalog, dict(owners) full = omissions._mapping(full_raw, 'sealed native full') receipt = omissions._mapping(receipt_raw, 'sealed native receipt') - bundle = selection.raw(chosen['bundle']['path']) - manifest = omissions._mapping(selection.raw(chosen['chunk_manifest']['path']), 'captured chunks') + manifest = omissions._mapping(manifest_raw, 'captured chunks') validate_manifest_mapping(manifest, bundle, manifest.get('bundle')) md5 = hashlib.md5(bundle).hexdigest() if receipt.get('bundle_md5') != md5: @@ -234,13 +286,16 @@ def _raw_inputs(selection, full_raw, receipt_raw, *, _catalogs=None): chunk_texts(bundle.decode('utf-8'), manifest['chunks']))) if len(receipts.populated_leaves(full)) > bounds['max_populated_paths']: raise ValueError('complete native populated inventory exceeds the global bound') - snapshot = schema_snapshot(selection) - bases, catalog = _catalog_context(_catalogs).schema_data(selection, snapshot) - context = omissions._mapping(selection.raw(chosen['context']['path']), 'scope context', json_only=True) + context = omissions._mapping(context_raw, 'scope context', json_only=True) owners = omissions._owners(full, catalog, context['vocabulary']) scopes = [row['owner'] for row in context['scopes']] if scopes.count('') != 1 or len(set(scopes)) != len(scopes) or any(p not in owners for p in scopes): raise ValueError('native scope must name distinct existing full-record owners and root once') + if raw_key is not None and catalog_reuse.runtime_token() is token: + ctx._raw_inputs[raw_key] = ( + deepcopy(full), deepcopy(receipt), deepcopy(manifest), dict(texts), md5, bases, deepcopy(catalog), dict(owners), + ) + ctx._prep_key_cache = ('raw', raw_key) return full, receipt, manifest, texts, md5, bases, catalog, owners @@ -281,6 +336,29 @@ def prepare(selection, execution, phase1): return _prepare(selection, execution, phase1, _ReceiptCatalogContext()) +def _prep_key(selection, execution, phase1, catalogs): + if catalogs is None: + return None, None, None + ctx = _catalog_context(catalogs) + token = catalog_reuse.runtime_token() + if token is None: + return ctx, None, None + snapshot = schema_snapshot(selection) + cat_key = _catalog_key(selection, snapshot) + fn_key = (receipts.uncovered_receiptable_leaves, receipts.check, receipts.apply_rereceipt, + policy, output_ceiling, _inputs, _raw_inputs) + key = ( + token, fn_key, cat_key, selection.registration.pin.sha256, selection.registration.raw, + selection.descriptor.raw, selection.receipt_policy.pin.sha256, selection.receipt_policy.raw, + selection.bounds_json, tuple((item.pin.role, item.pin.path, item.raw) for item in selection.authority), + execution.runtime_declaration.pin.sha256, execution.runtime_declaration.raw, + execution.runtime_declaration_sha256, phase1.seal.pin.sha256, + phase1.full.pin, phase1.full.raw, phase1.original_receipt.pin, phase1.original_receipt.raw, + ) + warm = (ctx._entry is not None and ctx._bytes <= _CATALOG_PAYLOAD_BYTES and ctx._entry[0] == cat_key) + return ctx, token, key if warm else (key,) + + def _prepare(selection, execution, phase1, catalogs): full, receipt, manifest, texts, md5, bases, catalog, owners = _inputs(selection, phase1, _catalogs=catalogs) chosen = selection.document()['inputs'] @@ -307,8 +385,19 @@ def complete(selection, execution, phase1, response_raw): def _complete(selection, execution, phase1, response_raw, catalogs): + ctx, token, prep_key = _prep_key(selection, execution, phase1, catalogs) + complete_key = None if ( + ctx is None or token is None or prep_key is None or len(prep_key) == 1 + or (response_raw is not None and type(response_raw) is not bytes) + ) else (token, prep_key, response_raw) + if complete_key is not None and complete_key in ctx._completed: + return ctx._completed[complete_key] request = _prepare(selection, execution, phase1, catalogs) - full, receipt, manifest, texts, md5, bases, _, _ = _inputs(selection, phase1, _catalogs=catalogs) + request_raw = contract.canonical(request) + if complete_key is None and ctx is not None and token is not None and (response_raw is None or type(response_raw) is bytes): + _, _, prep_key = _prep_key(selection, execution, phase1, catalogs) + if prep_key is not None and len(prep_key) > 1: + complete_key = (token, prep_key, response_raw) listed = request['requested_paths'] if response_raw is None: if listed: @@ -329,6 +418,7 @@ def _complete(selection, execution, phase1, response_raw, catalogs): raise ValueError('rereceipt must be a list') except ValueError as exc: answers, problem = [], str(exc) + full, receipt, manifest, texts, md5, bases, _, _ = _inputs(selection, phase1, _catalogs=catalogs) before = receipts.check(receipt, manifest, texts, full, md5, full, instrument_version=4, identifier_bases=bases) merged = receipts.apply_rereceipt(receipt, full, answers, texts, listed=listed, instrument_version=4) @@ -345,7 +435,7 @@ def _complete(selection, execution, phase1, response_raw, catalogs): result = {'protocol': contract.NAME, 'receipt_instrument_version': 4, 'selection_sha256': selection.registration.pin.sha256, 'policy_sha256': selection.receipt_policy.pin.sha256, - 'request_sha256': contract.sha(contract.canonical(request)), + 'request_sha256': contract.sha(request_raw), 'response_sha256': None if response_raw is None else contract.sha(response_raw), 'original_receipt_sha256': phase1.original_receipt.pin.sha256, 'effective_receipt_sha256': contract.sha(effective), @@ -370,17 +460,25 @@ def _complete(selection, execution, phase1, response_raw, catalogs): 'unanswered_paths': unanswered, 'rejected_paths': result['rejected_paths'], 'rejections': merged['rejections'], 'added_pairs': merged['added_pairs'], 'scientific_support': 'unverified; independent audit required, no deletion instruction'} - return ReceiptResult(tuple(listed), contract.canonical(request), effective, result_raw, contract.canonical(carry)) + built = ReceiptResult(tuple(listed), request_raw, effective, result_raw, contract.canonical(carry)) + if complete_key is not None and catalog_reuse.runtime_token() is token: + ctx._completed[complete_key] = built + return built def check_final(selection, execution, phase1, completion, *, final_full, final_receipt): """Check final coverage against the same selected original/effective bytes.""" + return _check_final(selection, execution, phase1, completion, + final_full=final_full, final_receipt=final_receipt, _catalogs=None) + + +def _check_final(selection, execution, phase1, completion, *, final_full, final_receipt, _catalogs=None): if (completion.selection_sha256 != selection.registration.pin.sha256 or completion.execution_sha256 != execution.execution.pin.sha256 or completion.phase1_seal_sha256 != phase1.seal.pin.sha256 or final_receipt != completion.effective_receipt.raw): raise ValueError('final native receipt lineage differs from checked stage completion') - original, _, manifest, texts, md5, bases, _, _ = _inputs(selection, phase1) + original, _, manifest, texts, md5, bases, _, _ = _inputs(selection, phase1, _catalogs=_catalogs) full = omissions._mapping(final_full, 'native final full') receipt = omissions._mapping(final_receipt, 'native effective receipt') block = receipts.check(receipt, manifest, texts, full, md5, original, @@ -397,3 +495,7 @@ def check_final(selection, execution, phase1, completion, *, final_full, final_r 'raw_json': selection.receipt_policy.raw.decode('utf-8')}, 'identity_rules': {'basis': 'captured native full schema', 'identifier_bases': [list(p) for p in bases]}, 'native_stage_completion_history_sha256': completion.history_sha256} + + +_DEFAULT_CHECK_FINAL = check_final + diff --git a/src/data_sheets_schema/native_shared_results.py b/src/data_sheets_schema/native_shared_results.py index 173c189a3..486ccb4a9 100644 --- a/src/data_sheets_schema/native_shared_results.py +++ b/src/data_sheets_schema/native_shared_results.py @@ -97,7 +97,7 @@ def _captured_selection(pool, value): return selection_api.rebuild(registration, authority, schemas, artifacts['receipt_policy']) -def rebuild(pool, value): +def rebuild(pool, value, *, _catalogs=None): """Reconstruct the captured run; this function performs no filesystem reads.""" from .api_runner import RunSpec selected = _captured_selection(pool, value) @@ -136,6 +136,9 @@ def rebuild(pool, value): phase1 = capture_api._phase1(selected, binding, history, reader) result = capture_api._CapturedRun(selected, value, composition, spec, reader, pool, transcript, control, binding, history, tuple(observations), phase1) + if _catalogs is not None: + result._catalogs = _catalogs + result._prefixes = _catalogs._prefixes capture_api._checked_observations(result) return result @@ -152,7 +155,8 @@ def capture(value, controls, *, authority_inputs, registration_raw): """One actual bounded capture, followed by the same saved-only reconstruction.""" composition = c.strict_json(value['composition_raw_json'].encode(), 'captured C', c.HARD_LIMITS['request_bytes']) selected_doc = c.parse_selection(composition['selection_raw_json'].encode('utf-8')) - live = capture_api._load(selected_doc['registration_path']) + live = capture_api._load(selected_doc['registration_path'], _catalogs=capture_api._LAST_VERIFIED_CATALOGS) + capture_api._LAST_VERIFIED_CATALOGS = live._catalogs snapshot = Snapshot(value['working_directory'], live.pool.members) selection = live.selection for item in (selection.registration, selection.receipt_policy, *selection.authority, @@ -210,7 +214,7 @@ def capture(value, controls, *, authority_inputs, registration_raw): raise ValueError('complete capture exceeds the selected evidence bound') snapshot.verify_unchanged() snapshot.sealed = True - return prepare(pool, value, controls, snapshot=snapshot) + return prepare(pool, value, controls, snapshot=snapshot, _catalogs=live._catalogs) def _readonly(command, policy, snapshot, controls): @@ -239,11 +243,11 @@ def _readonly(command, policy, snapshot, controls): return False -def prepare(pool, value, controls, *, snapshot=None): +def prepare(pool, value, controls, *, snapshot=None, _catalogs=None): from . import native_shared_attribution as attribution from . import native_shared_gates as gates from .audit_omissions import _mapping - run = rebuild(pool, value) + run = rebuild(pool, value, _catalogs=_catalogs) if snapshot is None: snapshot = Snapshot(value['working_directory'], pool.members) snapshot.sealed = True @@ -321,6 +325,7 @@ def classify(command, python, programs, supplied): problems = [*control['problems'], *lifecycle['problems'], *draft['problems']] from . import native_shared_policy, native_shared_effects draft_seen = False + previous_decision = None for line, _, frame in observed.rows(run.transcript): for block in frame.get('message', {}).get('content', ()): if (type(block) is dict and block.get('type') == 'tool_use' and block.get('name') == 'Bash' @@ -330,7 +335,19 @@ def classify(command, python, programs, supplied): continue data = frame['request']['input'] current = capture_api.at_callback(run, line) - view = capture_api.current_effect_view(current, correction_window=draft_seen, + key = (current.history.journal.pin.sha256, + None if current.phase1 is None else ( + current.phase1.seal.pin.sha256, + None if current.phase1.core_seal is None else current.phase1.core_seal.pin.sha256, + )) + if previous_decision is not None and previous_decision[0] == key: + decision = previous_decision[1] + else: + previous_decision = None + decision = current.decision() + if decision is None or decision.state != 'failed': + previous_decision = (key, decision) + view = capture_api._effect_view_from_decision(current, decision, correction_window=draft_seen, exclude_pending=data['tool_use_id']) if native_shared_policy.stage_overlay_governs(view, tool_name=data['tool_name'], tool_input=data['tool_input'], policy=policy): @@ -425,44 +442,47 @@ def require_saved_basis(value, result): raw = _read_pool(pin['path'], pin['bytes']) if len(raw) != pin['bytes'] or c.sha(raw) != pin['sha256']: raise ValueError('saved pool differs from its complete raw identity') - from .native_attribution_controller import load_controls - prepared = prepare(evidence.decode(raw), value, load_controls()) - if c.canonical(prepared['saved_result']) != c.canonical(summary['result']): - raise ValueError('saved native result differs from fresh byte-only reconstruction') - snapshot = prepared['snapshot'] - for actual, key in ((snapshot.identity(), 'captured_files'), (snapshot.metadata, 'captured_metadata'), - (snapshot.aliases, 'captured_aliases'), (snapshot.alias_metadata, 'captured_alias_metadata')): - if c.canonical(actual) != c.canonical(result[key]): - raise ValueError('flat captured pool differs from final captured file identities or metadata') - if result['runtime_gates_passed'] and not prepared['saved_result']['passed']: - raise ValueError('native completion contradicts fresh saved stage/control reconstruction') - # Re-run every original obligation. Existing exclusive projections are - # checked against complete captured bytes before the unchanged file-based - # observation/accounting consumers can open them. - projections = {} - for index, (original, body) in enumerate(sorted(snapshot.raw.items())): - projected = path.parent / 'captured' / str(index) / Path(original).name - actual = evidence.read_regular(str(projected), 'saved_projection', max_bytes=max(1, len(body))) - if actual.captured.raw != body: - raise ValueError('saved projection differs from its exact flat captured member') - projections[original] = projected + from .native_execution_authority import loaded_dependencies from . import native_shared_gates as gates from .native_execution import expected_runtime_observation - fresh_live = {**prepared['saved_result']['draft'], - 'recorder_completed': prepared['saved_result']['recorder_completed_in_trace'], - 'controller_stop': None if prepared['saved_result']['passed'] else 'saved reconstruction failed'} - cleanup = result['lifecycle_artifacts'].get('keep-awake.json') - cleanup_raw = (evidence.read_regular(cleanup['path'], 'cleanup', - max_bytes=c.HARD_LIMITS['input_bytes']).captured.raw if cleanup is not None else None) - fresh = gates.check(prepared, projections, - {**value['runtime'], 'attempt_directory': value['attempt_directory']}, prepared['controls'], - exit_code=result['child_exit_code'], shutdown=result['shutdown'], live=fresh_live, - first_stop=result['first_stop'], runtime_authority=result['runtime_observation'], - keep_awake=result['keep_awake'], keep_awake_raw=cleanup_raw, - runtime_authority_expected=expected_runtime_observation(value)) - if 'receipt_origin_reporting' in value: - from .native_receipt_origin import attach_prepared - fresh = attach_prepared(value, fresh, prepared) + with loaded_dependencies(value['dependencies']) as controls: + prepared = prepare(evidence.decode(raw), value, controls, + _catalogs=capture_api._LAST_VERIFIED_CATALOGS) + capture_api._LAST_VERIFIED_CATALOGS = prepared['run']._catalogs + if c.canonical(prepared['saved_result']) != c.canonical(summary['result']): + raise ValueError('saved native result differs from fresh byte-only reconstruction') + snapshot = prepared['snapshot'] + for actual, key in ((snapshot.identity(), 'captured_files'), (snapshot.metadata, 'captured_metadata'), + (snapshot.aliases, 'captured_aliases'), (snapshot.alias_metadata, 'captured_alias_metadata')): + if c.canonical(actual) != c.canonical(result[key]): + raise ValueError('flat captured pool differs from final captured file identities or metadata') + if result['runtime_gates_passed'] and not prepared['saved_result']['passed']: + raise ValueError('native completion contradicts fresh saved stage/control reconstruction') + # Re-run every original obligation. Existing exclusive projections are + # checked against complete captured bytes before the unchanged file-based + # observation/accounting consumers can open them. + projections = {} + for index, (original, body) in enumerate(sorted(snapshot.raw.items())): + projected = path.parent / 'captured' / str(index) / Path(original).name + actual = evidence.read_regular(str(projected), 'saved_projection', max_bytes=max(1, len(body))) + if actual.captured.raw != body: + raise ValueError('saved projection differs from its exact flat captured member') + projections[original] = projected + fresh_live = {**prepared['saved_result']['draft'], + 'recorder_completed': prepared['saved_result']['recorder_completed_in_trace'], + 'controller_stop': None if prepared['saved_result']['passed'] else 'saved reconstruction failed'} + cleanup = result['lifecycle_artifacts'].get('keep-awake.json') + cleanup_raw = (evidence.read_regular(cleanup['path'], 'cleanup', + max_bytes=c.HARD_LIMITS['input_bytes']).captured.raw if cleanup is not None else None) + fresh = gates.check(prepared, projections, + {**value['runtime'], 'attempt_directory': value['attempt_directory']}, prepared['controls'], + exit_code=result['child_exit_code'], shutdown=result['shutdown'], live=fresh_live, + first_stop=result['first_stop'], runtime_authority=result['runtime_observation'], + keep_awake=result['keep_awake'], keep_awake_raw=cleanup_raw, + runtime_authority_expected=expected_runtime_observation(value)) + if 'receipt_origin_reporting' in value: + from .native_receipt_origin import attach_prepared + fresh = attach_prepared(value, fresh, prepared) for name, recomputed in fresh.items(): old = result['gates'][name] if old['passed'] and (recomputed.get('checked') is not True or recomputed.get('passed') is not True): diff --git a/src/data_sheets_schema/native_shared_schema_gates.py b/src/data_sheets_schema/native_shared_schema_gates.py index 6ec862482..b069d0363 100644 --- a/src/data_sheets_schema/native_shared_schema_gates.py +++ b/src/data_sheets_schema/native_shared_schema_gates.py @@ -5,9 +5,15 @@ """ from __future__ import annotations +import yaml.error as _yaml_error + +_ORIG_YAML_MARK = getattr(_yaml_error, 'Mark', None) from linkml.validator import Validator from linkml.validator.plugins import JsonschemaValidationPlugin +if _ORIG_YAML_MARK is not None and getattr(_yaml_error, 'Mark', None) is not _ORIG_YAML_MARK: + _yaml_error.Mark = _ORIG_YAML_MARK + from . import audit_omissions, d4d_pair_consistency as pair from . import native_shared_contract as contract from .native_shared_receipts import schema_snapshot diff --git a/src/data_sheets_schema/native_shared_stage.py b/src/data_sheets_schema/native_shared_stage.py index 64ad2a1d8..7abfa2482 100644 --- a/src/data_sheets_schema/native_shared_stage.py +++ b/src/data_sheets_schema/native_shared_stage.py @@ -100,6 +100,10 @@ def record_bytes(*, record_type, sequence, previous_record_sha256, selection_sha return raw +_RECORD_VERIFY_CACHE = {} +_JOURNAL_VERIFY_CACHE = {} + + def _history(selection, execution, history): if type(history) is not c.RawHistory or history.journal.pin.path != selection.role('journal'): raise ValueError('native history does not name the selected journal') @@ -120,25 +124,49 @@ def _history(selection, execution, history): raise ValueError('native history byte bound exceeded') if sum(len(item.raw) for item in members) > bounds['max_evidence_bytes']: raise ValueError('native history evidence byte bound exceeded') + sel_sha = selection.registration.pin.sha256 + exe_sha = execution.execution.pin.sha256 + att_id = execution.attempt_id + sess_id = execution.session_id + limit_bytes = c.HARD_LIMITS['metadata_record_bytes'] rows, previous = [], None for sequence, artifact in enumerate(history.records): - if artifact.pin.role != 'record' or artifact.pin.path != _dynamic(selection, 'record', sequence): + rec_key = ( + sel_sha, exe_sha, att_id, sess_id, sequence, previous, + _dynamic(selection, 'record', sequence), limit_bytes, + artifact.pin, artifact.raw, + ) + cached_row = _RECORD_VERIFY_CACHE.get(rec_key) + if cached_row is not None: + rows.append(dict(cached_row)) + previous = artifact.pin.sha256 + continue + if artifact.pin.role != 'record' or artifact.pin.path != rec_key[6]: raise ValueError('native record path/ordinal differs from complete ordered history') - row = c.strict_json(artifact.raw, 'native stage record', c.HARD_LIMITS['metadata_record_bytes']) + row = c.strict_json(artifact.raw, 'native stage record', limit_bytes) c.exact(row, RECORD_KEYS, 'native stage record') expected = record_bytes(record_type=row['record_type'], sequence=sequence, - previous_record_sha256=previous, selection_sha256=selection.registration.pin.sha256, - execution_sha256=execution.execution.pin.sha256, attempt_id=execution.attempt_id, - session_id=None if sequence == 0 else execution.session_id, payload=row['payload']) + previous_record_sha256=previous, selection_sha256=sel_sha, + execution_sha256=exe_sha, attempt_id=att_id, + session_id=None if sequence == 0 else sess_id, payload=row['payload']) if artifact.raw != expected: raise ValueError('native record chronology/identity differs from the captured session') + if len(_RECORD_VERIFY_CACHE) > 512: + _RECORD_VERIFY_CACHE.clear() + _RECORD_VERIFY_CACHE[rec_key] = dict(row) rows.append(row) previous = artifact.pin.sha256 - expected = journal_bytes(selection_sha256=selection.registration.pin.sha256, - execution_sha256=execution.execution.pin.sha256, attempt_id=execution.attempt_id, - records=tuple(item.pin for item in history.records)) - if history.journal.raw != expected: - raise ValueError('native journal omits, reorders or changes captured records') + record_pins = tuple(item.pin for item in history.records) + j_key = (sel_sha, exe_sha, att_id, record_pins, c.HARD_LIMITS['metadata_record_bytes'], history.journal.raw) + if j_key not in _JOURNAL_VERIFY_CACHE: + expected = journal_bytes(selection_sha256=sel_sha, + execution_sha256=exe_sha, attempt_id=att_id, + records=record_pins) + if history.journal.raw != expected: + raise ValueError('native journal omits, reorders or changes captured records') + if len(_JOURNAL_VERIFY_CACHE) > 128: + _JOURNAL_VERIFY_CACHE.clear() + _JOURNAL_VERIFY_CACHE[j_key] = True artifacts = {} for item in (*history.artifacts, *history.observations): if item.pin.path in artifacts: @@ -203,19 +231,58 @@ def _envelope(raw): class _Replay: def __init__(self, selection, execution, phase1, history, *, _catalogs=None): + self._explicit_catalogs = _catalogs is not None self.catalogs = nr._catalog_context(_catalogs) + self._token = catalog_reuse.runtime_token() if self._explicit_catalogs else None + self._wkey_cache = {} self.s, self.e, self.p, self.h = selection, execution, phase1, history self.rows, self.artifacts = _history(selection, execution, history) self.at = 0 self.receipt = self.packet = self.index = self.assembly = None self.workers, self.omission, self.origins = {}, None, [] - self.cache = typed.DerivationCache() + self.cache = self.catalogs.derivations() self.options = {'captured_assets': nr.omission_assets(selection), 'derivations': self.cache} self.pending = None + self.pending_predecessor = None + self.pending_request_hash = None self.failure = None self.checked_outputs = {} self._base() + def _warm_key(self, *, include_core=True): + if (self._token is None or self.catalogs._entry is None + or self.catalogs._bytes > nr._CATALOG_PAYLOAD_BYTES): + return None + if include_core in self._wkey_cache: + return self._wkey_cache[include_core] + fn_token = (typed._derive, typed._derive_uncached, typed._seal, typed._open, + typed._result, typed.assemble, typed.check, typed.check_worker, + typed.index, typed.worker_request, typed.capture_response, + nr.omissions._prepare, nr._prepare, nr._complete, nr._inputs, nr._raw_inputs) + key = ( + self._token, fn_token, self.catalogs._entry[0], + self.s.registration.pin.sha256, self.s.registration.raw, + self.s.descriptor.raw, self.s.receipt_policy.raw, self.s.bounds_json, self.s.roles, + tuple((a.pin.role, a.pin.path, a.raw) for a in self.s.authority), + self.e.execution.pin.sha256, self.e.execution.raw, + self.e.runtime_declaration.pin.sha256, self.e.runtime_declaration.raw, + self.e.runtime_declaration_sha256, self.e.attempt_id, self.e.session_id, + self.p.seal.pin, self.p.full.pin, self.p.full.raw, + self.p.original_receipt.pin, self.p.original_receipt.raw, + None if (not include_core or self.p.core is None) else ( + self.p.core.pin, self.p.core.raw, self.p.core_seal.pin, self.p.core_seal_observation.pin), + ) + wcache = self.catalogs._warm_key_cache + if wcache is not None and include_core in wcache and wcache[include_core] == key: + key = wcache[include_core] + else: + if wcache is None: + wcache = {} + self.catalogs._warm_key_cache = wcache + wcache[include_core] = key + self._wkey_cache[include_core] = key + return key + def _expect(self, kind, payload): if self.at >= len(self.rows) or self.rows[self.at]['record_type'] != kind: raise ValueError('native history lacks ordered ' + kind) @@ -236,9 +303,18 @@ def _base(self): def _prefix(self, count=None): count = self.at if count is None else count - return c.sha(journal_bytes(selection_sha256=self.s.registration.pin.sha256, + pins = tuple(item.pin for item in self.h.records[:count]) + key = (self.s.registration.pin.sha256, self.e.execution.pin.sha256, + self.e.attempt_id, pins) + if self._explicit_catalogs and key in self.catalogs._prefixes: + return self.catalogs._prefixes[key][0] + raw = journal_bytes(selection_sha256=self.s.registration.pin.sha256, execution_sha256=self.e.execution.pin.sha256, attempt_id=self.e.attempt_id, - records=tuple(item.pin for item in self.h.records[:count]))) + records=pins) + digest = c.sha(raw) + if self._explicit_catalogs: + self.catalogs._prefixes[key] = (digest, raw) + return digest def _captured(self, pin, *, observation=False): selected = _pin(pin) @@ -292,6 +368,20 @@ def _inner(self, cursor): return self.index def _request(self, cursor, predecessor): + wkey = self._warm_key(include_core=(cursor.kind != 'receipt')) + rkey = None if wkey is None else ( + wkey, cursor, predecessor, + None if self.receipt is None else self.receipt.carry_json, + None if self.packet is None else self.packet['sha256'], + bool(self.workers) if cursor.kind == 'worker' else None, + tuple(sorted(self.workers.items())) if cursor.kind == 'integration' else (), + self.omission if cursor.kind == 'integration' else None, + ) + if rkey is not None and rkey in self.catalogs._requests: + artifact, response, inner, request_hash, cached_index, auxiliary = self.catalogs._requests[rkey] + if cursor.kind == 'integration': + self.index = cached_index + return artifact, response, inner, request_hash, auxiliary inner = self._inner(cursor) # Receipt precedes core. Replaying it after a genuine core has been # sealed must retain the original request's then-absent core context. @@ -324,19 +414,65 @@ def _request(self, cursor, predecessor): 'role_review': 'Review each whole containing entity and its owning role slot; an attested name alone does not establish its relationship. Scientific truth remains unverified.'}, 'inner_request': inner, 'response': asdict(response)} c.exact(payload, c.OUTER_REQUEST_KEYS, 'native outer request') - raw = c.canonical({'request_sha256': c.sha(c.canonical(payload)), 'payload': payload}) - c.strict_json(raw, 'complete native stage request', self.s.bounds()['max_request_bytes']) - return _artifact(raw, 'request', _dynamic(self.s, 'request', cursor.ordinal)), response, inner + payload_raw = c.canonical(payload) + request_hash = c.sha(payload_raw) + raw = b'{"payload":' + payload_raw + b',"request_sha256":"' + request_hash.encode('ascii') + b'"}' + if len(raw) > self.s.bounds()['max_request_bytes']: + raise ValueError('complete native stage request requires bounded nonempty immutable bytes') + c._JSON_VALIDATION.remember((raw, c.HARD_LIMITS['json_yaml_depth'], c.HARD_LIMITS['json_yaml_nodes_per_document'])) + artifact = _artifact(raw, 'request', _dynamic(self.s, 'request', cursor.ordinal)) + auxiliary = () + if cursor.kind == 'worker' and not self.workers: + auxiliary = (self._output('packet', c.canonical(self.packet)),) + elif cursor.kind == 'integration': + auxiliary = (self._output('typed_index', c.canonical(self.index)),) + if rkey is None: + wkey = self._warm_key(include_core=(cursor.kind != 'receipt')) + rkey = None if wkey is None else ( + wkey, cursor, predecessor, + None if self.receipt is None else self.receipt.carry_json, + None if self.packet is None else self.packet['sha256'], + bool(self.workers) if cursor.kind == 'worker' else None, + tuple(sorted(self.workers.items())) if cursor.kind == 'integration' else (), + self.omission if cursor.kind == 'integration' else None, + ) + if rkey is not None and catalog_reuse.runtime_token() is self._token: + self.catalogs._requests[rkey] = (artifact, response, inner, request_hash, self.index, auxiliary) + return artifact, response, inner, request_hash, auxiliary def _derive_response(self, cursor, inner, raw): envelope, outputs = None, () + wkey = self._warm_key(include_core=(cursor.kind != 'receipt')) if cursor.kind == 'receipt': + resp_key = None if wkey is None else (wkey, cursor, raw) + if resp_key is not None and resp_key in self.catalogs._responses: + value, outputs = self.catalogs._responses[resp_key] + self.receipt = value + return envelope, outputs value = nr._complete(self.s, self.e, self.p, raw, self.catalogs) outputs = self._receipt_outputs(value) if c.strict_json(value.result_json)['state'] != 'answers_complete': raise _IncompleteReceipt(outputs) self.receipt = value - elif cursor.kind == 'worker': + if resp_key is not None and catalog_reuse.runtime_token() is self._token: + self.catalogs._responses[resp_key] = (value, outputs) + return envelope, outputs + resp_key = None if wkey is None else ( + wkey, cursor, raw, + None if self.packet is None else self.packet['sha256'], + tuple(sorted(self.workers.items())) if cursor.kind != 'worker' else (), + self.omission if cursor.kind == 'integration' else None, + ) + if resp_key is not None and resp_key in self.catalogs._responses: + envelope, outputs, cached_assembly = self.catalogs._responses[resp_key] + if cursor.kind == 'worker': + self.workers[cursor.target_id] = envelope + elif cursor.kind == 'omission': + self.omission = raw + else: + self.assembly = cached_assembly + return envelope, outputs + if cursor.kind == 'worker': envelope = typed.capture_response(inner, raw) if not typed.check_worker(self.packet, cursor.target_id, envelope, **self.options)['passed']: raise ValueError('native worker does not satisfy complete structural coverage') @@ -349,17 +485,16 @@ def _derive_response(self, cursor, inner, raw): self.assembly = typed.assemble(self.packet, self.workers, self.omission, envelope, **self.options) audit_raw = typed._unblob(self.assembly['audit'], typed.audit_grammar.MAX_BYTES) outputs = (self._output('typed_assembly', c.canonical(self.assembly)), self._output('audit', audit_raw)) + if resp_key is not None and catalog_reuse.runtime_token() is self._token: + self.catalogs._responses[resp_key] = (envelope, outputs, self.assembly) return envelope, outputs def _stage(self, cursor): predecessor = self._prefix() - request, destination, inner = self._request(cursor, predecessor) - auxiliary = () - if cursor.kind == 'worker' and not self.workers: - auxiliary = (self._output('packet', c.canonical(self.packet)),) - elif cursor.kind == 'integration': - auxiliary = (self._output('typed_index', c.canonical(self.index)),) + request, destination, inner, request_hash, auxiliary = self._request(cursor, predecessor) if self.at == len(self.rows): + self.pending_predecessor = predecessor + self.pending_request_hash = request_hash self.pending = ('request_ready', cursor, request, destination, inner, auxiliary, None) return False expected = {'cursor': asdict(cursor), 'request': c.pin_dict(request.pin), @@ -370,13 +505,14 @@ def _stage(self, cursor): raise ValueError('native admitted request differs from complete intended semantic content') self._compare_outputs(auxiliary) if self.at == len(self.rows): + self.pending_predecessor = predecessor + self.pending_request_hash = request_hash self.pending = ('awaiting_response', cursor, captured_request, destination, inner, (), None) return False row = self.rows[self.at] if row['record_type'] != 'response_consumed': raise ValueError('native stage requires first response consumption before checking') consumed = row['payload'] - request_hash = c.strict_json(request.raw, max_bytes=self.s.bounds()['max_request_bytes'])['request_sha256'] if consumed['cursor'] != asdict(cursor) or consumed['request_sha256'] != request_hash: raise ValueError('native consumed response belongs to another stage or complete request') response = self._captured(consumed['response']) @@ -388,6 +524,8 @@ def _stage(self, cursor): consumed_pin = self.h.records[self.at].pin self.at += 1 if self.at == len(self.rows): + self.pending_predecessor = predecessor + self.pending_request_hash = request_hash self.pending = ('awaiting_response', cursor, captured_request, destination, inner, (), (response, consumed_pin)) return False @@ -469,16 +607,18 @@ def run(self): raise ValueError('native stage history has effects after assembly completion') return self - def _completion_artifact(self): + def _completion_artifact(self, assembly_sha256=None): integration = self.origins[-1] consumed = next(index for index, item in enumerate(self.h.records) if item.pin.sha256 == integration['consumed_record_sha256']) + if assembly_sha256 is None: + assembly_sha256 = self.checked_outputs['typed_assembly'].pin.sha256 body = {'kind': c.KINDS['completion'], 'version': c.VERSION, 'selection_sha256': self.s.registration.pin.sha256, 'execution_sha256': self.e.execution.pin.sha256, 'attempt_id': self.e.attempt_id, 'session_id': self.e.session_id, 'predecessor_history_sha256': self._prefix(consumed + 1), 'phase1_seal_sha256': self.p.seal.pin.sha256, 'core_seal_sha256': self.p.core_seal.pin.sha256, - 'assembly_sha256': c.sha(c.canonical(self.assembly)), 'stage_origins': self.origins, + 'assembly_sha256': assembly_sha256, 'stage_origins': self.origins, 'acceptance': self.assembly['acceptance']} return self._output('completion', c.canonical(body)) @@ -520,7 +660,7 @@ def _prepare_next(selection, execution, phase1, history, catalogs): state, cursor, request, response, inner, outputs, consumed = replay.pending if state == 'awaiting_response': destination = response - predecessor = c.strict_json(request.raw, max_bytes=selection.bounds()['max_request_bytes'])['payload']['predecessor_history_sha256'] + predecessor = replay.pending_predecessor else: entries = () if state == 'request_ready': @@ -557,7 +697,7 @@ def _check_response(selection, execution, phase1, history, request_raw, response response, consumed_pin = consumed if request.raw != request_raw or response.raw != response_raw: raise ValueError('native helper answer differs from first consumed request/response bytes') - request_hash = c.strict_json(request_raw, max_bytes=selection.bounds()['max_request_bytes'])['request_sha256'] + request_hash = replay.pending_request_hash outputs, entries, disposition = (), (), 'checked' try: envelope, outputs = replay._derive_response(cursor, inner, response_raw) @@ -570,9 +710,9 @@ def _check_response(selection, execution, phase1, history, request_raw, response 'request_artifact_sha256': request.pin.sha256, 'response_sha256': response.pin.sha256, 'session_id': execution.session_id, 'consumed_record_sha256': consumed_pin.sha256, 'helper_derived_response_sha256': c.sha(envelope)}) - completed = replay._completion_artifact() - outputs += (completed,) by_role = {item.pin.role: item for item in outputs} + completed = replay._completion_artifact(by_role['typed_assembly'].pin.sha256) + outputs += (completed,) entries += (('assembly_checked', c.canonical({'assembly': c.pin_dict(by_role['typed_assembly'].pin), 'audit': c.pin_dict(by_role['audit'].pin), 'completion': c.pin_dict(completed.pin)})),) records, journal = append_records(selection, execution, history, entries) @@ -590,14 +730,24 @@ def _check_response(selection, execution, phase1, history, request_raw, response def _complete(replay): + wkey = replay._warm_key() + ckey = None if wkey is None else ( + wkey, replay.h.journal.pin.sha256, + c.canonical(replay.assembly), + tuple(sorted((k, v.pin, v.raw) for k, v in replay.checked_outputs.items())), + c.canonical(replay.origins), + ) + if ckey is not None and ckey in replay.catalogs._completions: + return replay.catalogs._completions[ckey] # A fresh cache forces an independent derivation, while deduplicating only # repeated pure opens within this final check. No earlier verdict is reused. checked = typed.check(replay.assembly, captured_assets=replay.options['captured_assets'], - derivations=typed.DerivationCache()) + derivations=typed.DerivationCache(), + _catalog_lookup=catalog_reuse._CatalogLookup(replay.catalogs, replay.s)) if not checked['passed']: raise ValueError('native final typed assembly did not pass reconstruction') out = replay.checked_outputs - return c.StageCompletion(selection_sha256=replay.s.registration.pin.sha256, + result = c.StageCompletion(selection_sha256=replay.s.registration.pin.sha256, execution_sha256=replay.e.execution.pin.sha256, attempt_id=replay.e.attempt_id, session_id=replay.e.session_id, history_sha256=replay.h.journal.pin.sha256, phase1_seal_sha256=replay.p.seal.pin.sha256, core_seal_sha256=replay.p.core_seal.pin.sha256, @@ -608,10 +758,21 @@ def _complete(replay): 'findings': checked['finding_counts'], 'omissions': checked['omission_counts'], 'receipt': c.strict_json(replay.receipt.result_json)['counts']}), stage_origins_json=c.canonical(replay.origins)) + if ckey is not None and catalog_reuse.runtime_token() is replay._token: + replay.catalogs._completions[ckey] = result + return result def check_assembly(selection, execution, phase1, history, assembly_raw): - replay = _Replay(selection, execution, phase1, history).run() + return _check_assembly(selection, execution, phase1, history, assembly_raw, nr._ReceiptCatalogContext()) + + +_DEFAULT_PREPARE_NEXT = prepare_next +_DEFAULT_CHECK_ASSEMBLY = check_assembly + + +def _check_assembly(selection, execution, phase1, history, assembly_raw, catalogs): + replay = _Replay(selection, execution, phase1, history, _catalogs=catalogs).run() if replay.pending is not None or replay.failure is not None or replay.assembly is None: raise ValueError('native assembly cannot complete before every mandatory consumed stage') if c.canonical(replay.assembly) != assembly_raw: @@ -626,15 +787,16 @@ def main(): advance.add_argument('--registration', required=True) args = parser.parse_args() from .native_shared_capture import capture_stage, publish_stage - invocation = capture_stage(args.registration) + catalogs = nr._catalog_context() + invocation = capture_stage(args.registration, _catalogs=catalogs) selected = (invocation.selection, invocation.execution, invocation.phase1, invocation.history) - replay = _Replay(*selected).run() + replay = _Replay(*selected, _catalogs=catalogs).run() if replay.pending is not None and replay.pending[-1] is not None: response, _ = replay.pending[-1] - proposal = check_response(*selected, replay.pending[2].raw, response.raw) + proposal = _check_response(*selected, replay.pending[2].raw, response.raw, catalogs) else: - proposal = prepare_next(*selected) - result = publish_stage(invocation, proposal) + proposal = _prepare_next(*selected, catalogs) + result = publish_stage(invocation, proposal, _catalogs=catalogs) print(c.canonical(result).decode('utf-8')) diff --git a/src/data_sheets_schema/native_shared_streams.py b/src/data_sheets_schema/native_shared_streams.py index 9b7d23d43..43d3ee1d5 100644 --- a/src/data_sheets_schema/native_shared_streams.py +++ b/src/data_sheets_schema/native_shared_streams.py @@ -8,11 +8,33 @@ import os from pathlib import Path +import time from . import native_shared_contract as c from . import native_shared_evidence as evidence ROLES = ('transcript', 'control') +MAX_DIAGNOSTIC_BYTES = 16 * 1024 + + +class _LiveStreamRefusal(ValueError): + """The original refusal arguments plus immutable, bounded diagnostic bytes.""" + + def __init__(self, original, diagnostic_json): + super().__init__(*original.args) + self._diagnostic_json = diagnostic_json + + @property + def diagnostic_json(self): + return self._diagnostic_json + + +def diagnostic_document(raw): + """Decode a detached diagnostic; its contents never grant stream admission.""" + value = c.strict_json(raw, 'native live-stream diagnostic', MAX_DIAGNOSTIC_BYTES) + if c.canonical(value) != raw: + raise ValueError('native live-stream diagnostic is not canonical') + return value def _identity(value): @@ -41,14 +63,28 @@ def stream_files(members): return identities(found) -def _progress(earlier, later, maximum): - evidence._metadata(later, later['size']) - if any(later[k] != earlier[k] for k in ('device', 'inode', 'regular', 'symlink', 'links')): - raise ValueError('live native stream physical identity changed') - if later['size'] < earlier['size'] or later['size'] > maximum: - raise ValueError('live native stream shrank or exceeded its byte bound') - if later['size'] == earlier['size'] and later['mtime_ns'] != earlier['mtime_ns']: - raise ValueError('live native stream changed without appending bytes') +def _progress(earlier, later, maximum, *, diagnostic=None): + try: + evidence._metadata(later, later['size']) + if any(later[k] != earlier[k] for k in ('device', 'inode', 'regular', 'symlink', 'links')): + raise ValueError('live native stream physical identity changed') + if later['size'] < earlier['size'] or later['size'] > maximum: + raise ValueError('live native stream shrank or exceeded its byte bound') + if later['size'] == earlier['size'] and later['mtime_ns'] != earlier['mtime_ns']: + raise ValueError('live native stream changed without appending bytes') + except ValueError as original: + # Diagnostic work is best effort and happens only after the unchanged + # refusal. Never acquire more metadata or bytes to explain a failure. + if diagnostic is not None: + try: + raw = c.canonical({'kind': 'native_live_stream_refusal', 'version': 1, + **diagnostic, 'max_bytes': maximum, 'earlier': earlier, 'later': later}) + diagnostic_document(raw) + enriched = _LiveStreamRefusal(original, raw) + except Exception: + raise original + raise enriched from original + raise return later @@ -71,7 +107,8 @@ def read_live_stream(path, role, *, max_bytes, previous=None, identity=None): target = Path(path) if str(target.resolve(strict=True)) != path: raise ValueError('live native stream is not its exact physical path') - before = evidence._stat(target.lstat()) + raw_stat = target.lstat() + before = evidence._stat(raw_stat) evidence._metadata(before, before['size']) if before['size'] > max_bytes: raise ValueError('live native stream exceeds its byte bound') @@ -79,7 +116,26 @@ def read_live_stream(path, role, *, max_bytes, previous=None, identity=None): raise ValueError('live native stream differs from its observed file identity') fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) try: - opened = _progress(before, evidence._stat(os.fstat(fd)), max_bytes) + def sample(get_info): + nonlocal raw_stat + info = get_info() + if (info.st_dev == raw_stat.st_dev and info.st_ino == raw_stat.st_ino + and info.st_size == raw_stat.st_size and info.st_mtime_ns != raw_stat.st_mtime_ns): + deadline = time.monotonic() + 0.02 + while info.st_size == raw_stat.st_size and time.monotonic() < deadline: + time.sleep(0.0001) + info = get_info() + raw_stat = info + return evidence._stat(info) + + def progress(earlier, later, stage, *, pass_number=None, offset=0, opened_extent=None): + return _progress(earlier, later, max_bytes, diagnostic={ + 'role': role, 'path': path, 'stage': stage, + 'pass_number': pass_number, 'offset': offset, 'opened_extent': opened_extent, + 'previous_prefix': None if previous is None else { + 'bytes': previous.bytes, 'sha256': previous.sha256}}) + + opened = progress(before, sample(lambda: os.fstat(fd)), 'opened') size = opened['size'] if previous is not None and previous.bytes > size: raise ValueError('live native stream lost an observed prefix') @@ -87,27 +143,33 @@ def read_live_stream(path, role, *, max_bytes, previous=None, identity=None): pieces = [] offset = 0 while offset < size: - last = _progress(last, evidence._stat(os.fstat(fd)), max_bytes) + last = progress(last, sample(lambda: os.fstat(fd)), 'before_read', + pass_number=1, offset=offset, opened_extent=size) count = min(65536, size - offset) part = os.pread(fd, count, offset) if not part or len(part) > count: raise ValueError('live native stream ended inside its opened extent') pieces.append(part) offset += len(part) - last = _progress(last, evidence._stat(os.fstat(fd)), max_bytes) + last = progress(last, sample(lambda: os.fstat(fd)), 'after_read', + pass_number=1, offset=offset, opened_extent=size) raw = b''.join(pieces) del pieces offset = 0 while offset < size: - last = _progress(last, evidence._stat(os.fstat(fd)), max_bytes) + last = progress(last, sample(lambda: os.fstat(fd)), 'before_read', + pass_number=2, offset=offset, opened_extent=size) count = min(65536, size - offset) part = os.pread(fd, count, offset) if not part or len(part) > count or part != raw[offset:offset + len(part)]: raise ValueError('live native stream prefix changed during capture') offset += len(part) - last = _progress(last, evidence._stat(os.fstat(fd)), max_bytes) - last = _progress(last, evidence._stat(os.fstat(fd)), max_bytes) - _progress(last, evidence._stat(target.lstat()), max_bytes) + last = progress(last, sample(lambda: os.fstat(fd)), 'after_read', + pass_number=2, offset=offset, opened_extent=size) + last = progress(last, sample(lambda: os.fstat(fd)), 'final_fd', + offset=offset, opened_extent=size) + progress(last, sample(target.lstat), 'final_path', + offset=offset, opened_extent=size) if str(target.resolve(strict=True)) != path: raise ValueError('live native stream path changed during capture') if previous is not None and raw[:previous.bytes] != previous.raw: diff --git a/src/data_sheets_schema/typed_audit.py b/src/data_sheets_schema/typed_audit.py index 35b0fff17..38f3e647f 100644 --- a/src/data_sheets_schema/typed_audit.py +++ b/src/data_sheets_schema/typed_audit.py @@ -40,8 +40,183 @@ def _sha(raw): return hashlib.sha256(raw).hexdigest() +_DEFAULT_CANONICAL_BYTES = batches.canonical_bytes +_BLOB_KEYS = {"base64", "bytes", "sha256"} +_SCHEMA_ROW_KEYS = {"name", "path", "content"} +_REQUESTS_KEYS = {"assembly_rule", "shared_context", "workers"} +_WORKER_PAYLOAD_KEYS = {"packet_sha256", "worker_id", "shared_context", "stage"} +_PACKET_UNSEALED_KEYS = { + "kind", "protocol", "inputs", "schema_sources", "project", "limits", "plan", + "omission_request", "contracts", "requests", "registered_provenance", "limitations", +} +_PACKET_SEALED_KEYS = _PACKET_UNSEALED_KEYS | {"sha256"} +_ASSEMBLY_UNSEALED_KEYS = { + "kind", "packet", "workers", "omission_response", "integration_response", + "index", "audit", "lineage", "acceptance", +} +_ASSEMBLY_SEALED_KEYS = _ASSEMBLY_UNSEALED_KEYS | {"sha256"} +_SUBTREE_JSON_CACHE = {} +_STR_JSON_CACHE = {} +_VALIDATED_BOUNDED_BYTES = set() + + +def _str_json_bytes(value): + hit = _STR_JSON_CACHE.get(value) + if hit is None: + hit = _DEFAULT_CANONICAL_BYTES(value) + if len(_STR_JSON_CACHE) >= 256: + _STR_JSON_CACHE.pop(next(iter(_STR_JSON_CACHE))) + _STR_JSON_CACHE[value] = hit + return hit + + +def _inputs_json_bytes(inputs): + if type(inputs) is not dict: + return None + items = [] + for key, blob in inputs.items(): + if (type(key) is not str or type(blob) is not dict or len(blob) != 3 + or set(blob) != _BLOB_KEYS): + return None + b64, size, sha = blob["base64"], blob["bytes"], blob["sha256"] + if type(b64) is not str or type(size) is not int or type(sha) is not str: + return None + items.append((key, b64, size, sha)) + items.sort() + cache_key = ("inputs", tuple(items)) + hit = _SUBTREE_JSON_CACHE.get(cache_key) + if hit is None: + hit = _DEFAULT_CANONICAL_BYTES(inputs) + if len(_SUBTREE_JSON_CACHE) >= 64: + _SUBTREE_JSON_CACHE.pop(next(iter(_SUBTREE_JSON_CACHE))) + _SUBTREE_JSON_CACHE[cache_key] = hit + return hit + + +def _schema_sources_json_bytes(rows): + if type(rows) is not list: + return None + items = [] + for row in rows: + if type(row) is not dict or len(row) != 3 or set(row) != _SCHEMA_ROW_KEYS: + return None + name, path, content = row["name"], row["path"], row["content"] + if (type(name) is not str or type(path) is not str or type(content) is not dict + or len(content) != 3 or set(content) != _BLOB_KEYS): + return None + b64, size, sha = content["base64"], content["bytes"], content["sha256"] + if type(b64) is not str or type(size) is not int or type(sha) is not str: + return None + items.append((name, path, b64, size, sha)) + cache_key = ("schema_sources", tuple(items)) + hit = _SUBTREE_JSON_CACHE.get(cache_key) + if hit is None: + hit = _DEFAULT_CANONICAL_BYTES(rows) + if len(_SUBTREE_JSON_CACHE) >= 64: + _SUBTREE_JSON_CACHE.pop(next(iter(_SUBTREE_JSON_CACHE))) + _SUBTREE_JSON_CACHE[cache_key] = hit + return hit + + +def _requests_json_bytes(requests): + if type(requests) is not dict or len(requests) != 3 or set(requests) != _REQUESTS_KEYS: + return None + rule, shared, workers = requests["assembly_rule"], requests["shared_context"], requests["workers"] + if type(rule) is not str or type(shared) is not str or type(workers) is not dict: + return None + worker_items = [] + for key, val in workers.items(): + if type(key) is not str or type(val) is not str: + return None + worker_items.append((key, val)) + worker_items.sort() + cache_key = ("requests", rule, shared, tuple(worker_items)) + hit = _SUBTREE_JSON_CACHE.get(cache_key) + if hit is None: + hit = _DEFAULT_CANONICAL_BYTES(requests) + if len(_SUBTREE_JSON_CACHE) >= 64: + _SUBTREE_JSON_CACHE.pop(next(iter(_SUBTREE_JSON_CACHE))) + _SUBTREE_JSON_CACHE[cache_key] = hit + return hit + + def _json(value): - return batches.canonical_bytes(value) + if batches.canonical_bytes is not _DEFAULT_CANONICAL_BYTES: + return batches.canonical_bytes(value) + if type(value) is dict: + if len(value) == 4 and set(value) == _WORKER_PAYLOAD_KEYS: + p_sha = value["packet_sha256"] + s_ctx = value["shared_context"] + stage = value["stage"] + w_id = value["worker_id"] + if type(p_sha) is str and type(s_ctx) is str and type(stage) is str and type(w_id) is str: + return ( + b'{"packet_sha256":' + _str_json_bytes(p_sha) + + b',"shared_context":' + _str_json_bytes(s_ctx) + + b',"stage":' + _str_json_bytes(stage) + + b',"worker_id":' + _str_json_bytes(w_id) + b'}' + ) + kind = value.get("kind") + if kind == PACKET and len(value) in (12, 13): + keys = set(value) + if keys in (_PACKET_UNSEALED_KEYS, _PACKET_SEALED_KEYS): + inputs_b = _inputs_json_bytes(value["inputs"]) + req_b = _requests_json_bytes(value["requests"]) + schema_b = _schema_sources_json_bytes(value["schema_sources"]) + if inputs_b is not None and req_b is not None and schema_b is not None: + parts = [ + b'{"contracts":' + _DEFAULT_CANONICAL_BYTES(value["contracts"]), + b'"inputs":' + inputs_b, + b'"kind":"typed_audit_packet_v1"', + b'"limitations":' + _DEFAULT_CANONICAL_BYTES(value["limitations"]), + b'"limits":' + _DEFAULT_CANONICAL_BYTES(value["limits"]), + b'"omission_request":' + _DEFAULT_CANONICAL_BYTES(value["omission_request"]), + b'"plan":' + _DEFAULT_CANONICAL_BYTES(value["plan"]), + b'"project":' + _DEFAULT_CANONICAL_BYTES(value["project"]), + b'"protocol":' + _DEFAULT_CANONICAL_BYTES(value["protocol"]), + b'"registered_provenance":' + _DEFAULT_CANONICAL_BYTES(value["registered_provenance"]), + b'"requests":' + req_b, + b'"schema_sources":' + schema_b, + ] + if "sha256" in value: + parts.append(b'"sha256":' + _DEFAULT_CANONICAL_BYTES(value["sha256"])) + return b",".join(parts) + b"}" + elif kind == ASSEMBLY and len(value) in (9, 10): + keys = set(value) + if keys in (_ASSEMBLY_UNSEALED_KEYS, _ASSEMBLY_SEALED_KEYS): + packet_b = _json(value["packet"]) + workers_b = _inputs_json_bytes(value["workers"]) + if workers_b is not None: + parts = [ + b'{"acceptance":' + _DEFAULT_CANONICAL_BYTES(value["acceptance"]), + b'"audit":' + _DEFAULT_CANONICAL_BYTES(value["audit"]), + b'"index":' + _DEFAULT_CANONICAL_BYTES(value["index"]), + b'"integration_response":' + _DEFAULT_CANONICAL_BYTES(value["integration_response"]), + b'"kind":"typed_audit_assembly_v1"', + b'"lineage":' + _DEFAULT_CANONICAL_BYTES(value["lineage"]), + b'"omission_response":' + _DEFAULT_CANONICAL_BYTES(value["omission_response"]), + b'"packet":' + packet_b, + b'"workers":' + workers_b, + ] + if "sha256" in value: + parts.append(b'"sha256":' + _DEFAULT_CANONICAL_BYTES(value["sha256"])) + return b",".join(parts) + b"}" + elif type(value) is list and len(value) in (2, 3): + selected = value[0] if len(value) == 2 else (value[1] if value[0] == "captured_logical_schema_v1" else None) + if type(selected) is list and len(selected) == 4: + inputs_b = _inputs_json_bytes(selected[0]) + schema_b = _schema_sources_json_bytes(selected[1]) + if inputs_b is not None and schema_b is not None: + sel_b = ( + b"[" + inputs_b + b"," + schema_b + b"," + + _DEFAULT_CANONICAL_BYTES(selected[2]) + b"," + + _DEFAULT_CANONICAL_BYTES(selected[3]) + b"]" + ) + assets_b = _DEFAULT_CANONICAL_BYTES(value[-1]) + if len(value) == 2: + return b"[" + sel_b + b"," + assets_b + b"]" + return b'["captured_logical_schema_v1",' + sel_b + b"," + assets_b + b"]" + return _DEFAULT_CANONICAL_BYTES(value) def _exact(value, keys, label): @@ -73,7 +248,31 @@ def _seal(value): return {**value, "sha256": _sha(_json(value))} +_DEFAULT_OMISSIONS_MAPPING = omissions._mapping +_DEFAULT_OMISSIONS_READ = omissions._read +_DEFAULT_OMISSIONS_VALIDATE_JSON = omissions._validate_json +_DEFAULT_EVIDENCE_LOAD_JSON = evidence.load_json + + def _bounded_json(raw, label, limit): + if (omissions._mapping is _DEFAULT_OMISSIONS_MAPPING + and omissions._read is _DEFAULT_OMISSIONS_READ + and omissions._validate_json is _DEFAULT_OMISSIONS_VALIDATE_JSON + and evidence.load_json is _DEFAULT_EVIDENCE_LOAD_JSON + and type(raw) is bytes and 0 < len(raw) <= limit): + key = (len(raw), _sha(raw), omissions.MAX_NODES, omissions.MAX_DEPTH) + if key in _VALIDATED_BOUNDED_BYTES: + return json.loads(raw) + try: + value = evidence.load_json(raw.decode("utf-8")) + omissions._validate_json(value, max_nodes=omissions.MAX_NODES, max_depth=omissions.MAX_DEPTH) + if isinstance(value, dict): + if len(_VALIDATED_BOUNDED_BYTES) >= 128: + _VALIDATED_BOUNDED_BYTES.clear() + _VALIDATED_BOUNDED_BYTES.add(key) + return value + except Exception: + pass return omissions._mapping(raw, label, json_only=True, limit=limit) @@ -340,16 +539,42 @@ def _prepare(*, protocol, original_full, bundle, manifest, receipt, context, sch return packet -def _open(packet, *, derivations=None, captured_assets=None): +def _open(packet, *, derivations=None, captured_assets=None, _catalog_lookup=None): _exact(packet, {"kind", "protocol", "inputs", "schema_sources", "project", "limits", "plan", "omission_request", "contracts", "requests", "registered_provenance", "limitations", "sha256"}, "packet") - if len(_json(packet)) > MAX_PACKET_BYTES or packet["kind"] != PACKET or _json(packet["protocol"]) != _json(audit_protocol.select(audit_protocol.TYPED)): + raw_packet = _json(packet) + if len(raw_packet) > MAX_PACKET_BYTES or packet["kind"] != PACKET or _json(packet["protocol"]) != _json(audit_protocol.select(audit_protocol.TYPED)): raise ValueError("packet protocol/size mismatch") - derived, raw, prepared = _derive(packet["inputs"], packet["schema_sources"], packet["project"], packet["limits"], derivations=derivations, captured_assets=captured_assets) + if type(derivations) is DerivationCache: + assets = tuple(sorted((name, _sha(raw)) for name, raw in omissions.captured_asset_bytes(captured_assets).items())) + key = (len(raw_packet), _sha(raw_packet), captured_assets is not None, assets) + opened = getattr(derivations, "_opened", None) + if opened is None: + opened = derivations._opened = {} + hit = opened.get(key) + if hit is not None: + raw_bytes, payload_json = hit + return dict(raw_bytes), omissions.Prepared(payload_json) + _schema_reuse = None + if captured_assets is not None and _catalog_lookup is not None: + from .native_catalog_reuse import _CatalogLookup + if type(_catalog_lookup) is _CatalogLookup: + try: + snapshot = _snapshot(packet["schema_sources"]) + catalog = _catalog_lookup.for_snapshot(snapshot.sources[0][1], snapshot) + if catalog is not None: + _schema_reuse = _SchemaReuse(snapshot.sources, catalog) + except Exception: + _schema_reuse = None + derived, raw, prepared = _derive(packet["inputs"], packet["schema_sources"], packet["project"], packet["limits"], derivations=derivations, captured_assets=captured_assets, _schema_reuse=_schema_reuse) expected = _seal({**{key: packet[key] for key in ("kind", "protocol", "inputs", "schema_sources", "project", "limits")}, **derived, "limitations": list(LIMITATIONS)}) - if _json(packet) != _json(expected): + if raw_packet != _json(expected): raise ValueError("packet request/plan/contract/input identity mismatch") + if type(derivations) is DerivationCache: + while len(opened) >= derivations.max_entries: + opened.pop(next(iter(opened))) + opened[key] = (dict(raw), prepared.payload_json) return raw, prepared @@ -363,21 +588,30 @@ def _omission_findings(findings, candidates=None): raise ValueError("finding references an unknown omission candidate") -def worker_request(packet, worker_id, *, derivations=None, captured_assets=None): - """Export a request after packet sealing; identity hashes its exact payload.""" - _open(packet, derivations=derivations, captured_assets=captured_assets) +def _worker_request(packet, worker_id, *, derivations=None, captured_assets=None, _opened=False): + if not _opened: + _open(packet, derivations=derivations, captured_assets=captured_assets) if type(worker_id) is not str or worker_id not in packet["requests"]["workers"]: raise ValueError("unknown worker request") payload = {"packet_sha256": packet["sha256"], "worker_id": worker_id, "shared_context": packet["requests"]["shared_context"], "stage": packet["requests"]["workers"][worker_id]} + raw_payload = _json(payload) request = {"kind": "typed_audit_worker_request_v1", "payload": payload, - "request_sha256": _sha(_json(payload))} - if len(_json(request)) > packet["limits"]["max_request_bytes"]: + "request_sha256": _sha(raw_payload)} + raw_len = (len(raw_payload) + 135 + if batches.canonical_bytes is _DEFAULT_CANONICAL_BYTES + else len(_json(request))) + if raw_len > packet["limits"]["max_request_bytes"]: raise ValueError("complete worker request exceeds max_request_bytes") return request +def worker_request(packet, worker_id, *, derivations=None, captured_assets=None): + """Export a request after packet sealing; identity hashes its exact payload.""" + return _worker_request(packet, worker_id, derivations=derivations, captured_assets=captured_assets) + + def capture_response(request, response): """Declare saved bytes' association with an exported request, without authentication. @@ -426,9 +660,14 @@ def _unwrap(saved, request): return raw -def _workers(packet, workers, *, derivations=None, captured_assets=None): +def _workers(packet, workers, *, derivations=None, captured_assets=None, _opened=False): if type(workers) is not dict: raise ValueError("saved workers must have an explicit id roster") + if workers and not _opened and worker_request is _DEFAULT_WORKER_REQUEST: + _open(packet, derivations=derivations, captured_assets=captured_assets) + _opened = True + if worker_request is _DEFAULT_WORKER_REQUEST: + return {key: _unwrap(raw, _worker_request(packet, key, derivations=derivations, captured_assets=captured_assets, _opened=_opened)) for key, raw in workers.items()} return {key: _unwrap(raw, worker_request(packet, key, derivations=derivations, captured_assets=captured_assets)) for key, raw in workers.items()} @@ -455,13 +694,12 @@ def _index(packet, workers): return index -def index(packet, workers, omission_response, *, derivations=None, captured_assets=None): - """Bind workers and complete omission response; render the integration tail.""" - _, prepared = _open(packet, derivations=derivations, captured_assets=captured_assets) +def _index_with_workers(packet, workers, omission_response, *, derivations=None, captured_assets=None, _opened=None): + _, prepared = _opened if _opened is not None else _open(packet, derivations=derivations, captured_assets=captured_assets) report = prepared.check(omission_response, saved_request=packet["omission_request"]) if not report["protocol_complete"]: raise ValueError("omission response does not satisfy complete captured inventory") - inner_workers = _workers(packet, workers, derivations=derivations, captured_assets=captured_assets) + inner_workers = _workers(packet, workers, derivations=derivations, captured_assets=captured_assets, _opened=True) result = _index(packet, inner_workers) candidates = {row["id"] for row in report["declared_candidates"]} for raw in inner_workers.values(): @@ -477,7 +715,16 @@ def index(packet, workers, omission_response, *, derivations=None, captured_asse "index": result, "omission_check": report, "integration_request": request}) if len(_json(result)) > packet["limits"]["max_request_bytes"]: raise ValueError("complete integration request exceeds max_request_bytes") - return result + return result, inner_workers + + +def index(packet, workers, omission_response, *, derivations=None, captured_assets=None): + """Bind workers and complete omission response; render the integration tail.""" + return _index_with_workers(packet, workers, omission_response, derivations=derivations, captured_assets=captured_assets)[0] + + +_DEFAULT_WORKER_REQUEST = worker_request +_DEFAULT_INDEX = index def _account(audit, delta, candidates): @@ -499,11 +746,16 @@ def _account(audit, delta, candidates): "final_finding_ordinal": links.get(row["id"])} for row in candidates] -def _result(packet, workers, omission_response, integration_response, *, derivations=None, captured_assets=None): - raw, _ = _open(packet, derivations=derivations, captured_assets=captured_assets) - bound_index = index(packet, workers, omission_response, derivations=derivations, captured_assets=captured_assets) +def _result(packet, workers, omission_response, integration_response, *, derivations=None, captured_assets=None, _catalog_lookup=None): + opened = _open(packet, derivations=derivations, captured_assets=captured_assets, _catalog_lookup=_catalog_lookup) + raw, _ = opened + if index is _DEFAULT_INDEX: + bound_index, inner_workers = _index_with_workers(packet, workers, omission_response, derivations=derivations, captured_assets=captured_assets, _opened=opened) + else: + bound_index = index(packet, workers, omission_response, derivations=derivations, captured_assets=captured_assets) + inner_workers = _workers(packet, workers, derivations=derivations, captured_assets=captured_assets, _opened=True) inner_integration = _unwrap(integration_response, bound_index) - audit_raw, lineage = batches.assemble(packet["plan"], _workers(packet, workers, derivations=derivations, captured_assets=captured_assets), inner_integration, version=2) + audit_raw, lineage = batches.assemble(packet["plan"], inner_workers, inner_integration, version=2) audit, delta = audit_grammar._load(audit_raw), audit_grammar._load(inner_integration) # Check candidates in replaced/dropped proposal entry paths too. The final # exact-once allocation is separate from proposal membership. @@ -548,9 +800,9 @@ def check_integration(packet, workers, omission_response, integration_response, return _result(packet, workers, omission_response, integration_response, derivations=derivations, captured_assets=captured_assets)[3] -def assemble(packet, workers, omission_response, integration_response, *, derivations=None, captured_assets=None): +def assemble(packet, workers, omission_response, integration_response, *, derivations=None, captured_assets=None, _catalog_lookup=None): """Seal a new self-contained assembly only when mechanical checks pass.""" - bound_index, audit_raw, lineage, report = _result(packet, workers, omission_response, integration_response, derivations=derivations, captured_assets=captured_assets) + bound_index, audit_raw, lineage, report = _result(packet, workers, omission_response, integration_response, derivations=derivations, captured_assets=captured_assets, _catalog_lookup=_catalog_lookup) if not report["passed"]: raise ValueError("source-evidence acceptance failed; use check-integration for bounded diagnostics") result = _seal({"kind": ASSEMBLY, "packet": packet, @@ -563,7 +815,7 @@ def assemble(packet, workers, omission_response, integration_response, *, deriva return result -def check(assembly, *, derivations=None, captured_assets=None): +def check(assembly, *, derivations=None, captured_assets=None, _catalog_lookup=None): """Rebuild independently from captured original/response bytes, not reports.""" _exact(assembly, {"kind", "packet", "workers", "omission_response", "integration_response", "index", "audit", "lineage", "acceptance", "sha256"}, "assembly") @@ -573,7 +825,7 @@ def check(assembly, *, derivations=None, captured_assets=None): raise ValueError("assembly exceeds byte bound") expected = assemble(assembly["packet"], {key: _unblob(value, MAX_SAVED_RESPONSE_BYTES) for key, value in assembly["workers"].items()}, _unblob(assembly["omission_response"], omissions.MAX_RESPONSE_BYTES), - _unblob(assembly["integration_response"], MAX_SAVED_RESPONSE_BYTES), derivations=derivations, captured_assets=captured_assets) + _unblob(assembly["integration_response"], MAX_SAVED_RESPONSE_BYTES), derivations=derivations, captured_assets=captured_assets, _catalog_lookup=_catalog_lookup) if _json(assembly) != _json(expected): raise ValueError("saved assembly differs from independent captured-byte reconstruction") return {**expected["acceptance"], "assembly_sha256": expected["sha256"], "independently_reconstructed": True} diff --git a/tests/test_native_ci_evidence.py b/tests/test_native_ci_evidence.py new file mode 100644 index 000000000..ae52b4d06 --- /dev/null +++ b/tests/test_native_ci_evidence.py @@ -0,0 +1,255 @@ +"""Offline evidence capture controls; no native peer, helpers or providers run.""" +from copy import deepcopy +import json +from pathlib import Path +import stat +import tarfile +from types import SimpleNamespace + +import pytest + +from utils import native_ci_evidence as evidence + + +NODE = next(iter(evidence.CASES)) +SOURCE = {'head': 'a' * 40, 'tree': 'b' * 40, 'files': {}, 'scope': 'invented unit fixture'} +FAILED = {'setup': {'outcome': 'passed', 'duration_seconds': 0.1}, + 'call': {'outcome': 'failed', 'duration_seconds': 900.0}, + 'teardown': {'outcome': 'passed', 'duration_seconds': 0.2}} + + +def case(tmp_path): + selected = tmp_path / evidence.CASES[NODE] + selected.mkdir() + (selected / 'actual-result.json').write_text(json.dumps({'state': 'failed', 'synthetic': True})) + (selected / '.hidden').write_bytes(b'exact invented bytes\x00') + (selected / '.hidden').chmod(0o600) + return selected + + +def capture(tmp_path, output, **kwargs): + return evidence.capture_case(tmp_path, output, node_id=NODE, worker='gw2', + source=SOURCE, outcome=deepcopy(FAILED), **kwargs) + + +def test_failed_case_retains_exact_bytes_modes_and_original_failure(tmp_path): + selected = case(tmp_path) + original = evidence.snapshot(selected) + result = capture(tmp_path, tmp_path / 'capture') + assert result['status'] == 'retained' and result['unchanged'] is True + assert result['before'] == result['archived'] == result['after'] == original + assert evidence.snapshot(selected) == original + assert result['test_outcome'] == FAILED and result['scientific_acceptance'] == 'not_assessed' + assert result['source'] == SOURCE + assert result['declared_limits']['attempt_seconds'] == 900 + assert result['declared_limits']['helper_seconds'] == 180 + assert result['declared_limits']['base_fixture_seconds'] == 120 + with tarfile.open(tmp_path / 'capture/case.tar.gz') as archive: + assert archive.extractfile('case/.hidden').read() == b'exact invented bytes\x00' + assert archive.getmember('case/.hidden').mode == 0o600 + assert archive.extractfile('case/actual-result.json').read() == (selected / 'actual-result.json').read_bytes() + assert result['archive'] == evidence.pin((tmp_path / 'capture/case.tar.gz').read_bytes()) + + +def test_early_partial_constructor_case_is_retained_without_inventing_a_result(tmp_path): + selected = tmp_path / evidence.CASES[NODE] + selected.mkdir() + (selected / 'authority').mkdir() + (selected / 'authority/first.json').write_text('{"incomplete":true}') + result = capture(tmp_path, tmp_path / 'partial') + assert result['status'] == 'retained' + assert 'actual-result.json' not in result['before'] + assert result['availability']['actual-result.json'] == 'absent' + assert result['availability']['native-attempt/transcript.jsonl'] == 'absent' + assert result['case_completeness'] == 'not_assessed' + assert result['test_outcome']['call']['outcome'] == 'failed' + assert result['before']['authority/first.json']['type'] == 'file' + + +@pytest.mark.parametrize('tmp_available', [False, True]) +def test_missing_case_is_explicit_and_never_an_empty_success_archive(tmp_path, tmp_available): + result = capture(tmp_path if tmp_available else None, tmp_path / 'missing') + assert result['status'] == 'case_unavailable' + assert result['before'] is result['after'] is result['archive'] is None + assert result['unchanged'] is False + assert not (tmp_path / 'missing/case.tar.gz').exists() + + +def test_output_collision_preserves_original_capture(tmp_path): + case(tmp_path) + destination = tmp_path / 'capture' + capture(tmp_path, destination) + before = {p.name: p.read_bytes() for p in destination.iterdir()} + with pytest.raises(FileExistsError): + capture(tmp_path, destination) + assert {p.name: p.read_bytes() for p in destination.iterdir()} == before + + +def test_external_symlinks_and_json_authority_paths_are_never_followed(tmp_path): + selected = case(tmp_path) + outside = tmp_path / 'private-outside' + outside.mkdir() + secret = outside / 'unrelated.txt' + secret.write_bytes(b'DO NOT CAPTURE THIS EXTERNAL CONTENT') + (selected / 'file-link').symlink_to(secret) + (selected / 'dir-link').symlink_to(outside, target_is_directory=True) + (selected / 'authority.json').write_text(json.dumps({'path': str(secret)})) + result = capture(tmp_path, tmp_path / 'capture') + assert result['status'] == 'retained' + link_mode = stat.S_IMODE((selected / 'file-link').lstat().st_mode) + assert result['before']['file-link'] == {'mode': link_mode, 'type': 'symlink', 'target': str(secret)} + assert result['before']['dir-link']['type'] == 'symlink' + with tarfile.open(tmp_path / 'capture/case.tar.gz') as archive: + assert archive.getmember('case/file-link').issym() + assert archive.getmember('case/file-link').mode == link_mode + assert archive.getmember('case/dir-link').issym() + assert not any('unrelated.txt' in name for name in archive.getnames()) + assert all(secret.read_bytes() not in archive.extractfile(member).read() + for member in archive.getmembers() if member.isfile()) + + +@pytest.mark.parametrize('which', ['case', 'parent']) +def test_symlink_case_or_ancestor_refuses_without_reading_target(tmp_path, which): + actual = tmp_path / 'actual' + actual.mkdir() + selected = case(actual) + if which == 'case': + (tmp_path / evidence.CASES[NODE]).symlink_to(selected, target_is_directory=True) + root = tmp_path + else: + root = tmp_path / 'alias' + root.symlink_to(actual, target_is_directory=True) + result = capture(root, tmp_path / 'capture') + assert result['status'] == 'capture_failed' + assert result['before'] is None and result['archive'] is None + assert not (tmp_path / 'capture/case.tar.gz').exists() + + +def test_mode_or_bytes_changed_during_archive_cannot_claim_preservation(tmp_path, monkeypatch): + selected = case(tmp_path) + original = evidence.snapshot + def changed(path, archive=None): + rows = original(path, archive) + if archive is not None: + (selected / '.hidden').write_bytes(b'changed after captured bytes') + (selected / '.hidden').chmod(0o644) + return rows + monkeypatch.setattr(evidence, 'snapshot', changed) + result = capture(tmp_path, tmp_path / 'capture') + assert result['status'] == 'capture_changed' and result['unchanged'] is False + assert result['before'] == result['archived'] + assert result['before'] != result['after'] + assert result['archive'] is not None + + +@pytest.mark.parametrize('limit', ['MAX_CASE_BYTES', 'MAX_CASE_ENTRIES']) +def test_capture_bounds_are_explicit_failure_and_preserve_case(tmp_path, monkeypatch, limit): + selected = case(tmp_path) + before = {p.name: p.read_bytes() for p in selected.iterdir()} + monkeypatch.setattr(evidence, limit, 1) + result = capture(tmp_path, tmp_path / 'capture') + assert result['status'] == 'capture_failed' + assert 'bound exceeded' in result['error']['message'] + assert {p.name: p.read_bytes() for p in selected.iterdir()} == before + + +def test_unapproved_node_or_overlapping_output_never_writes(tmp_path): + selected = case(tmp_path) + with pytest.raises(ValueError, match='two named'): + evidence.capture_case(tmp_path, tmp_path / 'other', node_id='tests/other.py::test', + worker='gw0', source=SOURCE, outcome=FAILED) + assert not (tmp_path / 'other').exists() + with pytest.raises(ValueError, match='separate'): + capture(tmp_path, selected / 'new-output') + assert not (selected / 'new-output').exists() + + +def config(path, *, worker=None): + value = SimpleNamespace(getoption=lambda name: None if path is None else str(path), + pluginmanager=SimpleNamespace(getplugin=lambda name: None)) + if worker is not None: + value.workerinput = {'workerid': worker} + return value + + +def report(item, when, outcome='passed'): + value = SimpleNamespace(when=when, outcome=outcome, duration=0.25) + hook = evidence.pytest_runtest_makereport(item, None) + next(hook) + with pytest.raises(StopIteration): + hook.send(SimpleNamespace(get_result=lambda: value)) + assert value.when == when and value.outcome == outcome + return value + + +def test_plugin_default_off_reads_no_source_and_writes_nothing(tmp_path, monkeypatch): + monkeypatch.setattr(evidence, 'source_identity', lambda: pytest.fail('unrequested identity read')) + selected = config(None) + evidence.pytest_configure(selected) + item = SimpleNamespace(config=selected, nodeid=NODE, funcargs={'tmp_path': tmp_path}) + report(item, 'call', 'failed') + evidence.pytest_sessionfinish(SimpleNamespace(config=selected), 1) + assert not hasattr(selected, '_native_ci_evidence') + assert list(tmp_path.iterdir()) == [] + + +def test_distinct_worker_node_paths_and_capture_only_after_teardown(tmp_path, monkeypatch): + monkeypatch.setattr(evidence, 'source_identity', lambda: deepcopy(SOURCE)) + destination = tmp_path / 'selected-evidence' + controller = config(destination) + evidence.pytest_configure(controller) + for worker in ('gw0', 'gw1'): + selected = config(destination, worker=worker) + evidence.pytest_configure(selected) + private = tmp_path / worker + private.mkdir() + case(private) + item = SimpleNamespace(config=selected, nodeid=NODE, funcargs={'tmp_path': private}) + report(item, 'setup') + report(item, 'call', 'failed') + assert list((destination / worker).iterdir()) == [] + # Real teardown may clear funcargs: retention keeps the prior owned path. + item.funcargs = None + report(item, 'teardown') + session = SimpleNamespace(config=selected, exitstatus=1) + evidence.pytest_sessionfinish(session, 1) + assert session.exitstatus == 1 + retained = list((destination / worker).glob('*/manifest.json')) + assert len(retained) == 1 + assert json.loads(retained[0].read_text())['test_outcome']['call']['outcome'] == 'failed' + assert json.loads((destination / worker / 'outcomes.json').read_text())['pytest_exitstatus'] == 1 + assert (destination / 'gw0').resolve() != (destination / 'gw1').resolve() + + +def test_capture_exception_cannot_replace_failed_test_or_pytest_exit(tmp_path, monkeypatch): + monkeypatch.setattr(evidence, 'source_identity', lambda: deepcopy(SOURCE)) + selected = config(tmp_path / 'evidence') + evidence.pytest_configure(selected) + def refuse(*args, **kwargs): + raise OSError('invented diagnostic write failure') + monkeypatch.setattr(evidence, 'capture_case', refuse) + item = SimpleNamespace(config=selected, nodeid=NODE, funcargs={'tmp_path': tmp_path}) + report(item, 'call', 'failed') + report(item, 'teardown') + session = SimpleNamespace(config=selected, exitstatus=1) + evidence.pytest_sessionfinish(session, 1) + assert session.exitstatus == 1 + saved = json.loads((tmp_path / 'evidence/controller/outcomes.json').read_text()) + assert saved['pytest_exitstatus'] == 1 + assert saved['events'][0]['capture_status'] == 'capture_failed' + assert saved['events'][0]['test_outcome']['call']['outcome'] == 'failed' + + +def test_unrun_nodes_are_explicit_and_setup_failure_has_no_invented_case(tmp_path, monkeypatch): + monkeypatch.setattr(evidence, 'source_identity', lambda: deepcopy(SOURCE)) + selected = config(tmp_path / 'evidence') + evidence.pytest_configure(selected) + item = SimpleNamespace(config=selected, nodeid=NODE, funcargs={}) + report(item, 'setup', 'failed') + report(item, 'teardown') + evidence.pytest_sessionfinish(SimpleNamespace(config=selected), 1) + saved = json.loads((tmp_path / 'evidence/controller/outcomes.json').read_text()) + assert saved['unobserved_nodes'] == [node for node in evidence.CASES if node != NODE] + assert saved['events'][0]['capture_status'] == 'case_unavailable' + assert saved['events'][0]['test_outcome']['setup']['outcome'] == 'failed' + assert 'call' not in saved['events'][0]['test_outcome'] diff --git a/tests/test_native_ci_evidence_adversarial.py b/tests/test_native_ci_evidence_adversarial.py new file mode 100644 index 000000000..78c23c417 --- /dev/null +++ b/tests/test_native_ci_evidence_adversarial.py @@ -0,0 +1,157 @@ +"""Independent archive and actual pytest-lifetime controls, with invented cases. + +The tiny child suite imports only the retention plugin and creates a few bytes. +It never imports or invokes native execution, a helper or a provider. +""" +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tarfile +from types import SimpleNamespace + +import pytest + +from utils import native_ci_evidence as evidence + + +ROOT = Path(__file__).resolve().parents[1] +NODE = next(iter(evidence.CASES)) +SOURCE = {'head': 'a' * 40, 'tree': 'b' * 40, + 'scope': 'Invented retention-only test source, not native run provenance.'} + + +@pytest.mark.parametrize('empty', [False, True]) +def test_archive_contains_exact_selected_root_mode_even_when_empty(tmp_path, empty): + case = tmp_path / evidence.CASES[NODE] + case.mkdir(mode=0o710) + case.chmod(0o710) + if not empty: + (case / 'partial.txt').write_bytes(b'invented early construction bytes') + output = tmp_path / 'capture' + result = evidence.capture_case(tmp_path, output, node_id=NODE, worker='gw0', + source=SOURCE, outcome={'call': {'outcome': 'failed', 'duration_seconds': 0.1}}) + assert result['unchanged'] is True + assert result['before']['.'] == {'type': 'directory', 'mode': 0o710} + with tarfile.open(output / 'case.tar.gz') as archive: + members = archive.getmembers() + assert len([member for member in members if member.name.rstrip('/') == 'case']) == 1 + root = archive.getmember('case') + assert root.isdir() and root.mode == 0o710 + assert len(members) == (1 if empty else 2) + inventory = {} + for member in members: + name = '.' if member.name.rstrip('/') == 'case' else member.name.removeprefix('case/') + if member.isdir(): + inventory[name] = {'type': 'directory', 'mode': member.mode} + else: + assert member.isfile() + raw = archive.extractfile(member).read() + inventory[name] = {'type': 'file', 'mode': member.mode, + 'bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()} + assert inventory == result['archived'] == result['before'] == result['after'] + + +def test_leaf_replaced_by_external_link_is_refused_before_target_read(tmp_path, monkeypatch): + case = tmp_path / evidence.CASES[NODE] + case.mkdir() + leaf = case / 'selected.bin' + leaf.write_bytes(b'original selected bytes') + outside = tmp_path / 'outside.bin' + outside.write_bytes(b'never archive or read this target') + original_open = evidence.os.open + replaced = False + + def swap_before_open(path, flags, *args, **kwargs): + nonlocal replaced + if path == leaf.name and not replaced: + replaced = True + leaf.unlink() + leaf.symlink_to(outside) + return original_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(evidence.os, 'open', swap_before_open) + output = tmp_path / 'capture' + result = evidence.capture_case(tmp_path, output, node_id=NODE, worker='gw0', + source=SOURCE, outcome={'call': {'outcome': 'failed', 'duration_seconds': 0.1}}) + assert replaced + assert result['status'] == 'capture_failed' + assert result['before'] is None and result['archive'] is None + assert result['unchanged'] is False + assert not (output / 'case.tar.gz').exists() + assert outside.read_bytes() == b'never archive or read this target' + + +def test_other_node_never_captures_even_with_matching_case_directory(tmp_path, monkeypatch): + def forbidden(*args, **kwargs): + pytest.fail('unselected test caused evidence collection') + + monkeypatch.setattr(evidence, 'capture_case', forbidden) + selected = SimpleNamespace(_native_ci_evidence={ + 'worker': 'gw0', 'worker_root': tmp_path, 'events': [], 'source': SOURCE}) + item = SimpleNamespace(config=selected, nodeid='tests/other.py::test_unselected', + funcargs={'tmp_path': tmp_path}) + hook = evidence.pytest_runtest_makereport(item, None) + next(hook) + report = SimpleNamespace(when='teardown', outcome='passed', duration=0.1) + with pytest.raises(StopIteration): + hook.send(SimpleNamespace(get_result=lambda: report)) + assert selected._native_ci_evidence['events'] == [] + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.parametrize('assertion_fails,capture_fails', [(True, False), (False, True)]) +def test_actual_pytest_teardown_and_exit_survive_retention(tmp_path, assertion_fails, capture_fails): + project = tmp_path / 'tiny-suite' + (project / 'tests').mkdir(parents=True) + shim = project / 'invented_retention_plugin.py' + shim.write_text( + 'from utils.native_ci_evidence import *\n' + 'from utils import native_ci_evidence as selected\n' + f'selected.source_identity = lambda: {SOURCE!r}\n' + + ('def fail_capture(*a, **k):\n' + ' raise OSError("invented retention failure")\n' + 'selected.capture_case = fail_capture\n' if capture_fails else ''), encoding='utf-8') + function = NODE.split('::')[1] + (project / 'tests/test_native_shared_execution.py').write_text( + f'def {function}(tmp_path, request):\n' + f' case = tmp_path / {evidence.CASES[NODE]!r}\n' + ' case.mkdir()\n' + ' (case / "call.txt").write_text("invented call bytes")\n' + ' request.addfinalizer(lambda: (case / "teardown.txt").write_text("finalizer completed"))\n' + + (' assert False, "original assertion retained"\n' if assertion_fails else + ' assert (case / "call.txt").is_file()\n'), encoding='utf-8') + destination = tmp_path / 'evidence' + environment = dict(os.environ, PYTHONDONTWRITEBYTECODE='1', + PYTEST_DISABLE_PLUGIN_AUTOLOAD='1', PYTHONPATH=os.pathsep.join((str(project), str(ROOT)))) + environment.pop('PYTEST_ADDOPTS', None) + completed = subprocess.run([ + sys.executable, '-B', '-m', 'pytest', '-q', '-p', 'no:cacheprovider', + '-p', 'invented_retention_plugin', '--basetemp', str(tmp_path / 'pytest-temp'), + '--native-ci-evidence-root', str(destination), NODE, + ], cwd=project, env=environment, capture_output=True, text=True, timeout=30) + assert completed.returncode == int(assertion_fails), completed.stdout + completed.stderr + outcomes = json.loads((destination / 'controller/outcomes.json').read_text()) + assert outcomes['pytest_exitstatus'] == int(assertion_fails) + assert len(outcomes['events']) == 1 + event = outcomes['events'][0] + assert event['node_id'] == NODE + assert event['test_outcome']['setup']['outcome'] == 'passed' + assert event['test_outcome']['call']['outcome'] == ('failed' if assertion_fails else 'passed') + assert event['test_outcome']['teardown']['outcome'] == 'passed' + if capture_fails: + assert event['capture_status'] == 'capture_failed' + assert event['error']['message'] == 'invented retention failure' + assert not (destination / 'controller' / event['directory']).exists() + else: + assert 'original assertion retained' in completed.stdout + manifest_path = destination / 'controller' / event['directory'] / 'manifest.json' + manifest = json.loads(manifest_path.read_text()) + assert manifest['source'] == SOURCE + assert manifest['status'] == 'retained' and manifest['case_completeness'] == 'not_assessed' + assert manifest['before']['teardown.txt']['type'] == 'file' + with tarfile.open(manifest_path.parent / 'case.tar.gz') as archive: + assert archive.extractfile('case/teardown.txt').read() == b'finalizer completed' + assert archive.extractfile('case/call.txt').read() == b'invented call bytes' diff --git a/tests/test_native_shared_execution_variant.py b/tests/test_native_shared_execution_variant.py index b9f98c24a..ee0f6d136 100644 --- a/tests/test_native_shared_execution_variant.py +++ b/tests/test_native_shared_execution_variant.py @@ -21,7 +21,7 @@ def _report_only_correction(case): transcript = Path(case['value']['attempt_directory']) / 'transcript.jsonl' calls, results = {}, {} for line, raw in enumerate(transcript.read_bytes().splitlines()): - event = c.strict_json(raw) + event = c.strict_json(raw, max_bytes=16 * 1024 * 1024) for block in event.get('message', {}).get('content', []): if block['type'] == 'tool_use': assert block['id'] not in calls diff --git a/tests/test_native_shared_stream_diagnostics.py b/tests/test_native_shared_stream_diagnostics.py new file mode 100644 index 000000000..3aea73f02 --- /dev/null +++ b/tests/test_native_shared_stream_diagnostics.py @@ -0,0 +1,159 @@ +"""Reader-only diagnostic schedules, not a reproduction of the hosted race.""" +from copy import deepcopy +import os + +import pytest + +from data_sheets_schema import native_shared_contract as c +from data_sheets_schema import native_shared_evidence as evidence +from data_sheets_schema import native_shared_streams as streams + + +@pytest.mark.parametrize('role', streams.ROLES) +@pytest.mark.parametrize('sample,stage,pass_number,at_end,reads', [ + (1, 'opened', None, False, 0), + (2, 'before_read', 1, False, 0), + (3, 'after_read', 1, True, 1), + (4, 'before_read', 2, False, 1), + (5, 'after_read', 2, True, 2), + (6, 'final_fd', None, True, 2), + (7, 'final_path', None, True, 2), +]) +def test_exact_failed_sample_retained_without_extra_stream_reads( + tmp_path, monkeypatch, role, sample, stage, pass_number, at_end, reads): + path = tmp_path / (role + '.jsonl') + raw = b'{"first":1}\n' + path.write_bytes(raw) + old = c.EvidencePrefix(role, str(path), raw, len(raw), c.sha(raw), 1) + original_stat, original_read = evidence._stat, os.pread + before = original_stat(path.stat()) + later = {**before, 'mtime_ns': before['mtime_ns'] + 10} + samples, read_calls = [], [] + + def sampled(info): + value = original_stat(info) + if len(samples) == sample: + value['mtime_ns'] += 10 + samples.append(deepcopy(value)) + return value + + def read(fd, count, offset): + read_calls.append((count, offset)) + return original_read(fd, count, offset) + + monkeypatch.setattr(evidence, '_stat', sampled) + monkeypatch.setattr(streams.os, 'pread', read) + with pytest.raises(streams._LiveStreamRefusal) as caught: + streams.read_live_stream(str(path), role, max_bytes=4096, previous=old) + exc = caught.value + assert isinstance(exc, ValueError) + assert exc.args == ('live native stream changed without appending bytes',) + assert type(exc.__cause__) is ValueError and exc.__cause__.args == exc.args + assert len(samples) == sample + 1 and len(read_calls) == reads + expected = { + 'kind': 'native_live_stream_refusal', 'version': 1, + 'role': role, 'path': str(path), 'stage': stage, 'pass_number': pass_number, + 'offset': len(raw) if at_end else 0, + 'opened_extent': None if stage == 'opened' else len(raw), + 'max_bytes': 4096, 'earlier': before, 'later': later, + 'previous_prefix': {'bytes': len(raw), 'sha256': c.sha(raw)}, + } + assert exc.diagnostic_json == c.canonical(expected) + assert len(exc.diagnostic_json) <= streams.MAX_DIAGNOSTIC_BYTES + assert path.read_bytes() == raw + detached = streams.diagnostic_document(exc.diagnostic_json) + detached['earlier']['size'] = 999 + assert streams.diagnostic_document(exc.diagnostic_json) == expected + with pytest.raises(AttributeError): + exc.diagnostic_json = b'{}' + + +@pytest.mark.parametrize('pass_number,failed_sample', [(1, 4), (2, 8)]) +def test_second_chunk_reports_exact_offset_and_fixed_extent(tmp_path, monkeypatch, pass_number, failed_sample): + path = tmp_path / 'transcript.jsonl' + raw = b'x' * 70000 + path.write_bytes(raw) + original = evidence._stat + count = 0 + + def sampled(info): + nonlocal count + value = original(info) + if count == failed_sample: + value['mtime_ns'] += 1 + count += 1 + return value + + monkeypatch.setattr(evidence, '_stat', sampled) + with pytest.raises(streams._LiveStreamRefusal) as caught: + streams.read_live_stream(str(path), 'transcript', max_bytes=100000) + data = streams.diagnostic_document(caught.value.diagnostic_json) + assert data['stage'] == 'before_read' and data['pass_number'] == pass_number + assert data['offset'] == 65536 and data['opened_extent'] == len(raw) + assert data['previous_prefix'] is None and count == failed_sample + 1 + + +def test_injected_split_field_ambiguity_is_recorded_not_diagnosed(tmp_path, monkeypatch): + """A synthetic metadata schedule cannot establish the hosted failure cause.""" + path = tmp_path / 'control.jsonl' + path.write_bytes(b'{}\n') + real = evidence._stat + before = real(path.stat()) + # The first fstat presents a larger extent but the original timestamp; + # the next presents that same extent and another timestamp. No retry is + # allowed and no byte read occurs. This models observations, not stat(2). + first = {**before, 'size': before['size'] + 4} + second = {**first, 'mtime_ns': before['mtime_ns'] + 1} + schedule = iter([before, first, second]) + monkeypatch.setattr(evidence, '_stat', lambda info: next(schedule)) + monkeypatch.setattr(streams.os, 'pread', lambda *args: pytest.fail('refusal must precede first read')) + with pytest.raises(streams._LiveStreamRefusal) as caught: + streams.read_live_stream(str(path), 'control', max_bytes=4096) + data = streams.diagnostic_document(caught.value.diagnostic_json) + assert data['earlier'] == first and data['later'] == second + assert data['opened_extent'] == first['size'] + assert data['stage'] == 'before_read' and data['pass_number'] == 1 and data['offset'] == 0 + assert 'cause' not in data and 'diagnosis' not in data + + +@pytest.mark.parametrize('change,message', [ + ({'inode': 999999}, 'live native stream physical identity changed'), + ({'device': 999999}, 'live native stream physical identity changed'), + ({'links': 2}, 'captured member requires exact regular single-link metadata'), + ({'regular': False}, 'captured member requires exact regular single-link metadata'), + ({'symlink': True}, 'captured member requires exact regular single-link metadata'), + ({'size': 0}, 'live native stream shrank or exceeded its byte bound'), + ({'size': 4097}, 'live native stream shrank or exceeded its byte bound'), +]) +def test_other_sample_guards_retain_original_refusal_and_exact_metadata(tmp_path, monkeypatch, change, message): + path = tmp_path / 'transcript.jsonl' + path.write_bytes(b'{}\n') + real = evidence._stat + before = real(path.stat()) + later = {**before, **change} + schedule = iter([before, later]) + monkeypatch.setattr(evidence, '_stat', lambda info: next(schedule)) + monkeypatch.setattr(streams.os, 'pread', lambda *args: pytest.fail('guard must precede read')) + with pytest.raises(streams._LiveStreamRefusal) as caught: + streams.read_live_stream(str(path), 'transcript', max_bytes=4096) + assert caught.value.args == (message,) + data = streams.diagnostic_document(caught.value.diagnostic_json) + assert data['earlier'] == before and data['later'] == later and data['stage'] == 'opened' + + +def test_no_diagnostic_compatibility_progress_call_keeps_exact_exception_type(tmp_path): + path = tmp_path / 'sample' + path.write_bytes(b'x') + before = evidence._stat(path.stat()) + with pytest.raises(ValueError) as caught: + streams._progress(before, {**before, 'mtime_ns': before['mtime_ns'] + 1}, 4096) + assert type(caught.value) is ValueError + + +def test_reader_guard_before_sample_has_no_invented_diagnostic(tmp_path): + path = tmp_path / 'transcript.jsonl' + path.write_bytes(b'{}\n') + with pytest.raises(ValueError, match='exceeds its byte bound') as caught: + streams.read_live_stream(str(path), 'transcript', max_bytes=1) + assert type(caught.value) is ValueError + assert not hasattr(caught.value, 'diagnostic_json') diff --git a/tests/test_native_shared_stream_diagnostics_adversarial.py b/tests/test_native_shared_stream_diagnostics_adversarial.py new file mode 100644 index 000000000..c6f3534a1 --- /dev/null +++ b/tests/test_native_shared_stream_diagnostics_adversarial.py @@ -0,0 +1,237 @@ +"""Independent diagnostic-only refusal controls; no native runtime or helper. + +Real reader/adapter methods run against tiny local streams. Only diagnostic +failure injection and the already-authorized lifetime/state plumbing are stubs. +""" +from copy import deepcopy +import json +from types import SimpleNamespace + +import pytest + +from data_sheets_schema import native_shared_contract as c +from data_sheets_schema import native_shared_evidence as evidence +from data_sheets_schema import native_shared_streams as streams +from data_sheets_schema.native_shared_controller import CallbackAdapter +from data_sheets_schema.native_shared_receipts import _ReceiptCatalogContext + + +TEXT = 'live native stream changed without appending bytes' +STOP = 'native shared observation failed: ValueError: ' + TEXT + + +class Stopped(Exception): + pass + + +def metadata(): + return dict(exists=True, regular=True, symlink=False, links=1, + device=1, inode=2, size=16, mtime_ns=3) + + +def context(path='/synthetic/transcript.jsonl'): + return dict(role='transcript', path=path, stage='before_read', + pass_number=2, offset=0, opened_extent=16, + previous_prefix={'bytes': 4, 'sha256': c.sha(b'old\n')}) + + +def refusal(*, selected=None): + earlier = metadata() + with pytest.raises(streams._LiveStreamRefusal) as caught: + streams._progress(earlier, {**earlier, 'mtime_ns': 4}, 4096, + diagnostic=context() if selected is None else selected) + return caught.value + + +def adapter(tmp_path): + """Exercise actual observation/report methods without launch admission.""" + selected = CallbackAdapter.__new__(CallbackAdapter) + selected.raw = b'invented test-only composition' + selected.execution = {'attempt_directory': str(tmp_path)} + selected.failure = selected.final_evidence = selected.recorder_identity = None + selected._stream_diagnostic_json = None + selected.recorder_done = False + selected._activated = selected._initialized = True + selected._stream_prefixes, selected._stream_files = {}, {} + selected._receipt_catalogs = _ReceiptCatalogContext() + selected.controls = {'budgeted_cborg': SimpleNamespace(BudgetStop=Stopped)} + selected.state = SimpleNamespace(report=lambda **kw: {'draft_gate_passed': True, 'problems': []}) + selected.phase = SimpleNamespace(report=lambda **kw: {'synthetic_phase': 'unchanged'}) + return selected + + +def seed(tmp_path): + first, second = {'first': 1}, {'second': 2} + raw = c.canonical(first) + b'\n' + transcript = tmp_path / 'transcript.jsonl' + control = tmp_path / 'control.jsonl' + transcript.write_bytes(raw) + control.write_bytes(b'{"control":1}\n') + selected = adapter(tmp_path) + selected._observed_endpoint(first) + old = deepcopy(selected._stream_prefixes), deepcopy(selected._stream_files) + with transcript.open('ab') as out: + out.write(c.canonical(second) + b'\n') + return selected, old, second, transcript, control + + +def test_exception_owns_complete_metadata_bytes_and_decodes_detached(): + earlier, later, detail = metadata(), metadata(), context() + later['mtime_ns'] += 1 + expected = {'kind': 'native_live_stream_refusal', 'version': 1, + **deepcopy(detail), 'max_bytes': 4096, + 'earlier': deepcopy(earlier), 'later': deepcopy(later)} + with pytest.raises(streams._LiveStreamRefusal) as caught: + streams._progress(earlier, later, 4096, diagnostic=detail) + error = caught.value + assert isinstance(error, ValueError) and error.args == (TEXT,) + assert type(error.__cause__) is ValueError and error.__cause__.args == error.args + raw = error.diagnostic_json + assert type(raw) is bytes and raw == c.canonical(expected) + earlier['size'] = 900 + later['mtime_ns'] = -1 + detail['previous_prefix']['sha256'] = 'mutated' + detached = streams.diagnostic_document(raw) + detached['earlier']['size'] = 0 + detached['previous_prefix']['bytes'] = 0 + assert streams.diagnostic_document(error.diagnostic_json) == expected + with pytest.raises(AttributeError): + error.diagnostic_json = b'{}' + + +@pytest.mark.parametrize('change', [dict(mtime_ns=4), dict(inode=7), dict(size=0), dict(links=2)]) +def test_existing_progress_reason_and_arguments_are_unchanged(change, monkeypatch): + before = metadata(); after = {**before, **change} + def forbidden(*args, **kwargs): + pytest.fail('diagnostic attempted additional filesystem acquisition') + with monkeypatch.context() as patch: + for name in ('open', 'stat', 'lstat', 'fstat', 'pread', 'read', 'write'): + patch.setattr(streams.os, name, forbidden) + with pytest.raises(ValueError) as original: + streams._progress(before, after, 4096) + with pytest.raises(streams._LiveStreamRefusal) as enriched: + streams._progress(before, after, 4096, diagnostic=context()) + assert type(original.value) is ValueError + assert enriched.value.args == original.value.args + assert str(enriched.value) == str(original.value) + saved = streams.diagnostic_document(enriched.value.diagnostic_json) + assert saved['earlier'] == before and saved['later'] == after + + +@pytest.mark.parametrize('failure', ['encoding', 'decoding', 'byte_bound']) +def test_diagnostic_failure_preserves_plain_original_valueerror(monkeypatch, failure): + before = metadata() + def broken(*args, **kwargs): + raise RuntimeError('synthetic diagnostic failure') + if failure == 'encoding': + monkeypatch.setattr(streams.c, 'canonical', broken) + elif failure == 'decoding': + monkeypatch.setattr(streams, 'diagnostic_document', broken) + else: + monkeypatch.setattr(streams, 'MAX_DIAGNOSTIC_BYTES', 1) + with pytest.raises(ValueError) as caught: + streams._progress(before, {**before, 'mtime_ns': 4}, 4096, diagnostic=context()) + assert type(caught.value) is ValueError and caught.value.args == (TEXT,) + assert not hasattr(caught.value, 'diagnostic_json') + + +@pytest.mark.parametrize('site', ['endpoint_control', 'recapture']) +def test_first_stream_failure_keeps_existing_cursor_timing_and_stop(tmp_path, monkeypatch, site): + selected, old, event, transcript, control = seed(tmp_path) + source_bytes = transcript.read_bytes(), control.read_bytes() + progress = streams._progress + if site == 'endpoint_control': + def changed(before, after, maximum, *, diagnostic=None): + if diagnostic and diagnostic['role'] == 'control' and diagnostic['stage'] == 'opened': + after = {**after, 'size': before['size'], 'mtime_ns': before['mtime_ns'] + 1} + return progress(before, after, maximum, diagnostic=diagnostic) + monkeypatch.setattr(streams, '_progress', changed) + monkeypatch.setattr(selected, '_run', lambda *a: pytest.fail('failed endpoint reached recapture')) + else: + error = refusal(selected=context(str(transcript))) + def recapture(endpoints=None): + assert endpoints == (len(source_bytes[0]), len(source_bytes[1])) + raise error + monkeypatch.setattr(selected, '_run', recapture) + with pytest.raises(Stopped, match=TEXT): + selected.observe(event) + assert selected.failure == STOP + if site == 'endpoint_control': + assert (selected._stream_prefixes, selected._stream_files) == old + else: + assert selected._stream_prefixes['transcript'].raw == source_bytes[0] + assert selected._stream_prefixes['transcript'] != old[0]['transcript'] + assert selected._stream_files == old[1] + first = selected.report(complete=True) + assert first['controller_stop'] == STOP and first['draft_gate_passed'] is False + assert first['phase_history'] == {'synthetic_phase': 'unchanged'} + assert first['stream_diagnostic']['role'] == ('control' if site == 'endpoint_control' else 'transcript') + raw = c.canonical(first['stream_diagnostic']) + first['stream_diagnostic']['earlier']['size'] = 0 + first['stream_diagnostic']['path'] = 'caller mutation' + monkeypatch.setattr(selected, '_observed_endpoint', lambda *a: pytest.fail('stopped adapter reacquired streams')) + monkeypatch.setattr(selected, '_run', lambda *a: pytest.fail('failed report recovered live history')) + with pytest.raises(Stopped) as later: + selected.observe({'third': 3}) + assert str(later.value) == STOP + report = selected.report(complete=True) + assert c.canonical(report['stream_diagnostic']) == raw + assert json.loads(c.canonical(report))['controller_stop'] == STOP + assert (transcript.read_bytes(), control.read_bytes()) == source_bytes + + +def test_unrelated_first_stop_cannot_later_acquire_stream_diagnostic(tmp_path, monkeypatch): + selected = adapter(tmp_path) + def unrelated(*args): + raise ValueError('ordinary existing refusal') + monkeypatch.setattr(selected, '_observed_endpoint', unrelated) + with pytest.raises(Stopped): + selected.observe({}) + original = selected.failure + selected._failure_detail(refusal()) + monkeypatch.setattr(selected, '_run', lambda *a: pytest.fail('prior failure entered complete replay')) + result = selected.report(complete=True) + assert result['controller_stop'] == original + assert 'stream_diagnostic' not in result + assert selected._stream_diagnostic_json is None + + +@pytest.mark.parametrize('site', ['latching', 'report']) +def test_adapter_decode_failure_cannot_replace_or_hide_stop(tmp_path, monkeypatch, site): + selected = adapter(tmp_path); error = refusal() + def failed(*args): + raise error + monkeypatch.setattr(selected, '_observed_endpoint', failed) + def bad_decode(*args): + raise ValueError('diagnostic decode deliberately unavailable') + if site == 'latching': + monkeypatch.setattr(streams, 'diagnostic_document', bad_decode) + with pytest.raises(Stopped) as caught: + selected.observe({}) + assert str(caught.value) == STOP + if site == 'report': + assert selected._stream_diagnostic_json == error.diagnostic_json + monkeypatch.setattr(streams, 'diagnostic_document', bad_decode) + report = selected.report(complete=True) + assert report['controller_stop'] == STOP and report['draft_gate_passed'] is False + assert 'stream_diagnostic' not in report + + +def test_complete_report_first_failure_preserves_valueerror_qualified_text(tmp_path, monkeypatch): + selected = adapter(tmp_path); error = refusal() + def fail(*args): + raise error + monkeypatch.setattr(selected, '_run', fail) + report = selected.report(complete=True) + expected = 'completed native shared history cannot be verified: ValueError: ' + TEXT + assert report['controller_stop'] == expected and selected.failure == expected + assert report['stream_diagnostic'] == streams.diagnostic_document(error.diagnostic_json) + monkeypatch.setattr(selected, '_run', lambda *a: pytest.fail('failed completion was retried')) + assert selected.report(complete=True) == report + + +def test_successful_ordinary_report_shape_has_no_invented_diagnostic(tmp_path): + selected = adapter(tmp_path) + report = selected.report() + assert report['controller_stop'] is None and report['draft_gate_passed'] is True + assert 'stream_diagnostic' not in report diff --git a/tests/test_native_shared_streams.py b/tests/test_native_shared_streams.py index aa01bb1d3..bb8c96d8c 100644 --- a/tests/test_native_shared_streams.py +++ b/tests/test_native_shared_streams.py @@ -254,3 +254,51 @@ def live(*args, **kwargs): monkeypatch.setattr(capture, '_load_live', live) with pytest.raises(ValueError, match='parent-observed'): adapter._run((adapter._stream_prefixes['transcript'].bytes, adapter._stream_prefixes['control'].bytes)) + + +def test_inflight_append_mtime_before_size_write_settles_without_false_refusal(tmp_path, monkeypatch): + path, _ = files(tmp_path) + initial = path.read_bytes() + later = b'{"next":2}\n' + old = prefix(path, initial) + actual_open, actual_stat, actual_read = os.open, os.fstat, os.pread + target_fd = None + first_read_done = False + transient_samples = 0 + + def opening(name, flags, *args, **kwargs): + nonlocal target_fd + fd = actual_open(name, flags, *args, **kwargs) + if name == str(path): + target_fd = fd + return fd + + def reading(fd, count, offset): + nonlocal first_read_done + raw = actual_read(fd, count, offset) + if fd == target_fd and not first_read_done: + first_read_done = True + return raw + + def statting(fd): + nonlocal transient_samples + info = actual_stat(fd) + if fd == target_fd and first_read_done and transient_samples == 0: + transient_samples += 1 + append(path, later) + updated = actual_stat(fd) + return os.stat_result(( + info.st_mode, info.st_ino, info.st_dev, info.st_nlink, + info.st_uid, info.st_gid, info.st_size, + info.st_atime, updated.st_mtime, info.st_ctime, + )) + return info + + monkeypatch.setattr(streams.os, 'open', opening) + monkeypatch.setattr(streams.os, 'pread', reading) + monkeypatch.setattr(streams.os, 'fstat', statting) + member = streams.read_live_stream(str(path), 'transcript', max_bytes=4096, previous=old) + assert transient_samples == 1 + assert member.captured.raw == initial + assert path.read_bytes() == initial + later + diff --git a/tests/test_native_trace_single_decode.py b/tests/test_native_trace_single_decode.py new file mode 100644 index 000000000..9da488256 --- /dev/null +++ b/tests/test_native_trace_single_decode.py @@ -0,0 +1,217 @@ +"""#4537: complete constructor parity with the exact literal 0125 source. + +Reversing only the reviewed factoring reconstructs the complete original module. +Its fixed SHA256 is checked before compilation, so the oracle neither requires +Git/history at test time nor silently follows an unrelated production change. +The oracle uses the canonical module name; no type-name normalization is needed. +""" +from copy import deepcopy +from dataclasses import fields, is_dataclass, replace +import hashlib +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from data_sheets_schema import native_shared_observations as observed +from tests.test_native_shared_observations import POLICY, SESSION, events, initial, prefix + + +PARENT_SHA256 = "3bc93c2d1720d4e41ee83e7be7d8758a56836062c316f75ecc4b505e80b6d3ab" +CONSTRUCTOR_REUSE = ''' # Decode in initialization's original order, then reuse only this + # constructor's rows for indexing. No parsed state survives in a cache. + native=list(rows(transcript));parent=list(rows(control)) + initialized,*_=_initialization_from_rows(transcript,control,native,parent,policy=policy,runtime=runtime) +''' +HELPER_SPLIT = ''' return _initialization_from_rows(transcript,control,native,parent,policy=policy,runtime=runtime) + + +def _initialization_from_rows(transcript,control,native,parent,*,policy,runtime): + """Validate already decoded rows owned by this one caller.""" +''' + + +@pytest.fixture(scope="module") +def literal_parent(): + source = Path(observed.__file__).read_text(encoding="utf-8") + substitutions = ( + (CONSTRUCTOR_REUSE, " initialized,*_=initialization(transcript,control,policy=policy,runtime=runtime)\n"), + (" for number,raw,value in native:\n", " for number,raw,value in rows(transcript):\n"), + (" for number,raw,value in parent:\n", " for number,raw,value in rows(control):\n"), + (HELPER_SPLIT, ""), + ) + for candidate, original in substitutions: + assert source.count(candidate) == 1 + source = source.replace(candidate, original) + assert hashlib.sha256(source.encode()).hexdigest() == PARENT_SHA256 + namespace = {"__name__": observed.__name__} + exec(compile(source, "literal0125/native_shared_observations.py", "exec"), namespace) + return SimpleNamespace(**namespace) + + +def typed(value): + """Retain every field, scalar type, sequence type and mapping order.""" + identity = type(value).__module__ + "." + type(value).__qualname__ + if is_dataclass(value): + return (identity, tuple((field.name, typed(getattr(value, field.name))) for field in fields(value))) + if type(value) is dict: + return (identity, tuple((typed(key), typed(item)) for key, item in value.items())) + if type(value) in (list, tuple): + return (identity, tuple(map(typed, value))) + if value is None or type(value) in (str, bytes, int, bool, float): + return (identity, value) + raise AssertionError(f"uncovered semantic type {identity}") + + +def exception_graph(error, seen=None): + seen = {} if seen is None else seen + if error is None: + return None + if id(error) in seen: + return ("reference", seen[id(error)]) + number = len(seen) + seen[id(error)] = number + return (number, type(error).__module__, type(error).__qualname__, typed(error.args), + error.__suppress_context__, typed(getattr(error, "__notes__", None)), + exception_graph(error.__cause__, seen), exception_graph(error.__context__, seen)) + + +def outcome(call): + try: + return ("returned", typed(call())) + except Exception as error: + return ("refused", exception_graph(error)) + + +def constructor(module, transcript, control, *, runtime, session=SESSION, policy=POLICY): + trace = module.Trace.__new__(module.Trace) + try: + trace.__init__(transcript, control, session_id=session, policy=deepcopy(policy), runtime=deepcopy(runtime)) + except Exception as error: + return ("refused", exception_graph(error), typed(vars(trace))) + methods = [("pending", outcome(trace.pending))] + for identity in (*trace.calls, "missing"): + for method in ("request", "admitted", "settled"): + methods.append(((identity, method), outcome(lambda: getattr(trace, method)(identity)))) + return ("returned", typed(vars(trace)), tuple(methods)) + + +def inputs(tool="Read", raw=b'{"value":false,"missing":null,"list":[1,"x"]}\n'): + native, parent, _ = events(tool, raw) + initialized, sent, runtime = initial() + return initialized + native, sent + parent, runtime + + +@pytest.mark.parametrize("tool", ["Read", "Write"]) +@pytest.mark.parametrize("raw", [b"{}", b'{"long":"' + b"a" * 100000 + b'"}\n', + '{"text":"α\u2028β","integer":1,"float":1.0}\n'.encode()]) +def test_complete_trace_state_and_public_joins_match_literal_parent(literal_parent, tool, raw): + native, parent, runtime = inputs(tool, raw) + args = prefix("transcript", native), prefix("control", parent) + expected = constructor(literal_parent, *args, runtime=runtime) + assert expected[0] == "returned" + assert constructor(observed, *args, runtime=runtime) == expected + assert outcome(lambda: observed.initialization(*args, policy=POLICY, runtime=runtime)) == outcome( + lambda: literal_parent.initialization(*args, policy=POLICY, runtime=runtime)) + + +@pytest.mark.parametrize("change", [ + "missing_init", "duplicate_init", "late_init", "wrong_model", "wrong_runtime", + "wrong_policy", "wrong_session", "empty_session", "child_call", "wrong_role", + "duplicate_call", "duplicate_result", "duplicate_callback", "foreign_callback", + "callback_after_result", "different_decision", "contradictory_admission", + "terminal_then_event", "cancel", "bad_content", "bad_block", "bad_tool", + "malformed_callback", "untyped_callback", "wrong_streams", +]) +def test_complete_constructor_refusal_and_partial_state_match_literal_parent(literal_parent, change): + native, parent, runtime = inputs() + policy, session = deepcopy(POLICY), SESSION + if change == "missing_init": native = native[2:] + elif change == "duplicate_init": native.insert(1, deepcopy(native[1])) + elif change == "late_init": native = native[2:] + native[:2] + elif change == "wrong_model": native[1]["model"] = "foreign" + elif change == "wrong_runtime": runtime["model"] = "foreign" + elif change == "wrong_policy": policy["pretool_control"]["matcher"] = "Read" + elif change == "wrong_session": session = "foreign" + elif change == "empty_session": session = "" + elif change == "child_call": native[2]["parent_tool_use_id"] = "parent" + elif change == "wrong_role": native[2]["message"]["role"] = "user" + elif change == "duplicate_call": native.insert(3, deepcopy(native[2])) + elif change == "duplicate_result": native.append(deepcopy(native[-1])) + elif change == "duplicate_callback": native.insert(4, deepcopy(native[3])) + elif change == "foreign_callback": native[3]["request"]["input"]["tool_input"]["file_path"] = "/other" + elif change == "callback_after_result": native[3], native[4] = native[4], native[3] + elif change == "different_decision": parent[-1]["request"]["request"]["input"]["tool_name"] = "Write" + elif change == "contradictory_admission": parent[-1]["response"]["response"]["response"] = {"deny": True} + elif change == "terminal_then_event": native.insert(2, {"type": "result", "session_id": SESSION}) + elif change == "cancel": native.append({"type": "control_cancel_request"}) + elif change == "bad_content": native[2]["message"]["content"] = None + elif change == "bad_block": native[2]["message"]["content"] = [None] + elif change == "bad_tool": native[2]["message"]["content"][0]["name"] = "WebFetch" + elif change == "malformed_callback": native[3]["request"]["input"] = None + elif change == "untyped_callback": native[3]["request"] = None + args = prefix("transcript", native), prefix("control", parent) + if change == "wrong_streams": args = args[::-1] + expected = constructor(literal_parent, *args, runtime=runtime, session=session, policy=policy) + assert expected[0] == "refused" + assert constructor(observed, *args, runtime=runtime, session=session, policy=policy) == expected + + +def with_raw(value, raw): + return replace(value, raw=raw, bytes=len(raw), sha256=observed.c.sha(raw), lines=raw.count(b"\n")) + + +@pytest.mark.parametrize("native_raw,parent_raw", [ + (b"not JSON\n", b"also invalid\n"), (b"[]\n", b"not JSON\n"), + (b"\n", b"not JSON\n"), (b'{"a":1,"a":2}\n', b"not JSON\n"), + (None, b"not JSON\n"), (b"{}\n", b"not JSON\n"), +]) +def test_decode_and_initialization_error_precedence(literal_parent, native_raw, parent_raw): + native, parent, runtime = inputs() + transcript, control = prefix("transcript", native), prefix("control", parent) + if native_raw is not None: transcript = with_raw(transcript, native_raw) + control = with_raw(control, parent_raw) + expected = constructor(literal_parent, transcript, control, runtime=runtime, session="foreign") + assert expected[0] == "refused" + assert constructor(observed, transcript, control, runtime=runtime, session="foreign") == expected + assert outcome(lambda: observed.initialization(transcript, control, policy=POLICY, runtime=runtime)) == outcome( + lambda: literal_parent.initialization(transcript, control, policy=POLICY, runtime=runtime)) + + +def test_each_constructor_decodes_each_stream_once_in_the_original_order(monkeypatch, literal_parent): + native, parent, runtime = inputs() + args = prefix("transcript", native), prefix("control", parent) + calls, original = [], observed.rows + def counted(value): + calls.append(value.stream) + return original(value) + monkeypatch.setattr(observed, "rows", counted) + assert constructor(observed, *args, runtime=runtime) == constructor(literal_parent, *args, runtime=runtime) + assert calls == ["transcript", "control"] + calls.clear() + observed.initialization(*args, policy=POLICY, runtime=runtime) + assert calls == ["transcript", "control"] + + +def test_changed_capture_and_independent_mutable_trace_state(literal_parent): + native, parent, runtime = inputs() + args = prefix("transcript", native), prefix("control", parent) + first = observed.Trace(*args, session_id=SESSION, policy=deepcopy(POLICY), runtime=runtime) + first.frames[3]["message"]["content"][0]["input"]["file_path"] = "/mutated" + first.callbacks["tool-1"][1]["request"]["input"]["tool_input"].clear() + first.decisions["tool-1"][1]["response"].clear() + assert constructor(observed, *args, runtime=runtime) == constructor(literal_parent, *args, runtime=runtime) + native[2]["message"]["model"] = "changed" + changed = prefix("transcript", native), args[1] + expected = constructor(literal_parent, *changed, runtime=runtime) + assert expected[0] == "refused" + assert constructor(observed, *changed, runtime=runtime) == expected + + +@pytest.mark.parametrize("parent_decision", [False, True]) +def test_pending_call_join_results_match(literal_parent, parent_decision): + native, parent, runtime = inputs() + args = prefix("transcript", native[:-1]), prefix("control", parent if parent_decision else parent[:-1]) + expected = constructor(literal_parent, *args, runtime=runtime) + assert expected[0] == "returned" + assert constructor(observed, *args, runtime=runtime) == expected diff --git a/tests/test_shared_generation_batch_authority.py b/tests/test_shared_generation_batch_authority.py index 6f529369d..b4aa5fab4 100644 --- a/tests/test_shared_generation_batch_authority.py +++ b/tests/test_shared_generation_batch_authority.py @@ -11,6 +11,7 @@ from tests.test_generation_manifest_identity import external # noqa: F401 from tests.test_shared_generation_cli import (batch, module, registered, roster, stub_plan) # noqa: F401 +from tests.test_shared_generation_selection import registration_for @pytest.fixture(autouse=True) @@ -148,19 +149,39 @@ def test_roster_hardlink_to_later_output_is_protected(registered, monkeypatch): assert {p: p.read_bytes() for p in before} == before -def test_actual_schema_closure_hardlink_remains_protected(registered, monkeypatch): +def test_actual_schema_closure_hardlink_remains_protected(registered, monkeypatch, tmp_path): base, reg = registered + originals = [Path(pin['path']) for pin in reg['inputs']['full_schema']['sources']] + original_state = {p: (p.read_bytes(), p.stat().st_nlink) for p in originals} + # #4665: linking a repository schema changes its inode for every xdist + # worker, even if the alias is eventually removed. Copy before linking. + # The shipped full schema is flattened; fail before linking if that changes. + assert originals == [Path(reg['inputs']['full_schema']['root'])] + private_schema = tmp_path / 'private-schema' / originals[0].name + private_schema.parent.mkdir() + private_schema.write_bytes(original_state[originals[0]][0]) + private_schema = private_schema.resolve() + monkeypatch.setattr(api, 'FULL_SCHEMA_PATH', str(private_schema)) + reg = registration_for(base) + assert [Path(pin['path']) for pin in reg['inputs']['full_schema']['sources']] == [private_schema] path, rows = roster(base, reg) schema = Path(reg['inputs']['full_schema']['sources'][-1]['path']) target = specs_for(rows)[1].report_path target.parent.mkdir(parents=True) + assert schema.resolve().is_relative_to(tmp_path.resolve()) + assert target.resolve().is_relative_to(tmp_path.resolve()) + assert not schema.samefile(originals[0]) and schema.stat().st_nlink == 1 os.link(schema, target) + assert schema.samefile(target) and schema.stat().st_nlink == 2 + assert {p: (p.read_bytes(), p.stat().st_nlink) for p in originals} == original_state before = authority_bytes(path, rows) seen = dispatch_traps(monkeypatch) result = batch(reg, path) assert result.exit_code == 1 and 'aliases a run output' in result.output, result.output assert seen == {'plan': [], 'lock': [], 'execute': []} assert {p: p.read_bytes() for p in before} == before + assert schema.samefile(target) and schema.stat().st_nlink == 2 + assert {p: (p.read_bytes(), p.stat().st_nlink) for p in originals} == original_state def test_current_recheck_rejects_new_alias_without_recapturing(registered): diff --git a/utils/native_ci_evidence.py b/utils/native_ci_evidence.py new file mode 100644 index 000000000..eb1286881 --- /dev/null +++ b/utils/native_ci_evidence.py @@ -0,0 +1,257 @@ +"""Opt-in, post-teardown retention of two explicitly synthetic native cases. + +This plugin does not select tests, run a native/provider operation, change their +limits or alter pytest outcomes. It never follows a path declared inside a case. +""" +from contextlib import contextmanager +import hashlib +import io +import json +import os +from pathlib import Path +import re +import stat +import tarfile + +import pytest + +from utils.native_shared_acceptance import git + + +ROOT = Path(__file__).resolve().parents[1] +CASES = { + 'tests/test_native_shared_execution.py::test_actual_public_correction_three_workers_and_saved_completion': 'native26', + 'tests/test_native_shared_execution_variant.py::test_actual_public_merged_omissions_and_all_chunk_statuses': 'native26-omissions', +} +MAX_CASE_BYTES = 512 * 1024 * 1024 +MAX_CASE_ENTRIES = 10000 +SOURCE_FILES = ( + 'utils/native_ci_evidence.py', 'utils/native_shared_acceptance.py', + 'tests/test_native_shared_execution.py', 'tests/test_native_shared_execution_variant.py', + 'tests/native_shared_fixture.py', 'tests/native_shared_omission_fixture.py', + 'tests/fixtures/native_shared_execution/fake_cli.py', '.github/workflows/main.yaml', +) +LIMITS = {'attempt_seconds': 900, 'helper_seconds': 180, 'base_fixture_seconds': 120, + 'basis': 'Selected test/peer source declarations, not measured operation durations.'} +KNOWN_ARTIFACTS = ( + 'actual-result.json', 'authority/runtime.json', 'authority/runtime-execution.json', + 'authority/selection.json', 'authority/composition-execution.json', + 'native-attempt/registration.json', 'native-attempt/started.json', + 'native-attempt/transcript.jsonl', 'native-attempt/control.jsonl', + 'native-attempt/stderr.txt', 'fresh-attempt/stages/journal.json', +) + + +def pin(raw): + return {'bytes': len(raw), 'sha256': hashlib.sha256(raw).hexdigest()} + + +def save_new(path, value): + raw = (json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + '\n').encode() + with path.open('xb') as handle: + handle.write(raw) + + +def source_identity(): + return {'head': git(ROOT, 'rev-parse', 'HEAD').strip(), + 'tree': git(ROOT, 'rev-parse', 'HEAD^{tree}').strip(), + 'files': {name: pin((ROOT / name).read_bytes()) for name in SOURCE_FILES}, + 'scope': 'Checkout Git identity and selected working source bytes; not a full imported closure.'} + + +def _identity(info): + return (info.st_dev, info.st_ino, info.st_mode, info.st_size, + info.st_mtime_ns, info.st_ctime_ns) + + +@contextmanager +def directory(path): + """Open every absolute directory component without following a symlink.""" + absolute = Path(os.path.abspath(path)) + descriptor = os.open(absolute.anchor, os.O_RDONLY | os.O_DIRECTORY) + try: + for part in absolute.parts[1:]: + child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=descriptor) + os.close(descriptor) + descriptor = child + yield descriptor + finally: + os.close(descriptor) + + +def snapshot(case, archive=None): + """Hash mode/bytes; optionally archive those exact bytes via held directory FDs.""" + entries, total = {}, 0 + with directory(case) as descriptor: + root_info = os.fstat(descriptor) + entries['.'] = {'type': 'directory', 'mode': stat.S_IMODE(root_info.st_mode)} + if archive is not None: + root_member = tarfile.TarInfo('case') + root_member.type = tarfile.DIRTYPE + root_member.mode, root_member.mtime = entries['.']['mode'], root_info.st_mtime + archive.addfile(root_member) + def visit(parent, prefix): + nonlocal total + with os.scandir(parent) as scan: + names = sorted(entry.name for entry in scan) + for name in names: + relative = str(Path(prefix) / name) + if len(entries) >= MAX_CASE_ENTRIES: + raise ValueError('synthetic case entry bound exceeded') + info = os.stat(name, dir_fd=parent, follow_symlinks=False) + row = {'mode': stat.S_IMODE(info.st_mode)} + member = tarfile.TarInfo('case/' + relative) + member.mode, member.mtime = row['mode'], info.st_mtime + if stat.S_ISLNK(info.st_mode): + target = os.readlink(name, dir_fd=parent) + row.update(type='symlink', target=target) + member.type, member.linkname = tarfile.SYMTYPE, target + if archive is not None: + archive.addfile(member) + elif stat.S_ISDIR(info.st_mode): + child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + try: + if _identity(os.fstat(child)) != _identity(info): + raise ValueError('case directory changed during capture') + row['type'] = 'directory' + entries[relative] = row + member.type = tarfile.DIRTYPE + if archive is not None: + archive.addfile(member) + visit(child, relative) + finally: + os.close(child) + elif stat.S_ISREG(info.st_mode): + # Do not read special files or follow a replaced leaf link. + fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent) + with os.fdopen(fd, 'rb') as handle: + opened = os.fstat(handle.fileno()) + if not stat.S_ISREG(opened.st_mode) or _identity(opened) != _identity(info): + raise ValueError('case file changed before capture') + if total + info.st_size > MAX_CASE_BYTES: + raise ValueError('synthetic case byte bound exceeded') + raw = handle.read(MAX_CASE_BYTES - total + 1) + if len(raw) != info.st_size or _identity(os.fstat(handle.fileno())) != _identity(info): + raise ValueError('case file changed during capture') + total += len(raw) + row.update(type='file', **pin(raw)) + member.size = len(raw) + if archive is not None: + archive.addfile(member, io.BytesIO(raw)) + else: + raise ValueError('nonregular synthetic case entry') + if _identity(os.stat(name, dir_fd=parent, follow_symlinks=False)) != _identity(info): + raise ValueError('case entry changed during capture') + entries[relative] = row + visit(descriptor, '') + return entries + + +def capture_case(tmp_path, destination, *, node_id, worker, source, outcome): + """Publish a new diagnostic only; errors never replace an existing artifact.""" + if node_id not in CASES: + raise ValueError('only the two named synthetic cases may be retained') + destination = Path(destination) + case = None if tmp_path is None else Path(tmp_path) / CASES[node_id] + if case is not None: + a, b = destination.resolve(), case.resolve() + if a == b or a.is_relative_to(b) or b.is_relative_to(a): + raise ValueError('diagnostic output must be separate from the case') + destination.mkdir(exist_ok=False) + manifest = {'format': 'native_ci_synthetic_case_v1', 'node_id': node_id, + 'worker': worker, 'source': source, 'declared_limits': dict(LIMITS), + 'test_outcome': outcome, 'original_case': None if case is None else str(case), + 'status': 'capture_failed', 'before': None, 'archived': None, 'after': None, + 'unchanged': False, 'archive': None, + 'availability': {name: 'not_inspected' for name in KNOWN_ARTIFACTS}, + 'case_completeness': 'not_assessed', + 'scientific_acceptance': 'not_assessed', + 'scope': 'Synthetic test files only; retention does not establish acceptance.'} + try: + if case is None or not os.path.lexists(case): + manifest['status'] = 'case_unavailable' + manifest['availability'] = {name: 'case_unavailable' for name in KNOWN_ARTIFACTS} + else: + manifest['before'] = snapshot(case) + manifest['availability'] = { + name: manifest['before'].get(name, {}).get('type', 'absent') for name in KNOWN_ARTIFACTS} + with tarfile.open(destination / 'case.tar.gz', 'x:gz', dereference=False) as archive: + manifest['archived'] = snapshot(case, archive) + manifest['after'] = snapshot(case) + manifest['unchanged'] = manifest['before'] == manifest['archived'] == manifest['after'] + manifest['archive'] = pin((destination / 'case.tar.gz').read_bytes()) + manifest['status'] = 'retained' if manifest['unchanged'] else 'capture_changed' + except Exception as exc: + manifest['error'] = {'type': type(exc).__name__, 'message': str(exc)} + save_new(destination / 'manifest.json', manifest) + return manifest + + +def pytest_addoption(parser): + parser.addoption('--native-ci-evidence-root', default=None, + help='Fresh external directory for two synthetic native cases only') + + +def pytest_configure(config): + selected = config.getoption('--native-ci-evidence-root') + if selected is None: + return + root = Path(selected).resolve() + if root == ROOT or root.is_relative_to(ROOT) or ROOT.is_relative_to(root): + raise pytest.UsageError('native CI evidence must be outside the checkout') + worker = getattr(config, 'workerinput', {}).get('workerid', 'controller') + if worker != 'controller' and not re.fullmatch(r'gw[0-9]+', worker): + raise pytest.UsageError('unexpected native CI evidence worker identity') + if worker == 'controller': + root.mkdir(exist_ok=False) + save_new(root / 'selection.json', {'format': 'native_ci_selection_v1', 'cases': CASES, + 'source': source_identity(), 'scope': 'Only these nodes if collected; no tests selected by this plugin.'}) + worker_root = root / worker + worker_root.mkdir(exist_ok=False) + config._native_ci_evidence = {'root': root, 'worker_root': worker_root, + 'worker': worker, 'source': source_identity(), 'events': []} + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + state = getattr(item.config, '_native_ci_evidence', None) + if state is None or item.nodeid not in CASES: + return + observed = getattr(item, '_native_ci_outcome', {}) + observed[report.when] = {'outcome': report.outcome, 'duration_seconds': report.duration} + item._native_ci_outcome = observed + if report.when != 'teardown': + # Keep the fixture-owned identity even if teardown clears funcargs. + item._native_ci_tmp_path = (getattr(item, 'funcargs', None) or {}).get('tmp_path') + return + relative = hashlib.sha256(item.nodeid.encode()).hexdigest() + destination = state['worker_root'] / relative + event = {'node_id': item.nodeid, 'directory': relative, 'test_outcome': observed} + try: + captured = capture_case(getattr(item, '_native_ci_tmp_path', None), destination, + node_id=item.nodeid, worker=state['worker'], source=state['source'], outcome=observed) + event['capture_status'] = captured['status'] + except Exception as exc: + event.update(capture_status='capture_failed', error={'type': type(exc).__name__, 'message': str(exc)}) + state['events'].append(event) + + +def pytest_sessionfinish(session, exitstatus): + state = getattr(session.config, '_native_ci_evidence', None) + if state is None: + return + # Preserve pytest's own failure/skip/exit information; capture errors are + # separate diagnostics, never a replacement success or exception. + try: + save_new(state['worker_root'] / 'outcomes.json', { + 'pytest_exitstatus': int(exitstatus), 'events': state['events'], + 'unobserved_nodes': [node for node in CASES + if not any(row['node_id'] == node for row in state['events'])], + 'unobserved_meaning': 'Not observed in this worker; may be unselected, unrun, or run elsewhere.'}) + except Exception as exc: + reporter = session.config.pluginmanager.getplugin('terminalreporter') + if reporter is not None: + reporter.write_line('Native synthetic evidence summary failed: ' + type(exc).__name__)