diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a3cf8ae8..31445211 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -48,7 +48,7 @@ jobs:
uses: ./.github/workflows/topic06-benchmark.yml
w4:
- name: W4 small QEMU checks
+ name: W4/W5 small QEMU checks
needs: scope
if: github.event_name == 'pull_request' && needs.scope.outputs.w4 == 'true'
uses: ./.github/workflows/w4-full-numeric.yml
diff --git a/.github/workflows/llm-deploy.yml b/.github/workflows/llm-deploy.yml
index 3b20f7ce..eeaa3937 100644
--- a/.github/workflows/llm-deploy.yml
+++ b/.github/workflows/llm-deploy.yml
@@ -37,6 +37,11 @@ on:
required: false
default: false
+ run_w5_validation:
+ description: W5 only; mutually exclusive with W3/W4 and skips legacy W1/W2 full runs
+ type: boolean
+ required: false
+ default: false
run_w4_validation:
description: Also build and execute one complete W4 RV64 QEMU forward
type: boolean
@@ -69,20 +74,51 @@ env:
QWEN3_PY: output/qwen3-probe-venv/bin/python
jobs:
+ validation-selection:
+ name: Validate explicit full-run selection
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+ steps:
+ - name: Reject conflicting W5 and legacy full selections
+ env:
+ W5_REQUESTED: ${{ inputs.run_w5_validation }}
+ W3_REQUESTED: ${{ inputs.run_w3_validation }}
+ W4_REQUESTED: ${{ inputs.run_w4_validation }}
+ run: |
+ python3 - <<'PY'
+ import os
+ selected = lambda name: os.environ.get(name, "").casefold() == "true"
+ if selected("W5_REQUESTED") and (os.environ["GITHUB_EVENT_NAME"] != "workflow_dispatch"
+ or selected("W3_REQUESTED") or selected("W4_REQUESTED")):
+ raise SystemExit("W5 requires a manual W5-only run; disable W3 and W4 selections")
+ PY
+
+ w5-validation:
+ name: "W5 / explicit container generation only"
+ needs: validation-selection
+ if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w5_validation }}
+ uses: ./.github/workflows/w4-full-numeric.yml
+ with:
+ run_w5: true
+
w3-validation:
+ needs: validation-selection
name: "W3 / explicit full validation"
- if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w3_validation }}
+ if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w3_validation && !inputs.run_w5_validation }}
uses: ./.github/workflows/w3-nightly.yml
w4-validation:
+ needs: validation-selection
name: "W4 / explicit full QEMU validation"
- if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w4_validation }}
+ if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w4_validation && !inputs.run_w5_validation }}
uses: ./.github/workflows/w4-full-numeric.yml
with:
run_full: true
case: ${{ inputs.w4_case || 'full_seed_0' }}
llm-deploy:
+ needs: validation-selection
+ if: ${{ !inputs.run_w5_validation }}
name: "LLM Deploy v1.0 / W1 + W2"
runs-on: ubuntu-24.04
timeout-minutes: 60
@@ -129,6 +165,31 @@ jobs:
python3 -m pip install --upgrade pip
python3 -m pip install -v -e ".[all,llm-deploy]"
+ - name: W5 official three-round conversation template
+ run: |
+ set -euo pipefail
+ mkdir -p output/w5-multiturn
+ python3 -B -m pytest \
+ tests/test_conversation.py::test_official_three_round_rendering_matches_transformers \
+ -q -p no:cacheprovider --junitxml output/w5-multiturn/tests.xml
+ python3 - <<'PY'
+ import xml.etree.ElementTree as ET
+ root = ET.parse("output/w5-multiturn/tests.xml").getroot()
+ for suffix in ("None", "Give brief answers."):
+ name = f"test_official_three_round_rendering_matches_transformers[{suffix}]"
+ rows = [node for node in root.iter("testcase") if node.get("name") == name]
+ if len(rows) != 1 or any(rows[0].find(tag) is not None for tag in ("skipped", "error", "failure")):
+ raise SystemExit(f"Official three-round template must execute and pass: {name}")
+ PY
+ - name: Upload W5 conversation template evidence
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: w5-multiturn-${{ github.run_id }}-${{ github.run_attempt }}
+ if-no-files-found: warn
+ retention-days: 14
+ path: output/w5-multiturn/tests.xml
+
# ── docs:interfaces / docs:risks ──────────────────────────────────
- name: "docs:interfaces + docs:risks"
id: docs
@@ -461,10 +522,11 @@ jobs:
}
full-qwen3-onnx:
+ needs: validation-selection
name: "W1 / full Qwen3 ONNX release validation"
# This fetches the pinned LFS objects and executes the complete FP32 model. It is not
# counted as a passing gate on PRs where it did not execute.
- if: github.event_name != 'pull_request'
+ if: github.event_name != 'pull_request' && !inputs.run_w5_validation
runs-on: ubuntu-24.04
timeout-minutes: 60
steps:
@@ -522,6 +584,8 @@ jobs:
full-qwen3-frontend:
+ needs: validation-selection
+ if: ${{ !inputs.run_w5_validation }}
name: "W2 / complete Qwen3 frontend"
# Required on PRs as well as manual/nightly runs. No full-model ORT or IR inference.
runs-on: ubuntu-24.04
@@ -580,7 +644,7 @@ jobs:
w2-acceptance:
name: "W2 / required acceptance"
needs: [llm-deploy, full-qwen3-frontend]
- if: always()
+ if: always() && !inputs.run_w5_validation
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
@@ -694,3 +758,35 @@ jobs:
if not (passed or not_run):
raise SystemExit("Explicit W4 validation must actually run and pass")
PY
+
+ w5-execution-summary:
+ name: "W5 / explicit execution scope"
+ needs: [validation-selection, w5-validation, llm-deploy, full-qwen3-onnx, full-qwen3-frontend, w3-validation, w4-validation]
+ if: always()
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+ steps:
+ - name: Require the requested W5-only execution
+ env:
+ W5_REQUESTED: ${{ inputs.run_w5_validation }}
+ W5_JOB_RESULTS: ${{ toJSON(needs) }}
+ run: |
+ python3 - <<'PY'
+ import json, os
+ requested = (os.environ["GITHUB_EVENT_NAME"] == "workflow_dispatch"
+ and os.environ.get("W5_REQUESTED", "").casefold() == "true")
+ results = {name: item.get("result") for name, item in json.loads(os.environ["W5_JOB_RESULTS"]).items()}
+ selected = results.get("validation-selection") == "success"
+ passed = requested and selected and results.get("w5-validation") == "success"
+ if requested:
+ passed = passed and all(results.get(name) == "skipped" for name in (
+ "llm-deploy", "full-qwen3-onnx", "full-qwen3-frontend", "w3-validation", "w4-validation"))
+ not_run = not requested and selected and results.get("w5-validation") == "skipped"
+ label = "PASS" if passed else "NOT RUN" if not_run else "FAIL"
+ with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream:
+ stream.write(f"## W5 execution scope\n\n**{label}** — explicit container generation, three frozen prompts and 24 forwards.\n\n")
+ stream.write("Only the selected W5 path may execute; the reusable workflow requires real small QEMU, "
+ "official three-round template and full-generation evidence. NOT RUN is not W5 acceptance.\n")
+ if not (passed or not_run):
+ raise SystemExit("Requested W5-only validation must pass without concurrent legacy full runs")
+ PY
diff --git a/.github/workflows/w4-full-numeric.yml b/.github/workflows/w4-full-numeric.yml
index b39efc7f..c6c95fb1 100644
--- a/.github/workflows/w4-full-numeric.yml
+++ b/.github/workflows/w4-full-numeric.yml
@@ -1,8 +1,35 @@
-name: W4 full RV64 numeric
+name: W4 and W5 RV64 validation
on:
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: Small regressions by default; full modes are mutually exclusive
+ type: choice
+ default: small
+ options:
+ - small
+ - w4-forward
+ - w5-generation
+ case:
+ description: W4-only case; ignored by the frozen W5 three-prompt suite
+ type: choice
+ default: full_seed_0
+ options:
+ - full_seed_0
+ - full_seed_42
+ - one_token
+ - short_17
+ - short_255
+ - changed_future
+ - changed_padding
workflow_call:
inputs:
+ run_w5:
+ description: Explicit manual W5-only generation from an existing dispatch entry
+ type: boolean
+ required: false
+ default: false
run_full:
description: Explicitly execute one complete QEMU forward; false runs only unit and small QEMU regressions
type: boolean
@@ -18,7 +45,7 @@ permissions:
contents: read
concurrency:
- group: w4-full-numeric-${{ github.ref }}-${{ inputs.run_full && 'full' || 'unit' }}-${{ inputs.case || 'full_seed_0' }}
+ group: w4-full-numeric-${{ github.ref }}-${{ inputs.run_full && 'full' || 'unit' }}-${{ inputs.case || 'full_seed_0' }}-${{ inputs.run_w5 && 'w5' || inputs.mode || 'call' }}
cancel-in-progress: false
env:
@@ -33,6 +60,19 @@ jobs:
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
+ - name: Reject conflicting or nonmanual full-generation selections
+ env:
+ W5_REQUESTED: ${{ inputs.run_w5 }}
+ W4_REQUESTED: ${{ inputs.run_full }}
+ RUN_MODE: ${{ inputs.mode }}
+ run: |
+ python3 - <<'PY'
+ import os
+ w5 = os.environ.get("W5_REQUESTED", "").casefold() == "true" or os.environ.get("RUN_MODE") == "w5-generation"
+ w4 = os.environ.get("W4_REQUESTED", "").casefold() == "true" or os.environ.get("RUN_MODE") == "w4-forward"
+ if w5 and (w4 or os.environ["GITHUB_EVENT_NAME"] != "workflow_dispatch"):
+ raise SystemExit("W5 must be explicitly manual and cannot share a run with W4 full inference")
+ PY
- uses: actions/checkout@v4
with:
ref: ${{ github.sha }}
@@ -79,6 +119,8 @@ jobs:
tests/test_w4_fma_conformance.py tests/test_w4_fma_cross_backend.py \
tests/test_riscv_process_cleanup.py \
tests/test_riscv_weight_transport.py \
+ tests/test_qemu_generation.py::test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel \
+ tests/test_riscv_mean_policy.py tests/test_riscv_nonlinear_policy.py \
tests/test_w4_workflows.py tests/test_w4_ci_summary.py \
-q -p no:cacheprovider --basetemp output/w4-unit/pytest \
--junitxml output/w4-unit/tests.xml
@@ -96,7 +138,19 @@ jobs:
"test_post_execution_transport_corruption_fails_and_retains_copies",
"test_success_copy_cleanup_failure_is_not_a_pass",
"test_actual_shared_library_executes_kernel_calls_and_unloads[sequential]",
- "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]")
+ "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]",
+ "test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel[1]",
+ "test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel[4]",
+ "test_explicit_mean_order_rv64_and_context_reuse[13-1]",
+ "test_explicit_mean_order_rv64_and_context_reuse[13-4]",
+ "test_explicit_mean_order_rv64_and_context_reuse[128-1]",
+ "test_explicit_mean_order_rv64_and_context_reuse[128-4]",
+ "test_explicit_mean_order_rv64_and_context_reuse[1024-1]",
+ "test_explicit_mean_order_rv64_and_context_reuse[1024-4]",
+ "test_explicit_mean_overflow_propagates_and_build_remains_reusable[1]",
+ "test_explicit_mean_overflow_propagates_and_build_remains_reusable[4]",
+ "test_reference_nonlinear_rv64_contexts_and_error_recovery[1]",
+ "test_reference_nonlinear_rv64_contexts_and_error_recovery[4]")
for name in required:
real = [node for node in root.iter("testcase") if node.get("name") == name]
if len(real) != 1 or any(real[0].find(tag) is not None for tag in ("skipped", "error", "failure")):
@@ -128,8 +182,8 @@ jobs:
full-numeric:
name: One complete Qwen3 RV64 forward
needs: unit
- # Caller event alone never enables the expensive full-model forward.
- if: ${{ inputs.run_full }}
+ # Keep old reusable callers; a direct manual dispatch chooses only one full mode.
+ if: ${{ !inputs.run_w5 && inputs.mode != 'w5-generation' && (inputs.run_full || (github.event_name == 'workflow_dispatch' && inputs.mode == 'w4-forward')) }}
runs-on: ubuntu-24.04
timeout-minutes: 240
env:
@@ -393,16 +447,203 @@ jobs:
output/w4-ci/**/input.npy
output/w4-ci/evidence-retention.json
+ w5-generation:
+ name: Manual W5 three-prompt generation in a clean container
+ needs: unit
+ # Never automatic, and never paired with the W4 complete forward.
+ if: ${{ github.event_name == 'workflow_dispatch' && (inputs.run_w5 || inputs.mode == 'w5-generation') && !inputs.run_full && inputs.mode != 'w4-forward' }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 350
+ env:
+ SCRATCHV_W5_IMAGE: scratchv-w5:${{ github.run_id }}-${{ github.run_attempt }}
+ SCRATCHV_W5_ASSETS: ${{ github.workspace }}/output/w5-assets
+ SCRATCHV_W5_OUTPUT: ${{ github.workspace }}/output/w5-ci
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+ lfs: false
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+ with:
+ python-version: "3.12"
+ - name: Record W5 runner resources before installation
+ run: |
+ set -euo pipefail
+ mkdir -p output/w5-ci
+ {
+ printf 'commit=%s\nrun_id=%s\nattempt=%s\n' "$GITHUB_SHA" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT"
+ cat /etc/os-release
+ uname -a
+ lscpu
+ free -b
+ df -B1 .
+ docker version
+ } > output/w5-ci/runner-resources.txt
+ - name: Build the clean W5 deployment image
+ timeout-minutes: 25
+ run: |
+ set -euo pipefail
+ docker build --file deployment/w5/Dockerfile \
+ --build-arg BASE_IMAGE=ubuntu:24.04 --tag "$SCRATCHV_W5_IMAGE" .
+ docker image inspect "$SCRATCHV_W5_IMAGE" > output/w5-ci/container-image.json
+ docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" -m pip freeze \
+ > output/w5-ci/dependencies.txt
+ docker run --rm --entrypoint qemu-system-riscv64 "$SCRATCHV_W5_IMAGE" --version \
+ >> output/w5-ci/runner-resources.txt
+ - name: Acquire only the fixed W5 ONNX and tokenizer assets
+ timeout-minutes: 15
+ run: |
+ set -euo pipefail
+ git lfs install --local
+ git lfs pull --include="assets/qwen3-0.6b/tokenizer/*" --exclude=""
+ python - <<'PY'
+ import json, shutil
+ from pathlib import Path
+ from probes.w1_qwen3_export.run import acquire_model
+ assets = Path("output/w5-assets")
+ result = acquire_model(assets / "onnx")
+ shutil.copytree("assets/qwen3-0.6b/tokenizer", assets / "tokenizer")
+ Path("output/w5-ci/acquisition.json").write_text(json.dumps(result, indent=2) + "\n")
+ PY
+ - name: Require the official three-round template in the clean W5 image
+ timeout-minutes: 5
+ run: |
+ set -euo pipefail
+ docker run --rm --init --user "$(id -u):$(id -g)" --env HOME=/tmp \
+ --env SCRATCHV_QWEN3_TOKENIZER_DIR=/assets/tokenizer \
+ --mount "type=bind,src=$SCRATCHV_W5_ASSETS,dst=/assets,readonly" \
+ --mount "type=bind,src=$SCRATCHV_W5_OUTPUT,dst=/output" \
+ --mount "type=bind,src=$GITHUB_WORKSPACE/tests/test_conversation.py,dst=/tmp/test_conversation.py,readonly" \
+ --entrypoint python "$SCRATCHV_W5_IMAGE" \
+ -m pytest -q -c /dev/null -p no:cacheprovider \
+ /tmp/test_conversation.py::test_official_three_round_rendering_matches_transformers \
+ --junitxml /output/multiturn-tests.xml
+ python - <<'PY'
+ import xml.etree.ElementTree as ET
+ root = ET.parse("output/w5-ci/multiturn-tests.xml").getroot()
+ for suffix in ("None", "Give brief answers."):
+ name = f"test_official_three_round_rendering_matches_transformers[{suffix}]"
+ rows = [node for node in root.iter("testcase") if node.get("name") == name]
+ if len(rows) != 1 or any(rows[0].find(tag) is not None for tag in ("skipped", "error", "failure")):
+ raise SystemExit(f"Official three-round template must execute and pass: {name}")
+ PY
+ - name: Require four CPUs and capacity for 24 retained logits
+ run: |
+ set -euo pipefail
+ python - <<'PY'
+ import json, os, shutil
+ from pathlib import Path
+ mem = {line.split(":")[0]: int(line.split()[1]) * 1024
+ for line in Path("/proc/meminfo").read_text().splitlines()
+ if line.startswith(("MemTotal:", "MemAvailable:"))}
+ evidence = {"stage": "after_image_and_assets",
+ "available_cpus": len(os.sched_getaffinity(0)), "memory": mem,
+ "disk_free_bytes": shutil.disk_usage(".").free,
+ "minimum_available_cpus": 4,
+ "minimum_available_memory_bytes": 10 * 1024**3,
+ "minimum_disk_free_bytes": 10 * 1024**3,
+ "estimated_remaining_payload_bytes": 2 * 2384319432 + 24 * 256 * 151936 * 4,
+ "estimate_scope": "Bundle plus one temporary transport copy and all 24 logits, excluding build/cache/log overhead; image/assets already exist."}
+ evidence["passed"] = (evidence["available_cpus"] >= evidence["minimum_available_cpus"]
+ and mem.get("MemAvailable", 0) >= evidence["minimum_available_memory_bytes"]
+ and evidence["disk_free_bytes"] >= evidence["minimum_disk_free_bytes"])
+ Path("output/w5-ci/resource-preflight.json").write_text(json.dumps(evidence, indent=2) + "\n")
+ if not evidence["passed"]:
+ raise SystemExit("Insufficient W5 CPU/RAM/disk; generation NOT RUN. See resource-preflight.json")
+ PY
+ - name: Run the frozen W5 suite through the deployment script
+ id: generation
+ timeout-minutes: 285
+ run: |
+ set -euo pipefail
+ # Image preparation above owns dependency installation. The original
+ # deploy entry verifies its build using the same now-cached image.
+ set +e
+ timeout --signal=INT --kill-after=30s 280m \
+ /usr/bin/time -v -o output/w5-ci/process-time.txt \
+ bash scripts/deploy_w5.sh \
+ --model /assets/onnx/model.onnx --tokenizer-dir /assets/tokenizer \
+ --reference-suite --verify-onnx /assets/onnx/model.onnx \
+ --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \
+ --matmul-rhs-layout column-panel \
+ --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \
+ --timeout 1800 --total-timeout 16500 \
+ 2>&1 | tee output/w5-ci/runner.log
+ pipeline_status=("${PIPESTATUS[@]}")
+ set -e
+ printf '%s\n' "${pipeline_status[0]}" > output/w5-ci/runner-exit-code.txt
+ printf '%s\n' "${pipeline_status[1]}" > output/w5-ci/log-exit-code.txt
+ # Preserve generation failures (including timeout/cancellation) first.
+ if (( pipeline_status[0] != 0 )); then
+ exit "${pipeline_status[0]}"
+ fi
+ exit "${pipeline_status[1]}"
+ - name: Reap any container left by an interrupted Docker client
+ if: always()
+ timeout-minutes: 2
+ run: |
+ set -euo pipefail
+ # This tag belongs only to this run/attempt; never stop unrelated containers.
+ containers=$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE")
+ for container in $containers; do
+ docker stop --time 30 "$container"
+ done
+ test -z "$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE")"
+ - name: Require complete W5 generation evidence
+ id: w5_gate
+ if: always()
+ run: |
+ python -B -X utf8 scripts/check_w5_ci.py \
+ --report output/w5-ci/run/report.json \
+ --exit-code-file output/w5-ci/runner-exit-code.txt \
+ --log-exit-code-file output/w5-ci/log-exit-code.txt \
+ --output output/w5-ci/ci-gate.json
+ - name: Record final W5 resource and retention scope
+ if: always()
+ run: |
+ mkdir -p output/w5-ci
+ { free -b; df -B1 .; } > output/w5-ci/final-resources.txt
+ cat > output/w5-ci/evidence-retention.json <<'JSON'
+ {
+ "reports_retention_days": 30,
+ "raw_logits_uploaded": false,
+ "weights_uploaded": false,
+ "scope": "Three frozen raw-completion prompts, eight greedy tokens each; one ELF/bundle, 4 core, explicit Mean and nonlinear reference policies, padding off.",
+ "container_scope": "The original deployment script exit, CLI gate and workflow job together establish container acceptance. CLI container_reproduction=NOT_ASSERTED is preserved.",
+ "timing_scope": "GNU time measures the Docker client, not container CPU/RSS; per-forward QEMU elapsed/RSS evidence is in run.json. This CI run is not a calibrated speedup benchmark.",
+ "failure_scope": "Normal failures retain reports. Runner loss, hard job timeout or cancellation can prevent cleanup and artifact upload.",
+ "defaults_changed": false
+ }
+ JSON
+ - name: Upload W5 reports and failure diagnostics only
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: w5-generation-reports-${{ github.run_id }}-${{ github.run_attempt }}
+ if-no-files-found: warn
+ retention-days: 30
+ path: |
+ output/w5-ci/**/*.xml
+ output/w5-ci/**/*.json
+ output/w5-ci/**/*.txt
+ output/w5-ci/**/*.log
+ output/w5-ci/**/*.stderr
+ output/w5-ci/**/*.stdout
+ !output/w5-ci/.zig-cache/**
+
ci-status:
- name: Summarize W4 gates, including failed or skipped dependencies
+ name: Summarize W4/W5 gates, including failed or skipped dependencies
if: ${{ always() }}
- needs: [unit, full-numeric]
+ needs: [unit, full-numeric, w5-generation]
runs-on: ubuntu-24.04
timeout-minutes: 5
env:
UNIT_RESULT: ${{ needs.unit.result }}
FULL_NUMERIC_RESULT: ${{ needs.full-numeric.result }}
- FULL_REQUESTED: ${{ inputs.run_full }}
+ FULL_REQUESTED: ${{ !inputs.run_w5 && inputs.mode != 'w5-generation' && (inputs.run_full || (github.event_name == 'workflow_dispatch' && inputs.mode == 'w4-forward')) }}
+ W5_GENERATION_RESULT: ${{ needs.w5-generation.result }}
+ W5_REQUESTED: ${{ github.event_name == 'workflow_dispatch' && (inputs.run_w5 || inputs.mode == 'w5-generation') && !inputs.run_full && inputs.mode != 'w4-forward' }}
SELECTED_CASE: ${{ inputs.case || 'full_seed_0' }}
steps:
- name: Record W4 CI result without promoting skipped gates
@@ -413,29 +654,37 @@ jobs:
import json, os
from pathlib import Path
requested = os.environ.get("FULL_REQUESTED", "").casefold() == "true"
- unit, full = os.environ["UNIT_RESULT"], os.environ["FULL_NUMERIC_RESULT"]
- passed = unit == "success" and full == ("success" if requested else "skipped")
+ w5_requested = os.environ.get("W5_REQUESTED", "").casefold() == "true"
+ unit, full, w5 = (os.environ[key] for key in ("UNIT_RESULT", "FULL_NUMERIC_RESULT", "W5_GENERATION_RESULT"))
+ passed = (not (requested and w5_requested) and unit == "success"
+ and full == ("success" if requested else "skipped")
+ and w5 == ("success" if w5_requested else "skipped"))
row = {"schema_version": 1, "passed": passed, "status": "PASS" if passed else "FAIL",
- "scope": "One complete RV64 case plus retained-array audit" if requested else "Unit and small-graph regressions only",
+ "scope": ("Three frozen W5 cases and 24 complete QEMU forwards in the deployment container"
+ if w5_requested else "One complete RV64 case plus retained-array audit"
+ if requested else "Unit and small-graph regressions only"),
"commit": os.environ["GITHUB_SHA"], "run_id": os.environ["GITHUB_RUN_ID"],
"run_attempt": os.environ["GITHUB_RUN_ATTEMPT"], "case": os.environ["SELECTED_CASE"] if requested else None,
"full_numeric_requested": requested,
- "job_results": {"unit": unit, "full-numeric": full},
+ "w5_generation_requested": w5_requested,
+ "w5_generation_accepted": w5_requested and passed,
+ "job_results": {"unit": unit, "full-numeric": full, "w5-generation": w5},
"full_numeric_accepted": requested and passed, "team_accepted": False,
"independent_reproduction": False, "seven_case_coverage": False}
Path("output/w4-status/status.json").write_text(json.dumps(row, indent=2) + "\n")
labels = {"success": "PASS", "failure": "FAIL", "cancelled": "CANCELLED", "skipped": "NOT RUN (未运行)"}
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as stream:
- stream.write(f"## W4 CI {row['status']}\n\nCommit: `{row['commit']}`. {row['scope']}.\n\n"
+ stream.write(f"## W4/W5 CI {row['status']}\n\nCommit: `{row['commit']}`. {row['scope']}.\n\n"
"| 检查 | 本次结果 |\n| --- | --- |\n"
f"| 单测与真实小图 QEMU | {labels.get(unit, 'UNKNOWN')} |\n"
- f"| 完整 28 层 QEMU 与数组审计 | {labels.get(full, 'UNKNOWN')} |\n\n"
+ f"| W4 完整 28 层 QEMU 与数组审计 | {labels.get(full, 'UNKNOWN')} |\n"
+ f"| W5 容器三例 24 步生成 | {labels.get(w5, 'UNKNOWN')} |\n\n"
"数值和耗时见已执行任务的验收摘要；未运行项目没有本次数值结果。\n\n"
"验收范围
\n\n"
"PR 默认只执行单测与小图，完整前向通过手动工作流运行。"
"This does not approve team acceptance or independent reproduction.\n\n
+历史实跑的模型快照与工具版本
+
+- 模型快照：`c1899de289a04d12100db370d81485cdf75e47ca`，见
+ [模型清单](../../probes/w1_qwen3_export/manifest.json)及[冻结参考](../../probes/w5_generation/references.json)。
+- QEMU 8.2.2（Ubuntu 包 `1:8.2.2+ds-0ubuntu1.18`），Zig 0.14.1；
+ 目标为 `riscv64 / RV64GC / LP64D / QEMU virt` 裸机，数学库为 Zig 随附 musl。
+- Python 3.12.3、NumPy 2.2.6、ONNX Runtime 1.22.1、tokenizers 0.21.4。
+- 以上为历史记录中的版本；完整字段见
+ [数据摘要](W5/reports/validation-summary.json)的 `configuration`。不据此推断当前机器或最新依赖版本。
+
+
+
+
+环境准备完成后，按 [W5 报告中的生成命令与结果判定](W5/README.md#如何复现)执行；本节不把环境检查或镜像构建当作生成成功。
+
+
+
+## 6. W5 容器准备与独立复现
+
+入口为 [部署脚本](../../scripts/deploy_w5.sh)和 [Dockerfile](../../deployment/w5/Dockerfile)。需要已安装并可访问的 Docker Engine/Buildx；部署脚本会检查引擎和资产、构建镜像，再启动生成，不负责安装引擎或下载模型，也不更改 WSL 配置。
+
+**镜像、挂载与路径。** 容器只打包运行所需源码和依赖，资产只读挂载到 `/assets`，构建及报告
+写入 `/output`，以调用者 UID/GID 运行。默认镜像 `scratchv-w5:local`、基础镜像
+`ubuntu:24.04`；可显式设置 `SCRATCHV_CONTAINER_ENGINE`、`SCRATCHV_W5_IMAGE`、
+`SCRATCHV_W5_BASE_IMAGE`、`SCRATCHV_W5_ASSETS` 和 `SCRATCHV_W5_OUTPUT`。
+路径可含空格，挂载路径不能含逗号。脚本检查引擎和资产后构建、运行，不安装引擎，
+不下载模型，不更改 WSL 或 Windows。依赖来自
+[pyproject.toml](../../pyproject.toml) 的 `llm-deploy`，不另维护一份版本锁。
+
+容器中的 Python 与 QEMU 仍承担相同职责；容器不会替代 QEMU 执行模型。生成结果和计时见 [W5 实验结果](W5/README.md#实验结果)。
+
+
+独立容器复现：固定版本、环境准备、三例生成与报告回传
+
+以下是一套需要在同一 Bash 会话中顺序执行的完整流程。准备阶段设置的目录、镜像与输出变量会被后续运行和清理步骤继续使用。
+
+**固定版本的独立容器复现。** 以下步骤在同一 Bash 会话依次执行，使用独立的
+Ubuntu 24.04 x86_64 环境，不复制作者的虚拟环境、`output/`、ELF、bundle 或报告。
+三例共用一次新构建、执行 24 次前向，不先另跑 Paris，也不重新录制 ORT 参考。
+固定提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 是已完成完整数值实跑的功能基线；
+该提交用于复现既有完整数值结果，不含后续报告保存、SIGTERM 分类及 core 名称兼容修复；
+这些行为另用相关小图或 Python 测试验证，不将旧完整结果标为修改后的重跑。
+该固定提交也尚不支持 `--cores`，所以下面的历史复现命令保留 `--harts`；
+运行当前分支时优先使用 `--cores`。这次名称兼容修改没有重跑完整模型。
+
+| 条件 | 固定内容或要求 |
+|---|---|
+| 主机工具 | Bash、Git/Git LFS、Python 3.12、Docker Engine/Buildx、GNU time、timeout、setsid、tar |
+| 模型 | `Qwen/Qwen3-0.6B`，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 ONNX 及同版本 tokenizer |
+| 基础镜像 | `public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca` |
+| 原实跑工具 | Python 3.12.3、QEMU 8.2.2（Ubuntu `1:8.2.2+ds-0ubuntu1.18`）、Zig 0.14.1、ORT 1.22.1、NumPy 2.2.6、Torch 2.7.1+cpu |
+| 原容器引擎 | Docker Engine 29.9.2、Buildx 0.38.1 |
+| 准备后的资源 | 至少 4 个可用逻辑 CPU、10 GiB 可用内存、10 GiB 剩余磁盘；初始建议预留 25 GiB 磁盘 |
+| 时间预算 | 单步 1800 s，总预算 16500 s，外层 280 分钟，强制清理再留 30 s |
+
+镜像 digest 只固定基础镜像；apt 与未逐项锁定的间接依赖仍可能变化。记录实际镜像、
+工具和依赖版本，先核对差异是否影响结果比较，不默认为相同环境。这里沿用原资产
+身份检查，不增加源码、输出或报告哈希。保留部署前后镜像 ID 作为运行记录，
+不要求两者必须相等：Buildx 的缓存构建仍可能更新 provenance/attestation 元数据，
+使 manifest ID 改变。ID 变化本身不足以说明运行内容改变，也不代表版本差异可忽略；
+继续记录固定源码、模型和实际工具/依赖版本，复用既有 ELF/bundle 检查。
+
+**1. 新 clone 并固定提交。**
+
+以下命令在同一个 Bash 会话按顺序执行，从有足够空间的父目录开始。
+目录名 `scratchv-w5-repro` 必须不存在；重试时改用新的目录名，保留失败记录。
+
+```bash
+set -euo pipefail
+export W5_COMMIT=8ad347477f7f5ba15b2da7c91178b3c6668456cc
+test ! -e scratchv-w5-repro
+GIT_LFS_SKIP_SMUDGE=1 git clone --no-checkout --filter=blob:none \
+ https://github.com/ScratchV-Compiler/ScratchV.git scratchv-w5-repro
+cd scratchv-w5-repro
+git fetch --no-tags origin refs/pull/115/head
+GIT_LFS_SKIP_SMUDGE=1 git checkout --detach "$W5_COMMIT"
+test "$(git rev-parse HEAD)" = "$W5_COMMIT"
+test -z "$(git status --porcelain)"
+
+export W5_RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$"
+export SCRATCHV_CONTAINER_ENGINE=docker
+export SCRATCHV_W5_IMAGE="scratchv-w5:independent-$W5_RUN_ID"
+export SCRATCHV_W5_ASSETS="$PWD/output/w5-independent-assets"
+export SCRATCHV_W5_OUTPUT="$PWD/output/w5-independent-$W5_RUN_ID"
+export SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca
+test ! -e "$SCRATCHV_W5_ASSETS"
+test ! -e "$SCRATCHV_W5_OUTPUT"
+mkdir -p "$SCRATCHV_W5_OUTPUT"
+date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/start-utc.txt"
+{
+ git rev-parse HEAD
+ git status --porcelain
+ git remote -v
+} > "$SCRATCHV_W5_OUTPUT/source.txt"
+{
+ cat /etc/os-release
+ uname -a
+ lscpu
+ free -b
+ df -B1 .
+ python3 --version
+ git --version
+ git lfs version
+ docker version
+ docker buildx version
+ docker info --format 'OS={{.OSType}} Arch={{.Architecture}} CPUs={{.NCPU}} Memory={{.MemTotal}}'
+ command -v setsid timeout tar
+ /usr/bin/time --version
+} > "$SCRATCHV_W5_OUTPUT/runner-resources.txt" 2>&1
+docker info > /dev/null
+test "$(uname -m)" = x86_64
+python3 -c 'import sys; assert sys.version_info[:2] == (3, 12)'
+printf 'Run output: %s\n' "$SCRATCHV_W5_OUTPUT"
+```
+
+检查 `runner-resources.txt`：容器外的 Linux 环境应为 Ubuntu 24.04，Docker daemon 也运行在该 Linux 环境中；
+若使用远端 daemon、容器资源配额不足或没有引擎，先停止并回报，不用修改 WSL 来凑条件。
+挂载目录不能包含逗号。不要把私人凭据写入报告。
+
+**2. 构建镜像并记录实际环境。**
+
+复用现有 Dockerfile；随后部署脚本会再检查构建并复用 Docker 缓存。
+基础镜像和构建参数不变，不使用作者构建好的镜像。
+
+```bash
+docker build --file deployment/w5/Dockerfile \
+ --build-arg "BASE_IMAGE=$SCRATCHV_W5_BASE_IMAGE" \
+ --tag "$SCRATCHV_W5_IMAGE" . \
+ 2>&1 | tee "$SCRATCHV_W5_OUTPUT/image-build.log"
+docker image inspect "$SCRATCHV_W5_IMAGE" \
+ > "$SCRATCHV_W5_OUTPUT/container-image.json"
+docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \
+ > "$SCRATCHV_W5_OUTPUT/image-id-before.txt"
+docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" -m pip freeze \
+ > "$SCRATCHV_W5_OUTPUT/dependencies.txt"
+docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" \
+ -c 'import json; from probes.w1_qwen3_export.run import environment; print(json.dumps(environment(export=True), indent=2))' \
+ > "$SCRATCHV_W5_OUTPUT/image-environment.json"
+docker run --rm --entrypoint sh "$SCRATCHV_W5_IMAGE" -c \
+ 'python --version; python -m ziglang version; qemu-system-riscv64 --version; dpkg-query -W python3.12 qemu-system-misc' \
+ > "$SCRATCHV_W5_OUTPUT/image-tools.txt"
+cat "$SCRATCHV_W5_OUTPUT/image-tools.txt"
+```
+
+核对 `image-tools.txt` 与上表；环境差异应保留在报告中，部署脚本本身不锁 apt 补丁。
+
+**3. 独立获取固定资产。**
+
+下载只调用现有 `acquire_model`，沿用 W1 manifest 中已有的下载/分片完整性检查。
+它只用 Python 标准库，不执行模型、不重新导出、不在容器外的 Linux 环境安装推理依赖。
+Tokenizer 从同一提交的 Git LFS 获取。
+
+```bash
+git lfs install --local
+git lfs pull --include="assets/qwen3-0.6b/tokenizer/*" --exclude="" \
+ 2>&1 | tee "$SCRATCHV_W5_OUTPUT/tokenizer-acquisition.log"
+python3 - <<'PY' 2>&1 | tee "$SCRATCHV_W5_OUTPUT/model-acquisition.log"
+import json, os, shutil
+from pathlib import Path
+from probes.w1_qwen3_export.run import acquire_model
+assets = Path(os.environ["SCRATCHV_W5_ASSETS"])
+out = Path(os.environ["SCRATCHV_W5_OUTPUT"])
+result = acquire_model(assets / "onnx")
+shutil.copytree("assets/qwen3-0.6b/tokenizer", assets / "tokenizer")
+(out / "acquisition.json").write_text(json.dumps(result, indent=2) + "\n")
+PY
+test -z "$(git status --porcelain)"
+```
+
+不要另跑参考采集脚本或添加 `--record`：suite 内部已经逐步对照当轮 ORT 与冻结参考，
+无需再运行一遍完整 ORT 生成作为“预热”。
+
+**4. 资源检查。** 镜像和资产准备后核对可用资源，Docker 配额不能低于要求。
+
+```bash
+python3 - <<'PY'
+import json, os, shutil
+from pathlib import Path
+out = Path(os.environ["SCRATCHV_W5_OUTPUT"])
+mem = {line.split(":")[0]: int(line.split()[1]) * 1024
+ for line in Path("/proc/meminfo").read_text().splitlines()
+ if line.startswith(("MemTotal:", "MemAvailable:"))}
+evidence = {"stage": "after_image_and_assets",
+ "available_cpus": len(os.sched_getaffinity(0)), "memory": mem,
+ "disk_free_bytes": shutil.disk_usage(out).free,
+ "minimum_available_cpus": 4,
+ "minimum_available_memory_bytes": 10 * 1024**3,
+ "minimum_disk_free_bytes": 10 * 1024**3}
+evidence["passed"] = (evidence["available_cpus"] >= 4
+ and mem.get("MemAvailable", 0) >= 10 * 1024**3
+ and evidence["disk_free_bytes"] >= 10 * 1024**3)
+(out / "resource-preflight.json").write_text(json.dumps(evidence, indent=2) + "\n")
+if not evidence["passed"]:
+ raise SystemExit("Insufficient W5 CPU/RAM/disk; generation NOT RUN.")
+PY
+```
+
+资源不足时保留记录，不运行完整生成。
+
+**5. 一次运行三条完整用例并保存报告。**
+
+整块复制执行。`--reference-suite` 自带三条用例及每例 8-token 预算，
+不要同时传 `--max-new-tokens`，不要传 `--reuse-build`。
+代码从固定模型重新构建一次，跨用例自动复用同一 ELF/bundle。
+
+```bash
+test "$(git rev-parse HEAD)" = "$W5_COMMIT"
+test -z "$(git status --porcelain)"
+test ! -e "$SCRATCHV_W5_OUTPUT/run"
+date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-start-utc.txt"
+
+set +e
+timeout --signal=INT --kill-after=30s 280m \
+ /usr/bin/time -v -o "$SCRATCHV_W5_OUTPUT/process-time.txt" \
+ bash scripts/deploy_w5.sh \
+ --model /assets/onnx/model.onnx --tokenizer-dir /assets/tokenizer \
+ --reference-suite --verify-onnx /assets/onnx/model.onnx \
+ --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \
+ --matmul-rhs-layout column-panel \
+ --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \
+ --timeout 1800 --total-timeout 16500 \
+ 2>&1 | tee "$SCRATCHV_W5_OUTPUT/runner.log"
+pipeline_status=("${PIPESTATUS[@]}")
+printf '%s\n' "${pipeline_status[0]}" > "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt"
+printf '%s\n' "${pipeline_status[1]}" > "$SCRATCHV_W5_OUTPUT/log-exit-code.txt"
+date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-end-utc.txt"
+
+# 此唯一 tag 只供本次使用；不停止其他人的容器。
+{
+ containers=$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE")
+ list_status=$?
+ printf 'list_exit_code=%s\n' "$list_status"
+ for container in $containers; do
+ docker stop --time 30 "$container"
+ printf 'stop_exit_code=%s\n' "$?"
+ done
+ docker ps --filter "ancestor=$SCRATCHV_W5_IMAGE" \
+ --format '{{.ID}} {{.Status}}'
+ printf 'final_list_exit_code=%s\n' "$?"
+} > "$SCRATCHV_W5_OUTPUT/container-cleanup.log" 2>&1
+
+docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \
+ > "$SCRATCHV_W5_OUTPUT/image-id-after.txt"
+python3 -B -X utf8 scripts/check_w5_ci.py \
+ --report "$SCRATCHV_W5_OUTPUT/run/report.json" \
+ --reference probes/w5_generation/references.json \
+ --exit-code-file "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" \
+ --output "$SCRATCHV_W5_OUTPUT/result-check.json" \
+ > "$SCRATCHV_W5_OUTPUT/result-check.log" 2>&1
+printf '%s\n' "$?" > "$SCRATCHV_W5_OUTPUT/check-exit-code.txt"
+cat "$SCRATCHV_W5_OUTPUT/result-check.log"
+{
+ date -u +%FT%TZ
+ git rev-parse HEAD
+ git status --porcelain
+ free -b
+ df -B1 .
+} > "$SCRATCHV_W5_OUTPUT/final-resources.txt"
+set -e
+printf 'Preserved output: %s\n' "$SCRATCHV_W5_OUTPUT"
+```
+
+这里为失败留证临时关闭 `errexit`，**脚本走到最后不等于 PASS**。
+即使 suite 非零退出，仍运行容器清理和已有报告检查器；报告缺失时 checker 应失败，
+不能补造成功字段。`result-check.json` 在本地的 `commit` 可为 null；实际 SHA 由
+`source.txt` 和最终 Git 记录确认。
+
+若手动 Ctrl-C 使整段 shell 中断，重新进入同一 clone，恢复上面输出目录和镜像变量，
+执行本节从“此唯一 tag”注释之后的收尾段，再打包已有报告。不要直接重新跑模型。
+主机掉电、daemon 失联或强制终止可能阻止清理；如清理日志有非零码或最后仍列出容器，
+标为未完成并回报，不视为通过。
+
+**三条用例与结果判定。** 输入、应得到的文字、token 编号及停止原因统一见
+[三条实际输出与冻结参考对照](W5/README.md#fixed-case-results)。复现必须逐项匹配，不能只检查是否出现城市名。
+
+成功要求：三例、24 次前向全部完成，prompt IDs/新 IDs/文本/停止原因匹配；每步全部
+`256×151936=38,895,616` 个 logits 有限且严格 `max_abs < 1e-3`；28 层完整图检查、
+FMA 12 行 binary32 位模式与 FCSR 检查通过；同一 ELF/bundle 与所选策略匹配、
+padding `none`；每个 QEMU 正常退出并回收，没有清理错误。套件、日志管道、报告
+检查器退出码均为 0，最终无本次容器残留，源码没有改动。
+`container_reproduction=NOT_ASSERTED` 是内部 CLI 不会自行宣称容器复现的字段，
+不能手改；容器身份由外层构建、运行和退出记录说明。
+
+**回传报告。** 附 `handoff.txt` 说明执行人、UTC 时间、完整 SHA、独立 clone/构建、
+环境差异、三例 IDs/文本/误差、计时与缺失采样、清理结果以及失败阶段，保留：
+
+- 来源、主机资源、镜像/依赖/工具、下载记录、所有退出码与构建/运行/清理日志。
+- `run/report.json`、`result-check.json`、`run/executable.json`、`run/build/` 的 JSON。
+- `run/steps/step-0000…0023/run.json`、UART 完成帧 `uart.bin`、错误日志及
+ `run/fma-conformance/` 的报告。
+
+不传权重、ELF、原始 logits 或 Zig 缓存；失败时也回传已有报告并标明缺失项。
+原始产物保留至复现结果确认；重试使用新目录。沿用原小文件筛选方式打包：
+
+```bash
+python3 - <<'PY'
+import os, tarfile
+from pathlib import Path
+out = Path(os.environ["SCRATCHV_W5_OUTPUT"])
+allowed = {".json", ".xml", ".txt", ".log", ".stderr", ".stdout"}
+archive = out.parent / (out.name + "-reports.tar.gz")
+if archive.exists():
+ raise SystemExit(f"Keep earlier evidence; archive already exists: {archive}")
+with tarfile.open(archive, "w:gz") as bundle:
+ for path in sorted(out.rglob("*")):
+ relative = path.relative_to(out)
+ if (path.is_file() and not path.is_symlink()
+ and ".zig-cache" not in relative.parts
+ and (path.suffix in allowed or path.name == "uart.bin")):
+ bundle.add(path, arcname=str(Path(out.name) / relative), recursive=False)
+print(archive)
+PY
+```
+
+`process-time.txt` 的 CPU/RSS 属于 Docker 客户端，不是容器或 QEMU 总量。逐步耗时、
+RSS 使用 `run/steps/*/run.json`，partial 采样明确注明；未采集的 CPU 写“未独立采集”，
+不填作者历史数据，也不据不同机器的记录计算加速比。
+
+
diff --git a/docs/llm-deploy-v1.0/W4/README.md b/docs/llm-deploy-v1.0/W4/README.md
index 5f90c55f..3271661a 100644
--- a/docs/llm-deploy-v1.0/W4/README.md
+++ b/docs/llm-deploy-v1.0/W4/README.md
@@ -19,7 +19,7 @@ W3 [#96](https://github.com/ScratchV-Compiler/ScratchV/pull/96) 已于 2026-10-1
| [复现步骤](reproduction.md) | 固定依赖与模型、完整运行、诊断和原始数组复核 |
| [验收与 CI](validation.md) | 分层门槛、测试入口、默认关闭的 Nightly、证据边界 |
| [阶段衔接与后续准备](evolution.md) | W1–W4 接口对应、W5 前置工作、未来 MLIR 转换边界 |
-| [软仿优化实测报告](softsim-optimization-2026-10-22.md) | 多 hart、微内核、布局与运行器优化；完整模型对照和适用边界 |
+| [软仿优化实测报告](softsim-optimization-2026-10-22.md) | 多核、微内核、布局与运行器优化；完整模型对照和适用边界 |
## 验收标准
diff --git a/docs/llm-deploy-v1.0/W4/reports/full-hart-comparison-2026-10-22.json b/docs/llm-deploy-v1.0/W4/reports/full-hart-comparison-2026-10-22.json
index 3fb5ac5f..bfad256c 100644
--- a/docs/llm-deploy-v1.0/W4/reports/full-hart-comparison-2026-10-22.json
+++ b/docs/llm-deploy-v1.0/W4/reports/full-hart-comparison-2026-10-22.json
@@ -1,26 +1,26 @@
{
"status": "PASS",
"passed": true,
- "scope": "One ordered pair of complete full_seed_0 runs, harts 4 then 1. No old timing-anomaly baseline; not a repeated-run median or stability estimate.",
- "baseline": "1 hart, blocked_fma, 2x4, column-panel, padding none",
- "candidate": "4 harts, otherwise identical policy and fixed input",
+ "scope": "One ordered pair of complete full_seed_0 runs, 4 core then 1 core. No old timing-anomaly baseline; not a repeated-run median or stability estimate.",
+ "baseline": "1 core, blocked_fma, 2x4, column-panel, padding none",
+ "candidate": "4 core, otherwise identical policy and fixed input",
"identity_checks": [
- "1 hart retained QEMU report",
- "1 hart audit case",
- "1 hart audit numeric",
- "1 hart FMA actual tools",
- "1 hart codegen/build panels",
- "1 hart affinity unchanged",
- "1 hart measured forward wall",
- "1 hart runtime weights",
- "4 hart retained QEMU report",
- "4 hart audit case",
- "4 hart audit numeric",
- "4 hart FMA actual tools",
- "4 hart codegen/build panels",
- "4 hart affinity unchanged",
- "4 hart measured forward wall",
- "4 hart runtime weights",
+ "1 core retained QEMU report",
+ "1 core audit case",
+ "1 core audit numeric",
+ "1 core FMA actual tools",
+ "1 core codegen/build panels",
+ "1 core affinity unchanged",
+ "1 core measured forward wall",
+ "1 core runtime weights",
+ "4 core retained QEMU report",
+ "4 core audit case",
+ "4 core audit numeric",
+ "4 core FMA actual tools",
+ "4 core codegen/build panels",
+ "4 core affinity unchanged",
+ "4 core measured forward wall",
+ "4 core runtime weights",
"model_files",
"source_sha256",
"execution_tools",
@@ -315,17 +315,17 @@
"max_abs": 0.0
},
"limitations": [
- "One pair, ordered 4 harts then 1 hart; no repeatability/confidence estimate. Host load, temperature and filesystem-cache state were not held identically fixed.",
+ "One pair, ordered 4 core then 1 core; no repeatability/confidence estimate. Host load, temperature and filesystem-cache state were not held identically fixed.",
"Forward wall is QEMU launch through validated UART completion, not guest-only kernel time.",
"Whole-process CPU includes launch, dump and shutdown; compare against QEMU wall, not the forward wall or the approximate 0.5-second CPU sample.",
"RSS is a sampled QEMU process peak, separate from guest reserved RAM/workspace. Partial availability, sample counts and errors are retained for both runs.",
"Clock summaries span each entire wrapper. Raw/monotonic/UTC sampling skew or clock-frequency discipline is not automatically evidence of a time jump.",
- "ELFs, hart compiler definitions, per-hart stack reservations and parallel runtime can differ by design. Dynamic output paths, TCP ports and run timestamps are not identities.",
- "Generic Linux QEMU thread names cannot reliably identify individual hart threads.",
+ "ELFs, core compiler definitions, per-core stack reservations and parallel runtime can differ by design. Dynamic output paths, TCP ports and run timestamps are not identities.",
+ "Generic Linux QEMU thread names cannot reliably identify individual simulated CPU execution threads.",
"Existing report identities and freshly completed audit records are reused; no new hashes."
],
"publication": {
- "kind": "Publication copy of retained full-hart comparison; not a new model run",
+ "kind": "Publication copy of retained core-count comparison; not a new model run",
"date": "2026-10-22",
"source_result": "output/w4-harts-short-20261021/optimization-completion-20261022/full-hart-comparison/result.json",
"paths": "Paths under output/ are local retained-artifact indexes relative to the ScratchV development root, not files included in Git.",
diff --git a/docs/llm-deploy-v1.0/W4/runtime-contract.md b/docs/llm-deploy-v1.0/W4/runtime-contract.md
index 4ef42c86..ced771fd 100644
--- a/docs/llm-deploy-v1.0/W4/runtime-contract.md
+++ b/docs/llm-deploy-v1.0/W4/runtime-contract.md
@@ -63,6 +63,33 @@ int scratchv_run_external(const void *const inputs[],
blocked_fma；后者按 128 项 K 分块、块内显式融合乘加、块间 FP32 相加。保持非
fast-math、禁止隐式 FMA 合并，不能将两种累加顺序声称为逐位等价。
+`mean_policy` 默认为 `sequential`。W4/W5 CLI 可显式传
+`--mean-policy two-four-lane-last-axis`，只对 FP32 单一末轴 ReduceMean 使用已有
+W3 参考的双四路固定求和顺序；这是便携标量 C，不要求宿主或 guest 支持 SSE/RVV。
+非末轴归约、MatMul 和 Softmax 不随此选项改变。它可能改变 FP32 舍入结果，
+不能当作与旧顺序逐位等价，也不能标成完整的 `numpy-fp32-reference-v4` 实现。
+构建/运行报告记录该策略，旧构建缺字段只解释为 `sequential`；不同策略的 ELF
+不得混用。默认值、有限性检查、错误传播和工作区要求保持不变。
+
+`nonlinear_policy` 默认为 `libm`，保留原 Sin/Cos、Sigmoid 和 Softmax 路径。
+显式 `--nonlinear-policy fp32-reference` 使用已有参考的非融合三角多项式、
+Sigmoid 有理多项式及 Softmax 的 16 路求和/倒数乘法顺序；以标量 C 和显式 FP32 FMA
+实现，不要求 guest 支持 AVX512，也不按宿主 ISA 自动切换。三角函数在
+`abs(x) >= 18000` 时保留平台 libm 回退；不承诺该范围与所有平台逐位一致。
+它与 Mean 选项独立，既不改变 MatMul 策略，也不改变 Pow。两个选项都可能改变
+舍入结果；不能把组合称为完整 FP32 reference profile 的逐位实现或任意 ORT 版本的保证。
+原有限性检查、错误传播、Softmax 行并行/回调复用和工作区契约仍适用。
+
+参考 Sigmoid 将输入夹到 [-18, 18]、输出夹到 [0, 1]；Softmax 将减去行最大值后的
+指数输入下限固定为约 -103.972084。原非有限输入（包括 -inf mask）以及有限值相减
+产生的非有限结果仍被拒绝。这是固定数值策略，不代表通用精度提升；数值对照以
+IEEE FP32、最近偶数舍入（RNE）、保留次正规数及显式 FMA、禁止隐式融合/fast-math
+为前提。现有验证覆盖当前 Host 与 RV64GC/QEMU，不构成任意 ORT 版本、CPU dispatch、
+NPU 或 flush-to-zero 模式的等价保证。
+
+构建/运行证据记录此策略，缺字段的旧 ELF 只解释为 `libm`；CLI 复用必须显式匹配。
+数值策略需要重新生成 ELF，兼容的原权重 bundle 可以复用。
+
`riscv_external.py` 验证 ELF 架构、ABI、入口、装载段和布局，启动裸机 QEMU。guest
输出完成帧后，host 用 QMP 停止并转储精确大小输出，再退出；输出不是由 host C/NumPy
替算。实际 QEMU 在完整运行前须通过严格 FMA 位模式/FCSR 检查。
diff --git a/docs/llm-deploy-v1.0/W4/softsim-optimization-2026-10-22.md b/docs/llm-deploy-v1.0/W4/softsim-optimization-2026-10-22.md
index 5f1ae62c..dd2f7cc0 100644
--- a/docs/llm-deploy-v1.0/W4/softsim-optimization-2026-10-22.md
+++ b/docs/llm-deploy-v1.0/W4/softsim-optimization-2026-10-22.md
@@ -1,12 +1,14 @@
# QEMU 软仿优化报告（2026-10-22）
+本文的 core 指 QEMU 模拟的 CPU 核，不表示宿主的物理核心。当前命令使用 `--cores`，旧 `--harts` 仍兼容；历史数据和文件路径保持原样。
+
本报告汇总 PR104 已有的软仿优化实测与取舍。固定完整模型用例为 Qwen3-0.6B、FP32、batch=1、L=256、28 层、无 KV cache。测试在 4 个逻辑 CPU 的 WSL2/Linux 环境下进行；目标为 RV64GC/LP64D、QEMU virt 裸机。它不代表真实 RISC-V 或 NPU 性能。
-同配置完整模型单/四 hart 各执行一次，前向从 **1384.65 s 降到 410.53 s，3.373×**；CPU 总时间增加 **5.46%**，38,895,616 个输出逐位一致。小图的微内核、布局及等待优化收益分别统计，不能与该倍数相乘。
+同配置完整模型分别以 1 core 和 4 core 各执行一次，前向从 **1384.65 s 降到 410.53 s，3.373×**；CPU 总时间增加 **5.46%**，38,895,616 个输出逐位一致。小图的微内核、布局及等待优化收益分别统计，不能与该倍数相乘。
## 默认值与可选功能
-默认值保持 `1 hart / sequential / 1x4 / original / padding none`。手动选择 `--harts 1/2/4` 或 `--harts auto`；auto 只在启动/构建时按进程可用 CPU 选择 1/2/4，上限 4，不是运行中的动态调度，也不处理 cgroup CPU 带宽配额。
+默认值保持 `1 core / sequential / 1x4 / original / padding none`。手动选择 `--cores 1/2/4` 或 `--cores auto`；auto 只在启动/构建时按进程可用 CPU 选择 1/2/4，上限 4，不是运行中的动态调度，也不处理 cgroup CPU 带宽配额。
| 方向 | 已完成内容 | 适用边界 |
|---|---|---|
@@ -19,11 +21,11 @@
| 未默认启用 | 两缓冲 `two-buffer-pad-4k` 实验选项 | QEMU 专用、增加 4096 B，部分形状退化；下方主要性能对照均 padding none |
| 未采用 | 本轮 RVV 候选 | 正确性通过但 QEMU 实测负收益，不纳入正式实现 |
-## 完整模型：同配置 1/4 hart
+## 完整模型：同配置 1/4 core
-两边固定 `full_seed_0 / blocked_fma / 2x4 / column-panel / padding none`，按 4 hart、1 hart 的顺序各运行一次。编译器 Zig 0.14.1，QEMU 8.2.2（Ubuntu 包 1:8.2.2+ds-0ubuntu1.18），数学库为 Zig 自带 musl。
+两边固定 `full_seed_0 / blocked_fma / 2x4 / column-panel / padding none`，按 4 core、1 core 的顺序各运行一次。编译器 Zig 0.14.1，QEMU 8.2.2（Ubuntu 包 1:8.2.2+ds-0ubuntu1.18），数学库为 Zig 自带 musl。
-| 指标 | 1 hart | 4 hart |
+| 指标 | 1 core | 4 core |
|---|---:|---:|
| 前向耗时 | 1384.65 s | 410.53 s |
| QEMU 生命周期耗时 | 1385.97 s | 410.94 s |
@@ -33,9 +35,9 @@
| 对 ORT 最大绝对误差 | 8.58306884765625e-05 | 8.58306884765625e-05 |
| 对照输出逐位一致 | 是 | 是 |
-并行加速为 **3.372844654×**，前向耗时减少 **70.35%**；CPU 成本为单 hart 的 **1.05455 倍**。全部 logits 满足严格 `max_abs < 0.001`、`rtol=0`，12 项 FMA 检查及保留数组审计通过。
+并行加速为 **3.372844654×**，前向耗时减少 **70.35%**；CPU 成本为单核的 **1.05455 倍**。全部 logits 满足严格 `max_abs < 0.001`、`rtol=0`，12 项 FMA 检查及保留数组审计通过。
-完整数值、测量范围、采样缺失和来源索引见 [1/4 hart 对照数据](reports/full-hart-comparison-2026-10-22.json)。
+完整数值、测量范围、采样缺失和来源索引见 [1/4 core 对照数据](reports/full-hart-comparison-2026-10-22.json)。
测量口径：
@@ -44,13 +46,13 @@
- RSS 是进程观测峰值，不是 guest 预留 RAM；两侧分别有 3 次、1 次读取失败，均标为 partial，内存结论只称基本持平。
- 没有重复运行统计或置信区间；宿主负载、温度与文件缓存状态并未严格一致。没有使用早期跳时异常记录计算本次加速比。
- 执行时 Git 基线为 `af5768cbe49976896f88f10f8f0cc6ec08160f30`，工作树含未提交候选改动；不能仅凭该 commit 重建完整候选。两侧生产源码摘要比较通过，已有快照身份各自保留在对照 JSON 中。运行早于最终高 FD、Host C 清理等修复，不能视作当前 PR HEAD 的完整重测。
-- 完整模型本次没有可靠的 QMP hart/线程映射，不能将其线程采样精确标为主 hart 与各 worker。
+- 完整模型本次没有可靠的 QMP core/线程映射，不能将其线程采样精确标为主核与各 worker。
## 小图：微内核与布局
下表采用 2026-10-22 UART 事件计时后的同批对照中位数；每配置 3 次新进程运行，慢轮未剔除，padding 均关闭。微内核共 54 次、正式布局共 24 次性能运行，输出均逐位一致。
-| 对照 | 1 hart：前向耗时 | 4 hart：前向耗时 |
+| 对照 | 1 core：前向耗时 | 4 core：前向耗时 |
|---|---:|---:|
| Dense：1x4 → 2x4 | 5.8311 → 3.6155 s（1.613×） | 1.7681 → 1.1188 s（1.580×） |
| Dense：1x4 → 1x8 | 5.8311 → 3.6519 s（1.597×） | 1.7681 → 1.1211 s（1.577×） |
@@ -59,7 +61,7 @@
正式布局的 direct_dense 为 LHS `[256,1024]`、RHS `[1024,1024]`；batch_transpose_wide 为 LHS `[2,64,1024]`、存储 RHS `[2048,1024]`，转置后参与乘法。后者工作区由 9,437,184 → 1,048,576 bytes，减少 8 MiB。phase3 报告未直接记录 Dense 形状，本表仅按原报告用例名引用，不用旧阶段形状替代它。
-Dense 的 1x4 → 2x4 对照中，CPU 中位数分别从 5.8332 → 3.6183 s、6.8124 → 4.2183 s。Attention 的 4 hart 2x4 为 0.999×，说明更大微内核不保证所有形状受益。
+Dense 的 1x4 → 2x4 对照中，CPU 中位数分别从 5.8332 → 3.6183 s、6.8124 → 4.2183 s。Attention 的 4 core 2x4 为 0.999×，说明更大微内核不保证所有形状受益。
布局测试保留原布局对照与容量/重叠/溢出/错误恢复/不同输入及工作区复用检查。完整模型实际采用 196 个 panel，逻辑与物理权重均为 1,761,607,680 bytes；共享 embedding/lm_head 与动态 RHS 保留原布局。ELF 必须与对应物理布局的权重 bundle 配套。布局收益没有新的 SoftMMU 采样支撑，不能精确拆成硬件缓存、TLB 或地址生成各自贡献。
@@ -67,18 +69,18 @@ Dense 的 1x4 → 2x4 对照中，CPU 中位数分别从 5.8332 → 3.6183 s、6
这些是不同阶段的诊断或实现对照，不能组合成整体收益。早期 WFI/回调复用及 padding 对照仍使用约 50 ms 完成轮询，须与后续事件计时区分。
-- **空闲等待：** 在 4 hart、64 次串行 Softmax 的历史对照中，WFI 使 CPU 9.780 → 2.125 s（每配置仅 2 次）。它主要减少空转；后续 Softmax 已按独立行并行，此数字不是当前完整模型收益。
-- **回调复用：** 64 个同形状回调变为 1 个；末轴回调机器码 17,920 → 280 B，非末轴 19,456 → 304 B，工作区仍为 2 MiB。单 hart text 未变，不宣称单核代码体积收益。
-- **有限性检查融合：** 后续事件复测的 256×1024、4 hart 中位前向 0.676418 → 0.649905 s（−3.92%），CPU 2.281089 → 2.300152 s（+0.84%）。独立线程采样显示 worker 非运行时间减少，但该区间同时包含调度等影响，不等于纯 WFI/屏障等待。
+- **空闲等待：** 在 4 core、64 次串行 Softmax 的历史对照中，WFI 使 CPU 9.780 → 2.125 s（每配置仅 2 次）。它主要减少空转；后续 Softmax 已按独立行并行，此数字不是当前完整模型收益。
+- **回调复用：** 64 个同形状回调变为 1 个；末轴回调机器码 17,920 → 280 B，非末轴 19,456 → 304 B，工作区仍为 2 MiB。单核 text 未变，不宣称单核代码体积收益。
+- **有限性检查融合：** 后续事件复测的 256×1024、4 core 中位前向 0.676418 → 0.649905 s（−3.92%），CPU 2.281089 → 2.300152 s（+0.84%）。独立线程采样显示 worker 非运行时间减少，但该区间同时包含调度等影响，不等于纯 WFI/屏障等待。
- **UART 事件计时：** 观测后唤醒中位延迟 25.367 → 0.098 ms，减少轮询量化；这是宿主完成检测改进，不代表 guest 计算加速。
-- **QMP 尾延迟：** nodelay 对照中 stop 中位耗时由 42.617 → 0.621 ms（1 hart）、42.227 → 0.760 ms（4 hart）。STOP 事件已先到达，服务端 NODELAY 与独立客户端 QUICKACK 对照强烈支持 TCP Nagle/ACK 交互造成回复尾延迟；没有抓包，不声称逐包证明，也不把这约 42 ms 计为模型计算收益。
+- **QMP 尾延迟：** nodelay 对照中 stop 中位耗时由 42.617 → 0.621 ms（1 core）、42.227 → 0.760 ms（4 core）。STOP 事件已先到达，服务端 NODELAY 与独立客户端 QUICKACK 对照强烈支持 TCP Nagle/ACK 交互造成回复尾延迟；没有抓包，不声称逐包证明，也不把这约 42 ms 计为模型计算收益。
- 正式运行器已采用 UART 事件与 QMP nodelay；高 FD 边界及 Host C 超时/取消进程清理随后补齐相关回归，没有借此重跑完整模型。
## 退化用例与剩余问题
-- **Padding 有明确反例：** 256×1024 的实验中位耗时下降，但 255×1024 的 1/4 hart 耗时分别增加约 22.22%/23.14%；四 hart 收益组也保留慢运行。因此仅作默认关闭的 QEMU 两缓冲选项，不作为泛用硬件/NPU 缓存策略。
+- **Padding 有明确反例：** 256×1024 的实验中位耗时下降，但 255×1024 的 1/4 core 耗时分别增加约 22.22%/23.14%；4 core 收益组也保留慢运行。因此仅作默认关闭的 QEMU 两缓冲选项，不作为泛用硬件/NPU 缓存策略。
- **RVV 暂不采用：** 43 次边界正确性与 54 次性能运行逐位一致；本次 VLEN128/e32m1 一行四列候选，耗时为预选强标量方案的 2.31–2.92 倍，CPU 为 2.61–2.98 倍。标量与 RVV 的 tile/布局并非完全一致，这是实用候选比较，不是孤立的 ISA 因果试验，也不代表真实硬件或全部 RVV 实现。
-- 4 hart 剩余波动和额外 CPU 尚未完全分解为宿主调度、频率、硬件缓存、QEMU 浮点、SoftMMU/TB 等因素。独立采样的比例不能换算为无采样运行的 CPU 秒数。
+- 4 core 剩余波动和额外 CPU 尚未完全分解为宿主调度、频率、硬件缓存、QEMU 浮点、SoftMMU/TB 等因素。独立采样的比例不能换算为无采样运行的 CPU 秒数。
- 原始数组、ELF、权重、perf 采样与源码快照仍只保存在本地，公开摘要不等同于独立重建执行证据。E1/E3/E5、Nightly、团队验收和真实 RISC-V/NPU 部署另行完成。
- W5 token 循环、文本生成和会话部署不属于本报告的完成声明。
diff --git a/docs/llm-deploy-v1.0/W4/validation.md b/docs/llm-deploy-v1.0/W4/validation.md
index 0301d733..7b4a1590 100644
--- a/docs/llm-deploy-v1.0/W4/validation.md
+++ b/docs/llm-deploy-v1.0/W4/validation.md
@@ -1,6 +1,8 @@
# W4 分层验证与 CI
-## 日常最小验证与多 hart
+## 日常最小验证与多核
+
+本文的 core 指 QEMU 模拟的 CPU 核，不表示宿主的物理核心。当前命令参数使用 `--cores`，Python 参数与属性使用 `cores`。
先按改动选择直接相关的测试；文档和配置改动只做对应检查。矩阵生成或并行运行时
改动可先运行以下小图回归（按[复现步骤](reproduction.md)设置 Python、编译器和 QEMU）：
@@ -11,19 +13,33 @@ python -B -X utf8 -m pytest -q \
tests/test_riscv_external_harts.py tests/test_w4_fma_cross_backend.py
```
-外置运行时 `build_external(..., harts=1)`、完整探测及检查点诊断入口的
-`--harts {1,2,4,auto}` 均默认单核，手动 1/2/4 保持原值。`--harts auto`
-（或 `build_external(..., harts="auto")`）在构建前按当前进程可用 CPU 数向下选择：
-1 个选 1 hart，2–3 个选 2 hart，4 个及以上选 4 hart。Linux 优先使用 CPU affinity；
+外置运行时 `build_external(..., cores=1)`、完整探测及检查点诊断入口的
+`--cores {1,2,4,auto}` 均默认单核，手动 1/2/4 保持原值。`--cores auto`
+（或 `build_external(..., cores="auto")`）在构建前按当前进程可用 CPU 数向下选择：
+1 个选 1 core，2–3 个选 2 core，4 个及以上选 4 core。Linux 优先使用 CPU affinity；
不可用时回退进程/系统 CPU 数，无法确定时保守选 1。此口径不估计瞬时空闲率或
cgroup CPU 带宽配额，也不受 `OMP_NUM_THREADS` 等 BLAS 线程设置影响。
+旧 `--harts`、Python 的 `harts` 参数和属性仍兼容。CLI 解析结果的 `args.harts`
+是 `args.cores` 的可读写别名，修改任一属性都会同步；默认值、手动值和 `auto`
+在 `parse_args()` 返回时仍保持原字符串类型。命令行、普通构造和运行时 API 中，
+新旧名称同时传入时必须一致，否则在构建或运行前报错。新构建与报告以 `cores` 为主要字段，同时保留等值的
+`harts` 兼容字段；有请求值的报告同样保留 `cores_requested` 与 `harts_requested`。
+读取旧记录时接受仅有 `harts` 的格式；两种核数字段均缺失时按原规则视为单核，
+两种字段不一致或值无效时拒绝复用。历史报告和 ELF/bundle 不因改名而改写。
+
+`dataclasses.asdict(executable)` 同时输出整数 `cores` 和等值的旧 `harts` 字段，
+不输出私有核数字段；正式 `executable.json` 的保存和读取方式保持不变。
+`dataclasses.replace(executable, cores=2)` 和旧 `harts=2` 写法都会同步两个字段。
+`replace()` 会自动补入原值，因此一边为原值、另一边为新值时按新值更新；
+两边都是不同于原值的新值且互不相等时拒绝。建议一次只用一个名称更新核数。
+
选择只发生在启动/构建时，运行中不切换；CLI 会打印自动选择结果，报告用
-`harts_requested` 记录请求、`harts` 记录实际整数。当前数学策略的默认值仍是
+`cores_requested` 记录请求、`cores` 记录实际整数。当前数学策略的默认值仍是
`sequential`；需要矩阵并行时必须同时显式使用 `--matmul-policy blocked_fma`。
MatMul 的多核路径只用于 `blocked_fma` 中足够大的矩阵，按独立输出块分工；
每个元素的 K 顺序和 K=128 分组保持不变，算子结束后才复用工作区。
-`sequential` 下的 MatMul 仍串行。每个 hart 独立使用 1 MiB 栈。
+`sequential` 下的 MatMul 仍串行。每个 core 独立使用 1 MiB 栈。
外置运行器在 Linux 上用 inotify 与 pidfd 等待已有 UART 完成帧，启动前注册目录监视，
不再固定每 50ms 轮询；平台或内核不支持时保留轮询回退。`run.json` 的 `timing_mode`
@@ -32,14 +48,14 @@ MatMul 的多核路径只用于 `blocked_fma` 中足够大的矩阵，按独立
QMP 服务端连接启用 `nodelay=on`，减少 STOP 事件与命令回复之间的小包等待；仍等待
匹配回复，并在停止后复核 UART 帧，再转储输出、检查退出和清理资源。
-外置 Softmax 在实际 hart 数大于 1、总元素数至少为 32768，且独立行数不少于
-hart 数时可并行；它不依赖 `matmul_policy`，因此显式选择多 hart 后，即使 MatMul
-策略为 `sequential`，符合条件的 Softmax 也会并行。默认仍为单 hart、`sequential`。
+外置 Softmax 在实际 core 数大于 1、总元素数至少为 32768，且独立行数不少于
+core 数时可并行；它不依赖 `matmul_policy`，因此显式选择多核后，即使 MatMul
+策略为 `sequential`，符合条件的 Softmax 也会并行。默认仍为单核、`sequential`。
独立行对应固定的 `(outer, inner)` 坐标，每行完整保留原有求最大值、逐项 `expf`
与求和、逐项归一化顺序，不拆分行内归约。输出有限性检查紧跟每个 FP32 归一化值
-的写回，不再由主 hart 在算子结束后重复扫描整个输出；检查本身及输入、权重、其它
+的写回，不再由主核在算子结束后重复扫描整个输出；检查本身及输入、权重、其它
算子的校验仍保留。工作线程通过原子状态回传数值错误，`sv_parallel_for` 等全部回调
-完成后，主 hart 检查错误状态，再进行后续工作区复用。inline 存储、单 hart、小图
+完成后，主核检查错误状态，再进行后续工作区复用。inline 存储、单核、小图
及独立行数不足时仍走串行路径，并在同样的逐行写回位置检查输出。
32768 元素只是保守的启用门槛，尚未校准为各形状的最佳阈值；继续复用已有门铃/WFI
同步和 `expf` 页对齐。此代码生成改动需要重新构建 ELF，复用旧 ELF 不会加入
@@ -48,11 +64,11 @@ Softmax 并行或写回时的有限性检查融合。
同一次生成中，输入形状、dtype 和规范化 axis 相同的外置并行 Softmax 复用回调代码。
只共享函数与结构体定义；每次算子调用仍使用独立的输入、输出指针和初始化为零的
错误状态，完成同步后才检查状态和复用工作区。此复用不改变单核默认、手动/自动
-hart 选择、并行门槛或行内数值顺序；也不代表真实硬件缓存或完整模型的等比例收益。
+core 选择、并行门槛或行内数值顺序；也不代表真实硬件缓存或完整模型的等比例收益。
代码生成 API 可显式比较 `TensorCCodegen(..., constant_storage="external",
matmul_policy="blocked_fma", matmul_tile="2x4")` 或 `matmul_tile="1x8"`。
-`matmul_tile` 默认仍为 `"1x4"`；候选仅适用于外置 `blocked_fma`，不改变 hart 的
+`matmul_tile` 默认仍为 `"1x4"`；候选仅适用于外置 `blocked_fma`，不改变 core 的
默认、手动或自动选择方式。`2x4` 复用两行之间的 B 加载，`1x8` 复用八列之间的
A 加载；每个输出仍按原 K 顺序执行显式 FP32 FMA，保留 128 项分组和块末加法。
尾行、尾列有边界保护，不使用 RVV 或 fast-math；面板布局由下述独立实验选项控制。
@@ -66,7 +82,7 @@ A 加载；每个输出仍按原 K 顺序执行显式 FP32 FMA，保留 128 项
`matmul_rhs_layout="column-panel"`；默认 `original` 保留原布局。该选项只打包二维
常量 RHS，允许 LHS 带批维；每个面板的列宽沿用 `matmul_tile`，物理布局为
`[ceil(N/NR), K, NR]`。它复用相邻列的权重访存，仍按递增 K 顺序执行 FMA，保留
-128 项分组与块末加法，不改变默认单核或手动、自动 hart 选择。
+128 项分组与块末加法，不改变默认单核或手动、自动 core 选择。
只有常量的全部消费者都把它用作 MatMul RHS 时才选择面板。独占常量经过合法二维
转置后仅供 MatMul RHS 使用时，在局部复制的 IR 容器中折叠该转置，交由原有生命周期
@@ -82,19 +98,19 @@ A 加载；每个输出仍按原 K 顺序执行显式 FP32 FMA，保留 128 项
`matmul_rhs_layout` 与 `matmul_rhs_panels` 在报告和构建证据中记录逻辑形状、物理形状、
原权重、折叠的转置及视图/复制方式；复用同时校验布局和面板记录。旧记录缺字段只兼容
`original`，面板构建必须重新生成；选择相同选项不意味着可以绕过已有源码、权重和
-构建身份校验。小图收益需与同 hart、同微内核、同计时方式的原布局比较，常量转置链
+构建身份校验。小图收益需与相同 core 数量、同微内核、同计时方式的原布局比较，常量转置链
还包含省去转置复制的收益，不能把总差值全部归为缓存效果。
外置 ELF 构建将频繁调用的 `expf` 代码段起始地址按 4096 字节页对齐，避免当前
实现因生成图代码长度变化而跨页、引入额外的 QEMU TCG 开销。这里只调整链接布局，
-不改变 `expf` 的数值实现、默认单核或手动/自动 hart 选择。应用此修复必须重新构建
+不改变 `expf` 的数值实现、默认单核或手动/自动 core 选择。应用此修复必须重新构建
ELF；`--reuse-build` 复用旧 ELF 不会应用新布局。收益应通过相同条件的小图对照确认，
不能直接外推到完整模型。
QEMU 两缓冲工作区可显式尝试实验选项
`TensorCCodegen(..., constant_storage="external", qemu_workspace_layout="two-buffer-pad-4k")`。
完整探测及检查点 CLI 对应 `--qemu-workspace-layout {none,two-buffer-pad-4k}`，
-默认 `none` 完整保留原布局；该选项不改变默认单核或手动、自动 hart 选择。
+默认 `none` 完整保留原布局；该选项不改变默认单核或手动、自动 core 选择。
仅当原工作区的所有分配可归入恰好两个固定、相邻、互不重叠的区间时，实验布局才在
两者之间插入 4096 字节，工作区需求也增加 4096 字节。其他图或容量不足会明确报错，
不会静默忽略选项；完整 Qwen 图和检查点不保证满足两缓冲条件。
@@ -107,8 +123,8 @@ QEMU 两缓冲工作区可显式尝试实验选项
与地址变化仍存在，应独立评估，不能把它当作 NPU 内核、硬件缓存或通用内存布局优化。
以下只是参数示例，不是要求运行完整模型：
-`--qemu-workspace-layout two-buffer-pad-4k --harts 4`。
-先用相同输入的 1、4 hart 小图检查位级输出一致性、容量不足处理、耗时、累计 CPU 时间、
+`--qemu-workspace-layout two-buffer-pad-4k --cores 4`。
+先用相同输入的 1、4 core 小图检查位级输出一致性、容量不足处理、耗时、累计 CPU 时间、
RSS 和 SoftMMU 样本，覆盖收益与退化形状。报告的 `qemu_workspace_layout_requested`
保留请求，`codegen.qemu_workspace_layout` 和构建证据记录所用布局；构建证据的
`qemu_workspace_buffers` 记录缓冲区位置。修改布局须重新生成、构建 ELF；复用时请求
@@ -116,7 +132,7 @@ RSS 和 SoftMMU 样本，覆盖收益与退化形状。报告的 `qemu_workspace
多核任务、就绪和完成等待使用 QEMU virt 的 MSIP 软件中断门铃与 WFI。
等待端先清门铃，再检查任务代次、完成状态和错误；发送端先发布状态再通知，
-避免通知先到时丢失唤醒。空闲 worker 和等待完成的主 hart 可休眠，默认单核路径
+避免通知先到时丢失唤醒。空闲 worker 和等待完成的主核可休眠，默认单核路径
不启用此机制。启动阶段极短的 BSS 就绪轮询保留；其他板型需适配中断控制器。
应同时比较同一候选、相同小图的耗时、累计 CPU 时间和输出位模式。
@@ -128,9 +144,9 @@ QEMU 小图，必要时再用 `diagnostic --checkpoint layer_0.output`。检查
`COMPLETE` 仅表示执行完成，还须查看 `diagnostic_threshold_passed`。完整模型与
原数组 audit 留到候选稳定、明确需要完整验收时运行，不作为每次编辑的默认步骤。
-同一候选重复执行可复用已有 `--reuse-build`；改变源码、数学策略或 hart 数必须重新
+同一候选重复执行可复用已有 `--reuse-build`；改变源码、数学策略或 core 数必须重新
构建。`auto` 也必须与当前探测后选择的数量一致，复用数量不符时明确报错；
-不会修改旧 ELF 的 hart 数。旧的未记录 hart 数的构建按单核处理，不能直接以多核运行。
+不会修改旧 ELF 的 core 数。旧的未记录 core 数的构建按单核处理，不能直接以多核运行。
## 按风险阅读实现和测试
@@ -227,7 +243,7 @@ step 200 分钟、job 240 分钟。正常失败上传已有报告与日志；宿
复现者 / 日期：
源码：候选完整 SHA、工作树是否干净；若为未提交快照，附基线 SHA、源码清单和快照 SHA256：
环境：宿主 OS/架构、Python/依赖、Zig/QEMU 实际版本及二进制 SHA256（见主报告）：
-执行：完整命令、case、MatMul 策略、hart 数（默认 1）；是否独立重新构建并执行 QEMU：
+执行：完整命令、case、MatMul 策略、core 数（默认 1）；是否独立重新构建并执行 QEMU：
单测：通过/失败/跳过数，tests.xml 位置；每项相关跳过原因：
完整门槛：build:riscv-full / smoke:qemu-full / numeric:qemu-full-qwen3：
数值：elements_compared、max_abs、atol、rtol；FMA 精确检查和离线 audit 结果：
diff --git a/docs/llm-deploy-v1.0/W5/README.md b/docs/llm-deploy-v1.0/W5/README.md
new file mode 100644
index 00000000..4ab56d23
--- /dev/null
+++ b/docs/llm-deploy-v1.0/W5/README.md
@@ -0,0 +1,667 @@
+# W5：端到端文本生成
+
+## 改动与功能
+
+W5 把 W4 的“单次模型前向”接成了**逐 token 文本生成**：本机 Python 处理提示词、选择下一个 token 并校验；QEMU 中的 RISC-V 程序执行模型，返回用于选词的分数（logits）。
+
+| 改动 | 实现的功能 |
+|---|---|
+| 生成循环与 QEMU 适配 | 提示词转为 token 编号 → 模型计算 → greedy 选最高分 token → 加入历史并继续；同一轮生成复用同一 ELF 和权重文件，不逐 token 编译 |
+| 命令行与部署 | 提供单条提示词、三条固定参考用例、官方模板多轮文本和容器部署入口 |
+| 校验与失败处理 | 逐步对比完整模型输出、选出的 token 和最终文字；支持超时、取消、进程清理及失败后复用；报告或日志写入失败也会报错 |
+| 本轮命名与兼容修复 | 统一使用 `--cores` / `cores`；兼容旧 `--harts` / `harts`、`parse_args().harts`、dataclass 导出和更新 |
+| 本轮文档与数据整理 | 集中说明实跑结果、计时范围和复现方法，补齐可随仓库查看的[数据摘要](reports/validation-summary.json) |
+
+**默认行为不变：**默认单核，支持手动 `--cores 1/2/4` 和启动时 `--cores auto`，上限 4；padding 关闭。默认算术策略不变，下面完整模型结果使用显式参考配置。
+
+**目前的限制：**没有 KV cache，每生成一个新 token 都重新计算完整模型、启动一次新的 QEMU 并装载权重。完整模型多轮聊天、三例独立容器运行和第二人复现，尚无本文可用的完整实跑记录。
+
+## 实验结果
+
+**完整模型数据来自 2026-10-23 的历史实跑，基线为 `8ad3474`；不是本轮命名、兼容或文档修改后重新测量。**
+
+- **模型与输入：**Qwen3-0.6B，完整 28 层、FP32，batch=1、固定长度 256；原始文本补全，不套聊天模板。
+- **运行配置：**QEMU / RV64GC，4 core；`blocked_fma / 2x4 / column-panel`，Mean 为 `two-four-lane-last-axis`，非线性为 `fp32-reference`，padding 为 `none`。
+- **生成规模：**每条生成 8 个新 token、执行 8 次完整前向；三条共 24 次，复用同一 ELF 和权重文件。
+
+| 输入提示词 | 实际新生成文字 | 端到端生成与校验耗时（秒） | QEMU 仿真总耗时（秒） | 最大绝对误差 |
+|---|---|---:|---:|---:|
+| `The capital of France is` | ` Paris. The capital of Italy is Rome` | 3375.024 | 3199.695 | 0.000303983688 |
+| `The capital of Germany is` | ` Berlin. The capital of France is Paris` | 3668.852 | 3482.999 | 0.000262737274 |
+| `The capital of Japan is` | ` Tokyo. The capital of the United States` | 3489.655 | 3315.397 | 0.000197887421 |
+
+
+
+
+表中名称说明：
+
+- **端到端生成与校验耗时**：从开始处理这条提示词，到全部 8 个新 token 生成、转换为最终文字并完成主要校验，
+ 实际经过的时间。包含输入处理、每步 QEMU 启动与权重装载、模型计算、结果读回和进程清理，
+ 以及参考计算、逐步数值校验和最终文字与冻结参考的核对。
+ **最终文字已经生成并经过校验，这部分时间已包含在内；不含前置编译，也不含最后的报告整理、保存和终端显示。**
+ 计时顺序是：**处理提示词 → 逐个生成 8 个 token 并校验 → 转换为最终文字并核对 → 计时结束**；
+ 随后才整理、保存报告并打印文字。因此，它不是纯模型计算时间，也不是从部署到显示结果的全部链路耗时。
+- **QEMU 仿真总耗时**：同一条用例生成 8 个新 token 时，8 次 QEMU 仿真从启动到退出回收的
+ **实际经过时间之和**。包含模拟器启动、装载权重、模型计算、输出转储和退出清理；
+ 不包含前置编译、QEMU 启动前的输入准备和文件检查，也不包含进程外的 ORT 参考计算和完整结果校验。
+ **表中是 8 次仿真的累计耗时，不是单次前向耗时，也不是只执行模型算子的纯计算时间。**
+ 它已包含在“端到端生成与校验耗时”中，两者不能相加。
+- **最大绝对误差**：该用例中，模型输出分数与参考结果之间最大的绝对偏差，计算和判定方法见下文。
+
+
+
+
+8 次 QEMU 运行如何生成一条结果？
+
+三条提示词初始都为 5 个 token。第 1 步用这 5 个 token 选出第 1 个新 token；
+第 2 步用提示词和已选结果，共 6 个 token，选出第 2 个；依次继续，
+8 次前向的有效输入长度为 **5、6、7、8、9、10、11、12**，每次都右填充到 256。
+第 8 个新 token 产生后，历史长度变为 13，达到生成预算，不再执行第 9 次前向。
+
+每步用 greedy 选出下一个 token，最终将 8 个新 token 转成同一条输出，
+不是生成 8 个回答再择优。无 KV cache，复用的是同一 ELF 和权重文件；
+Python/QEMU 的具体职责见[流程表](#generation-flow)。
+
+
+
+
+
+
+最大绝对误差：和谁比、怎么算、能说明什么？
+
+1. **和谁比较。** 同一份 Qwen3 模型、相同的 token 输入及遮罩，分别由 ScratchV 编译后在 QEMU
+ 中执行，以及由 ONNX Runtime（ORT）参考程序计算。对比的是模型输出的 **logits**，
+ 即选下一个 token 时使用的候选分数；这些原始分数不是概率，也不是最终文字。
+ ORT 在这里提供数值基准，不是判断回答是否符合事实的裁判。
+2. **怎么算。** 每个对应位置的 QEMU 分数减去 ORT 分数，去掉正负号，再选出最大的差值：
+ `最大绝对误差 = max(|QEMU 分数 − ORT 分数|)`。
+ 例如参考分数为 1.2500、QEMU 为 1.2503，这个位置的误差约为 0.0003。
+ 每次前向比较全部 **38,895,616 个输出分数**（包括固定长度中补齐的位置），
+ 表中再取该用例 **8 次前向中的最大值**，不是平均误差，也不只检查最终选中的 token。
+3. **为什么小于 0.001 才通过。** 这是[开发计划](../开发计划.md)为完整 QEMU 前向规定的数值容差，
+ W5 沿用 W4 的 `ATOL = 1e-3`。它给不同浮点计算实现之间的偏差设定上限；
+ FP32 舍入、运算顺序和数学函数实现差异可能使两个后端的数值不完全一样。
+ 代码要求严格 **`max_abs < 0.001`**，等于 0.001 也不通过；
+ `rtol=0` 表示不再因分数较大而增加相对误差宽限。
+ **0.001 是项目约定的分数偏差上限，不是“错误率低于 0.1%”，也不是普遍适用的正确性证明。**
+4. **能否说明生成结果正确。** 单看误差小，只能说明这次计算的分数接近参考。
+ 当两个候选 token 的分数很接近时，小偏差也可能改变排名、选出另一个 token。
+ 因此，完整用例通过还要求：全部输出是有限数值，没有 NaN 或正负无穷；
+ 每一步实际选出的 token 与当步 ORT 结果及[冻结参考](../../../probes/w5_generation/references.json)一致；
+ 最终 token 序列、文字和停止原因也与冻结参考完全一致。不能仅凭误差小于 0.001 判整条用例通过。
+
+上述计算和检查见[生成入口](../../../scripts/run_inference.py)的 `compare_step` 与
+`reference_suite` 路径。**这里的三条固定用例共 24 步同时通过了数值与生成结果检查**，
+说明所列历史版本和显式配置在这些用例上与参考一致；不代表所有提示词都会得到相同结果，
+也不保证模型生成的内容在事实层面永远正确。本次只解释已有结果，没有重新执行模型。
+
+
+
+
+展开三例合计耗时、内存和采样限制
+
+整次测量 10540.190 s（约 2 小时 55 分 40 秒），包括准备、ORT 和数值比较。
+24 次前向的**启动至完成通知耗时**合计 9990.355 s，平均 **416.265 s/token**；
+**QEMU 仿真总耗时**合计 9998.091 s。这两个耗时指标的起止位置见[单次前向表说明](#container-results)。
+**采样到的内存峰值**：模拟器 5.406 GiB、主控程序 4.254 GiB、同时运行的进程合计 7.644 GiB。
+17/24 步各有一次内存采样读取失败，属于部分采样（`partial`），可能漏掉更高的瞬时占用。
+同时进程合计是在同一采样时刻求和后取最大值，不能直接相加各进程在不同时刻的峰值；
+按 RSS 求和也可能重复计算共享内存页。
+
+
+
+**结果：3/3 用例、24/24 步通过。** 每步完整 logits 与同模型、同输入的 ONNX Runtime（ORT）结果比较，均满足 `max(|QEMU − ORT|) < 0.001`，且全部数值有限；每步 token ID、最终文字和停止原因也与冻结参考一致。这说明所列用例与参考一致，不代表所有提示词的回答质量。
+
+**停止原因：**三条都是 `max_new_tokens`，即达到 8-token 预算，未遇到 EOS；Japan 的续写尚未成句，不能理解为模型已自然回答完毕。表中生成文字开头均有一个空格。
+
+**计时范围：**两列都是实际经过时间。端到端时间包括最终文字生成、ORT 参考计算和校验，不含前置编译、最终报告保存及终端显示；QEMU 时间是 8 次仿真从启动到退出回收的总和，已包含在端到端时间内，不能相加。
+
+**干净容器单次前向：**本机与容器都生成 ` Paris`，最大绝对误差均为 `0.000172138214`，两边完整输出逐字节相同。本机内部计时 **444.416 秒**且复用构建；容器 **547.454 秒**，其中本次 C 编译与链接 **93.868 秒**。构建条件不同，这不是容器性能对比，也不代表容器三例全部完成。
+
+本轮命名兼容修复另有真实 1/4 core 小图对照，逐步输出与 ORT 完全一致、最大误差为 **0**，取消和超时后的清理与复用通过；完整模型数据仍沿用上面的历史记录。
+
+数据入口：[逐 token 数值与资源摘要](reports/validation-summary.json) · [冻结参考](../../../probes/w5_generation/references.json) · [本轮相关验证记录](../../../tests/llm-deploy/w5/README.md)。计时、误差、逐 token 生成和内存说明见表格下方的折叠项；输入输出细节与历史 CI 记录见[其他](#其他)。
+
+## 如何复现
+
+**下面是在当前代码上运行三条固定用例的最短路径，不是本次已重新执行的记录。** 从仓库根目录执行；使用新输出目录，以免覆盖已有结果。
+
+1. **准备环境。** 按 [Linux 复现指南：W5 环境与固定资产](../LINUX_REPRODUCTION.md#w5-local-environment)完成准备；需要容器或第二人独立复现时，使用同一指南的[容器流程](../LINUX_REPRODUCTION.md#w5-container-reproduction)。已有环境可直接继续。
+2. **一次运行三例。** `--reference-suite` 已包含三条提示词及每条 8-token 预算，不再传 `--max-new-tokens`。只构建一次，跨用例复用同一 ELF 和权重文件。历史三例整次测量约 2 小时 56 分钟，实际时间取决于机器与环境。
+
+```bash
+python scripts/run_inference.py \
+ --model assets/qwen3-0.6b/onnx/model.onnx \
+ --tokenizer-dir assets/qwen3-0.6b/tokenizer \
+ --reference-suite --verify-onnx assets/qwen3-0.6b/onnx/model.onnx \
+ --cores 4 --matmul-policy blocked_fma --matmul-tile 2x4 \
+ --matmul-rhs-layout column-panel \
+ --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \
+ --timeout 1800 --total-timeout 16500 \
+ --output-dir output/w5-readme-repro-new
+```
+
+3. **核对结果。** 查看 `output/w5-readme-repro-new/report.json`：应有三条用例、24 次模型前向，文字与上表一致，逐步完整 logits 有限且最大绝对误差严格小于 0.001；token ID 与参考一致，停止原因均为 `max_new_tokens`，无进程清理错误。只出现城市名不能代替全部检查。构建描述在 `executable.json`，每步记录在 `steps/`。
+
+已有构建可用 `--reuse-build ` 替换 `--model ...`，保留原核数和全部算术配置，仍使用新的输出目录。上述完整三例命令不是小图测试，只有确实需要复现完整结果时才执行。
+
+若要**复现表中的历史版本**，使用[独立容器复现流程](../LINUX_REPRODUCTION.md#w5-container-reproduction)中的固定提交 `8ad3474` 和旧 `--harts` 参数；不要把旧版本复现结果当作本轮兼容修复的验证。
+
+## 其他
+
+
+实现细节、接口兼容与补充数据
+
+
+
+### 接口与默认配置
+
+W5 将 W4 的单次 QEMU 前向接成文本生成：本机 Python 准备输入、选择 token、控制循环并校验，
+QEMU 中的 RISC-V 程序执行模型。支持原始文本补全、三条固定参考用例及
+[官方模板多轮文本](../../../scratchv/runtime/conversation.py)；
+[命令行入口](../../../scripts/run_inference.py)、
+[部署脚本](../../../scripts/deploy_w5.sh)与[Dockerfile](../../../deployment/w5/Dockerfile)
+使用同一条生成流程。
+
+
+
+**先看已经取得的结果。** 2026-10-23 的历史实跑中，完整 28 层 Qwen3-0.6B 使用显式
+4 core 配置，三条用例各生成 8 个新 token，**24/24 步与冻结参考一致**；
+最大绝对误差 **0.000303984**，每条端到端生成与校验约 **56～61 分钟**。
+具体文字、计时范围和模型配置见[实际复现结果](#实际复现结果)。
+这些是 `8ad3474` 功能基线的历史数据，不是后续命名和兼容修复的新测量。
+
+**当前限制。** 无 KV cache，每个新 token 都执行一次完整前向并启动新的 QEMU。
+固定 8-token 预算不代表自然回答完毕；完整模型三轮聊天、三例独立容器运行及第二人复现
+尚无本报告可用的实跑记录，见[已验证范围与剩余问题](#remaining-scope)。
+
+| 想了解什么 | 阅读入口 |
+|---|---|
+| 输入如何流动、Python 和 QEMU 各做什么 | [实现说明](#整体怎么实现)、[逐步流程表](#generation-flow) |
+| 实际生成了什么、运行多快、如何判定正确 | [运行配置](#run-configuration)、[三条结果](#fixed-case-results)、[耗时表](#full-generation-results)、[误差判定](#max-absolute-error) |
+| 数据与历史 CI 在哪里 | [报告入口](#report-locations)、[CI/benchmark 快照](#ci-and-benchmark) |
+| 如何自己执行并核对 | [如何复现](#如何复现)，命令默认折叠 |
+
+**默认配置保持不变**：单核、MatMul `sequential / 1x4 / original`、Mean `sequential`、
+非线性 `libm`、padding `none`。可手动指定 `--cores 1/2/4` 或 `--cores auto`；
+auto 按进程可用 CPU 在构建或启动时选择 1/2/4，上限 4，不在 token 间动态调整。
+完整模型通过的记录使用下文显式参考策略，不代表默认算术策略也通过同一数值要求。
+W5 没有 `--qemu-workspace-layout` 参数，padding 始终关闭。
+
+旧名称与 Python 接口的兼容细节见[兼容说明](#core-compatibility)。
+
+
+
+
+展开 core/hart 名称与旧接口兼容说明
+
+本文的 core 指 QEMU 模拟的 CPU 核。
+[hart（hardware thread）](https://docs.riscv.org/reference/debug/introduction.html) 是 RISC-V 的正式术语，指硬件线程；
+本项目当前 QEMU 配置中每个模拟核对应一个 hart，因此改用更直观的 core 提高接口可读性，
+执行方式不变，旧名称仍兼容；在其他硬件上，hart 与物理核不一定一一对应。
+当前命令使用 `--cores`，Python 参数和属性使用 `cores`。
+
+旧 `--harts`、Python 的 `harts` 参数和属性仍兼容。CLI 解析结果的 `args.harts`
+是 `args.cores` 的可读写别名，修改任一属性都会同步；默认值、手动值和 `auto`
+在 `parse_args()` 返回时仍保持原字符串类型。命令行、普通构造和运行时 API 中，
+新旧名称同时传入时必须一致，否则在构建或运行前报错。新报告用 `cores_requested` 记录请求、`cores` 记录实际数量，
+并保留等值的 `harts_requested`、`harts` 兼容字段。旧构建记录只有 `harts` 时仍可读取；
+两种核数字段均缺失时按原规则视为单核，两种字段不一致或值无效时拒绝复用。
+历史报告不改写，复用已有 ELF/bundle 仍须通过原有配置和身份检查。
+
+`dataclasses.asdict(executable)` 同时输出整数 `cores` 和等值的旧 `harts` 字段，
+不输出私有核数字段；正式 `executable.json` 的保存和读取方式保持不变。
+`dataclasses.replace(executable, cores=2)` 和旧 `harts=2` 写法都会同步两个字段。
+`replace()` 会自动补入原值，因此一边为原值、另一边为新值时按新值更新；
+两边都是不同于原值的新值且互不相等时拒绝。建议一次只用一个名称更新核数。
+
+
+
+
+
+### 输入输出与生成流程
+
+
+
+**从提示词到文字结果。** 本节按三条固定用例的实际路径说明：本机 Python 控制生成循环，
+QEMU 内的 RISC-V 程序执行完整 Qwen3 模型，ONNX Runtime（ORT）在本机计算参考结果。
+`token` 是 tokenizer 划分和处理文字的单位，`token ID` 是它对应的编号；
+`logits` 是模型为各候选 token 给出的分数。
+QEMU 返回的是这些分数，最终文字由 Python 转换得到。
+
+**当前没有 KV cache（保存前面 token 的注意力中间结果的缓存）。**
+每生成一个新 token，都把“提示词 + 已生成 token”作为输入，重新执行完整 28 层模型。
+每条固定用例因此需要 **8 次完整前向和 8 次 QEMU 运行**，尚未采用基于缓存、
+只计算新 token 的增量 decode，也不是只执行“一次 prefill + 一次 decode”。
+这里的 8 次指完整模型前向，不包括运行前的浮点自检小程序。
+
+| 步骤 | 执行位置 | 输入、具体处理与输出 | 实现入口 |
+|---|---|---|---|
+| 运行前准备 | 本机 Python + 编译器；自检会调用 QEMU | 新构建时，将 ONNX 解析成中间表示，生成 C 程序、写权重包并编译链接出 ELF；复用时检查已有 ELF、权重包和配置。浮点自检与 ORT 参考程序初始化也在生成循环之前完成 | [CLI](../../../scripts/run_inference.py)的 `prepare_executable` / `main` |
+| 提示词编码 | 本机 Python | 原始文字经 tokenizer 转成 token 编号。三条用例是原始文本补全，不套聊天模板，不额外插入起止 token | [生成循环](../../../scratchv/runtime/generation.py)的 `generate` |
+| 准备本步输入 | 本机 Python | 将当前完整 token 历史右侧补齐到 256 个位置，并构造限制可关注位置的注意力遮罩，得到模型输入数组；有效长度另行保留，供选词使用 | [输入处理](../../../scratchv/runtime/llm_inputs.py)的 `prepare_inputs` |
+| 本步参考计算 | 本机 Python 调用 ORT | 固定用例先独立核对本步实际输入与冻结历史一致，再用这些输入运行同一份 ONNX 模型，得到 ORT 参考 logits | [CLI](../../../scripts/run_inference.py)的 `forward`；[参考输入检查](../../../probes/w2_runtime_model/run.py)的 `checked_inputs` |
+| 模型前向 | QEMU 模拟执行 RISC-V 程序 | QEMU 装载 ELF、输入二进制和权重；其中的 RISC-V 程序调用生成的模型函数，计算全部输出分数并写入输出内存，通过 UART 串口发送完成状态及输出字节数 | [运行器](../../../scratchv/runtime/riscv_external.py)的 `guest_harness` / `run_external` |
+| 读回与逐步校验 | 本机 Python | 验证 UART 成功完成帧后，用 QMP 暂停并复核完成帧，将输出内存保存为 `output.bin`，退出并回收进程；检查输出、退出码和清理结果，再与 ORT 比较，失败立即停止 | [QEMU 适配器](../../../scratchv/runtime/qemu_generation.py)；[CLI](../../../scripts/run_inference.py)的 `compare_step` |
+| 选择下一个 token | 本机 Python | 从 QEMU 输出中取最后有效位置的分数，用 greedy 选择最高分 token；同分取较小编号，无额外 Softmax。把实际选出的编号加入历史，未停止则回到“准备本步输入” | [输入与选词](../../../scratchv/runtime/llm_inputs.py)的 `last_valid_logits` / `greedy_token`；[生成循环](../../../scratchv/runtime/generation.py) |
+| 结束与结果输出 | 本机 Python | 满足停止条件后，把新生成的编号序列解码为文字；固定用例再核对完整编号、文字、停止原因与步数，保存报告并打印文字 | [生成循环](../../../scratchv/runtime/generation.py)；[CLI](../../../scripts/run_inference.py)的 `main` |
+
+**QEMU 不直接接收提示词字符串，也不直接打印模型回答。** UART 完成消息不是文本生成结果；
+模型输出是内存中的 logits，经 QMP 读回后，由 Python 选 token 和解码。
+表中的“本机 Python”表示 QEMU 外的控制程序；容器运行时，这套 Python 程序与 QEMU 一起在容器内运行，职责不变。
+
+同一条生成循环复用同一 ELF 和权重包，每步只更新输入；**没有逐 token 重复编译**。
+但每步仍启动新的 QEMU 进程并装载权重，不能把复用构建文件理解为 QEMU 或 KV cache 常驻。
+前置编译与初始化、生成阶段、最终报告保存各自的计时范围见[结果表说明](#full-generation-results)。
+
+
+展开模型输入输出的具体格式、遮罩和停止规则
+
+`generate(prompt, *, tokenizer, forward, max_new_tokens, cancelled=None, timeout_seconds=None)`
+返回 `GenerationResult`，包含提示词编号、新生成编号、最终文字和 `stop_reason`。
+
+| 项目 | 实际行为 |
+|---|---|
+| 提示词 | 原始文本模式直接编码，`add_special_tokens=False`；沿用 tokenizer 的 NFC 归一化 |
+| token 输入 | INT64 `input_ids[1,256]`；前面放有效编号，右侧用模型的 `pad_token_id` 补齐 |
+| 注意力遮罩 | FP32 `attention_mask[1,1,256,256]`；位置 `q` 只允许关注 `k <= q` 且 `k < valid_length` 的位置。允许处为 0，其余为 FP32 最小有限值；不是把补齐的 query 行全部屏蔽 |
+| 模型输出 | FP32 `logits[1,256,151936]`；包含全部位置的候选分数，所有元素都必须有限，包括补齐位置 |
+| 用于选词的输出 | 取 `logits[0, valid_length-1, :]`，即最后有效位置的完整词表分数；不是固定取第 256 行 |
+| 有效词表 | tokenizer 为 151669，模型输出宽度为 151936；若最高分落在不可解码 ID，报错，不过滤后重新选择 |
+| 停止顺序 | 新 token 生成后依次检查 EOS、上下文容量 256、新 token 预算；EOS IDs 为 `151645`、`151643` |
+| 空间和预算 | 提示词非空且不超过 256，不隐式截断；零预算或初始上下文已满时不调用模型前向 |
+| 失败与中断 | 不把异常、超时或取消的部分生成当作成功；适配器负责 QEMU 进程回收 |
+
+形状、遮罩和停止顺序见[输入实现](../../../scratchv/runtime/llm_inputs.py)；
+生成循环见[生成实现](../../../scratchv/runtime/generation.py)。
+
+
+
+
+展开如何校验：相同输入、完整分数、逐 token 选择与最终文字
+
+- **先确认模型程序成功执行。** UART 必须是完整合法帧，状态为 0、输出字节数正确；
+ 暂停 QEMU 后还会复核帧，并检查转储文件大小、输出有限性、退出码及清理结果。
+ 收到一条消息，或仅看到进程退出，都不能代替后续数值校验。
+- **同输入的数值参考。** `--verify-onnx` 使用相同输入运行 ORT，比较整个输出数组，
+ 包括补齐位置；要求类型、形状正确、所有值有限、严格 `max_abs < 1e-3`、`rtol=0`，
+ 并且最后有效位置的 greedy ID 相同。top-2 margin 只作诊断记录，不是额外通过阈值。
+ 误差如何计算及能说明什么见[误差解释](#max-absolute-error)。
+- **独立核对生成输入。** `--reference-suite` 从
+ [冻结参考](../../../probes/w5_generation/references.json)维护独立的参考历史，
+ `checked_inputs` 逐元素检查实际 token 输入和遮罩，再把核对后的实际输入交给 ORT。
+ QEMU 仍接收生成循环准备的实际输入。
+- **生成程序自己选择 token。** 每步要求 QEMU、ORT 和冻结参考选出的 ID 一致；
+ 只有通过后才推进独立参考历史。实际生成循环仍从 QEMU logits 中自行 greedy 选择，
+ 并把这个结果加入自己的历史，不能用冻结答案替代模型输出。
+- **最终用例检查。** 比较提示词 IDs、新 token IDs、解码文字、停止原因，以及前向次数和校验次数；
+ 同时检查要求出现的文字。只出现 `Paris` 等城市名，或只有最大误差达标，都不足以判整例通过。
+- **参考与实跑记录分开。** 冻结参考由[参考采集程序](../../../probes/w5_generation/run.py)运行 ORT 得到；
+ 其中 `baseline.qemu_e2e=NOT_RUN` 只表示采集参考时未运行 QEMU。之后的完整 QEMU 实跑
+ 见[实际结果](#fixed-case-results)和[历史数据摘要](reports/validation-summary.json)，不能混用两种记录。
+
+`--reference-suite` 必须使用 `raw` 模式并同时提供 `--verify-onnx` 参考模型，
+使用冻结的 token 预算，不接受另传 `--max-new-tokens`；它也不会自动选择算术策略。
+完整三例历史实跑显式使用以下选项，不代表默认算术策略也完成了相同的完整模型验证：
+
+```text
+--cores 4 --matmul-policy blocked_fma --matmul-tile 2x4
+--matmul-rhs-layout column-panel
+--mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference
+```
+
+
+
+
+多轮对话与失败处理
+
+**多轮文本。** `Conversation.from_directory(tokenizer_directory, system_prompt=None)`
+加载同一快照的 `tokenizer_config.json` 官方 `chat_template`，通过 Jinja2 不可变沙箱
+渲染，固定 `add_generation_prompt=True`、`enable_thinking=False`。生产路径使用
+tokenizers/Jinja2，不依赖 Transformers/Torch。仅支持文本 system/user/assistant，
+system 只在构造时提供；每轮重新渲染完整历史并编码，不截断旧消息，不缓存 KV。
+
+`turn(...)` 只有成功且至少生成一个 token，才同时保存 user 和 assistant；异常、取消、
+超时、零预算或没有生成空间时历史不变。EOS-only 可以保存合法的空 assistant 回复。
+`history` 是不可变快照，`reset()` 清除已完成轮次但保留初始 system。历史保存文本，
+不承诺下一轮 token IDs 直接拼接；超过 256-token 容量会在前向前失败。单个会话拒绝
+并发或重入 turn/reset。原始三例不使用聊天模板。
+
+**容器运行。** Python 与 QEMU 在容器内仍承担相同职责；镜像、资产挂载及运行准备见 [Linux 复现指南](../LINUX_REPRODUCTION.md#w5-container-reproduction)。
+
+部署预检和构建客户端各有独立进程组，SIGINT/SIGTERM 时转发终止并等待清理，必要时
+5 秒后强制结束，返回 130/143。容器运行阶段保留 `exec ... run --init` 信号转发，
+QEMU 适配器负责其子进程回收。CLI 最终保存报告失败时向 stderr 说明保存错误，
+保留已有生成失败或取消的退出码 1/130/143；原本成功或仅构建成功时改为退出 1。
+数值、参考文本等原始错误不会被保存错误覆盖，也不打印误导性的成功报告路径。
+运行中的 SIGINT/SIGTERM 均记录为取消，分别保留退出码 130/143。
+
+本分支的报告检查器支持 `--log-exit-code-file`：指定后要求生成与日志退出码均为 0，
+日志状态缺失、不可读或内容无效也返回 FAIL。外层 Bash 分别保存
+`runner-exit-code.txt` 和 `log-exit-code.txt`；生成失败优先保留原退出码，
+生成成功而 `tee` 失败时返回日志退出码。环境指南中固定 `8ad3474` 的复现命令保留原版
+参数，仍须按其成功要求单独核对日志退出码，不向旧检查器传入新参数。
+
+
+
+
+
+### 详细测量与验证记录
+
+下文使用**“本机直接运行”**和**“容器内运行”**区分是否通过 Docker；
+具体执行链路、运行环境和计时范围见单次前向表后的说明。
+
+
+
+**历史完整模型运行配置。** 以下数字来自 **2026-10-23 的实跑**，代码功能基线为
+`8ad347477f7f5ba15b2da7c91178b3c6668456cc`。后续报告、取消、日志修复及 core 命名兼容
+工作只做相关验证；本次整理文档没有重跑模型，不能把旧数字当作当前版本的新测量。
+
+| 项目 | 这次三条固定用例的配置 |
+|---|---|
+| 模型 | `Qwen/Qwen3-0.6B`，完整 28 层，FP32 |
+| 输入与生成 | batch=1，固定序列长度 256；原始文本补全，不使用聊天模板；每条生成 8 个新 token |
+| 生成方式 | greedy，每次取最高分 token；无 KV cache，每步执行完整前向 |
+| 模拟器与核心数 | QEMU 模拟 RV64GC，显式使用 4 core；三条用例复用同一 ELF 和权重包 |
+| 算术与布局 | `blocked_fma / 2x4 / column-panel`；Mean `two-four-lane-last-axis`；非线性 `fp32-reference`；工作区 padding `none` |
+| 参考与环境 | 本机 WSL2 Ubuntu，4 个可用逻辑 CPU；ORT 通过 `CPUExecutionProvider` 计算参考输出 |
+
+模型快照与工具版本见 [Linux 复现指南](../LINUX_REPRODUCTION.md#w5-measured-environment)。
+
+
+
+**三条实际输出与停止原因。** 下表来自实际 QEMU 运行报告，并已与冻结参考逐项核对，
+不是只展示预设的期望答案。每条均为 8 个新 token、8 次完整前向，`reference_passed=true`；
+全部 24 次前向的数值和生成结果通过，QEMU 进程及进程组均已回收。
+
+| 用例 | 原始输入提示词 | 实际新生成文字 | 实际停止原因 |
+|---|---|---|---|
+| France | `The capital of France is` | ` Paris. The capital of Italy is Rome` | `max_new_tokens`（达到 8 个 token 预算） |
+| Germany | `The capital of Germany is` | ` Berlin. The capital of France is Paris` | `max_new_tokens`（达到 8 个 token 预算） |
+| Japan | `The capital of Japan is` | ` Tokyo. The capital of the United States` | `max_new_tokens`（达到 8 个 token 预算） |
+
+三条输出开头均保留一个空格。**此次没有遇到 EOS，停止是因为预算用完，不代表自然回答完毕。**
+例如 Japan 的输出停在 `the United States`，后续句子尚未补全；
+这些是原始提示词的续写，不是聊天模板下的问答质量评测。
+
+
+展开三条用例的实际输入及新生成 token 编号
+
+| 用例 | 提示词 token IDs | 实际新生成 token IDs |
+|---|---|---|
+| France | `[785,6722,315,9625,374]` | `[12095,13,576,6722,315,15344,374,21718]` |
+| Germany | `[785,6722,315,9856,374]` | `[19846,13,576,6722,315,9625,374,12095]` |
+| Japan | `[785,6722,315,6323,374]` | `[26194,13,576,6722,315,279,3639,4180]` |
+
+
+
+对应的预期结果见[冻结参考](../../../probes/w5_generation/references.json)，
+实际结果与 24 步误差见[历史数据摘要](reports/validation-summary.json)的 `three_case_generation`。
+[报告入口](#report-locations)说明哪些文件随仓库保存、哪些原始数据只保留在本机。
+
+
+
+**三条固定用例的耗时与数值误差。** 对应上述实际文字结果，均为本机直接运行。
+
+**每行对应同一条提示词生成 8 个新 token：串行执行 8 次前向，每次使用 4 core。**
+本次按固定 token 预算结束，不代表让模型自由回答到自然结束。
+“QEMU 仿真总耗时”是这 8 次模拟器运行的实际经过时间相加。
+**本节各表的耗时均按实际经过时间展示，不把同时工作的多个线程的 CPU 用时相加。**
+累计 CPU 用时仍保留在[原始数据摘要](reports/validation-summary.json)中，不作为运行速度指标列入表格。
+
+| 用例 | 端到端生成与校验耗时（秒） | QEMU 仿真总耗时（秒） | 最大绝对误差 |
+|---|---:|---:|---:|
+| France | 3375.024 | 3199.695 | 0.000303983688 |
+| Germany | 3668.852 | 3482.999 | 0.000262737274 |
+| Japan | 3489.655 | 3315.397 | 0.000197887421 |
+
+
+
+
+
+
+
+
+
+
+
+
+
+**独立单次完整前向与干净容器 Paris。** 两次都只生成 `[12095]`、文本 ` Paris`，
+完整输出分数的最大绝对误差同为 `0.000172138214`；比较对象和通过条件与
+[上面的误差说明](#max-absolute-error)相同，这里每边只生成 1 个新 token。
+容器内重新构建一次，没有重复三例套件；
+155,582,464 字节完整输出与本机单次前向逐字节相同。
+
+**两列采用相同的程序内部计时方式，但构建方式不同；这是运行记录，不是容器性能开销对比。**
+
+| 测量项 | 本机直接运行（复用构建） | 容器内运行（重新编译） |
+|---|---:|---:|
+| 推理程序内部耗时（秒） | 444.416 | 547.454 |
+| C 程序编译与链接耗时（秒） | 复用已有构建，本次未重新编译完整模型 | 93.868 |
+| 启动至完成通知耗时（秒） | 418.954 | 415.058 |
+| 模拟器进程总耗时（秒） | 419.383 | 415.478 |
+| 采样到的模拟器内存峰值（GiB） | 5.398 | 5.398 |
+
+**本机直接运行**表示整条推理链路在 Docker 容器外执行，**容器内运行**表示同一链路在容器内执行。
+两边的模型计算都由 QEMU 完成。本次均在同一台电脑的 WSL2 Ubuntu 环境中，
+不是 Windows 原生与另一台电脑的对比；也不表示每次都重新下载模型、安装依赖或编译。
+
+**这张表的计时范围。** 以下时间项均为实际经过时间，包含关系不能相加。
+本机复用构建，容器的 **93.868 秒编译已经包含在 547.454 秒内部耗时中**；
+构建工作不同，不能把两列总时间之差视为容器性能损耗。
+
+| 名称 | 从哪里计到哪里 | 包含与排除 |
+|---|---|---|
+| 推理程序内部耗时 | 程序开始准备 → 按需构建、自检、参考程序初始化 → 生成与校验 → 程序内收尾 | 使用 `run/report.json` 的 `elapsed_seconds`；只计本次实际发生的工作，不计此前构建，也不含内部计时开始前的 Docker 镜像准备或容器启动 |
+| C 程序编译与链接耗时 | 启动编译工具 → 生成并检查模型 ELF | `compile_seconds` 包含编译器启动、C 编译、汇编、链接、日志保存及 ELF 检查；不含此前的模型解析、C 源码生成、权重打包和之后的推理 |
+| 端到端生成与校验耗时 | 处理提示词 → 得到最终文字并完成主要校验 | 是内部耗时的中间阶段，不含前置编译、浮点自检、ORT 初始化及最后报告保存、终端显示；含每步 ORT 参考计算。三例各生成 8 个 token，本表每边只生成 1 个 |
+| 启动至完成通知耗时（原 UART 耗时） | QEMU 启动 → Python 确认合法的成功完成帧 | 包含启动、权重装载、模型计算和通知接收；不是串口传输时间，也不是纯算子时间。输出读回和数值校验尚未完成 |
+| 模拟器进程总耗时 | 同一次 QEMU 启动 → 进程退出并回收 | 包含上项，以及暂停、输出转储和退出清理；不含进程外编译、ORT 参考计算和完整结果校验 |
+| 采样到的模拟器内存峰值 | 定时读取 QEMU 驻留内存 RSS，取成功采样的最大值 | 包含模拟器、权重和运行缓冲；不是模型文件大小、配置内存上限或整机占用。可能漏掉瞬时峰值；1 GiB = 1024³ 字节 |
+
+UART 是串口通信接口；本项目用它发送完成状态，完整分数经 QMP 读回。
+关于 QEMU 的输入输出和通知顺序，见[流程表](#generation-flow)。
+
+
+展开单次前向的历史计时细节与可比性限制
+
+- 本机表中采用内部计时 **444.416 秒**。此前外层测量的 **448.263 秒**仍在原始记录中，
+ 不混入本表；其中端到端生成与校验为 **441.033 秒**。
+ 容器没有独立记录同口径的生成与校验耗时，不能用其他计时项代替。
+- 本机复用产物中记录的 **85.680 秒**是历史模型编译时间，不计入本次。
+ 小型浮点自检程序仍需编译，本机为 **0.110 秒**、容器为 **0.047 秒**，
+ 已包含在各自内部耗时中，与完整模型编译分开。自检用于检查工具链浮点乘加行为，不是重跑完整模型。
+- 两列内部耗时相差 **103.038 秒**，容器本次模型编译 **93.868 秒**是主要的已测量差别；
+ 其他准备、缓存和运行开销未完整分段计时，不能把差值全归于编译或 Docker。
+ 模拟器进程耗时 **419.383 / 415.478 秒**接近，单次记录不能用于认定容器更快或更慢。
+- 本机从完成通知到退出回收约 **0.429 秒**，是 `419.383 - 418.954`；
+ 这是进程总耗时的一部分，不能再次累加。
+- 容器另有运行器计时 **430.481 秒**，包含权重传输、QEMU、后检查和清理，
+ 不含更早的输入打包、写入、ELF/bundle 检查及完整 ORT 参考计算，因此不是端到端生成与校验耗时。
+- 本机主控程序/同时进程采样内存峰值为 **4.119 / 7.210 GiB**。
+ 容器 QEMU 有 **2071 个有效样本、1 次读取失败**，标为 `partial`；
+ 未清空本机 Linux 页缓存，不宣称冷缓存性能。
+- 容器单 prompt 的 `full_model_gate=NOT_RUN` 表示没有运行三例套件，
+ 不能据此说没有运行完整模型。
+
+
+
+
+
+**小图与多轮行为。** 历史真实容器默认单核、手动 4 core、auto 解析为 4 的小图均
+生成 IDs `[2,3,4,6]`、文本 `2 3 4`，EOS 停止，逐步完整 logits 对 ORT 的最大误差为 0，
+padding `none`。不同上下文和中断后的再次生成可复用同一配置 ELF/bundle；
+历史 QEMU 超时/取消及 Docker 构建 SIGINT/SIGTERM 后未遗留目标进程。
+官方聊天模板的有/无 system 三轮渲染和 token 序列与 Transformers 一致；这些回复由
+模板验证预设，未表示完整模型三轮聊天实跑。
+
+
+展开 2026-10-25 小图验证细节：报告保存、Docker 取消、清理与复用
+
+**2026-10-25 局部修复验证。** 基于 `e93233143d2323de7de6888ff7d73bdbf50784c3`
+加入报告保存与取消处理修复后，注入权限、磁盘满、I/O 和序列化失败，CLI 均保留原始生成错误
+或取消退出码；原本成功的运行在最终报告无法保存时返回失败，不误报成功。后续调用
+仍可正常保存。2026-10-25 的该项验证没有重新执行完整模型，上方完整数值与性能数据仍属于历史基线。
+
+真实 Docker 小图在默认 1 core 和手动 4 core 下，正常生成均得到 `[2,3,4,6]`、
+文本 `2 3 4`，逐步 ORT 最大误差为 0。向 Docker 客户端发送 SIGINT/SIGTERM 后，
+报告均为 `CANCELLED`，退出码分别为 130/143；容器外的客户端进程已回收、QEMU 进程组为空、
+`--rm` 容器消失，0.3 秒后复查仍无残留。中断检查以临时 `-S` 包装器控制真实 guest
+的暂停时机，不修改生产 harness；正常生成和恢复使用实际小图前向。
+
+两种信号之后均复用原 ELF/bundle 恢复生成，IDs、文本及全部逐步 logits 仍与基线一致；
+4 core 的恢复还实测 `auto` 解析为 4，两组 padding 均为 `none`。这些结果说明小图
+取消、清理和复用正常，不扩大为完整模型重跑或完整多轮聊天结果。
+
+
+
+
+
+**报告放在哪里。**
+
+| 要找的内容 | 固定入口 | 是否能随 PR 查看 |
+|---|---|---|
+| W5 功能、复现、数值、耗时、内存及未完成项 | 本 README；[完整结果](#实际复现结果) | 是 |
+| W5 三例逐 token 数值和资源、本机/容器运行记录、小图与清理结论 | [validation-summary.json](reports/validation-summary.json) | 是，随仓库保存的历史数据摘要 |
+| W4 软仿优化过程与单核/4 core 完整前向对照 | [W4 软仿报告](../W4/softsim-optimization-2026-10-22.md)及[汇总数据](../W4/reports/full-hart-comparison-2026-10-22.json) | 是，已有仓库文件 |
+| CI 原始测试明细、benchmark 详细数据和诊断 | [固定历史 run 的附件](#ci-and-benchmark) | 在 Actions 中可查看/下载，有保留期限 |
+| 本地交互页面 `comparison.html` 与大体积原始产物 | 忽略的 `output/`；[本地文件定位](#local-raw-artifacts) | 否；不能把本机文件链接当作 PR 报告链接 |
+
+公开 JSON 是现有记录的白名单摘录，保留测量单位、历史版本、逐步数值及采样缺失；
+不含权重、ELF、大数组、机器绝对路径或进程 ID。它不是本次重新执行的结果。
+本地交互报告不再作为维护者查找 W5 结果的必经入口。
+
+
+
+[公开数据摘要](reports/validation-summary.json)收录上述关键数值及逐 token 结果，可随仓库保存。
+下面只是本机诊断文件定位，不是 GitHub 下载链接。路径均相对于执行该次记录的 W5
+工作树根目录；原始产物在其 `output/` 中，不提交权重、ELF 和逐步 logits：
+
+
+展开本地原始文件清单（仅供本机定位，不是 GitHub 下载链接）
+
+- `output/w5-followup-20261023/qemu-reference-4hart/run/report.json`：
+ 完整三例；`run/` 的上一级目录内 `steps.json`、`numeric-steps.json`、`resources.json` 为计时与资源细节。
+- `output/w5-followup-20261023/qemu-final-single-forward-4hart/`：本机独立单次前向。
+- `output/w5-container-20261023/paris/run/report.json`：容器完整 Paris 单 token。
+- `output/w5-container-20261023/small-summary.json`、`small-runs/` 与 `build-cancel/`：
+ 容器小图、构建取消和进程清理记录。
+- `output/w5-hardening-20261025/docker-final/test_external_signals_reap_gue{0,1}/docker-cancellation-summary.json`：
+ 2026-10-25 的 1/4 core 小图外部取消、进程清理和恢复结果。
+
+
+
+
+
+完整三例的独立容器运行以及其他执行人的独立复现记录仍未取得；不能把本机三例、
+容器单 token 或后续局部验证合计成这些结果。现有实现仍是无 KV cache、逐 token
+重启 QEMU 与装载权重；常驻运行、多轮完整模型语义和其他算术策略的完整结果未在
+上述实跑范围内。
+
+完整 W5 专项成功记录也尚未收录于上述历史快照；本次未检查远端是否已有后续运行。
+普通 CI 与 benchmark 的独立范围见下面折叠内容。
+
+
+
+
+展开历史 CI、benchmark 与 W4 对照（不是本次完整模型运行结果）
+
+**版本与验证范围。** 以下固定保留 2026-10-27 整理的历史记录，不代表后续推送的 CI 状态。
+PR head `e8c94a7`、测试合并候选 `fa79ae4` 的普通 CI 为
+**14 个 job 成功、7 个按条件跳过**。本轮 core 命名兼容的相关验证范围与结果见
+[测试说明](../../../tests/llm-deploy/w5/README.md)。旧远端 CI 不能证明后续改动已通过验证，
+局部验证也不表示当前版本重跑了完整模型。
+
+**历史 CI 与 benchmark 快照（2026-10-27 核对）。** [普通 CI #40173064525](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525)
+对应 PR head `e8c94a71becfae96c4171674e332ed1239335c4d`，
+实际执行 `fa79ae4d0f697f968574d647347076a013429684`（与当时 main `735dd00` 的合并候选）。
+下表是该次运行的快照，不是每次打开 README 都会刷新的实时状态。
+
+**CI 各项结果与附件下载（完整 W4/W5 本次未运行）**
+
+| 数据 | 该次结果与范围 | 直接查看 / 下载 |
+|---|---|---|
+| 通用测试 | **8321 passed、93 skipped**；额外模拟器检查 **7 passed** | [test job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121107820034)、[test-reports](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12591298436) |
+| W4/W5 单测与真实小图 | **1769 passed、4 skipped**；不代表完整 28 层模型通过 | [小图 job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121107851526)、[w4-unit 附件](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12591112082)中的 `tests.xml` |
+| 官方多轮模板 | **2 passed**；验证模板及 token 序列，不是完整模型三轮生成 | [w5-multiturn 附件](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12590439548) |
+| 编译器 benchmark | job **SUCCESS**，但对照存在无效旧基线，不能表述为全项通过，详见下表 | [benchmark job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121107819853)、[benchmark-reports](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12590609469) |
+| 课题 06 benchmark | job **SUCCESS**；全量报告实际为 **12 PASS / 11 FAIL**，CI 严格检查的范围更小 | [Topic 06 job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121107819972)、[topic06-benchmark-reports](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12591146751) |
+| 完整 W4/W5 | **NOT RUN**：普通 PR 未显式启用完整前向和三例容器生成 | [范围汇总 job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121108398602)；本次没有完整模型性能附件 |
+
+
+**已核对的 benchmark 数字与限制。** 这些是上述 CI 附件中已有的结果，2026-10-27
+只下载并整理，未重新执行。它们测试解释器/编译器等范围，不能替代 W5 完整生成计时。
+
+**注意：Topic 06 为 12 PASS / 11 FAIL，单算子对照的旧基线无效；job 成功不代表所有基准通过。**
+
+**benchmark 详细数字、失败原因与统计范围**
+
+| 附件内的报告 | 实际数据 | 如何理解 |
+|---|---|---|
+| `benchmark-reports / dsl_bench` | **23/23**；总耗时 **0.088 s**，总指令数 **153** | 仅该 DSL 基准的结果 |
+| `benchmark-reports / ir_interpreter` | **6/6**，最大误差 **4.6566e-10** | 仅解释器用例的数值对照 |
+| `topic06-benchmark-reports / report.md` | **12/23 PASS（52.2%），11 FAIL** | 8 项 TinyFive 输入 ABI 不支持；3 项 `dynamic_add` 计数增加，即使总动态指令减少也被标为回退 |
+| `benchmark-reports / single_op_bench.md` | 表中 **30 行 FAIL**；当前版本 **78/78** 组合有效，旧基线 **78/78** 无效 | 旧基线有标签解析错误，不能用旧版 0 指令数算加速比，也不能把表中 FAIL 都解释为当前代码失败 |
+
+Topic 06 的 CI 只对 activation/elementwise/loop 三组使用失败退出选项；
+全量报告未设置该选项，所以“job 成功”和“全量 11 项 FAIL”可以同时出现。
+这些是当时核对发现的报告/基准限制，尚未定位其引入版本，不据此断言由 W5 改动造成。
+当前仅整理可见结果，没有修改 CI 或修复这些基准。
+
+
+GitHub 运行页的“Load 摘要”加载的是工作流写入 `GITHUB_STEP_SUMMARY` 的已保存 Markdown；
+点击它不会重跑测试或模型。附件下载通常需登录 GitHub。上述小图/多轮附件预计
+2026-11-08 到期，通用测试/benchmark 附件预计 2026-11-24 到期；平台删除或权限变化也会
+使下载失效。本文保留核心结果，完整诊断日志仍以附件为准。
+
+**历史 W4 单次前向对照。** 2026-10-22 的同配置记录中，1 core 为 **1384.65 s**，
+4 core 为 **410.53 s**，约 **3.37×**；CPU 总时间 **+5.46%**，输出逐位相同。
+见[W4 软仿报告](../W4/softsim-optimization-2026-10-22.md)及
+[原始汇总 JSON](../W4/reports/full-hart-comparison-2026-10-22.json)。
+这只有一组有序对照，未使用旧计时异常记录，不能当作 W5 三例端到端生成的加速比。
+
+
+
+### 运行示例
+
+
+单条提示词、构建复用与多轮运行示例
+
+**单条提示词。** 完成 [W5 环境与固定资产准备](../LINUX_REPRODUCTION.md#w5-local-environment)后，从仓库根目录执行：
+
+```bash
+python scripts/run_inference.py \
+ --model assets/qwen3-0.6b/onnx/model.onnx \
+ --tokenizer-dir assets/qwen3-0.6b/tokenizer \
+ --prompt 'The capital of France is' --max-new-tokens 1 \
+ --verify-onnx assets/qwen3-0.6b/onnx/model.onnx \
+ --cores 4 --matmul-policy blocked_fma --matmul-tile 2x4 \
+ --matmul-rhs-layout column-panel \
+ --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \
+ --timeout 1800 --total-timeout 3600 --output-dir output/w5-paris-new
+```
+
+这会执行一次完整 28 层模型。只构建可加 `--build-only`，其 `BUILD_ONLY` 不表示
+完成生成。复用时以 `--reuse-build output/w5-paris-new/executable.json` 替换
+`--model ...`，仍传相同 tokenizer 和所有原有算术/core 配置，并使用新的输出目录。
+元数据可能含绝对路径，不直接把本机构建的产物搬进容器复用；跨环境从模型重新构建。
+
+聊天模式将上例 `--prompt ...` 换为下列参数，其他模型、参考策略及输出目录参数保留：
+
+```text
+--mode chat --chat-turn 'Hello.' --chat-turn 'What is 1+1?'
+--chat-turn 'Reply with one word.' --system-prompt 'Give brief answers.'
+```
+
+这会按顺序运行三轮，前两轮回答也计入后续 256-token 容量。模板一致性检查不等于
+完整模型三轮生成已实跑。相关测试命令集中在
+[W5 测试说明](../../../tests/llm-deploy/w5/README.md)。
+
+
+
+**容器运行与独立复现。** 固定提交、环境准备、资产获取、三例生成、清理及报告回传步骤集中在 [Linux 复现指南](../LINUX_REPRODUCTION.md#w5-container-reproduction)，按同一 Bash 会话中的顺序执行。
+
+
+
+
diff --git a/docs/llm-deploy-v1.0/W5/reports/validation-summary.json b/docs/llm-deploy-v1.0/W5/reports/validation-summary.json
new file mode 100644
index 00000000..f33c13b8
--- /dev/null
+++ b/docs/llm-deploy-v1.0/W5/reports/validation-summary.json
@@ -0,0 +1,826 @@
+{
+ "schema_version": 1,
+ "title": "W5 历史实跑数据摘要",
+ "scope": {
+ "historical_result": true,
+ "new_model_execution": false,
+ "current_commit_ci_result": false,
+ "full_model_measurement_date": "2026-10-23",
+ "full_model_baseline_commit": "8ad347477f7f5ba15b2da7c91178b3c6668456cc",
+ "baseline_source": "baseline_and_scope",
+ "baseline_note": "完整模型数据沿用 README 记录的历史功能基线，不代表后续命名、兼容修复或最新 PR 提交已经重跑完整模型。",
+ "sources_base": "所有来源路径相对于当时运行所在的 W5 仓库工作树根目录；output 原始文件保留本地，本摘要公开可读数据。",
+ "raw_report_schema_version": 1,
+ "field_mapping": "本摘要 cores/cores_requested 对应历史原始 schema 1 的 harts/harts_requested；没有重写原始记录。",
+ "excluded": "不复制机器绝对路径、PID、编译命令、制品哈希、源码、权重、ELF 或张量数组。token ID 是冻结生成用例结果。"
+ },
+ "units": {
+ "time": "带 _s 的字段单位为秒；wall 是经过时间，CPU 是线程累计处理器时间，不能互相相加。",
+ "memory": "带 _bytes 的字段单位为 byte；RSS 是采样观测值，不是内存硬上限；GiB 换算为 bytes / 1073741824。",
+ "cpu_percent": "QEMU CPU / QEMU process wall * 100；100% 表示一个逻辑 CPU。",
+ "numerical": "max_abs 为所比较 logits 元素最大绝对误差；*_frozen_selected_abs 为选中 token logit 相对冻结参考的绝对误差。",
+ "step": "numeric_steps 的 step 是各用例内部前向序号，从 0 到 7；case 与 step 共同定位一条记录。"
+ },
+ "measurement_definitions": {
+ "generation_wall_s": "单条用例生成调用经过时间，包含逐 token QEMU 调用与 Host 参考数值检查，不是编译耗时。",
+ "uart_completion_wall_s": "QEMU launch through host validation of the UART completion frame",
+ "qemu_process_wall_s": "QEMU 子进程寿命；包括 UART 完成后的停止、输出转储与退出清理。",
+ "runner_elapsed_s": "运行器调用总耗时，包含权重传输、QEMU、完成后检查和清理；不等同于纯算子执行。",
+ "measurement_total_wall_s": "外层测量全过程，包含准备、ORT 参考和数值检查。",
+ "qemu_cpu_s": "每步 RUSAGE_CHILDREN CPU 差值由测量工具归入 QEMU，三例汇总取 cases[*].qemu_cpu；不使用包含其他子进程的 total_child_cpu。",
+ "host_cpu_s": "Host 进程 CPU，含 ORT 及验证开销；case 与整个测量的范围分别记录。",
+ "rss": "Host+live descendants simultaneous RSS sum；共享页可能重复计数。各进程独立峰值不能相加当作同一时刻峰值。",
+ "cache": "复用相同 ELF/bundle；每次前向启动新 QEMU，未清空宿主页缓存，不能作为冷缓存结果。",
+ "performance_comparison": "这些数据没有单核完整生成对照，不据此计算 W5 1/4 核生成加速比。",
+ "inference_program_elapsed_s": "推理程序开始准备至程序内部计时结束，取对应 run/report.json 的 elapsed_seconds；包含当次实际发生的构建、自检、参考程序初始化、生成与校验及程序内收尾，不包含此前构建、Docker 镜像准备或容器启动。",
+ "compile_wall_s": "本次 C 程序编译与链接经过时间，取新构建的 build.compile_seconds；包含编译器启动、C 编译、汇编、链接、日志保存和 ELF 检查，不含此前模型解析、C 源码生成及权重打包。复用构建时为 null，不能计入旧构建记录中的编译时间。",
+ "deployment_elapsed_s": "保留的历史字段名；容器记录中与 inference_program_elapsed_s 同取 run/report.json.elapsed_seconds，并非 Docker 镜像准备到退出的外层全过程。"
+ },
+ "configuration": {
+ "model_id": "Qwen/Qwen3-0.6B",
+ "layers": 28,
+ "generation": {
+ "batch_size": 1,
+ "sequence_length": 256,
+ "dtype": "float32",
+ "use_cache": false,
+ "prompt_format": "raw_completion",
+ "add_special_tokens": false,
+ "sampling": "greedy_argmax_lowest_id"
+ },
+ "cores_requested": 4,
+ "cores": 4,
+ "matmul_policy": "blocked_fma",
+ "matmul_tile": "2x4",
+ "matmul_rhs_layout": "column-panel",
+ "mean_policy": "two-four-lane-last-axis",
+ "nonlinear_policy": "fp32-reference",
+ "qemu_workspace_layout": "none",
+ "execution": "one ELF/bundle; a fresh QEMU process per forward; no KV cache",
+ "versions": {
+ "compiler": "0.14.1",
+ "qemu": "QEMU emulator version 8.2.2 (Debian 1:8.2.2+ds-0ubuntu1.18)",
+ "target": "riscv64 / RV64GC / LP64D / QEMU virt / bare metal",
+ "math_library": "Zig-bundled musl"
+ },
+ "host_environment": {
+ "platform": "Linux-6.18.40.1-microsoft-standard-WSL2-x86_64-with-glibc2.39",
+ "python": "3.12.3",
+ "packages": {
+ "numpy": "2.2.6",
+ "onnxruntime": "1.22.1",
+ "tokenizers": "0.21.4"
+ },
+ "available_logical_cpus": 4,
+ "clocksource": "tsc",
+ "host_tree_sample_interval_s": 0.5,
+ "reference_provider": "CPUExecutionProvider"
+ },
+ "scope_note": "显式优化策略的历史完整模型结果；不是默认算术策略完整验收。默认单核与手动/auto 的覆盖另见小图记录。"
+ },
+ "three_case_generation": {
+ "sources": [
+ "three_case_report",
+ "three_case_resources",
+ "three_case_metadata"
+ ],
+ "status": "PASS",
+ "passed": true,
+ "full_model_gate": "PASS",
+ "container_reproduction": "NOT_ASSERTED",
+ "forward_count": 24,
+ "cases": [
+ {
+ "name": "france",
+ "prompt": "The capital of France is",
+ "max_new_tokens": 8,
+ "generated_token_ids": [
+ 12095,
+ 13,
+ 576,
+ 6722,
+ 315,
+ 15344,
+ 374,
+ 21718
+ ],
+ "generated_text": " Paris. The capital of Italy is Rome",
+ "stop_reason": "max_new_tokens",
+ "forward_count": 8,
+ "reference_passed": true,
+ "generation_wall_s": 3375.0240403530006,
+ "qemu_cpu_s": 11517.789080999999,
+ "host_cpu_s": 167.656669,
+ "sampled_host_peak_rss_bytes": 4568129536,
+ "sampled_simultaneous_tree_peak_rss_bytes": 8123035648,
+ "max_abs": 0.0003039836883544922,
+ "numeric_steps_passed": true
+ },
+ {
+ "name": "germany",
+ "prompt": "The capital of Germany is",
+ "max_new_tokens": 8,
+ "generated_token_ids": [
+ 19846,
+ 13,
+ 576,
+ 6722,
+ 315,
+ 9625,
+ 374,
+ 12095
+ ],
+ "generated_text": " Berlin. The capital of France is Paris",
+ "stop_reason": "max_new_tokens",
+ "forward_count": 8,
+ "reference_passed": true,
+ "generation_wall_s": 3668.8521105910004,
+ "qemu_cpu_s": 12574.174352,
+ "host_cpu_s": 190.67315399999998,
+ "sampled_host_peak_rss_bytes": 4404678656,
+ "sampled_simultaneous_tree_peak_rss_bytes": 8207597568,
+ "max_abs": 0.0002627372741699219,
+ "numeric_steps_passed": true
+ },
+ {
+ "name": "japan",
+ "prompt": "The capital of Japan is",
+ "max_new_tokens": 8,
+ "generated_token_ids": [
+ 26194,
+ 13,
+ 576,
+ 6722,
+ 315,
+ 279,
+ 3639,
+ 4180
+ ],
+ "generated_text": " Tokyo. The capital of the United States",
+ "stop_reason": "max_new_tokens",
+ "forward_count": 8,
+ "reference_passed": true,
+ "generation_wall_s": 3489.654915645,
+ "qemu_cpu_s": 11958.880560000001,
+ "host_cpu_s": 178.75917299999992,
+ "sampled_host_peak_rss_bytes": 4400611328,
+ "sampled_simultaneous_tree_peak_rss_bytes": 7957729280,
+ "max_abs": 0.00019788742065429688,
+ "numeric_steps_passed": true
+ }
+ ],
+ "totals": {
+ "measurement_total_wall_s": 10540.189622884001,
+ "generation_wall_s": 10533.531066589001,
+ "uart_completion_wall_s": 9990.355179675997,
+ "qemu_process_wall_s": 9998.090941066994,
+ "qemu_cpu_s": 36050.843993,
+ "qemu_average_cpu_percent": 360.58,
+ "host_measurement_cpu_s": 540.716702,
+ "host_generation_cpu_s": 537.088996,
+ "sampled_qemu_peak_rss_bytes": 5804662784,
+ "sampled_host_peak_rss_bytes": 4568129536,
+ "sampled_simultaneous_tree_peak_rss_bytes": 8207597568,
+ "numeric_max_abs": 0.0003039836883544922,
+ "all_qemu_reaped": true
+ },
+ "sampling": {
+ "qemu_rss_partial_steps": [
+ 0,
+ 1,
+ 5,
+ 6,
+ 7,
+ 9,
+ 11,
+ 12,
+ 14,
+ 15,
+ 16,
+ 17,
+ 18,
+ 20,
+ 21,
+ 22,
+ 23
+ ],
+ "qemu_rss_sample_errors": 17,
+ "host_tree_sample_count": 21032,
+ "host_tree_sample_errors": 0,
+ "max_abs_perf_minus_raw_s": 0.0007727560023340629,
+ "max_abs_utc_minus_perf_s": 0.0005040991982241394,
+ "note": "17/24 步 QEMU RSS 各失读一次，峰值属于 partial；Host/tree 零采样错误不能替代 QEMU 采样完整性。"
+ },
+ "numeric_comparison": {
+ "atol": 0.001,
+ "rtol": 0.0,
+ "elements_compared_per_step": 38895616,
+ "reference": "冻结 reference token 与本次 ORT/QEMU greedy token 逐步一致，并检查 logits 数值误差；不含原始张量数组。",
+ "all_24_steps": {
+ "selected_row_passed": true,
+ "frozen_step_passed": true,
+ "reference_token_passed": true
+ }
+ },
+ "numeric_steps": [
+ {
+ "case": "france",
+ "step": 0,
+ "valid_length": 5,
+ "qemu_token_id": 12095,
+ "ort_token_id": 12095,
+ "frozen_token_id": 12095,
+ "max_abs": 0.00017213821411132812,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "france",
+ "step": 1,
+ "valid_length": 6,
+ "qemu_token_id": 13,
+ "ort_token_id": 13,
+ "frozen_token_id": 13,
+ "max_abs": 9.34600830078125e-05,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "france",
+ "step": 2,
+ "valid_length": 7,
+ "qemu_token_id": 576,
+ "ort_token_id": 576,
+ "frozen_token_id": 576,
+ "max_abs": 0.00011712312698364258,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "france",
+ "step": 3,
+ "valid_length": 8,
+ "qemu_token_id": 6722,
+ "ort_token_id": 6722,
+ "frozen_token_id": 6722,
+ "max_abs": 0.0003039836883544922,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "france",
+ "step": 4,
+ "valid_length": 9,
+ "qemu_token_id": 315,
+ "ort_token_id": 315,
+ "frozen_token_id": 315,
+ "max_abs": 0.00013625621795654297,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "france",
+ "step": 5,
+ "valid_length": 10,
+ "qemu_token_id": 15344,
+ "ort_token_id": 15344,
+ "frozen_token_id": 15344,
+ "max_abs": 6.151199340820312e-05,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "france",
+ "step": 6,
+ "valid_length": 11,
+ "qemu_token_id": 374,
+ "ort_token_id": 374,
+ "frozen_token_id": 374,
+ "max_abs": 0.00016880035400390625,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 1.9073486328125e-06,
+ "passed": true
+ },
+ {
+ "case": "france",
+ "step": 7,
+ "valid_length": 12,
+ "qemu_token_id": 21718,
+ "ort_token_id": 21718,
+ "frozen_token_id": 21718,
+ "max_abs": 0.0001971721649169922,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 1.9073486328125e-06,
+ "passed": true
+ },
+ {
+ "case": "germany",
+ "step": 0,
+ "valid_length": 5,
+ "qemu_token_id": 19846,
+ "ort_token_id": 19846,
+ "frozen_token_id": 19846,
+ "max_abs": 0.00018739700317382812,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "germany",
+ "step": 1,
+ "valid_length": 6,
+ "qemu_token_id": 13,
+ "ort_token_id": 13,
+ "frozen_token_id": 13,
+ "max_abs": 5.5789947509765625e-05,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "germany",
+ "step": 2,
+ "valid_length": 7,
+ "qemu_token_id": 576,
+ "ort_token_id": 576,
+ "frozen_token_id": 576,
+ "max_abs": 0.0001690387725830078,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "germany",
+ "step": 3,
+ "valid_length": 8,
+ "qemu_token_id": 6722,
+ "ort_token_id": 6722,
+ "frozen_token_id": 6722,
+ "max_abs": 0.0002627372741699219,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 1.9073486328125e-06,
+ "passed": true
+ },
+ {
+ "case": "germany",
+ "step": 4,
+ "valid_length": 9,
+ "qemu_token_id": 315,
+ "ort_token_id": 315,
+ "frozen_token_id": 315,
+ "max_abs": 0.0002231597900390625,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 5.7220458984375e-06,
+ "passed": true
+ },
+ {
+ "case": "germany",
+ "step": 5,
+ "valid_length": 10,
+ "qemu_token_id": 9625,
+ "ort_token_id": 9625,
+ "frozen_token_id": 9625,
+ "max_abs": 0.00014162063598632812,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "germany",
+ "step": 6,
+ "valid_length": 11,
+ "qemu_token_id": 374,
+ "ort_token_id": 374,
+ "frozen_token_id": 374,
+ "max_abs": 0.00011169910430908203,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 1.9073486328125e-06,
+ "passed": true
+ },
+ {
+ "case": "germany",
+ "step": 7,
+ "valid_length": 12,
+ "qemu_token_id": 12095,
+ "ort_token_id": 12095,
+ "frozen_token_id": 12095,
+ "max_abs": 0.00017213821411132812,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 1.9073486328125e-06,
+ "passed": true
+ },
+ {
+ "case": "japan",
+ "step": 0,
+ "valid_length": 5,
+ "qemu_token_id": 26194,
+ "ort_token_id": 26194,
+ "frozen_token_id": 26194,
+ "max_abs": 3.3289194107055664e-05,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "japan",
+ "step": 1,
+ "valid_length": 6,
+ "qemu_token_id": 13,
+ "ort_token_id": 13,
+ "frozen_token_id": 13,
+ "max_abs": 0.00019788742065429688,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "japan",
+ "step": 2,
+ "valid_length": 7,
+ "qemu_token_id": 576,
+ "ort_token_id": 576,
+ "frozen_token_id": 576,
+ "max_abs": 0.0001494884490966797,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "japan",
+ "step": 3,
+ "valid_length": 8,
+ "qemu_token_id": 6722,
+ "ort_token_id": 6722,
+ "frozen_token_id": 6722,
+ "max_abs": 8.016824722290039e-05,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "japan",
+ "step": 4,
+ "valid_length": 9,
+ "qemu_token_id": 315,
+ "ort_token_id": 315,
+ "frozen_token_id": 315,
+ "max_abs": 0.00010776519775390625,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "japan",
+ "step": 5,
+ "valid_length": 10,
+ "qemu_token_id": 279,
+ "ort_token_id": 279,
+ "frozen_token_id": 279,
+ "max_abs": 9.918212890625e-05,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 0.0,
+ "passed": true
+ },
+ {
+ "case": "japan",
+ "step": 6,
+ "valid_length": 11,
+ "qemu_token_id": 3639,
+ "ort_token_id": 3639,
+ "frozen_token_id": 3639,
+ "max_abs": 5.27501106262207e-05,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 5.7220458984375e-06,
+ "passed": true
+ },
+ {
+ "case": "japan",
+ "step": 7,
+ "valid_length": 12,
+ "qemu_token_id": 4180,
+ "ort_token_id": 4180,
+ "frozen_token_id": 4180,
+ "max_abs": 5.817413330078125e-05,
+ "ort_frozen_selected_abs": 0.0,
+ "qemu_frozen_selected_abs": 1.9073486328125e-06,
+ "passed": true
+ }
+ ]
+ },
+ "single_full_forward_comparison": {
+ "date": "2026-10-23",
+ "scope": "相同完整 28 层模型及显式策略，各生成第一个 Paris token；本机复用构建，容器重新编译，不能用总时间差判断容器额外开销；不是三例完整生成的容器对照。",
+ "host": {
+ "sources": [
+ "host_single_report",
+ "host_single_resources",
+ "repeat_and_cleanup"
+ ],
+ "passed": true,
+ "forward_count": 1,
+ "full_reference_suite_evaluated": false,
+ "case": {
+ "prompt": "The capital of France is",
+ "max_new_tokens": 1,
+ "generated_token_ids": [
+ 12095
+ ],
+ "generated_text": " Paris",
+ "stop_reason": "max_new_tokens"
+ },
+ "measurement_total_wall_s": 448.2632728360004,
+ "generation_wall_s": 441.0328804979981,
+ "uart_completion_wall_s": 418.95395780800027,
+ "qemu_process_wall_s": 419.38253989200166,
+ "qemu_cpu_s": 1498.851419,
+ "host_measurement_cpu_s": 25.33218,
+ "host_generation_cpu_s": 21.604175,
+ "sampled_host_peak_rss_bytes": 4423188480,
+ "sampled_simultaneous_tree_peak_rss_bytes": 7741919232,
+ "qemu_rss": {
+ "sampled_peak_bytes": 5796397056,
+ "availability": "available",
+ "successful_samples": 2091,
+ "sample_errors": 0,
+ "sample_interval_s": 0.2
+ },
+ "max_abs": 0.00017213821411132812,
+ "qemu_reaped": true,
+ "inference_program_elapsed_s": 444.4163706699983,
+ "build_reused": true,
+ "compile_wall_s": null,
+ "compile_note": "本次复用已有 ELF 和权重包，没有重新编译完整模型；原始报告 build 中保留的旧 compile_seconds 不计入此次运行。"
+ },
+ "container": {
+ "sources": [
+ "container_paris_report",
+ "container_paris_summary"
+ ],
+ "passed": true,
+ "forward_count": 1,
+ "case": {
+ "prompt": "The capital of France is",
+ "max_new_tokens": 1,
+ "generated_token_ids": [
+ 12095
+ ],
+ "generated_text": " Paris",
+ "stop_reason": "max_new_tokens"
+ },
+ "full_model_gate": "NOT_RUN",
+ "container_reproduction": "NOT_ASSERTED",
+ "acceptance": "PASS: clean-container single Paris forward; existing host 3x8 gate is separate",
+ "deployment_elapsed_s": 547.4544949770025,
+ "runner_elapsed_s": 430.48115924100057,
+ "uart_completion_wall_s": 415.05810176999876,
+ "qemu_process_wall_s": 415.47787108400007,
+ "qemu_cpu_s": null,
+ "cpu_note": "No independent container CPU-total measurement in this deployment acceptance; Docker client GNU time is not container CPU time.",
+ "qemu_rss": {
+ "sampled_peak_bytes": 5795934208,
+ "availability": "partial",
+ "successful_samples": 2071,
+ "sample_errors": 1,
+ "sample_interval_s": 0.2
+ },
+ "max_abs": 0.00017213821411132812,
+ "qemu_reaped": true,
+ "process_exit_code": 0,
+ "matches_host_full_logits_byte_for_byte": true,
+ "compared_bytes": 155582464,
+ "scope_note": "full_model_gate=NOT_RUN 表示未执行容器三例 24 步整套验收；本次确实执行了完整模型一个前向。容器 CPU 未独立采集，不能使用 Docker 客户端 CPU 代替。",
+ "inference_program_elapsed_s": 547.4544949770025,
+ "build_reused": false,
+ "compile_wall_s": 93.86813328300195,
+ "compile_note": "本次在容器内重新编译完整模型；compile_wall_s 已包含在 inference_program_elapsed_s 中，不能再次相加。"
+ }
+ },
+ "container_small_graph": {
+ "date": "2026-10-23",
+ "source": "container_small_summary",
+ "passed": true,
+ "scope": "独立的小型确定性图；不代表完整模型生成性能。",
+ "cases": [
+ {
+ "mode": "default",
+ "cores_requested": 1,
+ "cores": 1,
+ "passed": true,
+ "tokens": [
+ 2,
+ 3,
+ 4,
+ 6
+ ],
+ "max_abs": 0,
+ "padding": "none"
+ },
+ {
+ "mode": "manual",
+ "cores_requested": 4,
+ "cores": 4,
+ "passed": true,
+ "tokens": [
+ 2,
+ 3,
+ 4,
+ 6
+ ],
+ "max_abs": 0,
+ "padding": "none"
+ },
+ {
+ "mode": "auto_reuse",
+ "cores_requested": "auto",
+ "cores": 4,
+ "passed": true,
+ "tokens": [
+ 2,
+ 3,
+ 4,
+ 6
+ ],
+ "max_abs": 0,
+ "padding": "none"
+ }
+ ]
+ },
+ "process_cleanup": {
+ "full_model_history": {
+ "date": "2026-10-23",
+ "source": "repeat_and_cleanup",
+ "formal_suite_passed": true,
+ "suite_forwards": 24,
+ "single_forward_passed": true,
+ "same_existing_elf_and_bundle_identity": true,
+ "repeat_full_output_byte_identical": true,
+ "compared_bytes": 155582464,
+ "qemu_processes_and_groups_absent": 25,
+ "all_recorded_processes_absent": true
+ },
+ "docker_external_cancellation": {
+ "date": "2026-10-25",
+ "base_commit": "e93233143d2323de7de6888ff7d73bdbf50784c3",
+ "version_note": "README 记录为该基线上加入报告保存与取消修复后的局部验证，不是 2026-10-23 完整模型重跑。",
+ "scope": "真实 Docker 小图；取消时用临时 -S 包装器控制 guest 暂停时机。0.3 秒后复查清理；恢复沿用相同 ELF/bundle。",
+ "cases": [
+ {
+ "source": "docker_cancellation_1_core",
+ "cores": 1,
+ "passed": true,
+ "available_auto_cores": 4,
+ "baseline": {
+ "status": "PASS",
+ "passed": true,
+ "cores_requested": 1,
+ "cores": 1,
+ "padding": "none",
+ "generated_token_ids": [
+ 2,
+ 3,
+ 4,
+ 6
+ ],
+ "generated_text": "2 3 4",
+ "max_abs": 0.0,
+ "all_numeric_steps_passed": true
+ },
+ "signals": [
+ {
+ "signal": "SIGINT",
+ "exit_code": 130,
+ "container_removed": true,
+ "host_processes_reaped": true,
+ "qemu_group_empty": true,
+ "delayed_recheck_seconds": 0.3,
+ "report_status": "CANCELLED",
+ "report_passed": false,
+ "recovery": {
+ "status": "PASS",
+ "passed": true,
+ "cores_requested": 1,
+ "cores": 1,
+ "max_abs": 0.0,
+ "all_numeric_steps_passed": true,
+ "output_matches_baseline": true,
+ "padding": "none"
+ }
+ },
+ {
+ "signal": "SIGTERM",
+ "exit_code": 143,
+ "container_removed": true,
+ "host_processes_reaped": true,
+ "qemu_group_empty": true,
+ "delayed_recheck_seconds": 0.3,
+ "report_status": "CANCELLED",
+ "report_passed": false,
+ "recovery": {
+ "status": "PASS",
+ "passed": true,
+ "cores_requested": 1,
+ "cores": 1,
+ "max_abs": 0.0,
+ "all_numeric_steps_passed": true,
+ "output_matches_baseline": true,
+ "padding": "none"
+ }
+ }
+ ]
+ },
+ {
+ "source": "docker_cancellation_4_core",
+ "cores": 4,
+ "passed": true,
+ "available_auto_cores": 4,
+ "baseline": {
+ "status": "PASS",
+ "passed": true,
+ "cores_requested": 4,
+ "cores": 4,
+ "padding": "none",
+ "generated_token_ids": [
+ 2,
+ 3,
+ 4,
+ 6
+ ],
+ "generated_text": "2 3 4",
+ "max_abs": 0.0,
+ "all_numeric_steps_passed": true
+ },
+ "signals": [
+ {
+ "signal": "SIGINT",
+ "exit_code": 130,
+ "container_removed": true,
+ "host_processes_reaped": true,
+ "qemu_group_empty": true,
+ "delayed_recheck_seconds": 0.3,
+ "report_status": "CANCELLED",
+ "report_passed": false,
+ "recovery": {
+ "status": "PASS",
+ "passed": true,
+ "cores_requested": "auto",
+ "cores": 4,
+ "max_abs": 0.0,
+ "all_numeric_steps_passed": true,
+ "output_matches_baseline": true,
+ "padding": "none"
+ }
+ },
+ {
+ "signal": "SIGTERM",
+ "exit_code": 143,
+ "container_removed": true,
+ "host_processes_reaped": true,
+ "qemu_group_empty": true,
+ "delayed_recheck_seconds": 0.3,
+ "report_status": "CANCELLED",
+ "report_passed": false,
+ "recovery": {
+ "status": "PASS",
+ "passed": true,
+ "cores_requested": "auto",
+ "cores": 4,
+ "max_abs": 0.0,
+ "all_numeric_steps_passed": true,
+ "output_matches_baseline": true,
+ "padding": "none"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ },
+ "remaining_scope": {
+ "current_head_full_model": "未因本次摘要整理重跑；历史完整模型结果不得描述为当前代码的新执行结果。",
+ "container_three_cases_24_steps": "上述容器数据只覆盖 Paris 单步及小图，未提供容器三例 24 步完整运行结果。",
+ "second_person_reproduction": "这些作者本机记录不构成第二人独立复现。",
+ "ci_and_benchmark": "本文件不包含最新 Actions CI 或 benchmark 状态；它们与历史 W5 模型测量分开查看。",
+ "chat_and_cache": "raw completion 与无 KV cache；三例文本不是完整多轮聊天质量验收。"
+ },
+ "sources": {
+ "three_case_report": "output/w5-followup-20261023/qemu-reference-4hart/run/report.json",
+ "three_case_resources": "output/w5-followup-20261023/qemu-reference-4hart/resources.json",
+ "three_case_metadata": "output/w5-followup-20261023/qemu-reference-4hart/metadata.json",
+ "host_single_report": "output/w5-followup-20261023/qemu-final-single-forward-4hart/run/report.json",
+ "host_single_resources": "output/w5-followup-20261023/qemu-final-single-forward-4hart/resources.json",
+ "repeat_and_cleanup": "output/w5-followup-20261023/qemu-final-single-forward-4hart/repeat-and-cleanup-check.json",
+ "container_paris_report": "output/w5-container-20261023/paris/run/report.json",
+ "container_paris_summary": "output/w5-container-20261023/paris-summary.json",
+ "container_small_summary": "output/w5-container-20261023/small-summary.json",
+ "docker_cancellation_1_core": "output/w5-hardening-20261025/docker-final/test_external_signals_reap_gue0/docker-cancellation-summary.json",
+ "docker_cancellation_4_core": "output/w5-hardening-20261025/docker-final/test_external_signals_reap_gue1/docker-cancellation-summary.json",
+ "frozen_reference": "probes/w5_generation/references.json",
+ "baseline_and_scope": "docs/llm-deploy-v1.0/W5/README.md"
+ }
+}
diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md"
index be84d5e5..4827c38e 100644
--- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md"
+++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md"
@@ -157,7 +157,7 @@ Host 端循环：
| Nightly 完整门禁 | 每日夜间 | 完整 ONNX、完整 IR 数值、QEMU 单次前向、端到端 | 2–4 h | 阻塞里程碑 |
| Weekly/Release 门禁 | 每周五 / tag | 多轮、文档复现、性能回归、交付物 | 4–8 h | 阻塞发布 |
-上表是分层目标，不代表所有未来 job 已交付。当前 `.github/workflows/llm-deploy.yml` 使用 GitHub 托管 Ubuntu runner：PR 小模型任务和手动/定时完整 ONNX 任务各有 60 分钟 job 上限，QEMU 小模型步骤另有超时。W3 已提供 `w3-preparation.yml`、`w3-full-numeric.yml` 和显式启用的 `w3-nightly.yml`，并可通过既有 `llm-deploy.yml` 手动设置 `run_w3_validation=true`，从同一提交调用 W3 编排。作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 通过同提交两套 W3 gate 及汇总，完整 raw 已下载验真、audit 通过，定时 schedule 未启用。通用 PR CI 全绿不能替代这些专项记录。W4 的 PR 小图回归由统一 CI 调用 `w4-full-numeric.yml`；`llm-deploy.yml` 手动设置 `run_w4_validation=true` 才调用完整前向，另有默认关闭的 `w4-nightly.yml`；所有 Nightly 入口需仓库变量 `W4_NIGHTLY_ENABLED=true`。配置和本地检查不代表已取得远端成功记录，完整门槛、独立复现与团队验收分别记录。生成门禁仍属 W5 待实现。实际工具为 Zig 0.14.1、`qemu-system-riscv64`（裸机）和固定 Python 依赖；旧独立测试仍使用 `qemu-riscv32`，不能互换。更大内存的自托管 runner、容器封装与缓存策略是完整模型阶段的资源计划，不宣称已经部署。
+上表是分层目标，不代表所有未来 job 已交付。当前 `.github/workflows/llm-deploy.yml` 使用 GitHub 托管 Ubuntu runner：PR 小模型任务和手动/定时完整 ONNX 任务各有 60 分钟 job 上限，QEMU 小模型步骤另有超时。W3 已提供 `w3-preparation.yml`、`w3-full-numeric.yml` 和显式启用的 `w3-nightly.yml`，并可通过既有 `llm-deploy.yml` 手动设置 `run_w3_validation=true`，从同一提交调用 W3 编排。作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 通过同提交两套 W3 gate 及汇总，完整 raw 已下载验真、audit 通过，定时 schedule 未启用。通用 PR CI 全绿不能替代这些专项记录。W4 的 PR 小图回归由统一 CI 调用 `w4-full-numeric.yml`；`llm-deploy.yml` 手动设置 `run_w4_validation=true` 才调用完整前向，另有默认关闭的 `w4-nightly.yml`；所有 Nightly 入口需仓库变量 `W4_NIGHTLY_ENABLED=true`。配置和本地检查不代表已取得远端成功记录，完整门槛、独立复现与团队验收分别记录。W5 已实现 Host token 循环、QEMU 生成适配、多轮文本和容器部署；显式参考算术策略的宿主完整三例 24 步及容器 Paris 单 token 已有历史实跑，功能、复现命令和结果集中在 [W5 开发说明](W5/README.md)、[独立复现步骤](W5/README.md#如何复现)与[实际复现结果](W5/README.md#实际复现结果)。完整三例的独立容器运行与其他执行人的独立复现仍待取得，不由现有局部结果替代。实际工具为 Zig 0.14.1、`qemu-system-riscv64`（裸机）和固定 Python 依赖；旧独立测试仍使用 `qemu-riscv32`，不能互换。工作流配置与运行范围见 [CI 说明](../CI.md)，开发文档不维护随运行变化的远端状态。
### 4.2 硬门槛 CI Job
diff --git a/probes/w4_qwen3_full/diagnostic.py b/probes/w4_qwen3_full/diagnostic.py
index 1f007ad0..71c5ad31 100644
--- a/probes/w4_qwen3_full/diagnostic.py
+++ b/probes/w4_qwen3_full/diagnostic.py
@@ -28,10 +28,10 @@
from probes.w3_qwen3_full.cases import CASE_NAMES, input_cases
from probes.w3_qwen3_full.comparison import compare_positions
from probes.w3_qwen3_full.worker import checkpoint_schema, checked_tensor, save_array
-from probes.w4_qwen3_full.run import ATOL, save_executable, spec_json
+from probes.w4_qwen3_full.run import ATOL, CoreNamespace, add_core_argument, save_executable, spec_json
from scratchv.backend.tensor_c_codegen import TensorCCodegen
from scratchv.ir.types import BasicBlock, DataType, Function, Instruction, OpCode, Program
-from scratchv.runtime.riscv_harts import resolve_harts
+from scratchv.runtime.riscv_harts import resolve_cores
from scratchv.runtime.riscv_external import build_external, run_external
from scratchv.runtime.riscv_tensor import discover_toolchain
from scratchv.runtime.weight_bundle import write_weight_bundle
@@ -196,9 +196,7 @@ def main(argv=None):
default="none",
help="QEMU-only experimental two-buffer workspace layout (default: none); "
"padding adds 4 KiB and may reduce performance; unsupported graphs fail")
- cli.add_argument("--harts", choices=("1", "2", "4", "auto"), default="1",
- help="RV64 harts (default: 1); auto selects 1/2/4 from available CPUs, "
- "up to 4; parallel MatMul requires blocked_fma")
+ add_core_argument(cli)
cli.add_argument("--matmul-policy", choices=("sequential", "blocked_fma"), default="sequential")
cli.add_argument("--matmul-tile", choices=("1x4", "2x4", "1x8"), default="1x4",
help="MatMul output tile (default: 1x4); 2x4/1x8 require blocked_fma")
@@ -209,15 +207,15 @@ def main(argv=None):
cli.add_argument("--matmul-rhs-layout", choices=("original", "column-panel"), default="original",
help="Experimental immutable MatMul RHS packing (default: original); "
"column-panel requires blocked_fma")
- args = cli.parse_args(argv)
+ args = cli.parse_args(argv, namespace=CoreNamespace())
if args.matmul_tile != "1x4" and args.matmul_policy != "blocked_fma":
cli.error("--matmul-tile 2x4/1x8 requires --matmul-policy blocked_fma")
if args.matmul_rhs_layout != "original" and args.matmul_policy != "blocked_fma":
cli.error("--matmul-rhs-layout column-panel requires --matmul-policy blocked_fma")
- requested_harts = args.harts if args.harts == "auto" else int(args.harts)
- args.harts = resolve_harts(requested_harts)
- if requested_harts == "auto":
- print(f"[W4] auto selected {args.harts} hart(s)", flush=True)
+ requested_cores = args.cores if args.cores == "auto" else int(args.cores)
+ args.cores = resolve_cores(requested_cores)
+ if requested_cores == "auto":
+ print(f"[W4] auto selected {args.cores} core(s)", flush=True)
if any(not math.isfinite(limit) or limit <= 0 for limit in (args.timeout, args.build_timeout)):
cli.error("Timeouts must be finite and positive")
out = new_output_dir(args.output_dir)
@@ -226,7 +224,8 @@ def main(argv=None):
"passed": False, "status": "FAIL", "stage": "initialization",
"stages_seconds": {}, "checkpoint": args.checkpoint, "case": args.case,
"matmul_policy": args.matmul_policy, "kernel_calls": args.kernel_calls,
- "harts_requested": requested_harts, "harts": args.harts,
+ "cores_requested": requested_cores, "cores": args.cores,
+ "harts_requested": requested_cores, "harts": args.cores,
"matmul_tile": args.matmul_tile,
"matmul_rhs_layout": args.matmul_rhs_layout,
"qemu_workspace_layout_requested": args.qemu_workspace_layout,
@@ -290,9 +289,9 @@ def stage(name, operation):
artifact.external_initializers, out / "weights"))
tools = discover_toolchain(args.cc, args.qemu)
exe = stage("build_riscv_checkpoint", lambda: build_external(artifact, out / "weights",
- out / "build", tools, timeout=args.build_timeout, harts=args.harts))
+ out / "build", tools, timeout=args.build_timeout, cores=args.cores))
save_executable(exe, out)
- report.update(build=exe.evidence, memory_layout=exe.layout, harts=args.harts)
+ report.update(build=exe.evidence, memory_layout=exe.layout, cores=args.cores)
report["derived_onnx"] = stage("slice_onnx", lambda: slice_onnx(model, row, model_dir, out / "checkpoint.onnx"))
del artifact, bindings, sliced, program, parser, target, model
gc.collect()
diff --git a/probes/w4_qwen3_full/run.py b/probes/w4_qwen3_full/run.py
index e9eee48b..0d21f9e4 100644
--- a/probes/w4_qwen3_full/run.py
+++ b/probes/w4_qwen3_full/run.py
@@ -26,7 +26,7 @@
from probes.w4_qwen3_full.fma_conformance import run_probe as run_fma_probe
from scratchv.backend.tensor_c_codegen import TensorCCodegen, TensorSpec, _shape
from scratchv.ir.types import DataType
-from scratchv.runtime.riscv_harts import resolve_harts
+from scratchv.runtime.riscv_harts import read_cores, resolve_cores
from scratchv.runtime.riscv_external import build_external, run_external, ExternalExecutable, validate_external_elf
from scratchv.runtime.riscv_tensor import discover_toolchain, RiscVToolchain, input_layout, toolchain_versions
from scratchv.runtime.weight_bundle import write_weight_bundle, validate_weight_bundle, plan_guest_memory
@@ -103,11 +103,16 @@ def spec_json(spec):
def save_executable(exe, out):
+ cores = read_cores(exe.evidence)
+ if cores != exe.cores:
+ raise ValueError("Executable core count does not match build evidence")
+ # Keep equal legacy fields so existing readers can consume newly saved builds.
+ evidence = {**exe.evidence, "cores": cores, "harts": cores}
record = {"elf": str(exe.elf_path), "bundle_dir": str(exe.bundle_dir), "manifest": exe.manifest,
"layout": exe.layout, "inputs": [spec_json(s) for s in exe.inputs],
"weights": [spec_json(s) for s in exe.weights], "output": spec_json(exe.output),
- "cc": list(exe.toolchain.cc), "qemu": exe.toolchain.qemu, "evidence": exe.evidence,
- "harts": exe.harts}
+ "cc": list(exe.toolchain.cc), "qemu": exe.toolchain.qemu, "evidence": evidence,
+ "cores": cores, "harts": cores}
atomic_text(out / "executable.json", json.dumps(record, indent=2) + "\n")
return sha256_file(out / "executable.json")
@@ -138,18 +143,22 @@ def spec(value):
exe = ExternalExecutable(Path(record["elf"]), Path(record["bundle_dir"]), record["manifest"],
record["layout"], tuple(map(spec, record["inputs"])), spec(record["output"]),
tuple(map(spec, record["weights"])), RiscVToolchain(tuple(record["cc"]), record["qemu"]),
- record["evidence"], harts=record.get("harts", 1))
- if (type(exe.harts) is not int or exe.harts not in (1, 2, 4)
- or type(exe.evidence.get("harts", 1)) is not int
- or exe.evidence.get("harts", 1) != exe.harts):
- raise ValueError("Reused hart count does not match build evidence")
+ record["evidence"], cores=read_cores(record))
+ if read_cores(exe.evidence) != exe.cores:
+ raise ValueError("Reused core count does not match build evidence")
+ mean_policy = exe.evidence.get("mean_policy", "sequential")
+ if not isinstance(mean_policy, str) or mean_policy not in ("sequential", "two-four-lane-last-axis"):
+ raise ValueError("Unsupported reused Mean policy")
+ nonlinear_policy = exe.evidence.get("nonlinear_policy", "libm")
+ if not isinstance(nonlinear_policy, str) or nonlinear_policy not in ("libm", "fp32-reference"):
+ raise ValueError("Unsupported reused nonlinear policy")
manifest = validate_weight_bundle(exe.bundle_dir, exe.weights)
if manifest != exe.manifest:
raise ValueError("Reused weights do not match build manifest")
_, input_bytes = input_layout(exe.inputs)
layout = plan_guest_memory(workspace_bytes=exe.layout["regions"]["workspace"]["nbytes"],
weight_bytes=manifest["total_bytes"], input_bytes=input_bytes, output_bytes=exe.output.nbytes,
- harts=exe.harts)
+ cores=exe.cores)
if layout != exe.layout:
raise ValueError("Reused memory layout is not the canonical non-overlapping plan")
evidence = validate_external_elf(exe.elf_path, layout)
@@ -160,7 +169,7 @@ def spec(value):
for name, digest in exe.evidence["sources"].items():
if Path(name).name != name or sha256_file(exe.elf_path.parent / name) != digest:
raise ValueError("Reused generated source changed after build")
- return exe
+ return replace(exe, evidence={**exe.evidence, "cores": exe.cores, "harts": exe.cores})
def ort_reference(model_dir, feed, output):
@@ -179,6 +188,39 @@ def ort_reference(model_dir, feed, output):
gc.collect()
+class CoreNamespace(argparse.Namespace):
+ """Keep the legacy attribute as a live alias, including caller updates."""
+
+ @property
+ def harts(self):
+ return self.cores
+
+ @harts.setter
+ def harts(self, value):
+ self.cores = value
+
+
+class _CoreCountAction(argparse.Action):
+ """One count for both spellings; never silently let one override the other."""
+
+ def __call__(self, parser, namespace, values, option_string=None):
+ provided = getattr(namespace, "_core_count_arguments", {})
+ if any(name != option_string and value != values for name, value in provided.items()):
+ parser.error("--cores and --harts must specify the same value")
+ # Preserve argparse's last-value behavior when a wrapper repeats one spelling.
+ provided[option_string] = values
+ namespace._core_count_arguments = provided
+ setattr(namespace, self.dest, values)
+
+
+def add_core_argument(parser):
+ parser.add_argument("--cores", "--harts", dest="cores",
+ choices=("1", "2", "4", "auto"), default="1",
+ action=_CoreCountAction,
+ help="QEMU CPU cores (default: 1); auto selects 1/2/4 from available CPUs, "
+ "up to 4; --harts is a compatibility alias; parallel MatMul requires blocked_fma")
+
+
def main(argv=None):
cli = argparse.ArgumentParser(description=__doc__)
cli.add_argument("--model-dir", type=Path, required=True)
@@ -190,35 +232,39 @@ def main(argv=None):
default="none",
help="QEMU-only experimental two-buffer workspace layout (default: none); "
"padding adds 4 KiB and may reduce performance; unsupported graphs fail")
- cli.add_argument("--harts", choices=("1", "2", "4", "auto"), default="1",
- help="RV64 harts (default: 1); auto selects 1/2/4 from available CPUs, "
- "up to 4; parallel MatMul requires blocked_fma")
+ add_core_argument(cli)
cli.add_argument("--timeout", type=float, default=10800)
cli.add_argument("--build-timeout", type=float, default=900)
cli.add_argument("--build-only", action="store_true")
cli.add_argument("--matmul-policy", choices=("sequential", "blocked_fma"), default="sequential")
+ cli.add_argument("--mean-policy", choices=("sequential", "two-four-lane-last-axis"), default="sequential",
+ help="FP32 last-axis Mean order; explicit W3 reference component, not its full profile")
+ cli.add_argument("--nonlinear-policy", choices=("libm", "fp32-reference"), default="libm",
+ help="Explicit FP32 Trig/Sigmoid/Softmax order; default preserves platform libm")
cli.add_argument("--matmul-tile", choices=("1x4", "2x4", "1x8"), default="1x4",
help="MatMul output tile (default: 1x4); 2x4/1x8 require blocked_fma")
cli.add_argument("--reuse-build", type=Path, help="Local executable.json; verified before every run")
cli.add_argument("--matmul-rhs-layout", choices=("original", "column-panel"), default="original",
help="Experimental immutable MatMul RHS packing (default: original); "
"column-panel requires blocked_fma")
- args = cli.parse_args(argv)
+ args = cli.parse_args(argv, namespace=CoreNamespace())
if args.matmul_tile != "1x4" and args.matmul_policy != "blocked_fma":
cli.error("--matmul-tile 2x4/1x8 requires --matmul-policy blocked_fma")
if args.matmul_rhs_layout != "original" and args.matmul_policy != "blocked_fma":
cli.error("--matmul-rhs-layout column-panel requires --matmul-policy blocked_fma")
- requested_harts = args.harts if args.harts == "auto" else int(args.harts)
- args.harts = resolve_harts(requested_harts)
- if requested_harts == "auto":
- print(f"[W4] auto selected {args.harts} hart(s)", flush=True)
+ requested_cores = args.cores if args.cores == "auto" else int(args.cores)
+ args.cores = resolve_cores(requested_cores)
+ if requested_cores == "auto":
+ print(f"[W4] auto selected {args.cores} core(s)", flush=True)
out = args.output_dir.resolve()
out.mkdir(parents=True, exist_ok=False)
start = time.perf_counter()
report = {"schema_version": 1, "gate": "numeric:qemu-full-qwen3", "passed": False,
"status": "FAIL", "stage": "initialization", "stages_seconds": {}, "cases": [],
- "harts_requested": requested_harts, "harts": args.harts,
- "matmul_tile": args.matmul_tile,
+ "cores_requested": requested_cores, "cores": args.cores,
+ "harts_requested": requested_cores, "harts": args.cores,
+ "matmul_tile": args.matmul_tile, "mean_policy": args.mean_policy,
+ "nonlinear_policy": args.nonlinear_policy,
"matmul_rhs_layout": args.matmul_rhs_layout,
"qemu_workspace_layout_requested": args.qemu_workspace_layout,
"gates": {"build:riscv-full": False, "smoke:qemu-full": False,
@@ -247,7 +293,6 @@ def stage(name, operation):
model_dir = args.model_dir.resolve()
report["model_dir"] = str(model_dir)
report["matmul_policy"] = args.matmul_policy
- report["harts"] = args.harts
report["model_files"] = stage("asset_hashes", lambda: assets.verify_files(model_dir))
import onnx
model = stage("metadata_load", lambda: onnx.load(str(model_dir / "model.onnx"), load_external_data=False))
@@ -259,8 +304,19 @@ def stage(name, operation):
if sha256_file(args.reuse_build) != previous.get("executable_sha256"):
raise ValueError("Reused executable metadata hash is missing or changed")
exe = stage("load_existing_build", lambda: load_executable(args.reuse_build))
- if exe.harts != args.harts:
- raise ValueError("Reused build hart count differs from requested --harts")
+ if (("cores" in previous or "harts" in previous)
+ and read_cores(previous) != exe.cores):
+ raise ValueError("Reused report core count differs from executable")
+ if exe.cores != args.cores:
+ raise ValueError("Reused build core count differs from requested --cores")
+ if (exe.evidence.get("mean_policy", "sequential") != args.mean_policy
+ or previous.get("mean_policy", "sequential") != args.mean_policy):
+ raise ValueError("Reused build Mean policy differs from requested --mean-policy; "
+ "rebuild the executable")
+ if (exe.evidence.get("nonlinear_policy", "libm") != args.nonlinear_policy
+ or previous.get("nonlinear_policy", "libm") != args.nonlinear_policy):
+ raise ValueError("Reused build nonlinear policy differs from requested --nonlinear-policy; "
+ "rebuild the executable")
if (exe.evidence.get("matmul_tile", "1x4") != args.matmul_tile
or previous.get("matmul_tile", "1x4") != args.matmul_tile):
raise ValueError("Reused build MatMul tile differs from requested --matmul-tile; "
@@ -304,6 +360,7 @@ def stage(name, operation):
initializers=parser.initializers, constant_storage="external",
max_constant_bytes=3 * 1024**3, max_workspace_bytes=1024**3,
kernel_calls=True, matmul_policy=args.matmul_policy, matmul_tile=args.matmul_tile,
+ mean_policy=args.mean_policy, nonlinear_policy=args.nonlinear_policy,
matmul_rhs_layout=args.matmul_rhs_layout,
qemu_workspace_layout=args.qemu_workspace_layout).generate())
report["codegen"] = {"source_bytes": len(artifact.source.encode()),
@@ -313,6 +370,8 @@ def stage(name, operation):
"qemu_workspace_buffers": artifact.qemu_workspace_buffers,
"weights": len(artifact.external_weights),
"kernel_calls": artifact.kernel_calls,
+ "mean_policy": artifact.mean_policy,
+ "nonlinear_policy": artifact.nonlinear_policy,
"matmul_tile": artifact.matmul_tile,
"matmul_rhs_layout": artifact.matmul_rhs_layout,
"matmul_rhs_panels": list(artifact.matmul_rhs_panels),
@@ -321,7 +380,7 @@ def stage(name, operation):
artifact.external_initializers, out / "weights"))
toolchain = discover_toolchain(args.cc, args.qemu)
exe = stage("build_riscv_full", lambda: build_external(artifact, out / "weights",
- out / "build", toolchain, timeout=args.build_timeout, harts=args.harts))
+ out / "build", toolchain, timeout=args.build_timeout, cores=args.cores))
del artifact, program, parser
gc.collect()
report["executable_sha256"] = save_executable(exe, out)
diff --git a/probes/w5_generation/README.md b/probes/w5_generation/README.md
new file mode 100644
index 00000000..20cf039f
--- /dev/null
+++ b/probes/w5_generation/README.md
@@ -0,0 +1,42 @@
+# W5 Host 生成参考
+
+本探针验证 Host token 循环；实际推理后端是固定完整 Qwen3-0.6B ONNX 的 ONNX Runtime CPU。它不启动 QEMU，不能代替 W5 的 QEMU Paris / 三 prompt 验收。生成合同见 [W5 合同](../../docs/llm-deploy-v1.0/W5/README.md)。
+
+## 固定参考
+
+[references.json](references.json) 固定以下三个原始 completion prompt；它们在第一次运行前选定，每条预算 8 个新 token，分别检查输出包含 Paris、Berlin、Tokyo。
+
+- `The capital of France is`
+- `The capital of Germany is`
+- `The capital of Japan is`
+
+prompt 按 UTF-8 原文编码，`add_special_tokens=False`，不套 chat template。模型身份引用既有 [W1 manifest](../w1_qwen3_export/manifest.json)，不增加另一套哈希清单。运行前检查 ONNX 文件长度，避免把 LFS pointer 当模型；资产完整性沿用已有下载/验收流程。本探针不宣称长度检查等价于 SHA256 校验。
+
+参考保存 prompt 字节及 token IDs、所有生成 IDs、文本、EOS 集、停止原因和逐步 top-1/top-2 间距。使用 full FP32 logits 的最后有效行；检查完整输出（包括 padding 行）有限性。argmax 覆盖整个模型词表；采样到 tokenizer 不支持的 padded ID 会失败，不做过滤。
+
+## 运行
+
+先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 安装选定环境并获取固定模型、tokenizer 资产。在仓库根目录运行：
+
+```bash
+python probes/w5_generation/run.py \
+ --assets assets/qwen3-0.6b \
+ --output-dir output/w5-generation-reference
+```
+
+正常运行只读冻结参考，写 `output/w5-generation-reference/reference-run.json`。`--assets` 可指向已有完整资产目录，避免工作树重复下载。默认模式要求生成 token IDs、文本和停止原因与冻结参考完全一致；浮点 top-2 间距用于诊断，不把它当跨后端数值误差标准。公开 CLI 的 `--record` 只向 output 写候选参考，**不会覆盖 references.json**；更换冻结参考必须单独审查 prompt、模型身份及验收条件的变化。
+
+一个 ORT session 顺序处理三个用例，沿用 W4 参考设置：CPU provider、intra/inter-op 均为 1、禁用 graph optimization 和 CPU memory arena。每次 forward 都由独立 oracle 检查：
+
+1. 当前 token 历史、最后有效位置、右 padding 和 causal/key padding mask。
+2. 完整 logits 的 FP32 ABI 与有限性，直接索引最后有效行并用 NumPy argmax 计算 token。
+3. 独立的 EOS → 容量 → token 预算停止规则、生成 IDs、解码文本和调用次数。
+
+这样一次真实 ORT 运行同时验证 Host 循环和参考输出，不再重复跑同一套完整参考。结果中 `qemu_e2e: NOT_RUN` 明确区分本轮覆盖范围。
+
+## 计时与内存
+
+每步同时记录 ORT forward wall/CPU 时间；每条用例另记包括 Host 输入准备、检查、采样及解码的 wall/CPU 时间。session 加载单独计时。CPU 时间含进程内所有线程；Linux `ru_maxrss` 是进程截至当时的累计峰值，不能作为每个用例的独立内存增量。这些是 ORT/Host 数据，不能与既有 QEMU 计时混用或推算 QEMU 加速比。
+
+本探针不读写 core 数、padding、CI 或 WSL 设置。QEMU 的默认 1 core、手动 1/2/4、启动时 auto 和默认关闭的 padding 仍由既有后端负责。
+QEMU 生成入口使用 `--cores`，仍接受旧 `--harts`；它们不属于本 Host 参考探针的参数，详见 [W5 开发说明](../../docs/llm-deploy-v1.0/W5/README.md)。
diff --git a/probes/w5_generation/references.json b/probes/w5_generation/references.json
new file mode 100644
index 00000000..35aaea10
--- /dev/null
+++ b/probes/w5_generation/references.json
@@ -0,0 +1,422 @@
+{
+ "schema_version": 1,
+ "status": "frozen",
+ "model_manifest": "probes/w1_qwen3_export/manifest.json",
+ "model_id": "Qwen/Qwen3-0.6B",
+ "revision": "c1899de289a04d12100db370d81485cdf75e47ca",
+ "contract": {
+ "batch_size": 1,
+ "sequence_length": 256,
+ "dtype": "float32",
+ "use_cache": false,
+ "prompt_format": "raw_completion",
+ "add_special_tokens": false,
+ "sampling": "greedy_argmax_lowest_id",
+ "skip_special_tokens": true,
+ "clean_up_tokenization_spaces": false,
+ "stop_priority": [
+ "eos",
+ "capacity",
+ "max_new_tokens"
+ ]
+ },
+ "cases": [
+ {
+ "name": "france",
+ "prompt": "The capital of France is",
+ "prompt_utf8_hex": "546865206361706974616c206f66204672616e6365206973",
+ "required_substring": "Paris",
+ "max_new_tokens": 8,
+ "prompt_token_ids": [
+ 785,
+ 6722,
+ 315,
+ 9625,
+ 374
+ ],
+ "generated_token_ids": [
+ 12095,
+ 13,
+ 576,
+ 6722,
+ 315,
+ 15344,
+ 374,
+ 21718
+ ],
+ "generated_text": " Paris. The capital of Italy is Rome",
+ "stop_reason": "max_new_tokens",
+ "semantic_passed": true,
+ "host_oracle_passed": true,
+ "steps": [
+ {
+ "step": 0,
+ "valid_length": 5,
+ "token_id": 12095,
+ "decoded_token": " Paris",
+ "top1_logit": 17.498851776123047,
+ "top2_logit": 14.337191581726074,
+ "top1_top2_margin": 3.1616601943969727,
+ "forward_wall_seconds": 5.923962489000019,
+ "forward_cpu_seconds": 5.923590624
+ },
+ {
+ "step": 1,
+ "valid_length": 6,
+ "token_id": 13,
+ "decoded_token": ".",
+ "top1_logit": 19.274662017822266,
+ "top2_logit": 19.083740234375,
+ "top1_top2_margin": 0.19092178344726562,
+ "forward_wall_seconds": 5.199123996000026,
+ "forward_cpu_seconds": 5.199001791
+ },
+ {
+ "step": 2,
+ "valid_length": 7,
+ "token_id": 576,
+ "decoded_token": " The",
+ "top1_logit": 16.967761993408203,
+ "top2_logit": 15.490779876708984,
+ "top1_top2_margin": 1.4769821166992188,
+ "forward_wall_seconds": 5.950034952000067,
+ "forward_cpu_seconds": 5.946767153999998
+ },
+ {
+ "step": 3,
+ "valid_length": 8,
+ "token_id": 6722,
+ "decoded_token": " capital",
+ "top1_logit": 18.299823760986328,
+ "top2_logit": 14.856491088867188,
+ "top1_top2_margin": 3.4433326721191406,
+ "forward_wall_seconds": 5.595218937000027,
+ "forward_cpu_seconds": 5.592634477000001
+ },
+ {
+ "step": 4,
+ "valid_length": 9,
+ "token_id": 315,
+ "decoded_token": " of",
+ "top1_logit": 20.05550193786621,
+ "top2_logit": 17.614667892456055,
+ "top1_top2_margin": 2.4408340454101562,
+ "forward_wall_seconds": 5.342139397999972,
+ "forward_cpu_seconds": 5.338031255000001
+ },
+ {
+ "step": 5,
+ "valid_length": 10,
+ "token_id": 15344,
+ "decoded_token": " Italy",
+ "top1_logit": 16.641294479370117,
+ "top2_logit": 16.617063522338867,
+ "top1_top2_margin": 0.02423095703125,
+ "forward_wall_seconds": 5.288755011000035,
+ "forward_cpu_seconds": 5.285648705
+ },
+ {
+ "step": 6,
+ "valid_length": 11,
+ "token_id": 374,
+ "decoded_token": " is",
+ "top1_logit": 22.20791244506836,
+ "top2_logit": 16.946107864379883,
+ "top1_top2_margin": 5.261804580688477,
+ "forward_wall_seconds": 5.632552263999969,
+ "forward_cpu_seconds": 5.624142145
+ },
+ {
+ "step": 7,
+ "valid_length": 12,
+ "token_id": 21718,
+ "decoded_token": " Rome",
+ "top1_logit": 20.907331466674805,
+ "top2_logit": 17.204931259155273,
+ "top1_top2_margin": 3.7024002075195312,
+ "forward_wall_seconds": 5.547119829000053,
+ "forward_cpu_seconds": 5.546546575000001
+ }
+ ],
+ "wall_seconds": 44.67187527400006,
+ "cpu_seconds": 44.649298945,
+ "process_peak_rss_kib": 5244604
+ },
+ {
+ "name": "germany",
+ "prompt": "The capital of Germany is",
+ "prompt_utf8_hex": "546865206361706974616c206f66204765726d616e79206973",
+ "required_substring": "Berlin",
+ "max_new_tokens": 8,
+ "prompt_token_ids": [
+ 785,
+ 6722,
+ 315,
+ 9856,
+ 374
+ ],
+ "generated_token_ids": [
+ 19846,
+ 13,
+ 576,
+ 6722,
+ 315,
+ 9625,
+ 374,
+ 12095
+ ],
+ "generated_text": " Berlin. The capital of France is Paris",
+ "stop_reason": "max_new_tokens",
+ "semantic_passed": true,
+ "host_oracle_passed": true,
+ "steps": [
+ {
+ "step": 0,
+ "valid_length": 5,
+ "token_id": 19846,
+ "decoded_token": " Berlin",
+ "top1_logit": 16.521493911743164,
+ "top2_logit": 14.624658584594727,
+ "top1_top2_margin": 1.8968353271484375,
+ "forward_wall_seconds": 5.496049604999939,
+ "forward_cpu_seconds": 5.495774664999999
+ },
+ {
+ "step": 1,
+ "valid_length": 6,
+ "token_id": 13,
+ "decoded_token": ".",
+ "top1_logit": 19.563419342041016,
+ "top2_logit": 19.295778274536133,
+ "top1_top2_margin": 0.2676410675048828,
+ "forward_wall_seconds": 5.400778742000057,
+ "forward_cpu_seconds": 5.400431184999995
+ },
+ {
+ "step": 2,
+ "valid_length": 7,
+ "token_id": 576,
+ "decoded_token": " The",
+ "top1_logit": 16.98853874206543,
+ "top2_logit": 15.723114013671875,
+ "top1_top2_margin": 1.2654247283935547,
+ "forward_wall_seconds": 5.322351349999963,
+ "forward_cpu_seconds": 5.322156993
+ },
+ {
+ "step": 3,
+ "valid_length": 8,
+ "token_id": 6722,
+ "decoded_token": " capital",
+ "top1_logit": 18.370086669921875,
+ "top2_logit": 14.746591567993164,
+ "top1_top2_margin": 3.623495101928711,
+ "forward_wall_seconds": 4.931711173000053,
+ "forward_cpu_seconds": 4.928969291000001
+ },
+ {
+ "step": 4,
+ "valid_length": 9,
+ "token_id": 315,
+ "decoded_token": " of",
+ "top1_logit": 20.154003143310547,
+ "top2_logit": 18.079017639160156,
+ "top1_top2_margin": 2.0749855041503906,
+ "forward_wall_seconds": 5.378948186000002,
+ "forward_cpu_seconds": 5.3775705759999965
+ },
+ {
+ "step": 5,
+ "valid_length": 10,
+ "token_id": 9625,
+ "decoded_token": " France",
+ "top1_logit": 18.578645706176758,
+ "top2_logit": 17.20067024230957,
+ "top1_top2_margin": 1.3779754638671875,
+ "forward_wall_seconds": 5.244032164000032,
+ "forward_cpu_seconds": 5.239271584999997
+ },
+ {
+ "step": 6,
+ "valid_length": 11,
+ "token_id": 374,
+ "decoded_token": " is",
+ "top1_logit": 21.884960174560547,
+ "top2_logit": 16.38650894165039,
+ "top1_top2_margin": 5.498451232910156,
+ "forward_wall_seconds": 5.602864303999922,
+ "forward_cpu_seconds": 5.602449051000008
+ },
+ {
+ "step": 7,
+ "valid_length": 12,
+ "token_id": 12095,
+ "decoded_token": " Paris",
+ "top1_logit": 21.248014450073242,
+ "top2_logit": 16.446701049804688,
+ "top1_top2_margin": 4.801313400268555,
+ "forward_wall_seconds": 5.380494970000086,
+ "forward_cpu_seconds": 5.373863341999993
+ }
+ ],
+ "wall_seconds": 42.910345698000015,
+ "cpu_seconds": 42.893603793,
+ "process_peak_rss_kib": 5244636
+ },
+ {
+ "name": "japan",
+ "prompt": "The capital of Japan is",
+ "prompt_utf8_hex": "546865206361706974616c206f66204a6170616e206973",
+ "required_substring": "Tokyo",
+ "max_new_tokens": 8,
+ "prompt_token_ids": [
+ 785,
+ 6722,
+ 315,
+ 6323,
+ 374
+ ],
+ "generated_token_ids": [
+ 26194,
+ 13,
+ 576,
+ 6722,
+ 315,
+ 279,
+ 3639,
+ 4180
+ ],
+ "generated_text": " Tokyo. The capital of the United States",
+ "stop_reason": "max_new_tokens",
+ "semantic_passed": true,
+ "host_oracle_passed": true,
+ "steps": [
+ {
+ "step": 0,
+ "valid_length": 5,
+ "token_id": 26194,
+ "decoded_token": " Tokyo",
+ "top1_logit": 16.132610321044922,
+ "top2_logit": 15.243704795837402,
+ "top1_top2_margin": 0.8889055252075195,
+ "forward_wall_seconds": 5.337803195999982,
+ "forward_cpu_seconds": 5.3336678580000125
+ },
+ {
+ "step": 1,
+ "valid_length": 6,
+ "token_id": 13,
+ "decoded_token": ".",
+ "top1_logit": 18.725910186767578,
+ "top2_logit": 18.47475242614746,
+ "top1_top2_margin": 0.2511577606201172,
+ "forward_wall_seconds": 5.40962941600003,
+ "forward_cpu_seconds": 5.409200784999996
+ },
+ {
+ "step": 2,
+ "valid_length": 7,
+ "token_id": 576,
+ "decoded_token": " The",
+ "top1_logit": 16.345678329467773,
+ "top2_logit": 15.184026718139648,
+ "top1_top2_margin": 1.161651611328125,
+ "forward_wall_seconds": 5.2799776159999965,
+ "forward_cpu_seconds": 5.279397790000004
+ },
+ {
+ "step": 3,
+ "valid_length": 8,
+ "token_id": 6722,
+ "decoded_token": " capital",
+ "top1_logit": 17.322513580322266,
+ "top2_logit": 14.836996078491211,
+ "top1_top2_margin": 2.4855175018310547,
+ "forward_wall_seconds": 5.683535881000012,
+ "forward_cpu_seconds": 5.678628095000008
+ },
+ {
+ "step": 4,
+ "valid_length": 9,
+ "token_id": 315,
+ "decoded_token": " of",
+ "top1_logit": 19.100204467773438,
+ "top2_logit": 17.763046264648438,
+ "top1_top2_margin": 1.337158203125,
+ "forward_wall_seconds": 5.363878753999984,
+ "forward_cpu_seconds": 5.363313796
+ },
+ {
+ "step": 5,
+ "valid_length": 10,
+ "token_id": 279,
+ "decoded_token": " the",
+ "top1_logit": 16.005170822143555,
+ "top2_logit": 15.628267288208008,
+ "top1_top2_margin": 0.3769035339355469,
+ "forward_wall_seconds": 5.153919003999931,
+ "forward_cpu_seconds": 5.153602902000003
+ },
+ {
+ "step": 6,
+ "valid_length": 11,
+ "token_id": 3639,
+ "decoded_token": " United",
+ "top1_logit": 16.810474395751953,
+ "top2_logit": 15.270747184753418,
+ "top1_top2_margin": 1.5397272109985352,
+ "forward_wall_seconds": 5.270241868000085,
+ "forward_cpu_seconds": 5.270055357000004
+ },
+ {
+ "step": 7,
+ "valid_length": 12,
+ "token_id": 4180,
+ "decoded_token": " States",
+ "top1_logit": 20.563379287719727,
+ "top2_logit": 17.63241195678711,
+ "top1_top2_margin": 2.930967330932617,
+ "forward_wall_seconds": 4.756456833000016,
+ "forward_cpu_seconds": 4.755680379999987
+ }
+ ],
+ "wall_seconds": 42.40610255299998,
+ "cpu_seconds": 42.394194961,
+ "process_peak_rss_kib": 5244696
+ }
+ ],
+ "tokenizer": {
+ "pad_token_id": 151643,
+ "generation_eos_token_ids": [
+ 151645,
+ 151643
+ ],
+ "model_vocab_size": 151936,
+ "tokenizer_vocab_size": 151669
+ },
+ "baseline": {
+ "recorded_at_utc": "2026-10-23T00:42:50.852643+00:00",
+ "acquisition": "One ORT run with independent per-step Host/input/argmax/stop oracle; not a second frozen replay.",
+ "environment": {
+ "python": "3.12.3",
+ "platform": "Linux-6.18.40.1-microsoft-standard-WSL2-x86_64-with-glibc2.39",
+ "packages": {
+ "numpy": "2.2.6",
+ "onnxruntime": "1.22.1",
+ "tokenizers": "0.21.4"
+ },
+ "provider": "CPUExecutionProvider",
+ "intra_op_threads": 1,
+ "inter_op_threads": 1,
+ "graph_optimization": "ORT_DISABLE_ALL",
+ "cpu_memory_arena": false
+ },
+ "session_load_wall_seconds": 0.3070653280000215,
+ "session_load_cpu_seconds": 0.29269802199999995,
+ "wall_seconds": 130.67631491099996,
+ "cpu_seconds": 130.61070972899998,
+ "process_peak_rss_kib": 5244696,
+ "memory_note": "Linux ru_maxrss is cumulative process peak, not per-case allocation.",
+ "qemu_e2e": "NOT_RUN"
+ }
+}
diff --git a/probes/w5_generation/run.py b/probes/w5_generation/run.py
new file mode 100644
index 00000000..8281ba52
--- /dev/null
+++ b/probes/w5_generation/run.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""Run three fixed W5 Host generation references with the pinned full ORT model.
+
+Default: compare token IDs/text/stop reasons with references.json.
+--record: collect a candidate reference in output only; never rewrite the fixture.
+This is an ORT/Host test, not the QEMU W5 end-to-end gate.
+"""
+from __future__ import annotations
+
+import argparse
+from datetime import datetime, timezone
+import importlib.metadata
+import json
+from pathlib import Path
+import platform
+import resource
+import sys
+import time
+
+import numpy as np
+
+ROOT = Path(__file__).resolve().parents[2]
+if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+
+from probes.w2_runtime_model.run import checked_inputs
+from scratchv.runtime.generation import generate
+from scratchv.runtime.llm_inputs import SEQ_LENGTH
+from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer
+
+FIXTURE = Path(__file__).with_name("references.json")
+
+
+def reference_stop(length, generated, budget, eos_ids):
+ """Independent stop oracle; the prompt EOS is deliberately ignored."""
+ if generated and generated[-1] in eos_ids:
+ return "eos"
+ if length == SEQ_LENGTH:
+ return "capacity"
+ if len(generated) >= budget:
+ return "max_new_tokens"
+ return None
+
+
+def session_for(assets, manifest):
+ import onnxruntime as ort
+
+ # Reuse the existing artifact identity/size manifest, without rehashing GBs.
+ # Size is an availability/LFS-pointer check, not a replacement integrity hash.
+ for item in manifest["files"]:
+ path = assets / "onnx" / item["name"]
+ if not path.is_file() or path.stat().st_size != item["bytes"]:
+ raise ValueError(f"Missing/incomplete fixed ONNX asset: {item['name']}")
+ options = ort.SessionOptions()
+ options.intra_op_num_threads = options.inter_op_num_threads = 1
+ options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL
+ options.enable_cpu_mem_arena = False
+ session = ort.InferenceSession(
+ str(assets / "onnx/model.onnx"), options, providers=["CPUExecutionProvider"])
+ inputs = [(item.name, item.type, item.shape) for item in session.get_inputs()]
+ outputs = [(item.name, item.type, item.shape) for item in session.get_outputs()]
+ if inputs != [("input_ids", "tensor(int64)", [1, 256]),
+ ("attention_mask", "tensor(float)", [1, 1, 256, 256])]:
+ raise ValueError(f"Unexpected model input ABI: {inputs}")
+ if outputs != [("logits", "tensor(float)", [1, 256, 151936])]:
+ raise ValueError(f"Unexpected model output ABI: {outputs}")
+ return session
+
+
+def run_case(case, tokenizer, session):
+ """Exercise Host generate, checked by independent per-call and final oracles."""
+ prompt = case["prompt"]
+ budget = case["max_new_tokens"]
+ prompt_ids = tokenizer.encode(prompt, add_special_tokens=False)
+ history = list(prompt_ids)
+ independently_generated = []
+ steps = []
+ eos_ids = tokenizer.generation_eos_token_ids
+ start_wall, start_cpu = time.perf_counter(), time.process_time()
+
+ def forward(prepared):
+ # A wrong append, stale input, padding value, mask, or extra call fails.
+ if reference_stop(len(history), independently_generated, budget, eos_ids):
+ raise AssertionError("Host called forward after the independent stop condition")
+ feed = checked_inputs(history, prepared, tokenizer.pad_token_id)
+ begin_wall, begin_cpu = time.perf_counter(), time.process_time()
+ logits = session.run(["logits"], feed)[0]
+ elapsed_wall = time.perf_counter() - begin_wall
+ elapsed_cpu = time.process_time() - begin_cpu
+ if (logits.shape != (1, SEQ_LENGTH, tokenizer.model_vocab_size)
+ or logits.dtype != np.float32 or not np.isfinite(logits).all()):
+ raise ValueError("ORT returned invalid complete FP32 logits")
+ # Deliberately independent of last_valid_logits/greedy_token.
+ vector = logits[0, len(history) - 1, :]
+ token = int(np.argmax(vector))
+ tokenizer.validate_token_ids([token])
+ top_two = np.partition(vector, -2)[-2:]
+ steps.append({
+ "step": len(steps), "valid_length": len(history),
+ "token_id": token,
+ "decoded_token": tokenizer.decode([token], skip_special_tokens=False,
+ clean_up_tokenization_spaces=False),
+ "top1_logit": float(vector[token]),
+ "top2_logit": float(top_two[0]),
+ "top1_top2_margin": float(vector[token]) - float(top_two[0]),
+ "forward_wall_seconds": elapsed_wall, "forward_cpu_seconds": elapsed_cpu,
+ })
+ independently_generated.append(token)
+ history.append(token)
+ return logits
+
+ actual = generate(prompt, tokenizer=tokenizer, forward=forward, max_new_tokens=budget)
+ wall_seconds = time.perf_counter() - start_wall
+ cpu_seconds = time.process_time() - start_cpu
+ expected_stop = reference_stop(len(history), independently_generated, budget, eos_ids)
+ expected_text = tokenizer.decode(independently_generated, skip_special_tokens=True,
+ clean_up_tokenization_spaces=False)
+ if tuple(actual.prompt_token_ids) != tuple(prompt_ids):
+ raise AssertionError("Host prompt IDs differ from the independent tokenizer result")
+ if tuple(actual.generated_token_ids) != tuple(independently_generated):
+ raise AssertionError("Host sampled IDs differ from independent ORT argmax")
+ if actual.stop_reason != expected_stop or actual.generated_text != expected_text:
+ raise AssertionError("Host stop reason/text differs from the independent oracle")
+ if len(steps) != len(independently_generated):
+ raise AssertionError("Forward count differs from generated token count")
+ return {
+ "name": case["name"], "prompt": prompt,
+ "prompt_utf8_hex": prompt.encode("utf-8").hex(),
+ "required_substring": case["required_substring"], "max_new_tokens": budget,
+ "prompt_token_ids": prompt_ids, "generated_token_ids": independently_generated,
+ "generated_text": expected_text, "stop_reason": expected_stop,
+ "semantic_passed": case["required_substring"] in expected_text,
+ "host_oracle_passed": True, "steps": steps,
+ "wall_seconds": wall_seconds, "cpu_seconds": cpu_seconds,
+ "process_peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
+ }
+
+
+def compare_frozen(actual, expected):
+ keys = ("prompt", "prompt_utf8_hex", "required_substring", "max_new_tokens",
+ "prompt_token_ids", "generated_token_ids", "generated_text", "stop_reason")
+ differences = [key for key in keys if actual[key] != expected[key]]
+ return {"passed": not differences, "different_fields": differences}
+
+
+def main(argv=None):
+ cli = argparse.ArgumentParser(description=__doc__)
+ cli.add_argument("--assets", type=Path, default=ROOT / "assets/qwen3-0.6b")
+ cli.add_argument("--output-dir", type=Path, required=True)
+ cli.add_argument("--record", action="store_true",
+ help="Record a candidate reference in output; do not edit frozen references")
+ args = cli.parse_args(argv)
+ fixture = json.loads(FIXTURE.read_text(encoding="utf-8"))
+ manifest = json.loads((ROOT / fixture["model_manifest"]).read_text(encoding="utf-8"))
+ if (manifest["revision"] != fixture["revision"]
+ or manifest["model_id"] != fixture["model_id"]):
+ raise ValueError("Reference fixture does not match the fixed model manifest")
+ if not args.record and fixture["status"] != "frozen":
+ raise ValueError("Freeze and review the initial reference before verification mode")
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ output = args.output_dir / "reference-run.json"
+ report = {
+ "schema_version": 1, "status": "running",
+ "started_at_utc": datetime.now(timezone.utc).isoformat(),
+ "mode": "record_candidate" if args.record else "verify_frozen",
+ "model_manifest": fixture["model_manifest"],
+ "model_id": manifest["model_id"], "revision": manifest["revision"],
+ "asset_identity": "existing manifest/revision; files checked for expected size, not rehashed",
+ "contract": fixture["contract"],
+ "environment": {
+ "python": platform.python_version(), "platform": platform.platform(),
+ "packages": {name: importlib.metadata.version(name)
+ for name in ("numpy", "onnxruntime", "tokenizers")},
+ "provider": "CPUExecutionProvider", "intra_op_threads": 1,
+ "inter_op_threads": 1, "graph_optimization": "ORT_DISABLE_ALL",
+ "cpu_memory_arena": False,
+ },
+ "memory_note": "Linux ru_maxrss is cumulative process peak, not per-case allocation.",
+ "qemu_e2e": "NOT_RUN", "cases": [],
+ }
+ started_wall, started_cpu = time.perf_counter(), time.process_time()
+
+ def save():
+ output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+
+ save()
+ try:
+ tokenizer = Qwen3Tokenizer.from_directory(args.assets / "tokenizer")
+ report["tokenizer"] = {
+ "pad_token_id": tokenizer.pad_token_id,
+ "generation_eos_token_ids": list(tokenizer.generation_eos_token_ids),
+ "model_vocab_size": tokenizer.model_vocab_size,
+ "tokenizer_vocab_size": tokenizer.tokenizer_vocab_size,
+ }
+ if not args.record and report["tokenizer"] != fixture["tokenizer"]:
+ raise ValueError("Tokenizer IDs/vocabulary changed from the frozen reference")
+ begin_wall, begin_cpu = time.perf_counter(), time.process_time()
+ session = session_for(args.assets, manifest)
+ report["session_load_wall_seconds"] = time.perf_counter() - begin_wall
+ report["session_load_cpu_seconds"] = time.process_time() - begin_cpu
+ for case in fixture["cases"]:
+ print(f"Running {case['name']}: {case['max_new_tokens']} token budget", flush=True)
+ result = run_case(case, tokenizer, session)
+ result["frozen_comparison"] = None if args.record else compare_frozen(result, case)
+ report["cases"].append(result)
+ save()
+ print(json.dumps({key: result[key] for key in
+ ("name", "generated_text", "stop_reason", "wall_seconds",
+ "cpu_seconds", "semantic_passed")}, ensure_ascii=False), flush=True)
+ report["status"] = "passed" if all(
+ case["semantic_passed"] and case["host_oracle_passed"]
+ and (args.record or case["frozen_comparison"]["passed"])
+ for case in report["cases"]) else "failed"
+ except BaseException as exc:
+ report["status"] = "failed"
+ report["error"] = f"{type(exc).__name__}: {exc}"
+ raise
+ finally:
+ report["wall_seconds"] = time.perf_counter() - started_wall
+ report["cpu_seconds"] = time.process_time() - started_cpu
+ report["process_peak_rss_kib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+ save()
+ print(f"{report['status'].upper()}: {output}", flush=True)
+ return 0 if report["status"] == "passed" else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/pyproject.toml b/pyproject.toml
index 076e3464..f35b4816 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,7 +35,7 @@ classifiers = [
riscv = ["tinyfive"]
llvm = ["llvmlite"] # optional: LLVM IR JIT execution
verify = ["onnxruntime"] # optional: ONNX Runtime comparison
-llm = ["tokenizers>=0.21,<0.22"] # optional: local Qwen3 text tokenizer; no Torch
+llm = ["tokenizers>=0.21,<0.22", "jinja2>=3.1.6,<4"] # optional: local Qwen3 text tokenizer; no Torch
# Pinned deployment environment: Ubuntu 24.04 x86_64 / CPython 3.12.
# Use the official CPU wheel so one install does not pull CUDA dependencies.
llm-deploy = [
@@ -50,6 +50,7 @@ llm-deploy = [
"safetensors==0.5.3",
"protobuf==5.29.5",
"tokenizers==0.21.4",
+ "jinja2==3.1.6",
"pytest==9.1.1",
"ziglang==0.14.1",
"PyYAML==6.0.3",
diff --git a/scratchv/backend/tensor_c_codegen.py b/scratchv/backend/tensor_c_codegen.py
index bd9f784b..00335304 100644
--- a/scratchv/backend/tensor_c_codegen.py
+++ b/scratchv/backend/tensor_c_codegen.py
@@ -19,6 +19,16 @@
Supported element types are FP32, INT32 and INT64. Control flow, dynamic shapes,
empty reductions and nonfinite arithmetic are rejected explicitly. This backend
does not silently lower tensor operations to scalar integer instructions.
+
+The default Mean policy preserves sequential FP32 accumulation. The explicit
+``mean_policy="two-four-lane-last-axis"`` option changes only single-last-axis
+FP32 ReduceMean to the existing reference's canonical two-four-lane order.
+It uses portable scalar C, preserves finite checks and requires no SIMD ISA.
+The separate nonlinear_policy="fp32-reference" option selects fixed FP32
+Sin/Cos, Sigmoid and Softmax arithmetic; its default "libm" is unchanged.
+The reference uses no host ISA dispatch, preserves all numeric checks, and
+retains libm trigonometry outside abs(x) < 18000. Pow and standalone Exp are
+unaffected. Neither explicit policy guarantees every ORT build's exact bits.
"""
from __future__ import annotations
@@ -30,6 +40,10 @@
import numpy as np
+from scratchv.backend.tensor_c_mean import TWO_FOUR_LANE_LAST_AXIS_C
+from scratchv.backend.tensor_c_nonlinear import (
+ TRIG_REFERENCE_C, SIGMOID_REFERENCE_C, SOFTMAX_REFERENCE_C,
+)
from scratchv.ir.types import DataType, OpCode, Program, Value
from scratchv.verification.ir_numpy_ops import OpError, check_instruction
@@ -82,6 +96,8 @@ class TensorCArtifact:
matmul_rhs_panels: tuple[dict[str, object], ...] = ()
qemu_workspace_layout: str = "none"
qemu_workspace_buffers: tuple[tuple[int, int], ...] = ()
+ mean_policy: str = "sequential"
+ nonlinear_policy: str = "libm"
def _shape(shape):
@@ -173,10 +189,23 @@ def _literal(value, dtype):
'''
_EXTERNAL_PRELUDE = r'''
+#if defined(SCRATCHV_CORES) && defined(SCRATCHV_HARTS) && SCRATCHV_CORES != SCRATCHV_HARTS
+#error "SCRATCHV_CORES and legacy SCRATCHV_HARTS disagree"
+#endif
+#ifndef SCRATCHV_CORES
+#ifdef SCRATCHV_HARTS
+#define SCRATCHV_CORES SCRATCHV_HARTS
+#else
+#define SCRATCHV_CORES 1
+#endif
+#endif
+#ifndef SCRATCHV_HARTS
+#define SCRATCHV_HARTS SCRATCHV_CORES
+#endif
/* External storage targets 64-bit hosts/guests; lengths are caller-validated. */
_Static_assert(sizeof(size_t) == 8 && sizeof(uintptr_t) == 8,
"external tensor ABI requires a 64-bit target");
-#if defined(SCRATCHV_HARTS) && SCRATCHV_HARTS > 1
+#if defined(SCRATCHV_CORES) && SCRATCHV_CORES > 1
/* Synchronous: all callbacks finish before workspace reuse or validation. */
extern void sv_parallel_for(size_t count,
void (*fn)(void *, size_t, size_t), void *context);
@@ -202,7 +231,8 @@ def __init__(self, program: Program, initializers: Mapping[str, np.ndarray] | No
*, max_workspace_bytes=256 * 1024 * 1024,
max_constant_bytes=256 * 1024 * 1024, constant_storage="inline",
kernel_calls=False, matmul_policy="sequential", matmul_tile="1x4",
- qemu_workspace_layout="none", matmul_rhs_layout="original"):
+ qemu_workspace_layout="none", matmul_rhs_layout="original",
+ mean_policy="sequential", nonlinear_policy="libm"):
self.program = program
self.initializers = {} if initializers is None else dict(initializers)
self.max_workspace_bytes = max_workspace_bytes
@@ -210,6 +240,8 @@ def __init__(self, program: Program, initializers: Mapping[str, np.ndarray] | No
self.constant_storage = constant_storage
self.kernel_calls = kernel_calls
self.matmul_policy = matmul_policy
+ self.mean_policy = mean_policy
+ self.nonlinear_policy = nonlinear_policy
self.matmul_tile = matmul_tile
self.matmul_rhs_layout = matmul_rhs_layout
self.qemu_workspace_layout = qemu_workspace_layout
@@ -222,6 +254,14 @@ def generate(self) -> TensorCArtifact:
raise TensorCCodegenError("kernel_calls must be boolean")
if self.kernel_calls and self.constant_storage != "external":
raise TensorCCodegenError("kernel_calls requires external constant storage")
+ if (not isinstance(self.mean_policy, str)
+ or self.mean_policy not in ("sequential", "two-four-lane-last-axis")):
+ raise TensorCCodegenError(
+ "mean_policy must be 'sequential' or 'two-four-lane-last-axis'")
+ if (not isinstance(self.nonlinear_policy, str)
+ or self.nonlinear_policy not in ("libm", "fp32-reference")):
+ raise TensorCCodegenError(
+ "nonlinear_policy must be 'libm' or 'fp32-reference'")
if self.matmul_policy not in ("sequential", "blocked_fma"):
raise TensorCCodegenError("matmul_policy must be 'sequential' or 'blocked_fma'")
if self.matmul_policy != "sequential" and self.constant_storage != "external":
@@ -269,6 +309,8 @@ def generate(self) -> TensorCArtifact:
self.external_scalar_checks = []
self.kernels = []
self._softmax_callbacks = {}
+ self._mean_last_used = False
+ self._nonlinear_used = set()
self.constant_bytes = 0
self._counter = 0
defined = set()
@@ -413,8 +455,12 @@ def generate(self) -> TensorCArtifact:
self.body += ["if (!output) return 1;",
f"for (size_t i=0; i<{output.size}ULL; ++i) (({ctype} *)output)[i] = {self.names[output.name]}[i];",
"return 0;"]
+ mean_prelude = TWO_FOUR_LANE_LAST_AXIS_C if self._mean_last_used else ""
+ nonlinear_prelude = "".join(source for name, source in (
+ ("trig", TRIG_REFERENCE_C), ("sigmoid", SIGMOID_REFERENCE_C),
+ ("softmax", SOFTMAX_REFERENCE_C)) if name in self._nonlinear_used)
if self.constant_storage == "external":
- source = (_PRELUDE + _EXTERNAL_PRELUDE + "".join(self.kernels)
+ source = (_PRELUDE + mean_prelude + nonlinear_prelude + _EXTERNAL_PRELUDE + "".join(self.kernels)
+ "#if defined(_WIN32)\n__declspec(dllexport)\n#endif\n"
+ "int scratchv_run_external(const void *const inputs[], "
"const void *const weights[], void *workspace, size_t workspace_bytes, "
@@ -428,18 +474,21 @@ def generate(self) -> TensorCArtifact:
external_initializers=tuple(self.external_initializers),
kernel_calls=self.kernel_calls,
matmul_policy=self.matmul_policy,
+ mean_policy=self.mean_policy,
+ nonlinear_policy=self.nonlinear_policy,
matmul_tile=self.matmul_tile,
matmul_rhs_layout=self.matmul_rhs_layout,
matmul_rhs_panels=tuple(self.matmul_rhs_panels),
qemu_workspace_layout=self.qemu_workspace_layout,
qemu_workspace_buffers=qemu_workspace_buffers,
)
- source = (_PRELUDE + "\n" + "\n".join(self.constants)
+ source = (_PRELUDE + mean_prelude + nonlinear_prelude + "\n" + "\n".join(self.constants)
+ f"\nstatic union {{ uint64_t align; unsigned char bytes[{max(high, 8)}]; }} sv_arena;\n"
+ "#if defined(_WIN32)\n__declspec(dllexport)\n#endif\n"
+ "int scratchv_run(const void *const inputs[], void *output) {\n "
+ "\n ".join(self.body) + "\n}\n")
- return TensorCArtifact(source, inputs, output, high, self.constant_bytes)
+ return TensorCArtifact(source, inputs, output, high, self.constant_bytes,
+ mean_policy=self.mean_policy, nonlinear_policy=self.nonlinear_policy)
def _panel_inputs(self, program, initializers):
"""Select immutable 2D RHS storage; copy only IR containers we change.
@@ -888,6 +937,13 @@ def _emit(self, instruction, output, details):
if op == OpCode.SQRT:
checks = f"if ({a}<0.0f) return 2; "
expression = f"{function}({a})"
+ if self.nonlinear_policy == "fp32-reference":
+ if op in (OpCode.SIN, OpCode.COS):
+ self._nonlinear_used.add("trig")
+ expression = f"sv_fp32_reference_trig({a}, {int(op == OpCode.SIN)})"
+ elif op == OpCode.SIGMOID:
+ self._nonlinear_used.add("sigmoid")
+ expression = f"sv_fp32_reference_sigmoid({a})"
self.body.append(f"for (size_t i=0; i<{size}ULL; ++i) {{ {checks}{dst}[i]={expression}; }}")
return
x, src = xs[0], names[0]
@@ -917,7 +973,7 @@ def _emit(self, instruction, output, details):
# Each output retains increasing K order, fused FP32 products,
# and a separate FP32 addition after each 128-element K block.
# Tiles only share input loads across independent outputs; they
- # never split a reduction between harts or change its grouping.
+ # never split a reduction between cores or change its grouping.
# __builtin_fmaf retains explicit FMA with -ffp-contract=off.
tile_rows, tile_columns = (int(part) for part in self.matmul_tile.split("x"))
lanes = tile_rows * tile_columns
@@ -996,8 +1052,8 @@ def _emit(self, instruction, output, details):
# Each task owns one (batch, row tile, column tile); its K loop
# and 128-element grouping are shared verbatim with serial.
if tasks >= 2 and math.prod(batch) * m * n * k >= 32768:
- condition = ("defined(SCRATCHV_HARTS) && SCRATCHV_HARTS > 1 "
- f"&& SCRATCHV_HARTS <= {tasks}")
+ condition = ("defined(SCRATCHV_CORES) && SCRATCHV_CORES > 1 "
+ f"&& SCRATCHV_CORES <= {tasks}")
context = f"sv_matmul_{dst}"
aliases = []
seen = set()
@@ -1036,6 +1092,18 @@ def _emit(self, instruction, output, details):
f"{dst}[batch*{m*n}ULL+m*{n}ULL+n]=acc; }}")
elif op == OpCode.REDUCE_MEAN:
axes = details["axes"]
+ if (self.mean_policy == "two-four-lane-last-axis"
+ and x.dtype == DataType.FLOAT32 and x.shape
+ and axes == (len(x.shape) - 1,)):
+ # Explicitly changes the reduction order; no SIMD ISA required.
+ # Shape, not the caller's address, determines the row prefix.
+ self._mean_last_used = True
+ width = x.shape[-1]
+ self.body.append(
+ f"for (size_t i=0; i<{size}ULL; ++i) "
+ f"{dst}[i]=sv_two_four_lane_mean_last("
+ f"{src}+i*{width}ULL, {width}ULL, i);")
+ return # Keep the existing finite check and error propagation.
remaining = tuple(a for a in range(len(x.shape)) if a not in axes)
outer_shape = tuple(x.shape[a] for a in remaining)
reduced_shape = tuple(x.shape[a] for a in axes)
@@ -1049,14 +1117,24 @@ def _emit(self, instruction, output, details):
axis = details["axis"]
outer, dimension, inner = math.prod(x.shape[:axis]), x.shape[axis], math.prod(x.shape[axis+1:])
off = f"o*{dimension*inner}ULL+j*{inner}ULL+t"
- # A row retains its original max, exp/sum and normalization order.
- # Only independent (outer, inner) rows may run on different harts.
- row = ("float maximum=-0x1.fffffep+127f, sum=0.0f; "
+ # Only independent (outer, inner) rows may run on different cores.
+ # The explicitly selected arithmetic is identical in both paths.
+ initialization, exponential = "sum=0.0f", "expf"
+ accumulate, finish, normalize = "sum+=v;", "", "/=sum"
+ if self.nonlinear_policy == "fp32-reference":
+ self._nonlinear_used.add("softmax")
+ initialization = "sv_reference_lanes[16]={0.0f}"
+ exponential = "sv_fp32_reference_exp_nonpositive"
+ accumulate = "sv_reference_lanes[j%16ULL]+=v;"
+ finish = ("float sum=sv_fp32_reference_sum_sixteen(sv_reference_lanes); "
+ "float sv_reference_reciprocal=1.0f/sum; ")
+ normalize = "*=sv_reference_reciprocal"
+ row = (f"float maximum=-0x1.fffffep+127f, {initialization}; "
f"for (size_t j=0; j<{dimension}ULL; ++j) if ({src}[{off}]>maximum) maximum={src}[{off}]; "
f"for (size_t j=0; j<{dimension}ULL; ++j) {{ float shifted={src}[{off}]-maximum; "
- "if (!sv_finite(shifted)) return 2; float v=expf(shifted); "
- f"{dst}[{off}]=v; sum+=v; }} "
- f"for (size_t j=0; j<{dimension}ULL; ++j) {{ {dst}[{off}]/=sum; "
+ f"if (!sv_finite(shifted)) return 2; float v={exponential}(shifted); "
+ f"{dst}[{off}]=v; {accumulate} }} "
+ f"{finish}for (size_t j=0; j<{dimension}ULL; ++j) {{ {dst}[{off}]{normalize}; "
f"if (!sv_finite({dst}[{off}])) return 2; }} ")
serial = (f"for (size_t o=0; o<{outer}ULL; ++o) "
f"for (size_t t=0; t<{inner}ULL; ++t) {{ " + row + "}")
@@ -1064,8 +1142,8 @@ def _emit(self, instruction, output, details):
# Keep small/empty graphs and inline storage on the original path.
# This floor avoids a barrier for tiny graphs; it is not autotuning.
if self.constant_storage == "external" and rows >= 2 and size >= 32768:
- condition = ("defined(SCRATCHV_HARTS) && SCRATCHV_HARTS > 1 "
- f"&& SCRATCHV_HARTS <= {rows}")
+ condition = ("defined(SCRATCHV_CORES) && SCRATCHV_CORES > 1 "
+ f"&& SCRATCHV_CORES <= {rows}")
# Reuse code only: each operation keeps its own input/output
# pointers and status, initialized below before the joined call.
key = (x.dtype, x.shape, axis)
diff --git a/scratchv/backend/tensor_c_mean.py b/scratchv/backend/tensor_c_mean.py
new file mode 100644
index 00000000..720932fd
--- /dev/null
+++ b/scratchv/backend/tensor_c_mean.py
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: MPL-2.0
+# Adapted from Eigen (https://gitlab.com/libeigen/eigen), commit
+# 1d8b82b0740839c0de7f1242a3585e3390ff5f33:
+# Eigen/src/Core/Redux.h
+# Eigen/src/Core/arch/SSE/PacketMath.h
+# Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h
+# Copyright (C) 2006-2008 Benoit Jacob
+# Copyright (C) 2007 Julien Pommier
+# Copyright (C) 2008-2009 Gael Guennebaud
+# Copyright (C) 2009-2019 Gael Guennebaud
+# Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com)
+#
+# This Source Code Form is subject to the terms of the Mozilla
+# Public License, v. 2.0. If a copy of the MPL was not distributed
+# with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
+# The full license is also distributed in LICENSES/Eigen-MPL-2.0.txt.
+"""C source for the explicitly selected FP32 last-axis Mean order.
+
+This portable scalar C follows the existing FP32 Eigen reference's two four-lane
+accumulators, canonical row prefix, pairwise lane fold and tail. It changes
+floating-point summation order; it is not a host SIMD or hardware requirement.
+Finite input/output checks remain the tensor backend's responsibility.
+"""
+
+TWO_FOUR_LANE_LAST_AXIS_C = r'''
+/*
+ * SPDX-License-Identifier: MPL-2.0
+ * Adapted from Eigen (https://gitlab.com/libeigen/eigen), commit
+ * 1d8b82b0740839c0de7f1242a3585e3390ff5f33:
+ * Eigen/src/Core/Redux.h
+ * Eigen/src/Core/arch/SSE/PacketMath.h
+ * Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h
+ * Copyright (C) 2006-2008 Benoit Jacob
+ * Copyright (C) 2007 Julien Pommier
+ * Copyright (C) 2008-2009 Gael Guennebaud
+ * Copyright (C) 2009-2019 Gael Guennebaud
+ * Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla
+ * Public License, v. 2.0. If a copy of the MPL was not distributed
+ * with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ * The full license is also distributed in LICENSES/Eigen-MPL-2.0.txt.
+ */
+
+/* Nonempty contiguous FP32 row. Canonical row alignment is independent of the
+ * caller's actual address. Compile without fast-math or contraction. */
+static float sv_two_four_lane_mean_last(const float *x, size_t width, size_t row_index) {
+ const size_t start = (width < 4 || width % 4 == 0)
+ ? 0 : (4 - ((row_index % 4) * (width % 4)) % 4) % 4;
+ float sum;
+ if (width - start < 4) {
+ sum = x[0];
+ for (size_t k=1; k 4) {
+ float b0=x[start+4], b1=x[start+5], b2=x[start+6], b3=x[start+7];
+ for (size_t k=start+8; k pair_end) {
+ a0=a0+x[pair_end]; a1=a1+x[pair_end+1];
+ a2=a2+x[pair_end+2]; a3=a3+x[pair_end+3];
+ }
+ }
+ sum = (a0+a2)+(a1+a3);
+ for (size_t k=0; k= 18000), fused bounded-rational
+Sigmoid and polynomial Softmax with sixteen accumulators, pairwise lane fold,
+ldexpf scaling and reciprocal multiplication. It requires no SIMD ISA.
+Input/output finiteness and error propagation remain the backend's contract.
+"""
+
+TRIG_REFERENCE_C = r'''
+/*
+ * SPDX-License-Identifier: MPL-2.0
+ * Adapted from Eigen (https://gitlab.com/libeigen/eigen), commit
+ * 1d8b82b0740839c0de7f1242a3585e3390ff5f33:
+ * Eigen/src/Core/Redux.h
+ * Eigen/src/Core/arch/SSE/PacketMath.h
+ * Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h
+ * Copyright (C) 2006-2008 Benoit Jacob
+ * Copyright (C) 2007 Julien Pommier
+ * Copyright (C) 2008-2009 Gael Guennebaud
+ * Copyright (C) 2009-2019 Gael Guennebaud
+ * Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com)
+ *
+ * This Source Code Form is subject to the terms of the Mozilla
+ * Public License, v. 2.0. If a copy of the MPL was not distributed
+ * with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ * The full license is also distributed in LICENSES/Eigen-MPL-2.0.txt.
+ */
+
+/* Explicit FP32 non-FMA trig order from fp32_eigen.trig.
+ * Outside the conservative interval, retain the original platform libm.
+ * Portable scalar C; no host SIMD or RVV requirement. */
+static float sv_fp32_reference_trig(float value, int sine) {
+ union {float f; uint32_t u;} input={.f=value}, absolute, rounded_bits, output;
+ absolute.u=input.u & 0x7fffffffU;
+ if (!(absolute.f < 18000.0f)) return sine ? sinf(value) : cosf(value);
+ float reduced=absolute.f;
+ float rounded=reduced * 0x1.45f3060000000p-1f;
+ rounded=rounded + 12582912.0f;
+ rounded_bits.f=rounded;
+ uint32_t quadrant=rounded_bits.u;
+ float multiple=rounded-12582912.0f;
+ reduced=multiple*-0x1.9200000000000p+0f+reduced;
+ reduced=multiple*-0x1.fb80000000000p-12f+reduced;
+ reduced=multiple*0x1.5dc0000000000p-23f+reduced;
+ reduced=multiple*0x1.e973dc0000000p-35f+reduced;
+ float squared=reduced*reduced;
+ float cosine=0x1.98e6160000000p-16f;
+ cosine=cosine*squared+-0x1.6c06dc0000000p-10f;
+ cosine=cosine*squared+0x1.55553c0000000p-5f;
+ cosine=cosine*squared-0.5f;
+ cosine=cosine*squared+1.0f;
+ float sinpoly=-0x1.9ae1940000000p-13f;
+ sinpoly=sinpoly*squared+0x1.110ba60000000p-7f;
+ sinpoly=sinpoly*squared+-0x1.5555500000000p-3f;
+ sinpoly=sinpoly*squared;
+ sinpoly=sinpoly*reduced+reduced;
+ int even=(quadrant & 1U)==0;
+ output.f=sine ? (even ? sinpoly : cosine) : (even ? cosine : sinpoly);
+ uint32_t sign=sine ? (input.u ^ (quadrant<<30)) & 0x80000000U
+ : ((quadrant+1U)<<30) & 0x80000000U;
+ output.u ^= sign;
+ return output.f;
+}
+'''
+
+SIGMOID_REFERENCE_C = r'''
+/*
+MLAS polynomial constants/evaluation adapted from ONNX Runtime v1.22.1
+core/mlas/lib/logistic.cpp, via scratchv.verification.fp32_reference.sigmoid.
+Copyright (c) Microsoft Corporation. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+static float sv_fp32_reference_sigmoid(float x) {
+ if (x < -18.0f) x=-18.0f; if (x > 18.0f) x=18.0f;
+ float squared=x*x;
+ float numerator=0x1.806aa20000000p-35f;
+ float denominator=0x1.5789ea0000000p-41f;
+ numerator=__builtin_fmaf(numerator,squared,0x1.f09d960000000p-24f);
+ numerator=__builtin_fmaf(numerator,squared,0x1.fe82760000000p-15f);
+ numerator=__builtin_fmaf(numerator,squared,0x1.16fab00000000p-7f);
+ numerator=__builtin_fmaf(numerator,squared,0x1.fc7e640000000p-3f);
+ denominator=__builtin_fmaf(denominator,squared,0x1.8be4f60000000p-28f);
+ denominator=__builtin_fmaf(denominator,squared,0x1.a62fba0000000p-18f);
+ denominator=__builtin_fmaf(denominator,squared,0x1.be2a7e0000000p-10f);
+ denominator=__builtin_fmaf(denominator,squared,0x1.de7c300000000p-4f);
+ denominator=__builtin_fmaf(denominator,squared,0x1.fc7e680000000p-1f);
+ float value=(numerator*x)/denominator+0.5f;
+ if (value < 0.0f) value=0.0f; if (value > 1.0f) value=1.0f;
+ return value;
+}
+'''
+
+SOFTMAX_REFERENCE_C = r'''
+/*
+ * MLAS polynomial constants/evaluation are adapted from ONNX Runtime v1.22.1:
+ * core/mlas/lib/logistic.cpp, compute.cpp, x86_64/TransKernelFma3.S and
+ * x86_64/TransKernelAvx512F.S (including their amd64 Windows equivalents).
+ * Copyright (c) Microsoft Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/* Fixed sixteen-lane reference arithmetic, scalar FP32 implementation.
+ * This chooses numerical order, not the runtime host ISA. */
+extern float fmaf(float,float,float);
+extern float ldexpf(float,int);
+static float sv_fp32_reference_exp_nonpositive(float x) {
+ const float lower=-103.9720840454f, bias=12582912.0f;
+ if (x < lower) x=lower;
+ float rounded=fmaf(x,1.44269504088896341f,bias);
+ float exponent=rounded-bias;
+ x=fmaf(exponent,-6.93145752e-1f,x);
+ x=fmaf(exponent,-1.42860677e-6f,x);
+ float p=0x1.694000p-10f;
+ p=fmaf(p,x,0x1.125edcp-7f);
+ p=fmaf(p,x,0x1.555b5ap-5f);
+ p=fmaf(p,x,0x1.555450p-3f);
+ p=fmaf(p,x,0x1.fffff6p-2f);
+ p=fmaf(p,x,0x1.000000p+0f);
+ p=fmaf(p,x,0x1.000000p+0f);
+ return ldexpf(p,(int)exponent);
+}
+static float sv_fp32_reference_sum_sixteen(const float *s) {
+ float a0=s[0]+s[8],a1=s[1]+s[9],a2=s[2]+s[10],a3=s[3]+s[11];
+ float a4=s[4]+s[12],a5=s[5]+s[13],a6=s[6]+s[14],a7=s[7]+s[15];
+ float first=(a0+a1)+(a2+a3);
+ float last=(a4+a5)+(a6+a7);
+ return first+last;
+}
+'''
diff --git a/scratchv/runtime/conversation.py b/scratchv/runtime/conversation.py
new file mode 100644
index 00000000..a84adf18
--- /dev/null
+++ b/scratchv/runtime/conversation.py
@@ -0,0 +1,153 @@
+"""Host text conversations using the supplied Qwen3 chat template.
+
+The template is loaded locally and rendered without Transformers or Torch. This
+layer owns text history only: the existing generation loop and forward backend
+continue to own token generation and execution resources.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from .generation import GenerationResult, generate
+from .llm_inputs import PreparedInputs
+
+if TYPE_CHECKING:
+ from .qwen3_tokenizer import Qwen3Tokenizer
+
+
+@dataclass(frozen=True)
+class ChatMessage:
+ """One text-only message; tools and multimodal content are unsupported."""
+
+ role: str
+ content: str
+
+ def __post_init__(self) -> None:
+ if self.role not in ("system", "user", "assistant"):
+ raise ValueError("role must be system, user or assistant")
+ if not isinstance(self.content, str):
+ raise TypeError("message content must be str")
+ self.content.encode("utf-8", errors="strict")
+
+
+class Conversation:
+ """Transactional text history with explicit non-thinking generation.
+
+ A successful turn with at least one generated token commits its user and
+ assistant messages together. Errors, cancellation, timeout, zero budget and
+ a full prompt with no room for output leave history unchanged. A response
+ stopped by token budget or capacity is retained as the returned partial
+ text; its ``GenerationResult.stop_reason`` remains explicit.
+
+ History is re-rendered and encoded for every turn, without truncation or
+ implicit removal of old messages. There is no guest session or KV cache.
+ The same instance must not be used concurrently or reentrantly.
+ """
+
+ def __init__(
+ self,
+ tokenizer: Qwen3Tokenizer,
+ chat_template: str,
+ *,
+ system_prompt: str | None = None,
+ ) -> None:
+ if not isinstance(chat_template, str):
+ raise TypeError("chat_template must be str")
+ if not chat_template.strip():
+ raise ValueError("chat_template must not be empty")
+ # Match the official template renderer's whitespace and sandbox rules;
+ # undefined optional message fields intentionally remain false/empty.
+ from jinja2 import TemplateError
+ from jinja2.sandbox import ImmutableSandboxedEnvironment
+
+ def raise_exception(message: str) -> None:
+ raise TemplateError(message)
+
+ environment = ImmutableSandboxedEnvironment(
+ trim_blocks=True, lstrip_blocks=True,
+ extensions=["jinja2.ext.loopcontrols"],
+ )
+ environment.globals["raise_exception"] = raise_exception
+ self._template = environment.from_string(chat_template)
+ self.tokenizer = tokenizer
+ self._initial = (() if system_prompt is None else
+ (ChatMessage("system", system_prompt),))
+ self._history = self._initial
+ self._busy = False
+
+ @classmethod
+ def from_directory(
+ cls, directory: str | Path, *, system_prompt: str | None = None,
+ ) -> Conversation:
+ """Load the official template and tokenizer from the same snapshot."""
+ from .qwen3_tokenizer import Qwen3Tokenizer
+
+ directory = Path(directory)
+ config = json.loads((directory / "tokenizer_config.json").read_text(encoding="utf-8"))
+ if not isinstance(config, dict):
+ raise ValueError("tokenizer_config.json must contain a JSON object")
+ template = config.get("chat_template")
+ if not isinstance(template, str) or not template.strip():
+ raise ValueError("tokenizer_config.json must contain one nonempty chat_template")
+ return cls(Qwen3Tokenizer.from_directory(directory), template,
+ system_prompt=system_prompt)
+
+ @property
+ def history(self) -> tuple[ChatMessage, ...]:
+ """An immutable snapshot of committed text messages."""
+ return self._history
+
+ def reset(self) -> None:
+ """Discard completed turns while retaining the initial system prompt."""
+ if self._busy:
+ raise RuntimeError("conversation has an active turn")
+ self._history = self._initial
+
+ def render_prompt(self, user_text: str) -> str:
+ """Render committed history plus a pending user turn without mutation."""
+ user = ChatMessage("user", user_text)
+ messages = [dict(role=item.role, content=item.content)
+ for item in (*self._history, user)]
+ return self._template.render(
+ messages=messages, tools=None,
+ add_generation_prompt=True, enable_thinking=False,
+ )
+
+ def turn(
+ self,
+ user_text: str,
+ *,
+ forward: Callable[[PreparedInputs], np.ndarray],
+ max_new_tokens: int,
+ cancelled: Callable[[], bool] | None = None,
+ timeout_seconds: float | None = None,
+ ) -> GenerationResult:
+ """Generate one reply and atomically commit it on success.
+
+ Timeout/cancellation semantics are those of :func:`generate`; a backend
+ remains responsible for interrupting and cleaning up blocking work.
+ The full rendered prompt must fit the fixed 256-token ABI.
+ """
+ if self._busy:
+ raise RuntimeError("conversation has an active turn")
+ self._busy = True
+ try:
+ prompt = self.render_prompt(user_text)
+ result = generate(
+ prompt, tokenizer=self.tokenizer, forward=forward,
+ max_new_tokens=max_new_tokens, cancelled=cancelled,
+ timeout_seconds=timeout_seconds,
+ )
+ if result.generated_token_ids:
+ self._history += (ChatMessage("user", user_text),
+ ChatMessage("assistant", result.generated_text))
+ return result
+ finally:
+ self._busy = False
diff --git a/scratchv/runtime/generation.py b/scratchv/runtime/generation.py
new file mode 100644
index 00000000..59cc63b5
--- /dev/null
+++ b/scratchv/runtime/generation.py
@@ -0,0 +1,133 @@
+"""Synchronous Host greedy generation for the fixed-length Qwen3 ABI.
+
+The caller owns the forward backend and its resources. This module neither
+builds a model nor starts a process, and never changes hart or workspace policy.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+import math
+from numbers import Real
+import time
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+from .llm_inputs import (
+ PreparedInputs,
+ StopReason,
+ generation_stop_reason,
+ greedy_token,
+ last_valid_logits,
+ prepare_inputs,
+)
+
+if TYPE_CHECKING:
+ from .qwen3_tokenizer import Qwen3Tokenizer
+
+
+class GenerationCancelledError(RuntimeError):
+ """The caller cancelled generation; no successful result is returned."""
+
+
+@dataclass(frozen=True)
+class GenerationResult:
+ """Owned token IDs and the generated continuation, excluding the prompt.
+
+ IDs retain generated EOS/special tokens. Display text skips special tokens
+ and disables whitespace cleanup; IDs are the authoritative comparison.
+ """
+
+ prompt_token_ids: tuple[int, ...]
+ generated_token_ids: tuple[int, ...]
+ generated_text: str
+ stop_reason: StopReason
+
+
+def generate(
+ prompt: str,
+ *,
+ tokenizer: Qwen3Tokenizer,
+ forward: Callable[[PreparedInputs], np.ndarray],
+ max_new_tokens: int,
+ cancelled: Callable[[], bool] | None = None,
+ timeout_seconds: float | None = None,
+) -> GenerationResult:
+ """Generate a raw completion without KV cache, templates or truncation.
+
+ ``forward`` receives fresh right-padded inputs at each step and returns the
+ full finite FP32 logits. It may reuse its output buffer between calls; this
+ loop retains no output arrays. Each call to ``generate`` has isolated state.
+
+ Cancellation and timeout are cooperative checks before/after forward and
+ before returning success. They cannot interrupt a blocking backend. The
+ backend must enforce its own execution timeout and clean up its processes;
+ backend exceptions (including KeyboardInterrupt) propagate unchanged.
+ A finite nonnegative timeout starts after input/configuration validation.
+ Zero expires immediately, even when the token budget is zero.
+ """
+ if not isinstance(prompt, str):
+ raise TypeError("prompt must be str")
+ if not callable(forward):
+ raise TypeError("forward must be callable")
+ if cancelled is not None and not callable(cancelled):
+ raise TypeError("cancelled must be callable or None")
+ if timeout_seconds is not None:
+ if isinstance(timeout_seconds, (bool, np.bool_)) or not isinstance(timeout_seconds, Real):
+ raise TypeError("timeout_seconds must be a real number or None")
+ timeout_seconds = float(timeout_seconds)
+ if not math.isfinite(timeout_seconds) or timeout_seconds < 0:
+ raise ValueError("timeout_seconds must be finite and nonnegative")
+
+ prompt_ids = tuple(tokenizer.validate_token_ids(
+ tokenizer.encode(prompt, add_special_tokens=False)
+ ))
+ vocab_size = tokenizer.model_vocab_size
+ pad_token_id = tokenizer.pad_token_id
+ eos_ids = tuple(tokenizer.validate_token_ids(tokenizer.generation_eos_token_ids))
+ history = list(prompt_ids)
+ prepared = prepare_inputs(history, pad_token_id=pad_token_id, vocab_size=vocab_size)
+ # Validate the budget even if capacity or EOS would otherwise end the loop.
+ stop = generation_stop_reason(
+ valid_length=len(history), generated_tokens=0,
+ max_new_tokens=max_new_tokens, eos_token_ids=eos_ids, vocab_size=vocab_size,
+ )
+ deadline = None if timeout_seconds is None else time.monotonic() + timeout_seconds
+
+ def check_abort() -> None:
+ if cancelled is not None and cancelled():
+ raise GenerationCancelledError("generation cancelled")
+ if deadline is not None and time.monotonic() >= deadline:
+ raise TimeoutError("generation timeout expired")
+
+ generated: list[int] = []
+ while True:
+ check_abort()
+ if stop is not None:
+ text = tokenizer.decode(
+ generated, skip_special_tokens=True, clean_up_tokenization_spaces=False
+ )
+ check_abort()
+ return GenerationResult(prompt_ids, tuple(generated), text, stop)
+
+ logits = forward(prepared)
+ check_abort()
+ row = last_valid_logits(logits, len(history), vocab_size=vocab_size)
+ # Release the full output before the next call allocates/reuses a buffer.
+ del logits
+ next_id = greedy_token(row)
+ del row
+ # The model has unused vocabulary rows: never mask or resample them.
+ tokenizer.validate_token_ids([next_id])
+ check_abort()
+ generated.append(next_id)
+ history.append(next_id)
+ stop = generation_stop_reason(
+ valid_length=len(history), generated_tokens=len(generated),
+ max_new_tokens=max_new_tokens, last_token_id=next_id,
+ eos_token_ids=eos_ids, vocab_size=vocab_size,
+ )
+ if stop is None:
+ prepared = prepare_inputs(history, pad_token_id=pad_token_id, vocab_size=vocab_size)
diff --git a/scratchv/runtime/qemu_generation.py b/scratchv/runtime/qemu_generation.py
new file mode 100644
index 00000000..49c152fb
--- /dev/null
+++ b/scratchv/runtime/qemu_generation.py
@@ -0,0 +1,140 @@
+"""Fixed-ABI Host generation over one reusable RV64 executable and bundle.
+
+Each token launches a fresh QEMU guest with updated inputs. The ELF and bundle
+are built once by the caller; this adapter neither recompiles nor changes hart
+selection, code generation policy, weights, or workspace layout.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable
+import json
+import math
+from numbers import Real
+from pathlib import Path
+import threading
+import time
+
+import numpy as np
+
+from .generation import GenerationCancelledError
+from .llm_inputs import PreparedInputs, SEQ_LENGTH
+from .riscv_external import ExternalExecutable, run_external
+
+
+class QemuForward:
+ """Callable backend for ``generation.generate`` using the fixed W5 ABI.
+
+ ``timeout`` bounds each guest invocation; ``deadline`` optionally bounds the
+ entire generation using an absolute ``time.monotonic()`` deadline. A shared
+ cancellation callback is checked during guest UART/QMP waits. Instances are
+ intentionally serial, but may be reused by independent generation contexts.
+ Step directories are unique and failed invocations retain diagnostic data.
+ """
+
+ def __init__(self, executable: ExternalExecutable, output_dir, *,
+ timeout: float = 3600.0, cancelled: Callable[[], bool] | None = None,
+ deadline: float | None = None):
+ for name, value in (("timeout", timeout), ("deadline", deadline)):
+ if value is None and name == "deadline":
+ continue
+ if isinstance(value, (bool, np.bool_)) or not isinstance(value, Real):
+ raise TypeError(f"{name} must be a real number")
+ if not math.isfinite(value) or (name == "timeout" and value <= 0):
+ raise ValueError(f"{name} must be finite" + (" and positive" if name == "timeout" else ""))
+ if cancelled is not None and not callable(cancelled):
+ raise TypeError("cancelled must be callable or None")
+ if executable.evidence.get("qemu_workspace_layout", "none") != "none":
+ raise ValueError("W5 generation requires qemu_workspace_layout=none")
+ specs = {spec.name: spec for spec in executable.inputs}
+ expected = {"input_ids": ((1, SEQ_LENGTH), np.dtype("int64")),
+ "attention_mask": ((1, 1, SEQ_LENGTH, SEQ_LENGTH), np.dtype("float32"))}
+ if len(specs) != len(executable.inputs) or set(specs) != set(expected):
+ raise ValueError("W5 requires exactly input_ids and attention_mask inputs")
+ for name, (shape, dtype) in expected.items():
+ if tuple(specs[name].shape) != shape or specs[name].numpy_dtype != dtype:
+ raise ValueError(f"W5 {name} shape/dtype differs from the fixed ABI")
+ output = executable.output
+ if (len(output.shape) != 3 or tuple(output.shape[:2]) != (1, SEQ_LENGTH)
+ or output.shape[2] < 1 or output.numpy_dtype != np.dtype("float32")):
+ raise ValueError("W5 output must be full FP32 [1,256,vocab] logits")
+ self.executable = executable
+ self.vocab_size = output.shape[2]
+ self.output_dir = Path(output_dir).resolve()
+ # Refuse stale step/report reuse instead of overwriting previous runs.
+ self.output_dir.mkdir(parents=True, exist_ok=False)
+ self.timeout = float(timeout)
+ self.cancelled = cancelled
+ self.deadline = None if deadline is None else float(deadline)
+ self.runs: list[dict] = []
+ self._lock = threading.Lock()
+ self._next_step = 0
+
+ def _cancelled(self):
+ if self.cancelled is not None and self.cancelled():
+ return True
+ if self.deadline is not None and time.monotonic() >= self.deadline:
+ raise TimeoutError("generation timeout expired")
+ return False
+
+ def __call__(self, prepared: PreparedInputs) -> np.ndarray:
+ if not self._lock.acquire(blocking=False):
+ raise RuntimeError("QemuForward does not support overlapping invocations")
+ try:
+ if not isinstance(prepared, PreparedInputs):
+ raise TypeError("QemuForward requires PreparedInputs")
+ if (isinstance(prepared.valid_length, (bool, np.bool_))
+ or not isinstance(prepared.valid_length, (int, np.integer))
+ or not 1 <= prepared.valid_length <= SEQ_LENGTH):
+ raise ValueError("valid_length must be in [1,256]")
+ if self._cancelled():
+ raise GenerationCancelledError("generation cancelled")
+ timeout = self.timeout
+ if self.deadline is not None:
+ timeout = min(timeout, self.deadline - time.monotonic())
+ if timeout <= 0:
+ raise TimeoutError("generation timeout expired")
+ step_dir = self.output_dir / f"step-{self._next_step:04d}"
+ self._next_step += 1
+ checkpoint = self._cancelled if self.cancelled is not None or self.deadline is not None else None
+ try:
+ output, report = run_external(
+ self.executable, prepared.as_feed(), step_dir,
+ timeout=timeout, cancelled=checkpoint,
+ )
+ # run_external already verifies every output element is finite;
+ # generate retains its full-output check before selecting a row.
+ if output.shape != (1, SEQ_LENGTH, self.vocab_size) or output.dtype != np.float32:
+ raise ValueError("QEMU returned logits outside the fixed ABI")
+ if self._cancelled():
+ raise GenerationCancelledError("generation cancelled")
+ self.runs.append(dict(report, step=len(self.runs), run_dir=str(step_dir),
+ valid_length=int(prepared.valid_length)))
+ return output
+ except BaseException as exc:
+ report_path = step_dir / "run.json"
+ report = {}
+ try:
+ def finite_number(value):
+ number = float(value)
+ if not math.isfinite(number):
+ raise ValueError("run.json must contain finite JSON numbers")
+ return number
+
+ candidate = json.loads(report_path.read_text(encoding="utf-8"),
+ parse_constant=finite_number, parse_float=finite_number)
+ if not isinstance(candidate, dict):
+ raise ValueError("run.json must contain a JSON object")
+ report = candidate
+ except FileNotFoundError:
+ # Failures before guest launch need not produce a report.
+ pass
+ except Exception as report_error:
+ # Diagnostic failures must not replace the backend error,
+ # including cancellation/interrupts, or lose this attempt.
+ report["report_read_error"] = f"{type(report_error).__name__}: {report_error}"
+ self.runs.append(dict(report, passed=False, status="FAIL", step=len(self.runs),
+ run_dir=str(step_dir), valid_length=int(prepared.valid_length),
+ error=f"{type(exc).__name__}: {exc}"))
+ raise
+ finally:
+ self._lock.release()
diff --git a/scratchv/runtime/riscv_external.py b/scratchv/runtime/riscv_external.py
index 3a9c6985..436f69f3 100644
--- a/scratchv/runtime/riscv_external.py
+++ b/scratchv/runtime/riscv_external.py
@@ -6,7 +6,7 @@
from __future__ import annotations
-from dataclasses import dataclass
+from dataclasses import InitVar, dataclass
import ctypes
import errno
import hashlib
@@ -14,6 +14,7 @@
import os
from pathlib import Path
import select
+import selectors
import signal
import socket
import stat
@@ -31,7 +32,10 @@
_terminate_invocation, _windows_job, input_layout, pack_inputs, toolchain_versions,
)
from .weight_bundle import MIB, plan_guest_memory, validate_weight_bundle
-from .riscv_harts import STARTUP_ASSEMBLY as MULTIHART_STARTUP, WORKER_SOURCE, resolve_harts, validate_harts
+from .riscv_harts import (
+ STARTUP_ASSEMBLY as MULTICORE_STARTUP, CORE_DEFINITIONS, WORKER_SOURCE, UNSET,
+ read_cores, resolve_cores, select_cores, validate_cores,
+)
from .weight_transport import (
stage_transport, validate_transport, remove_transport, save_transport_report,
refresh_transport_residuals,
@@ -110,7 +114,7 @@ def identity(info):
"sha256": hashlib.sha256(data).hexdigest()}
-@dataclass(frozen=True)
+@dataclass(frozen=True, init=False)
class ExternalExecutable:
elf_path: Path
bundle_dir: Path
@@ -121,7 +125,36 @@ class ExternalExecutable:
weights: tuple
toolchain: RiscVToolchain
evidence: dict
- harts: int = 1
+ cores: int
+ harts: int
+ _replace_cores: InitVar[int]
+
+ def __init__(self, elf_path, bundle_dir, manifest, layout, inputs, output,
+ weights, toolchain, evidence, cores=UNSET, *, harts=UNSET, _replace_cores=UNSET):
+ if _replace_cores is not UNSET:
+ # replace() supplies the original values for omitted fields. Its
+ # InitVar carries the baseline without adding a key to asdict().
+ # One changed alias updates both; two different new values fail.
+ baseline = validate_cores(_replace_cores)
+ if cores is not UNSET:
+ validate_cores(cores)
+ if harts is not UNSET:
+ validate_cores(harts)
+ if cores is not UNSET and harts is not UNSET and cores != harts:
+ if cores == baseline:
+ cores = harts
+ elif harts == baseline:
+ harts = cores
+ count = select_cores(cores, harts=harts)
+ for name, value in (("elf_path", elf_path), ("bundle_dir", bundle_dir),
+ ("manifest", manifest), ("layout", layout), ("inputs", inputs),
+ ("output", output), ("weights", weights), ("toolchain", toolchain),
+ ("evidence", evidence), ("cores", count), ("harts", count)):
+ object.__setattr__(self, name, value)
+
+ @property
+ def _replace_cores(self):
+ return self.cores
def guest_harness(artifact, manifest, layout):
@@ -132,7 +165,7 @@ def guest_harness(artifact, manifest, layout):
inputs = ",".join(f"(const void *)0x{layout['input_base'] + n:x}UL" for n in offsets) or "0"
weights = ",".join(f"(const void *)0x{layout['weights_base'] + t['offset']:x}UL"
for t in manifest["tensors"]) or "0"
- return r'''
+ return CORE_DEFINITIONS + r'''
typedef unsigned long usize;
typedef unsigned char u8;
typedef unsigned int u32;
@@ -154,7 +187,7 @@ def guest_harness(artifact, manifest, layout):
for (int i=0;i<4;++i) uart_byte((u8)(status>>(8*i)));
for (int i=0;i<8;++i) uart_byte((u8)(size>>(8*i)));
__asm__ volatile("fence rw,rw" ::: "memory");
-#if SCRATCHV_HARTS > 1
+#if SCRATCHV_CORES > 1
__asm__ volatile("csrw mie, zero" ::: "memory");
#endif
for (;;) __asm__ volatile("wfi");
@@ -172,15 +205,21 @@ def guest_harness(artifact, manifest, layout):
'''
-def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, harts=1):
- harts = resolve_harts(harts)
+def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, cores=UNSET, harts=UNSET):
+ cores = resolve_cores(select_cores(cores, harts=harts, allow_auto=True))
if artifact.constant_storage != "external":
raise ValueError("External runtime requires external code generation")
+ mean_policy = getattr(artifact, "mean_policy", "sequential")
+ if not isinstance(mean_policy, str) or mean_policy not in ("sequential", "two-four-lane-last-axis"):
+ raise ValueError("Unsupported external Mean policy")
+ nonlinear_policy = getattr(artifact, "nonlinear_policy", "libm")
+ if not isinstance(nonlinear_policy, str) or nonlinear_policy not in ("libm", "fp32-reference"):
+ raise ValueError("Unsupported external nonlinear policy")
manifest = validate_weight_bundle(bundle_dir, artifact.external_weights)
_, input_bytes = input_layout(artifact.inputs)
layout = plan_guest_memory(workspace_bytes=artifact.workspace_bytes,
weight_bytes=manifest["total_bytes"], input_bytes=input_bytes,
- output_bytes=artifact.output.nbytes, harts=harts)
+ output_bytes=artifact.output.nbytes, cores=cores)
build_dir = Path(build_dir).resolve()
build_dir.mkdir(parents=True, exist_ok=False)
linker = LINKER_SCRIPT.replace("0x1c000000", hex(layout["code_capacity"]))
@@ -193,11 +232,11 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0,
" .text.expf : ALIGN(4096) { *(.text.expf) } > RAM\n"
" .text : { *(.text .text.*) } > RAM",
)
- if harts != 1:
- linker = linker.replace("0x100000", hex(harts * MIB)).replace(
- "1 MiB guest stack", f"{harts} MiB guest stacks")
+ if cores != 1:
+ linker = linker.replace("0x100000", hex(cores * MIB)).replace(
+ "1 MiB guest stack", f"{cores} MiB guest stacks")
sources = {"model.c": artifact.source, "guest.c": guest_harness(artifact, manifest, layout),
- "start.S": STARTUP_ASSEMBLY if harts == 1 else MULTIHART_STARTUP, "link.ld": linker}
+ "start.S": STARTUP_ASSEMBLY if cores == 1 else MULTICORE_STARTUP, "link.ld": linker}
for name, source in sources.items():
(build_dir / name).write_text(source, encoding="utf-8")
target = (["-target", "riscv64-linux-musl", "-static"] if toolchain.is_zig else
@@ -208,7 +247,7 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0,
"-fno-stack-protector", "-fno-pie", "-fPIC" if toolchain.is_zig else "-fno-pic",
"-fno-fast-math", "-ffp-contract=off", "-fno-strict-aliasing",
"-ffunction-sections", "-fdata-sections", *artifact.compile_flags,
- f"-DSCRATCHV_HARTS={harts}",
+ f"-DSCRATCHV_CORES={cores}", f"-DSCRATCHV_HARTS={cores}",
"model.c", "guest.c", "start.S", "-Wl,-T,link.ld", "-Wl,-e,sv_boot",
"-Wl,--gc-sections", "-Wl,--build-id=none", *(["-lm"] if toolchain.is_zig else []),
"-o", "model.elf"]
@@ -232,7 +271,8 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0,
if result.returncode:
raise RuntimeError(result.stderr.decode(errors="replace")[-12000:])
evidence = validate_external_elf(build_dir / "model.elf", layout)
- evidence.update(compile_seconds=time.perf_counter() - start, command=command, harts=harts,
+ evidence.update(compile_seconds=time.perf_counter() - start, command=command, cores=cores, harts=cores,
+ mean_policy=mean_policy, nonlinear_policy=nonlinear_policy,
matmul_tile=artifact.matmul_tile,
matmul_rhs_layout=artifact.matmul_rhs_layout,
matmul_rhs_panels=list(artifact.matmul_rhs_panels),
@@ -245,12 +285,20 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0,
(build_dir / "build.json").write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8")
return ExternalExecutable(build_dir / "model.elf", Path(bundle_dir).resolve(), manifest, layout,
tuple(artifact.inputs), artifact.output, tuple(artifact.external_weights),
- toolchain, evidence, harts)
+ toolchain, evidence, cores)
+
+
+def _check_cancelled(cancelled):
+ if cancelled is not None and cancelled():
+ from .generation import GenerationCancelledError
+ raise GenerationCancelledError("generation cancelled")
class QMP:
"""A bounded localhost QMP client that ignores asynchronous events."""
- def __init__(self, port, deadline):
+ def __init__(self, port, deadline, *, cancelled=None):
+ self.cancelled = cancelled
+ self.receive_buffer = bytearray()
self.deadline = deadline
self.serial = 0
self.socket = socket.create_connection(("127.0.0.1", port), timeout=self.remaining())
@@ -268,12 +316,15 @@ def __init__(self, port, deadline):
raise
def remaining(self):
+ _check_cancelled(getattr(self, "cancelled", None))
value = self.deadline - time.monotonic()
if value <= 0:
raise TimeoutError("QMP deadline exceeded")
return min(value, 30.0)
def read(self):
+ if getattr(self, "cancelled", None) is not None:
+ return self._read_cancellable()
self.socket.settimeout(self.remaining())
line = self.stream.readline(1024 * 1024)
if not line:
@@ -282,6 +333,34 @@ def read(self):
raise RuntimeError("Truncated/oversized QMP response")
return json.loads(line)
+ def _read_cancellable(self):
+ # A timed-out makefile.readline cannot safely be resumed. Keep partial
+ # lines ourselves and use readiness notifications with bounded abort
+ # checkpoints; no successful frame is delayed by the checkpoint period.
+ limit = 1024 * 1024
+ with selectors.DefaultSelector() as selector:
+ selector.register(self.socket, selectors.EVENT_READ)
+ while True:
+ remaining = self.remaining()
+ newline = self.receive_buffer.find(b"\n")
+ if newline >= 0:
+ if newline + 1 >= limit:
+ raise RuntimeError("Truncated/oversized QMP response")
+ line = bytes(self.receive_buffer[:newline + 1])
+ del self.receive_buffer[:newline + 1]
+ return json.loads(line)
+ if len(self.receive_buffer) >= limit:
+ raise RuntimeError("Truncated/oversized QMP response")
+ if not selector.select(min(remaining, 0.05)):
+ continue
+ self.socket.settimeout(remaining)
+ chunk = self.socket.recv(min(65536, limit - len(self.receive_buffer)))
+ if not chunk:
+ if self.receive_buffer:
+ raise RuntimeError("Truncated/oversized QMP response")
+ raise EOFError("QMP connection closed before response")
+ self.receive_buffer.extend(chunk)
+
def command(self, name, arguments=None):
self.serial += 1
request = {"execute": name, "id": self.serial}
@@ -446,8 +525,9 @@ def _drain(self):
if mask & (0x400 | 0x800 | 0x8000):
raise RuntimeError("UART completion watch lost its directory")
- def wait(self, process, deadline, timeout):
+ def wait(self, process, deadline, timeout, *, cancelled=None):
while True:
+ _check_cancelled(cancelled)
if time.monotonic() >= deadline:
raise TimeoutError(f"RV64 forward exceeded {timeout}s")
# Inspect before checking exit: a valid final write may accompany exit.
@@ -483,6 +563,8 @@ def wait(self, process, deadline, timeout):
poller.register(self.fd, select.POLLIN)
poller.register(self.pidfd, select.POLLIN)
timeout_ms = min(2_147_483_647, max(0, deadline - time.monotonic()) * 1000)
+ if cancelled is not None:
+ timeout_ms = min(timeout_ms, 50)
for fd, events in poller.poll(timeout_ms):
if events & select.POLLNVAL:
raise OSError(errno.EBADF, "UART completion wait descriptor is invalid")
@@ -550,14 +632,29 @@ def report(self):
}
-def run_external(executable, inputs, run_dir, *, timeout=3600.0):
- """Run a fresh guest, require completion, stop, dump exact output, then quit."""
+def run_external(executable, inputs, run_dir, *, timeout=3600.0, cancelled=None):
+ """Run a fresh guest, require completion, stop, dump exact output, then quit.
+
+ An optional cancellation callback is observed during UART/QMP waits and
+ between transport stages. Raising or returning true unwinds process cleanup.
+ Existing event timing remains unchanged without a cancellation callback.
+ """
+ if cancelled is not None and not callable(cancelled):
+ raise TypeError("cancelled must be callable or None")
+ _check_cancelled(cancelled)
if not np.isfinite(timeout) or timeout <= 0:
raise ValueError("timeout must be positive and finite")
- harts = validate_harts(getattr(executable, "harts", 1))
- built_harts = validate_harts(executable.evidence.get("harts", 1))
- if harts != built_harts or executable.layout["regions"]["stack"]["nbytes"] != harts * MIB:
- raise ValueError("Executable hart count disagrees with its build or stack layout")
+ cores = select_cores(getattr(executable, "cores", UNSET),
+ harts=getattr(executable, "harts", UNSET))
+ built_cores = read_cores(executable.evidence)
+ mean_policy = executable.evidence.get("mean_policy", "sequential")
+ if not isinstance(mean_policy, str) or mean_policy not in ("sequential", "two-four-lane-last-axis"):
+ raise ValueError("Unsupported executable Mean policy")
+ nonlinear_policy = executable.evidence.get("nonlinear_policy", "libm")
+ if not isinstance(nonlinear_policy, str) or nonlinear_policy not in ("libm", "fp32-reference"):
+ raise ValueError("Unsupported executable nonlinear policy")
+ if cores != built_cores or executable.layout["regions"]["stack"]["nbytes"] != cores * MIB:
+ raise ValueError("Executable core count (legacy harts) disagrees with its build or stack layout")
if sha256_file(executable.elf_path) != executable.evidence["sha256"]:
raise ValueError("ELF changed after build")
manifest = validate_weight_bundle(executable.bundle_dir, executable.weights)
@@ -572,7 +669,7 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0):
port = reservation.getsockname()[1]
layout = executable.layout
command = [executable.toolchain.qemu, "-machine", "virt", "-cpu", "rv64", "-accel",
- "tcg" if harts == 1 else "tcg,thread=multi", "-smp", str(harts), "-m", str(layout["memory_mib"]), "-bios", "none", "-display", "none",
+ "tcg" if cores == 1 else "tcg,thread=multi", "-smp", str(cores), "-m", str(layout["memory_mib"]), "-bios", "none", "-display", "none",
"-monitor", "none", "-serial", "file:uart.bin", "-no-reboot",
"-qmp", f"tcp:127.0.0.1:{port},server=on,wait=off,nodelay=on",
"-kernel", str(executable.elf_path), "-device",
@@ -582,7 +679,7 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0):
options["start_new_session"] = True
started = time.perf_counter()
report = {"passed": False, "status": "FAIL", "command": command, "timeout_s": timeout,
- "harts": harts,
+ "cores": cores, "harts": cores, "mean_policy": mean_policy, "nonlinear_policy": nonlinear_policy,
"qemu_workspace_layout": executable.evidence.get("qemu_workspace_layout", "none"),
"transport": "QMP physical memory dump after guest completion", "guest_completed": False}
transport_dir = run_dir / "weight-transport"
@@ -604,6 +701,7 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0):
path = str(transport_dir / row["file"]).replace(",", ",,")
command += ["-device", f"loader,file={path},addr=0x{layout['weights_base'] + row['offset']:x},force-raw=on"]
# Empty bundles intentionally emit no raw loader: QEMU rejects 0-byte files.
+ _check_cancelled(cancelled)
report["stage"] = "qemu"
deadline = time.monotonic() + timeout
completion = _UartCompletionWaiter(run_dir, executable.output.nbytes)
@@ -611,12 +709,17 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0):
qemu_started = time.perf_counter()
process = subprocess.Popen(command, cwd=run_dir, stdout=stdout, stderr=stderr, **options)
job = _windows_job(process)
+ report["qemu_pid"] = process.pid
monitor = _QemuMemoryMonitor(process)
monitor.start()
- completion.wait(process, deadline, timeout)
+ if cancelled is None:
+ completion.wait(process, deadline, timeout)
+ else:
+ completion.wait(process, deadline, timeout, cancelled=cancelled)
report.update(guest_completed=True, forward_wall_s=time.perf_counter() - qemu_started,
forward_wall_scope="QEMU launch through host validation of the UART completion frame")
- qmp = QMP(port, deadline)
+ qmp = (QMP(port, deadline) if cancelled is None else
+ QMP(port, deadline, cancelled=cancelled))
qmp.command("stop")
# A fast event must not hide bytes appended after the first valid frame.
decode_completion((run_dir / "uart.bin").read_bytes(), executable.output.nbytes)
@@ -633,7 +736,19 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0):
raise ValueError("Output dump size mismatch")
report["output_dump_s"] = time.perf_counter() - dump_started
report["qmp_quit_reply_received"] = qmp.command("quit") is not None
- process.wait(timeout=min(10, max(0.1, deadline - time.monotonic())))
+ if cancelled is None:
+ process.wait(timeout=min(10, max(0.1, deadline - time.monotonic())))
+ else:
+ while True:
+ _check_cancelled(cancelled)
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError("QEMU shutdown deadline exceeded")
+ try:
+ process.wait(timeout=min(0.05, remaining))
+ break
+ except subprocess.TimeoutExpired:
+ continue
qemu_exited = time.perf_counter()
if process.returncode:
raise RuntimeError(f"QEMU shutdown failed ({process.returncode})")
@@ -645,9 +760,11 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0):
raise ValueError("Nonfinite RV64 output")
if validate_weight_bundle(executable.bundle_dir, executable.weights) != manifest:
raise ValueError("Weight bundle changed during execution")
+ _check_cancelled(cancelled)
report["stage"] = "weight_transport_postcheck"
report["weight_transport"]["post_validation_elapsed_s"] = validate_transport(
transport_dir, report["weight_transport"])
+ _check_cancelled(cancelled)
report.update(passed=True, status="PASS", exit_code=process.returncode,
output_sha256=sha256_file(output_path), elf_sha256=executable.evidence["sha256"],
weight_sha256=manifest["sha256"], input_sha256=sha256_file(run_dir / "inputs.bin"))
@@ -697,6 +814,7 @@ def cleanup(label, action):
cleanup("process.wait fallback", lambda: process.wait(timeout=5))
if process.returncode is not None and qemu_exited is None:
qemu_exited = time.perf_counter()
+ report["qemu_reaped"] = process is not None and process.returncode is not None
report["qemu_wall_s"] = (qemu_exited - qemu_started
if qemu_exited is not None and qemu_started is not None else None)
if job is not None:
diff --git a/scratchv/runtime/riscv_harts.py b/scratchv/runtime/riscv_harts.py
index 6f0b7e07..d6192bfe 100644
--- a/scratchv/runtime/riscv_harts.py
+++ b/scratchv/runtime/riscv_harts.py
@@ -2,20 +2,46 @@
import os
-def validate_harts(harts):
- if type(harts) is not int or harts not in (1, 2, 4):
- raise ValueError("harts must be an integer in {1, 2, 4}")
- return harts
+UNSET = object()
-def resolve_harts(harts):
+def validate_cores(cores):
+ if type(cores) is not int or cores not in (1, 2, 4):
+ raise ValueError("cores (legacy harts) must be an integer in {1, 2, 4}")
+ return cores
+
+
+def select_cores(cores=UNSET, *, harts=UNSET, allow_auto=False):
+ """Validate either spelling, rejecting conflicting explicit requests."""
+ def check(value):
+ return value if allow_auto and type(value) is str and value == "auto" else validate_cores(value)
+ if cores is not UNSET:
+ cores = check(cores)
+ if harts is not UNSET:
+ harts = check(harts)
+ if cores is not UNSET and harts is not UNSET and cores != harts:
+ raise ValueError("cores and legacy harts disagree")
+ return cores if cores is not UNSET else harts if harts is not UNSET else 1
+
+
+def read_cores(record, *, default=1):
+ """Read current/legacy metadata; UNSET requires an explicit count."""
+ cores, harts = record.get("cores", UNSET), record.get("harts", UNSET)
+ if cores is UNSET and harts is UNSET:
+ if default is UNSET:
+ raise ValueError("Missing cores (or legacy harts) field")
+ return validate_cores(default)
+ return select_cores(cores, harts=harts)
+
+
+def resolve_cores(cores):
"""Resolve an explicit auto request once, using process CPU availability.
Prefer Linux affinity over machine-wide counts. This is a startup choice,
not a load monitor or cgroup bandwidth-quota policy.
"""
- if harts != "auto":
- return validate_harts(harts)
+ if type(cores) is not str or cores != "auto":
+ return validate_cores(cores)
try:
available = len(os.sched_getaffinity(0))
except (AttributeError, OSError, NotImplementedError):
@@ -28,7 +54,35 @@ def resolve_harts(harts):
return 4 if available >= 4 else 2 if available >= 2 else 1
-STARTUP_ASSEMBLY = r"""
+def validate_harts(harts):
+ """Compatibility alias for validate_cores."""
+ return validate_cores(harts)
+
+
+def resolve_harts(harts):
+ """Compatibility alias for resolve_cores."""
+ return resolve_cores(harts)
+
+
+
+# C and preprocessed assembly both accept the old compiler definition.
+CORE_DEFINITIONS = r"""
+#if defined(SCRATCHV_CORES) && defined(SCRATCHV_HARTS) && SCRATCHV_CORES != SCRATCHV_HARTS
+#error "SCRATCHV_CORES and legacy SCRATCHV_HARTS disagree"
+#endif
+#ifndef SCRATCHV_CORES
+#ifdef SCRATCHV_HARTS
+#define SCRATCHV_CORES SCRATCHV_HARTS
+#else
+#define SCRATCHV_CORES 1
+#endif
+#endif
+#ifndef SCRATCHV_HARTS
+#define SCRATCHV_HARTS SCRATCHV_CORES
+#endif
+"""
+
+STARTUP_ASSEMBLY = CORE_DEFINITIONS + r"""
.section .data,"aw"
.balign 4
sv_boot_ready:
@@ -37,7 +91,7 @@ def resolve_harts(harts):
.global sv_boot
sv_boot:
csrr a0, mhartid
- li t0, SCRATCHV_HARTS
+ li t0, SCRATCHV_CORES
bgeu a0, t0, .Lpark
.option push
.option norelax
@@ -92,24 +146,24 @@ def resolve_harts(harts):
# Included after the external runtime's types and its one-owner UART frame().
-WORKER_SOURCE = r'''
+WORKER_SOURCE = CORE_DEFINITIONS + r'''
typedef void (*sv_task_fn)(void *, usize, usize);
-#if SCRATCHV_HARTS > 1
-static usize sv_generation, sv_done[SCRATCHV_HARTS];
+#if SCRATCHV_CORES > 1
+static usize sv_generation, sv_done[SCRATCHV_CORES];
static unsigned sv_workers_ready, sv_failure;
static usize sv_count;
static sv_task_fn sv_function;
static void *sv_context;
-/* QEMU virt CLINT/ACLINT MSWI: one 32-bit MSIP register per hart.
+/* QEMU virt CLINT/ACLINT MSWI: one 32-bit MSIP register per core.
* Publish RAM state before the MMIO doorbell. Notifications can coalesce:
* generation/ready/done/failure, rather than the interrupt, own the state. */
-static void sv_notify(usize hart) {
+static void sv_notify(usize core) {
__asm__ volatile("fence rw,ow" ::: "memory");
- ((volatile u32 *)0x02000000UL)[hart]=1;
+ ((volatile u32 *)0x02000000UL)[core]=1;
}
-static void sv_clear_doorbell(usize hart) {
- ((volatile u32 *)0x02000000UL)[hart]=0;
+static void sv_clear_doorbell(usize core) {
+ ((volatile u32 *)0x02000000UL)[core]=0;
__asm__ volatile("fence iorw,iorw" ::: "memory");
}
static void sv_park_worker(void) {
@@ -121,28 +175,28 @@ def resolve_harts(harts):
unsigned failure = __atomic_load_n(&sv_failure, __ATOMIC_ACQUIRE);
if (failure) frame(failure, 0);
}
-static void sv_execute(usize hart, usize count, sv_task_fn fn, void *context) {
- usize width=count/SCRATCHV_HARTS, remainder=count%SCRATCHV_HARTS;
- usize begin=hart*width+(hart str | None:
return None
+def plan_batch_size(plan: dict) -> int:
+ # Persisted ten-file plans must retain their original file-to-batch mapping.
+ size = plan.get("batch_size", BATCH_SIZE)
+ if type(size) is not int or size not in (BATCH_SIZE, 10):
+ raise ValueError("Invalid review batch size; expected 5 or legacy 10.")
+ return size
+
+
def make_plan(raw: dict) -> dict:
files = raw.get("files", [])
errors = list(raw.get("errors", []))
@@ -55,9 +64,14 @@ def make_plan(raw: dict) -> dict:
errors.append("Fetched file count differs from PR changed_files (including the API 3000-file limit).")
if not raw.get("head") or raw.get("head") != raw.get("head_after"):
errors.append("PR head changed while fetching files, or its identity is missing.")
- batches = max(1, (len(files) + BATCH_SIZE - 1) // BATCH_SIZE)
+ try:
+ batch_size = plan_batch_size(raw)
+ except ValueError as exc:
+ errors.append(str(exc))
+ batch_size = BATCH_SIZE
+ batches = max(1, (len(files) + batch_size - 1) // batch_size)
if batches > 256:
- errors.append("PR exceeds the 256-job matrix limit at 10 files per batch; review requires a separately sized run.")
+ errors.append(f"PR exceeds the 256-job matrix limit at {batch_size} files per batch; review requires a separately sized run.")
batches = 1 # Keep the failed plan valid for artifact publication; review jobs are skipped.
return {
"repository": raw.get("repository"),
@@ -66,12 +80,12 @@ def make_plan(raw: dict) -> dict:
"changed_files": raw.get("changed_files"),
"files": files,
"errors": errors,
- "batch_size": BATCH_SIZE,
+ "batch_size": batch_size,
"batches": batches,
}
-def request_review(payload: dict, key: str) -> tuple[str, int, str]:
+def request_review(payload: dict, key: str, *, timeout_seconds: int = REQUEST_TIMEOUT_SECONDS) -> tuple[str, int, str]:
"""Return content, HTTP status and Retry-After without logging secret-bearing data."""
req = urllib.request.Request(
API_URL, json.dumps(payload).encode(),
@@ -79,7 +93,7 @@ def request_review(payload: dict, key: str) -> tuple[str, int, str]:
method="POST",
)
try:
- response = urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SECONDS)
+ response = urllib.request.urlopen(req, timeout=timeout_seconds)
except urllib.error.HTTPError as exc:
return "", exc.code, exc.headers.get("Retry-After", "")
with response:
@@ -111,7 +125,7 @@ def is_timeout(exc: BaseException) -> bool:
def review_file(file: dict, prompt: str, key: str, request=request_review,
sleep=time.sleep, clock=time.monotonic) -> dict:
- record = {"filename": file["filename"], "status": "pending", "attempts": 0}
+ record = {"filename": file["filename"], "status": "pending", "attempts": 0, "request_attempts": []}
started = clock()
problem = patch_problem(file)
if problem:
@@ -137,10 +151,14 @@ def review_file(file: dict, prompt: str, key: str, request=request_review,
record["attempts"] = attempt
retry = True
delay = 5
+ attempt_started = clock()
+ outcome = "interrupted"
try:
content, status, retry_after = request(payload, key)
+ outcome = f"HTTP {status}"
if status == 200:
- record.update(status="reviewed", review=content, elapsed=round(clock() - started, 1))
+ record.pop("reason", None)
+ record.update(status="reviewed", review=content)
return record
record["reason"] = f"HTTP {status}"
retry = status == 429 or 500 <= status <= 599
@@ -150,14 +168,22 @@ def review_file(file: dict, prompt: str, key: str, request=request_review,
pass
except (OSError, ValueError, KeyError, IndexError, TypeError, http.client.HTTPException) as exc:
# Exception text or HTTP response bodies may echo credentials; record only the class.
- record["reason"] = f"No usable API result ({type(exc).__name__})."
- # A timeout already consumed the full 90s budget; retrying only doubles wall-clock.
+ outcome = type(exc).__name__
+ record["reason"] = f"No usable API result ({outcome})."
+ # A timeout has already consumed its request window; do not repeat it.
retry = not is_timeout(exc)
+ finally:
+ # Record only timing/status, never request/response bodies or exception text.
+ finished = clock()
+ record["elapsed"] = round(finished - started, 1)
+ record["request_attempts"].append({
+ "attempt": attempt, "outcome": outcome,
+ "elapsed_seconds": round(finished - attempt_started, 6),
+ })
if attempt == 2 or not retry:
break
sleep(delay)
record["status"] = "failed"
- record["elapsed"] = round(clock() - started, 1)
return record
@@ -167,7 +193,12 @@ def run_batch(plan: dict, batch: int, prompt: str, output: Path, key: str,
raise ValueError("Cannot review an invalid or stale plan.")
if not 0 <= batch < plan["batches"]:
raise ValueError("Invalid batch index.")
- files = plan["files"][batch * BATCH_SIZE:(batch + 1) * BATCH_SIZE]
+ batch_size = plan_batch_size(plan)
+ if request is request_review:
+ # Legacy ten-file plans keep their original window and batch time budget.
+ timeout_seconds = REQUEST_TIMEOUT_SECONDS if batch_size == BATCH_SIZE else 90
+ request = partial(request_review, timeout_seconds=timeout_seconds)
+ files = plan["files"][batch * batch_size:(batch + 1) * batch_size]
result = {
"head": plan["head"], "batch": batch,
"files": [{"filename": file["filename"], "status": "pending", "attempts": 0} for file in files],
@@ -187,6 +218,12 @@ def summarize(plan: dict, batches: list[dict], current_head: str | None = None)
if current_head is not None and current_head != plan.get("head"):
errors.append("PR head changed after planning; results are stale.")
expected = [file["filename"] for file in plan["files"]]
+ try:
+ batch_size = plan_batch_size(plan)
+ except ValueError as exc:
+ errors.append(str(exc))
+ batches = []
+ batch_size = BATCH_SIZE
records: dict[str, dict] = {}
seen = set()
for batch in batches:
@@ -201,7 +238,7 @@ def summarize(plan: dict, batches: list[dict], current_head: str | None = None)
errors.append("Invalid batch identity.")
continue
seen.add(number)
- allowed = set(expected[number * BATCH_SIZE:(number + 1) * BATCH_SIZE])
+ allowed = set(expected[number * batch_size:(number + 1) * batch_size])
for record in batch.get("files", []):
name = record.get("filename") if isinstance(record, dict) else None
if not isinstance(name, str) or name not in allowed:
@@ -324,7 +361,9 @@ def main(argv=None) -> int:
stream.write("matrix=" + json.dumps(matrix) + "\n")
return int(bool(plan["errors"]))
if args.command == "review":
- result = run_batch(json.loads(args.plan.read_text(encoding="utf-8")), args.batch,
+ plan = json.loads(args.plan.read_text(encoding="utf-8"))
+ timeout_seconds = REQUEST_TIMEOUT_SECONDS if plan_batch_size(plan) == BATCH_SIZE else 90
+ result = run_batch(plan, args.batch,
args.prompt.read_text(encoding="utf-8"), args.output,
os.environ.get("SENSENOVA_API_KEY", ""))
reviewable = [record for record in result["files"] if record["status"] != "unreviewable"]
@@ -335,7 +374,7 @@ def main(argv=None) -> int:
elif record.get("elapsed", 0) > SLOW_REVIEW_SECONDS:
emit_annotation("warning", "AI review slow",
f"{record['filename']}: {record['elapsed']}s of the "
- f"{REQUEST_TIMEOUT_SECONDS}s budget")
+ f"{timeout_seconds}s budget")
if reviewable and not any(record["status"] == "reviewed" for record in reviewable):
emit_annotation("error", "AI review batch failed",
f"batch {result['batch']}: 0 of {len(reviewable)} reviewable files reviewed")
diff --git a/scripts/check_w5_ci.py b/scripts/check_w5_ci.py
new file mode 100644
index 00000000..a888c869
--- /dev/null
+++ b/scripts/check_w5_ci.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""Check the existing W5 CLI report; never execute or rehash model artifacts."""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+from pathlib import Path
+import re
+import sys
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+
+from scratchv.runtime.riscv_harts import UNSET, read_cores
+
+POLICIES = {"cores": 4, "matmul_policy": "blocked_fma", "matmul_tile": "2x4",
+ "matmul_rhs_layout": "column-panel", "mean_policy": "two-four-lane-last-axis",
+ "nonlinear_policy": "fp32-reference", "qemu_workspace_layout": "none"}
+
+
+def require(condition, message):
+ if not condition:
+ raise ValueError(message)
+
+
+def validate_report(report, reference):
+ """Require all three frozen eight-token cases and their actual QEMU evidence."""
+ require(isinstance(report, dict) and isinstance(reference, dict), "Expected report objects")
+ require(reference.get("status") == "frozen", "Reference is not frozen")
+ cases = reference["cases"]
+ require([case["name"] for case in cases] == ["france", "germany", "japan"]
+ and all(case["max_new_tokens"] == len(case["generated_token_ids"]) == 8
+ for case in cases), "Expected the frozen three-by-eight contract")
+ require(report.get("passed") is True and report.get("status") == "PASS"
+ and report.get("stage") == "complete" and report.get("full_model_gate") == "PASS"
+ and report.get("mode") == "raw" and not report.get("error"),
+ "Generation did not complete the full reference gate")
+ build = report.get("build", {})
+ for key, value in POLICIES.items():
+ reported = read_cores(report, default=UNSET) if key == "cores" else report.get(key)
+ built = read_cores(build, default=UNSET) if key == "cores" else build.get(key)
+ require(type(reported) is type(value) and reported == value
+ and type(built) is type(value) and built == value,
+ f"Unexpected {key} in report/build")
+ audit = build.get("w5_graph_audit", {})
+ require(audit.get("passed") is True and type(audit.get("layer_count")) is int
+ and audit["layer_count"] == 28, "Missing complete 28-layer graph audit")
+ require(all(type(report.get(key)) is int and report[key] == 24
+ for key in ("forward_attempts", "qemu_forwards")), "Expected 24 executed forwards")
+ for key, size in (("cases", 3), ("numeric_steps", 24), ("runs", 24)):
+ require(isinstance(report.get(key), list) and len(report[key]) == size,
+ f"Incomplete or extra {key}")
+ fma = report.get("fma_conformance", {})
+ comparison = fma.get("comparison", {})
+ rows = comparison.get("rows", [])
+ require(fma.get("passed") is True and fma.get("status") == "PASS"
+ and comparison.get("passed") is True
+ and comparison.get("criterion") == "exact binary32 bits and FCSR"
+ and type(comparison.get("failed_rows")) is int and comparison["failed_rows"] == 0
+ and isinstance(rows, list) and len(rows) == 12
+ and all(all(row.get(key) is True for key in ("passed", "bits_match", "csr_match"))
+ for row in rows), "FMA exact bit/FCSR conformance did not pass")
+ # Compare existing identities only: no new hashes or extra artifact reads.
+ elf, weights = build.get("sha256"), report["runs"][0].get("weight_sha256")
+ require(all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value)
+ for value in (elf, weights)), "Missing existing ELF/bundle identity")
+ index = 0
+ for actual, frozen in zip(report["cases"], cases):
+ require(actual.get("reference_passed") is True
+ and all(actual.get(key) == frozen[key] for key in (
+ "name", "prompt", "max_new_tokens", "prompt_token_ids",
+ "generated_token_ids", "generated_text", "stop_reason"))
+ and frozen["required_substring"] in actual["generated_text"]
+ and all(type(actual.get(key)) is int and actual[key] == 8
+ for key in ("numeric_step_count", "forward_count")),
+ f"Frozen output mismatch: {frozen['name']}")
+ for local_step, token in enumerate(frozen["generated_token_ids"]):
+ numeric, run = report["numeric_steps"][index], report["runs"][index]
+ valid_length = len(frozen["prompt_token_ids"]) + local_step
+ require(numeric.get("case") == frozen["name"]
+ and type(numeric.get("step")) is int and numeric["step"] == local_step
+ and type(numeric.get("valid_length")) is int
+ and numeric["valid_length"] == valid_length
+ and all(numeric.get(key) is True for key in (
+ "passed", "selected_row_passed", "reference_token_passed"))
+ and all(type(numeric.get(key)) is int and numeric[key] == token
+ for key in ("qemu_token_id", "ort_token_id", "reference_token_id")),
+ f"Wrong or missing token evidence at {frozen['name']}:{local_step}")
+ error = numeric.get("max_abs")
+ require(type(error) in (int, float) and math.isfinite(error) and 0 <= error < 1e-3
+ and type(numeric.get("atol")) in (int, float) and numeric["atol"] == 1e-3
+ and type(numeric.get("rtol")) in (int, float) and numeric["rtol"] == 0
+ and type(numeric.get("elements_compared")) is int
+ and numeric["elements_compared"] == 256 * 151936,
+ f"Full-logits strict tolerance failed at {frozen['name']}:{local_step}")
+ require(run.get("passed") is True and run.get("status") == "PASS"
+ and run.get("guest_completed") is True and run.get("qemu_reaped") is True
+ and type(run.get("exit_code")) is int and run["exit_code"] == 0
+ and not run.get("cleanup_errors") and not run.get("error")
+ and type(run.get("step")) is int and run["step"] == index
+ and type(run.get("valid_length")) is int and run["valid_length"] == valid_length
+ and run.get("elf_sha256") == elf and run.get("weight_sha256") == weights
+ and read_cores(run, default=UNSET) == POLICIES["cores"]
+ and all(type(run.get(key)) is type(POLICIES[key]) and run[key] == POLICIES[key]
+ for key in ("mean_policy", "nonlinear_policy", "qemu_workspace_layout")),
+ f"QEMU completion, cleanup or reuse failed at forward {index}")
+ index += 1
+
+
+def main(argv=None):
+ cli = argparse.ArgumentParser(description=__doc__)
+ cli.add_argument("--report", type=Path, required=True)
+ cli.add_argument("--reference", type=Path, default=ROOT / "probes/w5_generation/references.json")
+ cli.add_argument("--exit-code-file", type=Path, required=True)
+ cli.add_argument("--log-exit-code-file", type=Path,
+ help="Also require the tee log capture to have exited successfully")
+ cli.add_argument("--output", type=Path, required=True)
+ args = cli.parse_args(argv)
+ result = {"status": "FAIL", "passed": False, "scope": "Three frozen cases, 24 full QEMU forwards",
+ "commit": os.environ.get("GITHUB_SHA"), "report": str(args.report),
+ "defaults_changed": False, "raw_logits_uploaded": False}
+ try:
+ require(args.exit_code_file.read_text().strip() == "0", "Generation process did not exit successfully")
+ if args.log_exit_code_file is not None:
+ require(args.log_exit_code_file.read_text().strip() == "0",
+ "Log capture process did not exit successfully")
+ report = json.loads(args.report.read_text(encoding="utf-8"))
+ validate_report(report, json.loads(args.reference.read_text(encoding="utf-8")))
+ result.update(status="PASS", passed=True, numeric_steps=24, cases=3,
+ max_abs=max(row["max_abs"] for row in report["numeric_steps"]),
+ elapsed_seconds=report.get("elapsed_seconds"))
+ except (OSError, ValueError, TypeError, KeyError, AttributeError) as exc:
+ result["error"] = f"{type(exc).__name__}: {exc}"
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n", encoding="utf-8")
+ text = (f"## W5 generation {result['status']}\n\n{result['scope']}. "
+ "4 core; explicit Mean/nonlinear reference policies; padding off. "
+ "This is not acceptance of the unchanged default arithmetic configuration.\n\n"
+ + (f"24/24 numeric steps, 3/3 frozen outputs; max_abs={result['max_abs']:.9g}; "
+ f"elapsed={result['elapsed_seconds']} s.\n" if result["passed"]
+ else f"Not accepted: {result['error']}\n"))
+ print(text)
+ if os.environ.get("GITHUB_STEP_SUMMARY"):
+ with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream:
+ stream.write(text)
+ return 0 if result["passed"] else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/ci_scopes.json b/scripts/ci_scopes.json
index 87b87c36..d0bf470e 100644
--- a/scripts/ci_scopes.json
+++ b/scripts/ci_scopes.json
@@ -30,7 +30,18 @@
"tests/test_w2_*.py",
"tests/test_numeric_metrics.py",
"tests/test_numeric_summary.py",
- ".github/workflows/llm-deploy.yml"
+ ".github/workflows/llm-deploy.yml",
+ "scripts/run_inference.py",
+ "scripts/deploy_w5.sh",
+ "scripts/check_w5_ci.py",
+ "deployment/w5/**",
+ "probes/w5_generation/**",
+ "tests/test_generation.py",
+ "tests/test_conversation.py",
+ "tests/test_qemu_generation.py",
+ "tests/test_w5_*.py",
+ "tests/test_riscv_mean_policy.py",
+ "tests/test_riscv_nonlinear_policy.py"
],
"w4": [
".github/workflows/w4-full-numeric.yml",
@@ -59,6 +70,21 @@
"tests/test_riscv_external*.py",
"tests/test_riscv_process_cleanup.py",
"tests/test_riscv_weight_transport.py",
- "requirements/qwen3-small-probe.txt"
+ "requirements/qwen3-small-probe.txt",
+ "scripts/run_inference.py",
+ "scripts/deploy_w5.sh",
+ "scripts/check_w5_ci.py",
+ "deployment/w5/**",
+ "probes/w5_generation/**",
+ "tests/test_generation.py",
+ "tests/test_conversation.py",
+ "tests/test_qemu_generation.py",
+ "tests/test_w5_*.py",
+ "tests/test_riscv_mean_policy.py",
+ "tests/test_riscv_nonlinear_policy.py",
+ "pyproject.toml",
+ "assets/qwen3-0.6b/**",
+ "tests/test_tensor_c_mean_policy.py",
+ "tests/test_tensor_c_nonlinear_policy.py"
]
}
diff --git a/scripts/deploy_w5.sh b/scripts/deploy_w5.sh
new file mode 100755
index 00000000..1c25c955
--- /dev/null
+++ b/scripts/deploy_w5.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# Thin clean-container entry point; generation arguments belong to run_inference.py.
+set -euo pipefail
+
+repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+engine="${SCRATCHV_CONTAINER_ENGINE:-docker}"
+image="${SCRATCHV_W5_IMAGE:-scratchv-w5:local}"
+base_image="${SCRATCHV_W5_BASE_IMAGE:-ubuntu:24.04}"
+assets="${SCRATCHV_W5_ASSETS:-$repo_root/assets/qwen3-0.6b}"
+output="${SCRATCHV_W5_OUTPUT:-$repo_root/output/w5-deployment}"
+
+if ! command -v -- "$engine" >/dev/null 2>&1; then
+ printf 'Container engine is unavailable: %s. Install/start it separately; no system configuration was changed.\n' "$engine" >&2
+ exit 127
+fi
+if ! command -v setsid >/dev/null 2>&1; then
+ printf 'Linux setsid is required to clean up cancelled container-client processes.\n' >&2
+ exit 127
+fi
+
+# Own each preflight/build client process group while Bash waits. Killing only
+# this script must not leave the client (or its helpers) building in background.
+# The container engine owns daemon-side cancellation; a real-engine check is
+# still required to verify that side of the protocol.
+run_engine_stage() {
+ local client_pid="" status attempt
+ cancel_engine_stage() {
+ local exit_status="$1" target="${client_pid:-${!:-}}"
+ trap '' INT TERM
+ if [[ -n "$target" ]]; then
+ # The direct-PID fallback covers a signal before setsid takes effect.
+ kill -TERM -- "-$target" 2>/dev/null || kill -TERM "$target" 2>/dev/null || true
+ for ((attempt = 0; attempt < 50; attempt++)); do
+ if ! kill -0 -- "-$target" 2>/dev/null; then
+ break
+ fi
+ sleep 0.1
+ done
+ kill -KILL -- "-$target" 2>/dev/null || true
+ wait "$target" 2>/dev/null || true
+ fi
+ exit "$exit_status"
+ }
+ trap 'cancel_engine_stage 130' INT
+ trap 'cancel_engine_stage 143' TERM
+ setsid -- "$engine" "$@" &
+ client_pid=$!
+ if wait "$client_pid"; then
+ status=0
+ else
+ status=$?
+ fi
+ trap - INT TERM
+ return "$status"
+}
+
+if ! run_engine_stage info >/dev/null 2>&1; then
+ printf 'Container engine is not ready: %s. No model execution was started.\n' "$engine" >&2
+ exit 1
+fi
+if [[ ! -d "$assets" ]]; then
+ printf 'Assets directory does not exist: %s\n' "$assets" >&2
+ exit 2
+fi
+assets="$(cd -- "$assets" && pwd)"
+mkdir -p -- "$output"
+output="$(cd -- "$output" && pwd)"
+# Docker/Podman --mount use comma-delimited fields.
+if [[ "$assets" == *","* || "$output" == *","* ]]; then
+ printf 'Container mount paths must not contain commas.\n' >&2
+ exit 2
+fi
+
+run_engine_stage build \
+ --file "$repo_root/deployment/w5/Dockerfile" \
+ --build-arg "BASE_IMAGE=$base_image" \
+ --tag "$image" "$repo_root"
+
+# Numeric IDs keep report/build files owned by the invoking Linux developer.
+# --init forwards signals/reaps children; the existing runner owns QEMU cleanup.
+exec "$engine" run --rm --init \
+ --user "$(id -u):$(id -g)" \
+ --env HOME=/tmp \
+ --env SCRATCHV_ZIG_CACHE=/output/.zig-cache \
+ --mount "type=bind,src=$assets,dst=/assets,readonly" \
+ --mount "type=bind,src=$output,dst=/output" \
+ "$image" --output-dir /output/run "$@"
diff --git a/scripts/run_inference.py b/scripts/run_inference.py
new file mode 100644
index 00000000..3ef4eb38
--- /dev/null
+++ b/scripts/run_inference.py
@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""Build/reuse one RV64 ELF and weight bundle for Host-driven QEMU generation.
+
+Raw completion is the default; chat and the frozen reference suite are explicit.
+Each forward uses the existing bounded QEMU runner; weights/ELF are not rebuilt.
+"""
+from __future__ import annotations
+
+import argparse
+from dataclasses import asdict, replace
+import gc
+import json
+from pathlib import Path
+import signal
+import sys
+import threading
+import time
+
+import numpy as np
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+
+from probes.w3_common import atomic_text
+from probes.w2_runtime_model.run import checked_inputs
+from probes.w4_qwen3_full.run import CoreNamespace, add_core_argument, load_executable, save_executable, ATOL
+from probes.w4_qwen3_full.fma_conformance import run_probe as run_fma_probe
+from scratchv.backend.tensor_c_codegen import TensorCCodegen
+from scratchv.runtime.generation import GenerationCancelledError, generate
+from scratchv.runtime.llm_inputs import last_valid_logits, greedy_token
+from scratchv.runtime.qemu_generation import QemuForward
+from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer
+from scratchv.runtime.riscv_external import build_external, sha256_file
+from scratchv.runtime.riscv_harts import UNSET, read_cores, resolve_cores, select_cores
+from scratchv.runtime.riscv_tensor import discover_toolchain
+from scratchv.runtime.weight_bundle import write_weight_bundle
+
+
+def parse_args(argv=None):
+ cli = argparse.ArgumentParser(description=__doc__)
+ source = cli.add_mutually_exclusive_group(required=True)
+ source.add_argument("--model", type=Path, help="Static ONNX graph; compile once")
+ source.add_argument("--reuse-build", type=Path, help="Existing W4/W5 executable.json")
+ cli.add_argument("--tokenizer-dir", type=Path, required=True)
+ cli.add_argument("--output-dir", type=Path, required=True, help="New output directory")
+ cli.add_argument("--mode", choices=("raw", "chat"), default="raw")
+ prompt = cli.add_mutually_exclusive_group()
+ prompt.add_argument("--prompt")
+ prompt.add_argument("--chat-turn", action="append", help="Repeat for ordered user turns")
+ prompt.add_argument("--reference-suite", action="store_true", help="Run all 3 frozen W5 prompts")
+ cli.add_argument("--system-prompt")
+ cli.add_argument("--max-new-tokens", type=int)
+ add_core_argument(cli)
+ cli.add_argument("--timeout", type=float, default=3600, help="Per-forward QEMU timeout, seconds")
+ cli.add_argument("--total-timeout", type=float, default=10800, help="Generation/suite timeout, seconds")
+ cli.add_argument("--build-timeout", type=float, default=900)
+ cli.add_argument("--build-only", action="store_true")
+ cli.add_argument("--cc")
+ cli.add_argument("--qemu")
+ cli.add_argument("--matmul-policy", choices=("sequential", "blocked_fma"), default="sequential")
+ cli.add_argument("--mean-policy", choices=("sequential", "two-four-lane-last-axis"), default="sequential",
+ help="FP32 last-axis Mean order; default preserves sequential reduction")
+ cli.add_argument("--nonlinear-policy", choices=("libm", "fp32-reference"), default="libm",
+ help="Explicit FP32 Trig/Sigmoid/Softmax order; default preserves platform libm")
+ cli.add_argument("--matmul-tile", choices=("1x4", "2x4", "1x8"), default="1x4")
+ cli.add_argument("--matmul-rhs-layout", choices=("original", "column-panel"), default="original")
+ cli.add_argument("--verify-onnx", type=Path, help="Optional ORT numeric/greedy oracle for every step")
+ args = cli.parse_args(argv, namespace=CoreNamespace())
+ if any(not np.isfinite(t) or t <= 0 for t in (args.timeout, args.total_timeout, args.build_timeout)):
+ cli.error("timeouts must be positive and finite")
+ if args.max_new_tokens is not None and args.max_new_tokens < 0:
+ cli.error("--max-new-tokens must be nonnegative")
+ if not args.build_only and args.prompt is None and not args.chat_turn and not args.reference_suite:
+ cli.error("generation requires --prompt, --chat-turn, or --reference-suite")
+ if (args.chat_turn or args.system_prompt is not None) and args.mode != "chat":
+ cli.error("--chat-turn/--system-prompt require --mode chat")
+ if args.reference_suite and (args.mode != "raw" or args.max_new_tokens is not None or not args.verify_onnx):
+ cli.error("--reference-suite requires raw mode, --verify-onnx and its frozen token budgets")
+ if (args.matmul_tile != "1x4" or args.matmul_rhs_layout != "original") and args.matmul_policy != "blocked_fma":
+ cli.error("nondefault tile/RHS layout requires --matmul-policy blocked_fma")
+ if args.reuse_build and args.cc:
+ cli.error("--cc cannot change a reused build")
+ return args
+
+
+def prepare_executable(args, out, cores=UNSET, *, harts=UNSET):
+ """Reuse existing metadata validation, or compile/write weights exactly once."""
+ cores = select_cores(cores, harts=harts)
+ if args.reuse_build:
+ previous = args.reuse_build.with_name("report.json")
+ record = json.loads(previous.read_text(encoding="utf-8"))
+ if record.get("executable_sha256") != sha256_file(args.reuse_build):
+ raise ValueError("Reused executable metadata identity is missing or changed")
+ exe = load_executable(args.reuse_build)
+ if (("cores" in record or "harts" in record)
+ and read_cores(record) != exe.cores):
+ raise ValueError("Reused report core count differs from executable")
+ # W4 stored arithmetic policy in the sibling report; W5 also keeps it
+ # in the validated executable metadata.
+ policy = exe.evidence.get("matmul_policy", record.get("matmul_policy"))
+ expected = {"matmul_tile": args.matmul_tile, "matmul_rhs_layout": args.matmul_rhs_layout,
+ "qemu_workspace_layout": "none"}
+ defaults = {"matmul_tile": "1x4", "matmul_rhs_layout": "original", "qemu_workspace_layout": "none"}
+ if exe.cores != cores or policy != args.matmul_policy:
+ raise ValueError("Reused build core count/arithmetic policy differs from requested configuration")
+ if (exe.evidence.get("mean_policy", "sequential") != args.mean_policy
+ or record.get("mean_policy", "sequential") != args.mean_policy):
+ raise ValueError("Reused build Mean policy differs from requested --mean-policy")
+ if (exe.evidence.get("nonlinear_policy", "libm") != args.nonlinear_policy
+ or record.get("nonlinear_policy", "libm") != args.nonlinear_policy):
+ raise ValueError("Reused build nonlinear policy differs from requested --nonlinear-policy")
+ if any(exe.evidence.get(k, defaults[k]) != value for k, value in expected.items()):
+ raise ValueError("Reused build layout/tile differs from requested configuration; padding must be none")
+ # Preserve validated W4 evidence when this executable is saved as W5
+ # metadata. A later reuse must not depend on the original W4 report.
+ evidence = dict(exe.evidence)
+ evidence.setdefault("matmul_policy", policy)
+ if "w5_graph_audit" not in evidence and isinstance(record.get("graph_audit"), dict):
+ evidence["w5_graph_audit"] = dict(record["graph_audit"])
+ if evidence != exe.evidence:
+ exe = replace(exe, evidence=evidence)
+ if args.qemu:
+ exe = replace(exe, toolchain=discover_toolchain(exe.toolchain.cc, args.qemu))
+ return exe
+ from scratchv.frontend.onnx_parser import ONNXParser
+ parser = ONNXParser()
+ program = parser.parse(str(args.model.resolve()), mmap_external_data=True)
+ artifact = TensorCCodegen(
+ program, initializers=parser.initializers, constant_storage="external",
+ max_constant_bytes=3 * 1024**3, max_workspace_bytes=1024**3, kernel_calls=True,
+ matmul_policy=args.matmul_policy, matmul_tile=args.matmul_tile, mean_policy=args.mean_policy,
+ nonlinear_policy=args.nonlinear_policy,
+ matmul_rhs_layout=args.matmul_rhs_layout, qemu_workspace_layout="none",
+ ).generate()
+ write_weight_bundle(artifact.external_weights, artifact.external_initializers, out / "weights")
+ exe = build_external(artifact, out / "weights", out / "build",
+ discover_toolchain(args.cc, args.qemu), timeout=args.build_timeout, cores=cores)
+ audit = {"passed": False, "layer_count": None}
+ if tuple(exe.output.shape) == (1, 256, 151936):
+ import onnx
+ from probes.w2_qwen3_parse.validation import audit_graph_structure, Qwen3StructureError
+ try:
+ found = audit_graph_structure(onnx.load(str(args.model), load_external_data=False))
+ audit = {key: found[key] for key in ("passed", "layer_count", "graph", "tied_embeddings")}
+ except Qwen3StructureError as exc:
+ audit["error"] = str(exc)
+ return replace(exe, evidence={**exe.evidence, "matmul_policy": args.matmul_policy,
+ "w5_graph_audit": audit})
+
+
+def require_full_executable(exe):
+ """Check executable evidence already validated/promoted by preparation."""
+ audit = exe.evidence.get("w5_graph_audit")
+ if (not isinstance(audit, dict) or audit.get("passed") is not True
+ or audit.get("layer_count") != 28 or tuple(exe.output.shape) != (1, 256, 151936)):
+ raise ValueError("Full reference suite requires complete graph evidence for this executable")
+
+
+def oracle_session(model_path):
+ import onnxruntime as ort
+ options = ort.SessionOptions()
+ options.intra_op_num_threads = options.inter_op_num_threads = 1
+ options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL
+ options.enable_cpu_mem_arena = False
+ return ort.InferenceSession(str(model_path), options, providers=["CPUExecutionProvider"])
+
+
+def reference_session(model_path, fixture):
+ """Require the fixed complete Qwen3 graph before claiming a W5 full gate."""
+ import onnx
+ from probes.w2_qwen3_parse.validation import audit_graph_structure
+ from probes.w5_generation.run import session_for
+ manifest = json.loads((ROOT / fixture["model_manifest"]).read_text(encoding="utf-8"))
+ if (manifest["model_id"] != fixture["model_id"] or manifest["revision"] != fixture["revision"]
+ or model_path.name != "model.onnx" or model_path.parent.name != "onnx"):
+ raise ValueError("Reference suite requires the fixed W1 ONNX asset layout/revision")
+ audit = audit_graph_structure(onnx.load(str(model_path), load_external_data=False))
+ if not audit["passed"] or audit["layer_count"] != 28:
+ raise ValueError("Reference suite requires the complete 28-layer Qwen3 graph")
+ return session_for(model_path.parent.parent, manifest)
+
+
+def compare_step(actual, expected, valid_length, vocab_size):
+ """Full finite-output check with the existing W4 absolute tolerance."""
+ left = last_valid_logits(actual, valid_length, vocab_size=vocab_size)
+ right = last_valid_logits(expected, valid_length, vocab_size=vocab_size)
+ maximum = 0.0
+ for row in range(actual.shape[1]):
+ delta = np.abs(actual[0, row].astype(np.float64) - expected[0, row].astype(np.float64))
+ maximum = max(maximum, float(delta.max()))
+ ids = greedy_token(left), greedy_token(right)
+ selected_row_passed = (np.array_equal(left, actual[0, valid_length - 1])
+ and np.array_equal(right, expected[0, valid_length - 1]))
+ top_two = np.partition(right, -2)[-2:] if right.size > 1 else None
+ # Finite FP32 endpoints can have a difference outside FP32 range.
+ return {"passed": maximum < ATOL and ids[0] == ids[1] and selected_row_passed,
+ "selected_row_passed": selected_row_passed, "max_abs": maximum,
+ "atol": ATOL, "rtol": 0.0, "qemu_token_id": ids[0], "ort_token_id": ids[1],
+ "ort_top2_margin": (float(top_two[1]) - float(top_two[0])) if top_two is not None else None,
+ "elements_compared": int(actual.size)}
+
+
+def main(argv=None):
+ args = parse_args(argv)
+ out = args.output_dir.resolve()
+ out.mkdir(parents=True, exist_ok=False)
+ requested = args.cores if args.cores == "auto" else int(args.cores)
+ cores = resolve_cores(requested)
+ report = {"schema_version": 1, "status": "FAIL", "passed": False, "stage": "prepare",
+ "mode": args.mode, "cores_requested": requested, "cores": cores,
+ "harts_requested": requested, "harts": cores,
+ "qemu_workspace_layout": "none", "matmul_policy": args.matmul_policy,
+ "mean_policy": args.mean_policy, "nonlinear_policy": args.nonlinear_policy,
+ "matmul_tile": args.matmul_tile, "matmul_rhs_layout": args.matmul_rhs_layout,
+ "cases": [], "runs": [], "numeric_steps": [], "qemu_forwards": 0,
+ "full_model_gate": "NOT_RUN", "container_reproduction": "NOT_ASSERTED",
+ "execution": "one ELF/bundle; a fresh QEMU process per forward; no KV cache"}
+ started = time.perf_counter()
+ adapter = None
+ previous = {}
+ exit_code = 1
+
+ def interrupted(signum, frame):
+ raise KeyboardInterrupt(f"generation interrupted by signal {signum}")
+
+ def save():
+ if adapter is not None:
+ report["runs"] = adapter.runs
+ report["forward_attempts"] = len(adapter.runs)
+ report["qemu_forwards"] = sum(bool(run.get("guest_completed")) for run in adapter.runs)
+ report["elapsed_seconds"] = time.perf_counter() - started
+ atomic_text(out / "report.json", json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n")
+
+ try:
+ if threading.current_thread() is threading.main_thread():
+ for sig in (signal.SIGINT, signal.SIGTERM):
+ previous[sig] = signal.getsignal(sig)
+ signal.signal(sig, interrupted)
+ tokenizer = Qwen3Tokenizer.from_directory(args.tokenizer_dir)
+ exe = prepare_executable(args, out, cores)
+ if args.reference_suite:
+ require_full_executable(exe)
+ report["executable_sha256"] = save_executable(exe, out)
+ adapter = QemuForward(exe, out / "steps", timeout=args.timeout)
+ if adapter.vocab_size != tokenizer.model_vocab_size:
+ raise ValueError("Tokenizer model vocabulary differs from the executable output")
+ report["build"] = exe.evidence
+ if args.build_only:
+ report.update(status="BUILD_ONLY", stage="build_complete")
+ else:
+ report["stage"] = "fma_preflight"
+ fma = run_fma_probe(out / "fma-conformance", cc=exe.toolchain.cc,
+ qemu=exe.toolchain.qemu, timeout=min(args.build_timeout, 120))
+ report["fma_conformance"] = fma
+ if not fma.get("passed") or fma.get("status") != "PASS":
+ raise ValueError("Actual RV64 execution toolchain failed FMA conformance")
+ fixture = None
+ if args.reference_suite:
+ report["stage"] = "reference_contract"
+ fixture = json.loads((ROOT / "probes/w5_generation/references.json").read_text(encoding="utf-8"))
+ meta = {"pad_token_id": tokenizer.pad_token_id,
+ "generation_eos_token_ids": list(tokenizer.generation_eos_token_ids),
+ "model_vocab_size": tokenizer.model_vocab_size,
+ "tokenizer_vocab_size": tokenizer.tokenizer_vocab_size}
+ if fixture["status"] != "frozen" or fixture["tokenizer"] != meta or len(fixture["cases"]) != 3:
+ raise ValueError("Reference suite/tokenizer differs from the frozen contract")
+ cases = fixture["cases"]
+ else:
+ texts = args.chat_turn or [args.prompt]
+ cases = [{"name": f"turn-{i}", "prompt": text,
+ "max_new_tokens": 8 if args.max_new_tokens is None else args.max_new_tokens}
+ for i, text in enumerate(texts)]
+ report["stage"] = "oracle_prepare"
+ session = (reference_session(args.verify_onnx, fixture) if fixture is not None else
+ oracle_session(args.verify_onnx) if args.verify_onnx else None)
+ conversation = None
+ if args.mode == "chat":
+ report["stage"] = "conversation_prepare"
+ from scratchv.runtime.conversation import Conversation
+ conversation = Conversation.from_directory(args.tokenizer_dir, system_prompt=args.system_prompt)
+ deadline = time.monotonic() + args.total_timeout
+ adapter.deadline = deadline
+ report["stage"] = "generation"
+ if fixture is not None:
+ report["full_model_gate"] = "RUNNING"
+ save()
+
+ def forward(prepared):
+ step = len(report["numeric_steps"]) - numeric_start
+ if fixture is not None:
+ if step >= len(case["generated_token_ids"]):
+ raise ValueError("Host called forward after the frozen reference ended")
+ feed = checked_inputs(reference_history, prepared, tokenizer.pad_token_id)
+ else:
+ feed = prepared.as_feed()
+ expected = session.run(["logits"], feed)[0] if session is not None else None
+ actual = adapter(prepared)
+ if expected is not None:
+ comparison = compare_step(actual, expected, prepared.valid_length, adapter.vocab_size)
+ comparison.update(case=case["name"], step=step, valid_length=prepared.valid_length)
+ if fixture is not None:
+ # Select independently from frozen history, never from a
+ # caller-declared row or the generation sampling helper.
+ position = len(reference_history) - 1
+ qemu_id = int(np.argmax(actual[0, position]))
+ ort_id = int(np.argmax(expected[0, position]))
+ frozen_id = case["generated_token_ids"][step]
+ comparison["reference_token_id"] = frozen_id
+ comparison["selected_row_passed"] = (
+ comparison["selected_row_passed"] and comparison["qemu_token_id"] == qemu_id
+ and comparison["ort_token_id"] == ort_id)
+ comparison["reference_token_passed"] = qemu_id == ort_id == frozen_id
+ comparison["passed"] = (comparison["passed"]
+ and comparison["selected_row_passed"] and comparison["reference_token_passed"])
+ report["numeric_steps"].append(comparison)
+ if not comparison["passed"]:
+ # The outer handler saves the failure after preserving its cause.
+ raise ValueError("QEMU/ORT full logits, selected row or frozen token differ; see numeric_steps")
+ if fixture is not None:
+ reference_history.append(frozen_id)
+ return actual
+
+ for case in cases:
+ numeric_start, run_start = len(report["numeric_steps"]), len(adapter.runs)
+ reference_history = list(case["prompt_token_ids"]) if fixture is not None else None
+ remaining = max(0.0, deadline - time.monotonic())
+ kwargs = {"forward": forward, "max_new_tokens": case["max_new_tokens"],
+ "timeout_seconds": remaining}
+ result = (conversation.turn(case["prompt"], **kwargs) if conversation else
+ generate(case["prompt"], tokenizer=tokenizer, **kwargs))
+ row = {"name": case["name"], "prompt": case["prompt"],
+ "max_new_tokens": case["max_new_tokens"], **asdict(result)}
+ if fixture is not None:
+ row["numeric_step_count"] = len(report["numeric_steps"]) - numeric_start
+ row["forward_count"] = len(adapter.runs) - run_start
+ row["reference_passed"] = (
+ row["numeric_step_count"] == row["forward_count"] == len(case["generated_token_ids"])
+ and list(result.prompt_token_ids) == case["prompt_token_ids"]
+ and list(result.generated_token_ids) == case["generated_token_ids"]
+ and result.generated_text == case["generated_text"]
+ and result.stop_reason == case["stop_reason"]
+ and case["required_substring"] in result.generated_text)
+ report["cases"].append(row)
+ if fixture is not None and not row["reference_passed"]:
+ raise ValueError(f"Frozen reference mismatch: {case['name']}")
+ save()
+ print(result.generated_text, flush=True)
+ if conversation:
+ report["history"] = [asdict(message) for message in conversation.history]
+ if fixture is not None:
+ report["full_model_gate"] = "PASS"
+ report.update(passed=True, status="PASS", stage="complete")
+ exit_code = 0
+ except (GenerationCancelledError, KeyboardInterrupt, SystemExit) as exc:
+ # The runner's temporary signal handler exits with 128 + signal.
+ # Other SystemExit exceptions retain their original propagation.
+ if isinstance(exc, SystemExit) and exc.code not in (130, 143):
+ raise
+ report.update(status="CANCELLED", error=f"{type(exc).__name__}: {exc}")
+ if report["full_model_gate"] == "RUNNING":
+ report["full_model_gate"] = "CANCELLED"
+ exit_code = exc.code if isinstance(exc, SystemExit) else 130
+ print(report["error"], file=sys.stderr, flush=True)
+ except Exception as exc:
+ report.update(error=f"{type(exc).__name__}: {exc}")
+ if report["full_model_gate"] == "RUNNING":
+ report["full_model_gate"] = "FAIL"
+ print(report["error"], file=sys.stderr, flush=True)
+ finally:
+ for sig, handler in previous.items():
+ signal.signal(sig, handler)
+ try:
+ save()
+ except Exception as exc:
+ # A secondary persistence error must not hide generation/cancellation.
+ # Even successful generation or build-only needs a usable final report.
+ exit_code = exit_code or 1
+ print(f"Report save failed: {type(exc).__name__}: {exc}; "
+ f"{out / 'report.json'} was not updated and may be missing or stale.",
+ file=sys.stderr, flush=True)
+ else:
+ print(f"{report['status']}: {out / 'report.json'}", flush=True)
+ finally:
+ gc.collect()
+ return exit_code
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/llm-deploy/w5/README.md b/tests/llm-deploy/w5/README.md
new file mode 100644
index 00000000..508fa0d3
--- /dev/null
+++ b/tests/llm-deploy/w5/README.md
@@ -0,0 +1,134 @@
+# W5 报告保存与容器取消测试
+
+本目录验证生成 CLI 的报告保存边界，以及真实 Docker 容器中的外部取消和进程清理。只使用小图，不下载或运行完整 Qwen3 模型。功能、实现和实跑说明见 [W5 开发说明](../../../docs/llm-deploy-v1.0/W5/README.md)。
+
+## 本轮 core 兼容与 main 整合验证（2026-10-30）
+
+本轮在整合 main 的 `013b3c7` 后，验证新旧核数接口、配置读写及相关真实小图。
+四组测试互不重叠，合计 **631 passed，0 failed / error / skipped**；
+核数测试文件中另有 **18 项未被本轮筛选选中**，没有将它们计入通过数。
+
+| 范围 | 本轮结果 | 本地日志与 JUnit 文件名前缀 |
+|---|---:|---|
+| W4/W5 CLI、报告检查、诊断、旧构建复用和假引擎部署编排 | 480 passed | `interfaces` |
+| 默认/手动/auto、新旧参数和元数据、`asdict/replace`、C/汇编宏兼容 | 82 passed，18 deselected | `cores` |
+| QEMU 适配器、真实 1/4 core 多 token、不同上下文、超时/取消后恢复 | 29 passed | `qemu-generation` |
+| main 带入的指令计数模块 | 40 passed | `main-integration` |
+
+真实 1/4 core 小图均生成 `[2,3,4,6]`，不同上下文生成 `[4]`，超时和取消后的恢复生成 `[6]`。
+小图完整输出与 ORT 逐元素相同，最大绝对误差为 0；复用同一 ELF/权重包，padding 关闭，进程均已回收。
+本轮没有运行完整 Qwen3 模型，也没有启动真实 Docker 验证；部署编排使用假引擎。
+
+日志、JUnit 和两份小图结果保存在本机 `output/w5-cores-publish-20261030/`，不随 Git 提交。
+本节与 PR 说明提供可随仓库查看的范围和结论。旧 CI
+[#40173064525](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525)
+对应 `e8c94a7`，不能代表本轮修改；远端检查应查看本轮提交对应的运行。
+
+
+展开本轮实际执行的相关验证命令
+
+在仓库根目录、已满足下方环境要求时执行。使用新的输出目录保留本轮 JUnit。
+
+```bash
+python -m pytest tests/test_w4_full_probe_contract.py tests/test_w5_inference_cli.py tests/test_w5_ci.py tests/test_w4_diagnostic.py tests/test_w5_deployment.py -q --junitxml=output/w5-cores-publish-20261030/interfaces.xml
+```
+
+```bash
+python -m pytest tests/test_riscv_external_harts.py -q -k 'core or auto or manual or each_hart or invalid_hart or macro' --junitxml=output/w5-cores-publish-20261030/cores.xml
+```
+
+```bash
+python -m pytest tests/test_qemu_generation.py -q --junitxml=output/w5-cores-publish-20261030/qemu-generation.xml
+```
+
+```bash
+python -B -m pytest tests/test_inst_counter.py -q -p no:cacheprovider --junitxml=output/w5-cores-publish-20261030/main-integration.xml
+```
+
+
+
+## 环境与约定
+
+本文的 core 指 QEMU 模拟的 CPU 核。当前 CLI 使用 `--cores`，旧 `--harts` 继续兼容。
+
+在仓库根目录执行命令，使用已安装 `.[llm-deploy]` 的 Ubuntu 24.04 / Python 3.12 虚拟环境。`conftest.py` 复用既有 CLI fixtures；Docker 测试复用 `tests/test_qemu_generation.py` 的小图生成器和原 `scripts/deploy_w5.sh`。
+
+真实容器测试要求本机 Linux Docker Engine 正常运行、当前用户有访问权限，且能够构建 `deployment/w5/Dockerfile`；模型和输出通过临时目录挂载。宿主不需要额外安装 QEMU，镜像提供它及编译工具链。可通过 `SCRATCHV_W5_BASE_IMAGE` 指定可访问的 Ubuntu 24.04 基础镜像，默认使用 `public.ecr.aws/ubuntu/ubuntu:24.04`。首次构建需要下载依赖，后续复用 Docker 缓存。
+
+测试不会安装或启动 Docker，也不修改 WSL 配置。每次使用独立镜像标签，仅清理自己启动的客户端、容器及标签。中断诊断使用临时 QEMU 启动包装器的 `-S` 暂停真实 guest，不修改生产 harness。默认单核、手动 4 core 和 padding 关闭均保持；4 core 的恢复阶段在容器可用 CPU 自动选择为 4 时仍使用旧别名 `--harts auto`，同时覆盖兼容调用。
+
+## 运行全部本目录测试
+
+显式启用真实容器检查；缺少引擎或权限时失败，不会将缺失条件当作通过：
+
+```bash
+SCRATCHV_W5_DOCKER_TESTS=1 python -B -m pytest tests/llm-deploy/w5 -q -p no:cacheprovider
+```
+
+未设置该变量时，报告保存测试照常执行，真实 Docker 两项会跳过；这不能作为容器清理验收结果。若要保留小图、日志与 JSON，可追加 `--basetemp=output/<新的空目录>`，注意 pytest 会清理指定的 basetemp，不要指向已有证据目录。
+
+## 分项运行
+
+| 文件 | 覆盖内容 | 通过标准 |
+| --- | --- | --- |
+| `test_report_persistence.py` | 权限、磁盘满、I/O/序列化失败；生成、超时、取消、数值/参考文本错误；中间保存恢复与下一次调用 | 原错误和退出码保留；保存失败不误报 PASS；信号处理恢复、清理执行；后续可正常保存 |
+| `test_docker_cancellation.py` | 默认 1 / 手动 4 core 的真实多 token、Docker 客户端 SIGINT/SIGTERM、同 ELF/bundle 恢复 | 正常输出 `[2,3,4,6]` / `2 3 4`，四步 ORT 误差为 0；SIGINT/SIGTERM 分别返回 130/143、报告 CANCELLED；容器、宿主 PID 和 QEMU 进程组消失；恢复结果一致 |
+
+仅报告保存边界：
+
+```bash
+python -B -m pytest tests/llm-deploy/w5/test_report_persistence.py -q -p no:cacheprovider
+```
+
+仅真实容器取消与恢复：
+
+```bash
+SCRATCHV_W5_DOCKER_TESTS=1 python -B -m pytest tests/llm-deploy/w5/test_docker_cancellation.py -q -p no:cacheprovider
+```
+
+Docker 测试在每项临时目录写入 `baseline.log`、`sigint.log`、`sigterm.log`、相应恢复日志和 `docker-cancellation-summary.json`；`deployment/<运行名>/report.json` 保留逐步数值及 QEMU 清理状态。清理断言来自独立的宿主进程和容器检查，失败后的兜底清理不能使测试变为通过。
+
+CLI 测试还检查 `--cores` 的默认、手动和自动选择，旧 `--harts` 别名，以及新旧参数冲突；
+新旧核数字段的读取与构建复用保持一致；还检查 `parse_args().harts` 的读取、双向赋值及旧包装脚本继续执行。
+CLI 与部署编排测试仍位于原位置，可按影响范围单独执行：
+
+```bash
+python -B -m pytest tests/test_w5_inference_cli.py -q -p no:cacheprovider
+```
+
+```bash
+python -B -m pytest tests/test_w5_deployment.py -q -p no:cacheprovider
+```
+
+后者覆盖引擎不可用、构建失败、参数透传，以及 info/build 阶段的取消清理；使用假引擎，不替代本目录的真实 Docker run 阶段检查。
+
+执行对象的构造、`asdict()` 和 `replace()` 兼容检查沿用已有运行时测试：确认新旧核数字段等值、
+没有私有字段进入结果、复制相互独立，替换核数时同步且拒绝无效值。下面只运行内存中的对象检查，不启动 QEMU：
+
+```bash
+python -B -m pytest tests/test_riscv_external_harts.py -k 'executable_constructor or executable_asdict or executable_replace' -q -p no:cacheprovider
+```
+
+
+W5 专项的日志退出状态沿用现有工作流测试，直接执行 YAML 中的 Bash，并以短小假生成器
+替代模型。需要 Bash、GNU tee、`/dev/full` 和 PyYAML，不需要 Docker 或模型资产：
+生成或日志任一失败必须失败，分别保留退出码；状态文件本身写失败也不能成功。
+
+```bash
+python -B -m pytest tests/test_w4_workflows.py -k 'w5_deployment_pipeline or w5_manual_job' -q -p no:cacheprovider
+```
+
+报告检查器的无模型测试覆盖有效、非零、缺失、不可读和无效的日志状态；即使生成报告
+为 PASS，也不能覆盖日志失败。还检查报告、构建记录和每次前向中的 `cores` / `harts` 字段：新格式、旧格式及混合格式均可读取，缺失、无效或冲突值必须失败。旧版未传日志检查参数的调用仍可用：
+
+```bash
+python -B -m pytest tests/test_w5_ci.py -q -p no:cacheprovider
+```
+
+原多轮文档中的模板对照入口保留如下，使用本地官方 tokenizer，不运行完整模型：
+
+```bash
+SCRATCHV_QWEN3_TOKENIZER_DIR=assets/qwen3-0.6b/tokenizer python -B -m pytest tests/test_conversation.py -q -p no:cacheprovider
+```
+
+它对照有/无 system 的三轮官方模板渲染和 token 序列；回复由测试预设，不能当作完整模型三轮聊天结果。依赖本地官方 tokenizer 及 Transformers。显式指定 tokenizer 目录时，缺资产会失败；未指定且默认位置缺资产时，对应官方资产测试会跳过。
diff --git a/tests/llm-deploy/w5/conftest.py b/tests/llm-deploy/w5/conftest.py
new file mode 100644
index 00000000..22417cf8
--- /dev/null
+++ b/tests/llm-deploy/w5/conftest.py
@@ -0,0 +1,19 @@
+"""Reuse the existing W5 CLI fixtures for focused persistence checks."""
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+
+import pytest
+
+_path = Path(__file__).resolve().parents[2] / "test_w5_inference_cli.py"
+_spec = spec_from_file_location("w5_existing_cli_fixtures", _path)
+_existing = module_from_spec(_spec)
+_spec.loader.exec_module(_existing)
+
+cli = _existing.cli
+runtime = _existing.runtime
+reference_suite = _existing.reference_suite
+
+
+@pytest.fixture
+def cli_helpers():
+ return _existing
diff --git a/tests/llm-deploy/w5/test_docker_cancellation.py b/tests/llm-deploy/w5/test_docker_cancellation.py
new file mode 100644
index 00000000..1724e051
--- /dev/null
+++ b/tests/llm-deploy/w5/test_docker_cancellation.py
@@ -0,0 +1,264 @@
+"""Opt-in real Docker signal propagation, guest cleanup, and executable reuse."""
+import json
+import os
+from pathlib import Path
+import runpy
+import shutil
+import signal
+import subprocess
+import time
+import uuid
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[3]
+pytestmark = pytest.mark.skipif(
+ os.environ.get("SCRATCHV_W5_DOCKER_TESTS") != "1",
+ reason="set SCRATCHV_W5_DOCKER_TESTS=1 for real Docker small-graph checks",
+)
+
+
+def _docker(*args, check=True):
+ return subprocess.run(
+ ["docker", *args], text=True, capture_output=True, timeout=30, check=check,
+ )
+
+
+@pytest.fixture(scope="module")
+def docker_assets(tmp_path_factory):
+ # Explicit opt-in must fail on an unavailable engine, never silently skip.
+ assert shutil.which("docker"), "Docker is required with SCRATCHV_W5_DOCKER_TESTS=1"
+ assert Path("/proc/self/stat").exists(), "These checks require a local Linux Docker engine"
+ ready = _docker("info", "--format", "{{.ServerVersion}}", check=False)
+ assert ready.returncode == 0, ready.stderr
+ assets = tmp_path_factory.mktemp("docker-small-assets")
+ helper = runpy.run_path(str(ROOT / "tests/test_qemu_generation.py"))
+ helper["make_small_onnx"](assets / "model.onnx")
+
+ from tokenizers import AddedToken, Tokenizer, models, pre_tokenizers
+ tokenizer_dir = assets / "tokenizer"
+ tokenizer_dir.mkdir()
+ tokenizer = Tokenizer(models.WordLevel({str(i): i for i in range(8)}, unk_token="5"))
+ tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
+ tokenizer.add_special_tokens([
+ AddedToken(value, special=True, normalized=False) for value in ("0", "6")
+ ])
+ tokenizer.save(str(tokenizer_dir / "tokenizer.json"))
+ added = {
+ str(token_id): {
+ name: getattr(token, name)
+ for name in ("content", "lstrip", "rstrip", "single_word", "normalized", "special")
+ }
+ for token_id, token in tokenizer.get_added_tokens_decoder().items()
+ }
+ metadata = {
+ "tokenizer_config.json": {
+ "tokenizer_class": "Qwen2Tokenizer", "pad_token": "0", "eos_token": "6",
+ "unk_token": None, "bos_token": None, "added_tokens_decoder": added,
+ "additional_special_tokens": [],
+ "chat_template": "{% for message in messages %}{{ message['content'] }} {% endfor %}",
+ },
+ "generation_config.json": {"pad_token_id": 0, "eos_token_id": [6]},
+ "config.json": {"model_type": "qwen3", "vocab_size": 8},
+ }
+ for name, value in metadata.items():
+ (tokenizer_dir / name).write_text(json.dumps(value), encoding="utf-8")
+
+ # Pause only actual generation guests. Tool discovery and FMA preflight run
+ # normally; no fake QEMU or production harness change is involved.
+ wrapper = assets / "qemu-control"
+ wrapper.write_text(
+ "#!/usr/bin/env python\n"
+ "import os, sys\n"
+ "args = sys.argv[1:]\n"
+ "if '/steps/' in os.getcwd() and os.path.exists('/assets/park'):\n"
+ " args += ['-S']\n"
+ "os.execv('/usr/bin/qemu-system-riscv64', ['qemu-system-riscv64', *args])\n",
+ encoding="utf-8",
+ )
+ wrapper.chmod(0o755)
+ image = f"scratchv-w5-cancellation:{uuid.uuid4().hex[:12]}"
+ yield assets, image
+ # Only this fixture's unique image is eligible for cleanup.
+ _docker("image", "rm", image, check=False)
+
+
+def _owned_containers(image, output):
+ found = []
+ listing = _docker("ps", "-aq", "--filter", f"ancestor={image}")
+ for cid in listing.stdout.split():
+ result = _docker("inspect", cid, check=False)
+ if result.returncode:
+ continue
+ record = json.loads(result.stdout)[0]
+ if any(m.get("Source") == str(output.resolve()) and m.get("Destination") == "/output"
+ for m in record["Mounts"]):
+ found.append(cid)
+ return found
+
+
+def _process_identity(pid):
+ try:
+ fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()
+ status = Path(f"/proc/{pid}/status").read_text()
+ except FileNotFoundError:
+ return None
+ nspid = next((line.split()[1:] for line in status.splitlines()
+ if line.startswith("NSpid:")), [str(pid)])
+ return {"pid": int(pid), "pgid": int(fields[2]), "starttime": fields[19],
+ "container_pid": int(nspid[-1])}
+
+
+def _wait_parked(client, image, output):
+ deadline = time.monotonic() + 600 # Includes the deployment script's image build.
+ while time.monotonic() < deadline:
+ assert client.poll() is None, f"Deployment exited before paused guest: {client.returncode}"
+ for cid in _owned_containers(image, output):
+ top = _docker("top", cid, "-eo", "pid,pgid,comm,args", check=False)
+ if top.returncode:
+ continue
+ rows = [line.split(None, 3) for line in top.stdout.splitlines()[1:]]
+ guests = [row for row in rows if len(row) == 4
+ and "qemu-system-riscv64" in row[3] and "-S" in row[3].split()]
+ if guests:
+ processes = [_process_identity(row[0]) for row in rows]
+ processes = [item for item in processes if item is not None]
+ guest = _process_identity(guests[0][0])
+ assert guest is not None, "Paused guest disappeared unexpectedly"
+ return cid, processes, guest
+ time.sleep(0.2)
+ pytest.fail("Timed out waiting for real QEMU -S inside the deployment container")
+
+
+def _assert_reaped(cid, processes, guest, evidence):
+ deadline = time.monotonic() + 15
+ while time.monotonic() < deadline:
+ live = [item for item in processes if _process_identity(item["pid"]) == item]
+ present = _docker("inspect", cid, check=False).returncode == 0
+ if not live and not present:
+ break
+ time.sleep(0.1)
+ assert not live, f"Container host processes survived cancellation: {live}"
+ assert not present, f"--rm container survived cancellation: {cid}"
+ time.sleep(0.3)
+ assert all(_process_identity(item["pid"]) != item for item in processes)
+ assert _docker("inspect", cid, check=False).returncode != 0
+ # Inspect the actual guest process group independently of report assertions.
+ group_members = []
+ for entry in Path("/proc").iterdir():
+ if entry.name.isdigit():
+ identity = _process_identity(entry.name)
+ if identity is not None and identity["pgid"] == guest["pgid"]:
+ group_members.append(identity)
+ assert not group_members, f"QEMU process group still has members: {group_members}"
+ evidence.update(container_removed=True, host_processes_reaped=True,
+ qemu_group_empty=True, delayed_recheck_seconds=0.3)
+
+
+def _report(output, name):
+ return json.loads((output / name / "report.json").read_text(encoding="utf-8"))
+
+
+def _assert_pass(record, harts):
+ assert record["status"] == "PASS" and record["passed"] is True
+ assert record["harts"] == harts and record["qemu_workspace_layout"] == "none"
+ assert record["qemu_forwards"] == 4
+ assert record["cases"][0]["generated_token_ids"] == [2, 3, 4, 6]
+ assert record["cases"][0]["generated_text"] == "2 3 4"
+ assert len(record["numeric_steps"]) == 4
+ assert all(step["passed"] and step["max_abs"] == 0 for step in record["numeric_steps"])
+ assert all(run["qemu_reaped"] and run["guest_completed"] for run in record["runs"])
+
+
+@pytest.mark.parametrize("harts", [1, 4], ids=["default-single", "manual-four"])
+def test_external_signals_reap_guest_and_reuse_build(tmp_path, docker_assets, harts):
+ assets, image = docker_assets
+ output = tmp_path / "deployment"
+ output.mkdir()
+ env = dict(os.environ, SCRATCHV_W5_ASSETS=str(assets), SCRATCHV_W5_OUTPUT=str(output),
+ SCRATCHV_W5_IMAGE=image,
+ SCRATCHV_W5_BASE_IMAGE=os.environ.get(
+ "SCRATCHV_W5_BASE_IMAGE", "public.ecr.aws/ubuntu/ubuntu:24.04"))
+ env["SCRATCHV_CONTAINER_ENGINE"] = "docker"
+ common = ["--tokenizer-dir", "/assets/tokenizer", "--prompt", "1",
+ "--max-new-tokens", "8", "--verify-onnx", "/assets/model.onnx",
+ "--qemu", "/assets/qemu-control", "--matmul-policy", "blocked_fma",
+ "--timeout", "90", "--total-timeout", "180", "--build-timeout", "180"]
+ # The single-core baseline really exercises the CLI default.
+ hart_args = [] if harts == 1 else ["--harts", "4"]
+ active = None
+ evidence = {"harts": harts, "image": image, "cancellations": []}
+
+ def launch(name, source, selection):
+ log = (tmp_path / f"{name}.log").open("w", encoding="utf-8")
+ try:
+ return subprocess.Popen(
+ ["bash", str(ROOT / "scripts/deploy_w5.sh"), *source, *common, *selection,
+ "--output-dir", f"/output/{name}"],
+ cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True,
+ )
+ finally:
+ log.close()
+
+ try:
+ active = launch("baseline", ["--model", "/assets/model.onnx"], hart_args)
+ assert active.wait(timeout=600) == 0, (tmp_path / "baseline.log").read_text()[-6000:]
+ baseline = _report(output, "baseline")
+ _assert_pass(baseline, harts)
+ original = json.loads((output / "baseline/executable.json").read_text())
+ evidence["baseline"] = baseline
+ reuse = ["--reuse-build", "/output/baseline/executable.json"]
+ auto = int(_docker("run", "--rm", "--entrypoint", "python", image, "-c",
+ "from scratchv.runtime.riscv_harts import resolve_harts; "
+ "print(resolve_harts('auto'))").stdout.strip())
+ evidence["available_auto_harts"] = auto
+ recovery_args = ["--harts", "auto"] if harts == 4 and auto == 4 else hart_args
+
+ for sig in (signal.SIGINT, signal.SIGTERM):
+ name = sig.name.lower()
+ (assets / "park").touch()
+ active = launch(name, reuse, hart_args)
+ cid, processes, guest = _wait_parked(active, image, output)
+ cancellation = {"signal": sig.name, "client_pid": active.pid, "container_id": cid,
+ "host_processes": processes, "qemu_host_process": guest}
+ evidence["cancellations"].append(cancellation)
+ # Exercise Docker's real signal proxy. killpg would bypass this chain.
+ active.send_signal(sig)
+ cancellation["exit_code"] = active.wait(timeout=30)
+ assert cancellation["exit_code"] == 128 + int(sig), (
+ tmp_path / f"{name}.log").read_text()[-6000:]
+ _assert_reaped(cid, processes, guest, cancellation)
+ cancelled = _report(output, name)
+ cancellation["report"] = cancelled
+ assert cancelled["status"] == "CANCELLED" and not cancelled["passed"]
+ assert cancelled["harts"] == harts and cancelled["qemu_workspace_layout"] == "none"
+ assert cancelled["qemu_forwards"] == 0 and len(cancelled["runs"]) == 1
+ run = cancelled["runs"][0]
+ assert run["qemu_reaped"] and not run["guest_completed"]
+ assert run["qemu_pid"] == guest["container_pid"]
+ (assets / "park").unlink()
+
+ recovery_name = f"{name}-recovery"
+ active = launch(recovery_name, reuse, recovery_args)
+ assert active.wait(timeout=600) == 0, (tmp_path / f"{recovery_name}.log").read_text()[-6000:]
+ recovered = _report(output, recovery_name)
+ _assert_pass(recovered, harts)
+ assert recovered["cases"][0]["generated_token_ids"] == baseline["cases"][0]["generated_token_ids"]
+ saved = json.loads((output / recovery_name / "executable.json").read_text())
+ assert all(saved[key] == original[key] for key in ("elf", "bundle_dir", "qemu", "harts"))
+ assert recovered["harts_requested"] == ("auto" if recovery_args == ["--harts", "auto"] else harts)
+ cancellation["recovery"] = recovered
+ evidence["passed"] = True
+ finally:
+ (assets / "park").unlink(missing_ok=True)
+ # Failure cleanup cannot satisfy the assertions above, and targets only
+ # our unique image plus output mount and our separately started client.
+ for cid in _owned_containers(image, output):
+ _docker("rm", "-f", cid, check=False)
+ if active is not None and active.poll() is None:
+ os.killpg(active.pid, signal.SIGKILL)
+ active.wait(timeout=15)
+ (tmp_path / "docker-cancellation-summary.json").write_text(
+ json.dumps(evidence, indent=2) + "\n", encoding="utf-8",
+ )
diff --git a/tests/llm-deploy/w5/test_report_persistence.py b/tests/llm-deploy/w5/test_report_persistence.py
new file mode 100644
index 00000000..cc49cfbc
--- /dev/null
+++ b/tests/llm-deploy/w5/test_report_persistence.py
@@ -0,0 +1,268 @@
+"""Report I/O failures retain the primary outcome; no model or QEMU runs."""
+import errno
+import json
+import signal
+
+import pytest
+
+from scratchv.runtime.generation import GenerationCancelledError
+
+
+@pytest.mark.parametrize("error", [
+ PermissionError(errno.EACCES, "report directory is read-only"),
+ OSError(errno.ENOSPC, "report disk is full"),
+ OSError(errno.EIO, "report device failed"),
+], ids=["permission", "disk-full", "io"])
+@pytest.mark.parametrize("build_only", [False, True], ids=["generation", "build-only"])
+def test_final_write_failure_never_reports_success(
+ cli, runtime, cli_helpers, tmp_path, monkeypatch, capsys, error, build_only,
+):
+ atomic = cli.atomic_text
+ attempted = []
+ collected = []
+ previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)}
+
+ def write(path, text):
+ value = json.loads(text)
+ attempted.append(value)
+ if value["status"] in {"PASS", "BUILD_ONLY"}:
+ raise error
+ atomic(path, text)
+
+ monkeypatch.setattr(cli, "atomic_text", write)
+ monkeypatch.setattr(cli.gc, "collect", lambda: collected.append(True))
+ options = ["--build-only"] if build_only else ["--prompt", "text", "--max-new-tokens", "1"]
+ assert cli.main(cli_helpers.argv(tmp_path, *options)) == 1
+ output = capsys.readouterr()
+ assert "Report save failed:" in output.err and str(error) in output.err
+ assert "missing or stale" in output.err
+ assert "PASS:" not in output.out and "BUILD_ONLY:" not in output.out
+ assert attempted[-1]["harts"] == 1 and attempted[-1]["qemu_workspace_layout"] == "none"
+ assert collected == [True]
+ assert all(signal.getsignal(sig) is handler for sig, handler in previous.items())
+ if build_only:
+ assert not runtime.fma and not runtime.adapters[0].runs
+ assert not (tmp_path / "run/report.json").exists()
+ else:
+ # The older atomic checkpoint cannot be mistaken for final success.
+ saved = cli_helpers.report(tmp_path)
+ assert not saved["passed"] and saved["status"] != "PASS"
+
+
+@pytest.mark.parametrize("failure,code", [
+ (RuntimeError("primary generation failure"), 1),
+ (TimeoutError("primary forward timed out"), 1),
+ (GenerationCancelledError("primary cancellation"), 130),
+ (KeyboardInterrupt("primary interrupt"), 130),
+ (SystemExit(130), 130),
+ (SystemExit(143), 143),
+], ids=["generation", "timeout", "cancel", "interrupt", "runner-sigint", "runner-sigterm"])
+@pytest.mark.parametrize("save_error", [
+ PermissionError(errno.EACCES, "secondary save permission failure"),
+ OSError(errno.ENOSPC, "secondary save disk full"),
+], ids=["permission", "disk-full"])
+def test_secondary_write_failure_keeps_primary_error_and_exit(
+ cli, runtime, cli_helpers, tmp_path, monkeypatch, capsys, failure, code, save_error,
+):
+ runtime.failure = failure
+ snapshots = []
+ collected = []
+ previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)}
+
+ def write(path, text):
+ snapshots.append(json.loads(text))
+ raise save_error
+
+ monkeypatch.setattr(cli, "atomic_text", write)
+ monkeypatch.setattr(cli.gc, "collect", lambda: collected.append(True))
+ assert cli.main(cli_helpers.argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1")) == code
+ output = capsys.readouterr()
+ assert str(failure) in output.err and type(failure).__name__ in output.err
+ assert str(save_error) in output.err and "Report save failed:" in output.err
+ assert "report.json" not in output.out
+ assert snapshots[-1]["error"] == f"{type(failure).__name__}: {failure}"
+ assert snapshots[-1]["status"] == ("CANCELLED" if code in (130, 143) else "FAIL")
+ assert snapshots[-1]["forward_attempts"] == 1 and not snapshots[-1]["passed"]
+ assert collected == [True]
+ assert all(signal.getsignal(sig) is handler for sig, handler in previous.items())
+
+
+def test_intermediate_write_failure_is_saved_as_failure_when_storage_recovers(
+ cli, runtime, cli_helpers, tmp_path, monkeypatch, capsys,
+):
+ atomic = cli.atomic_text
+ attempts = []
+
+ def write(path, text):
+ attempts.append(json.loads(text))
+ if len(attempts) == 1:
+ raise OSError(errno.ENOSPC, "checkpoint disk full")
+ atomic(path, text)
+
+ monkeypatch.setattr(cli, "atomic_text", write)
+ assert cli.main(cli_helpers.argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1")) == 1
+ saved = cli_helpers.report(tmp_path)
+ assert len(attempts) == 2
+ assert saved["status"] == "FAIL" and not saved["passed"]
+ assert saved["qemu_forwards"] == 1 and len(saved["cases"]) == 1
+ assert "checkpoint disk full" in saved["error"]
+ assert "checkpoint disk full" in capsys.readouterr().err
+
+
+def test_suite_initial_checkpoint_failure_prevents_generation(
+ cli, runtime, reference_suite, cli_helpers, tmp_path, monkeypatch,
+):
+ atomic = cli.atomic_text
+ attempts = []
+
+ def write(path, text):
+ attempts.append(json.loads(text))
+ if len(attempts) == 1:
+ raise PermissionError(errno.EACCES, "cannot save RUNNING")
+ atomic(path, text)
+
+ monkeypatch.setattr(cli, "atomic_text", write)
+ assert cli.main(cli_helpers.argv(
+ tmp_path, "--reference-suite", "--verify-onnx", "model.onnx",
+ )) == 1
+ saved = cli_helpers.report(tmp_path)
+ assert saved["full_model_gate"] == "FAIL" and saved["qemu_forwards"] == 0
+ assert saved["numeric_steps"] == [] and saved["cases"] == []
+ assert "cannot save RUNNING" in saved["error"]
+
+
+def test_numeric_failure_survives_report_write_failure(
+ cli, runtime, reference_suite, cli_helpers, tmp_path, monkeypatch, capsys,
+):
+ runtime.oracle = cli_helpers.logits(4)
+ atomic = cli.atomic_text
+ snapshots = []
+
+ def write(path, text):
+ value = json.loads(text)
+ snapshots.append(value)
+ if value["full_model_gate"] == "FAIL":
+ raise OSError(errno.ENOSPC, "numeric-report disk full")
+ atomic(path, text)
+
+ monkeypatch.setattr(cli, "atomic_text", write)
+ assert cli.main(cli_helpers.argv(
+ tmp_path, "--reference-suite", "--verify-onnx", "model.onnx",
+ )) == 1
+ output = capsys.readouterr()
+ assert "QEMU/ORT full logits, selected row or frozen token differ" in output.err
+ assert "numeric-report disk full" in output.err
+ assert snapshots[-1]["full_model_gate"] == "FAIL"
+ assert not snapshots[-1]["numeric_steps"][0]["passed"]
+ assert "QEMU/ORT" in snapshots[-1]["error"]
+ assert cli_helpers.report(tmp_path)["full_model_gate"] == "RUNNING"
+ assert "PASS:" not in output.out
+
+
+def test_reference_text_failure_survives_report_write_failure(
+ cli, runtime, reference_suite, cli_helpers, tmp_path, monkeypatch, capsys,
+):
+ reference_suite.fixture["cases"][0]["generated_text"] = "not-the-result"
+ fixture_path = cli.ROOT / "probes/w5_generation/references.json"
+ fixture_path.write_text(json.dumps(reference_suite.fixture))
+ atomic = cli.atomic_text
+ snapshots = []
+
+ def write(path, text):
+ value = json.loads(text)
+ snapshots.append(value)
+ if value["cases"]:
+ raise PermissionError(errno.EACCES, "reference-report denied")
+ atomic(path, text)
+
+ monkeypatch.setattr(cli, "atomic_text", write)
+ assert cli.main(cli_helpers.argv(
+ tmp_path, "--reference-suite", "--verify-onnx", "model.onnx",
+ )) == 1
+ output = capsys.readouterr()
+ assert "Frozen reference mismatch: france" in output.err
+ assert "reference-report denied" in output.err
+ assert snapshots[-1]["error"] == "ValueError: Frozen reference mismatch: france"
+ assert snapshots[-1]["full_model_gate"] == "FAIL"
+ assert snapshots[-1]["numeric_steps"][0]["passed"]
+ assert not snapshots[-1]["cases"][0]["reference_passed"]
+
+
+def test_serialization_failure_returns_failure_without_false_saved_message(
+ cli, runtime, cli_helpers, tmp_path, capsys,
+):
+ runtime.exe.evidence["invalid_measurement"] = float("nan")
+ assert cli.main(cli_helpers.argv(tmp_path, "--build-only")) == 1
+ output = capsys.readouterr()
+ assert "Report save failed: ValueError:" in output.err
+ assert "BUILD_ONLY:" not in output.out
+ assert not (tmp_path / "run/report.json").exists()
+
+
+def test_next_invocation_can_save_after_failed_final_report(
+ cli, runtime, cli_helpers, tmp_path, monkeypatch,
+):
+ atomic = cli.atomic_text
+
+ def deny(path, text):
+ raise PermissionError(errno.EACCES, "first output unavailable")
+
+ monkeypatch.setattr(cli, "atomic_text", deny)
+ assert cli.main(cli_helpers.argv(tmp_path, "--build-only")) == 1
+ monkeypatch.setattr(cli, "atomic_text", atomic)
+ next_root = tmp_path / "next"
+ assert cli.main(cli_helpers.argv(next_root, "--prompt", "text", "--max-new-tokens", "1")) == 0
+ saved = cli_helpers.report(next_root)
+ assert saved["status"] == "PASS" and saved["passed"]
+ assert saved["qemu_forwards"] == 1 and "error" not in saved
+ assert saved["harts"] == 1 and saved["qemu_workspace_layout"] == "none"
+
+
+@pytest.mark.parametrize("code", [130, 143], ids=["sigint", "sigterm"])
+@pytest.mark.parametrize("suite", [False, True], ids=["prompt", "reference-suite"])
+def test_runner_signal_exit_is_saved_as_cancellation(
+ cli, runtime, reference_suite, cli_helpers, tmp_path, monkeypatch, capsys, code, suite,
+):
+ runtime.failure = SystemExit(code)
+ collected = []
+ previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)}
+ monkeypatch.setattr(cli.gc, "collect", lambda: collected.append(True))
+ options = (["--reference-suite", "--verify-onnx", "model.onnx"] if suite else
+ ["--prompt", "text", "--max-new-tokens", "1"])
+ assert cli.main(cli_helpers.argv(tmp_path, *options)) == code
+ saved = cli_helpers.report(tmp_path)
+ assert saved["status"] == "CANCELLED" and not saved["passed"]
+ assert saved["error"] == f"SystemExit: {code}"
+ assert saved["full_model_gate"] == ("CANCELLED" if suite else "NOT_RUN")
+ assert saved["forward_attempts"] == 1 and saved["qemu_forwards"] == 0
+ assert f"SystemExit: {code}" in capsys.readouterr().err
+ assert collected == [True]
+ assert all(signal.getsignal(sig) is handler for sig, handler in previous.items())
+
+
+@pytest.mark.parametrize("code", [0, 2, "143"], ids=["success", "other-error", "message"])
+@pytest.mark.parametrize("save_fails", [False, True], ids=["saved", "save-fails"])
+def test_non_signal_system_exit_propagates_after_cleanup(
+ cli, runtime, cli_helpers, tmp_path, monkeypatch, capsys, code, save_fails,
+):
+ failure = SystemExit(code)
+ runtime.failure = failure
+ collected = []
+ previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)}
+ monkeypatch.setattr(cli.gc, "collect", lambda: collected.append(True))
+ if save_fails:
+ def deny(path, text):
+ raise OSError(errno.ENOSPC, "unrelated exit report disk full")
+ monkeypatch.setattr(cli, "atomic_text", deny)
+ with pytest.raises(SystemExit) as caught:
+ cli.main(cli_helpers.argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1"))
+ assert caught.value is failure and caught.value.code == code
+ output = capsys.readouterr()
+ assert "CANCELLED" not in output.out
+ if save_fails:
+ assert "Report save failed:" in output.err
+ assert "unrelated exit report disk full" in output.err
+ else:
+ assert cli_helpers.report(tmp_path)["status"] == "FAIL"
+ assert collected == [True]
+ assert all(signal.getsignal(sig) is handler for sig, handler in previous.items())
diff --git a/tests/test_ai_code_review.py b/tests/test_ai_code_review.py
index 0a62e617..07b15e7c 100644
--- a/tests/test_ai_code_review.py
+++ b/tests/test_ai_code_review.py
@@ -194,11 +194,11 @@ def interrupted(*_):
def test_missing_job_and_pending_record_fail_accounting():
- p = plan(11)
- batch = reviewed_batch(names=tuple(f"{i}.py" for i in range(9)))
- batch["files"].append({"filename": "9.py", "status": "pending"})
+ p = plan(6)
+ batch = reviewed_batch(names=tuple(f"{i}.py" for i in range(4)))
+ batch["files"].append({"filename": "4.py", "status": "pending"})
summary = review.summarize(p, [batch])
- assert summary["counts"] == {"reviewed": 9, "failed": 2, "unreviewable": 0}
+ assert summary["counts"] == {"reviewed": 4, "failed": 2, "unreviewable": 0}
assert not summary["complete"]
@@ -265,9 +265,10 @@ def test_failed_fetch_still_writes_plan_and_matrix(tmp_path, monkeypatch):
assert json.loads(output.read_text().split("=", 1)[1]) == {"include": [{"batch": 0}]}
def test_large_pr_reports_matrix_limit_without_generating_invalid_matrix():
- p = plan(2561)
+ assert plan(1280)["batches"] == 256
+ p = plan(1281)
assert p["errors"] and p["batches"] == 1
- assert "256-job" in p["errors"][0]
+ assert "256-job" in p["errors"][0] and "5 files per batch" in p["errors"][0]
@pytest.mark.parametrize("invalid", [[], None, {"head": "abc", "batch": 0, "files": [None]},
@@ -312,6 +313,119 @@ def request(*_):
assert not summary["complete"]
+def test_longer_request_window_uses_smaller_batches_with_job_margin(monkeypatch):
+ calls = []
+ class Response:
+ status = 200
+ def __enter__(self): return self
+ def __exit__(self, *_): return None
+ def read(self, _):
+ return b'{"choices":[{"finish_reason":"stop","message":{"content":"No issue."}}]}'
+ def urlopen(request, *, timeout):
+ calls.append(timeout)
+ return Response()
+ monkeypatch.setattr(review.urllib.request, "urlopen", urlopen)
+ assert review.request_review({}, "test-key") == ("No issue.", 200, "")
+ assert calls == [180]
+ p = plan(47)
+ assert p["batch_size"] == 5 and p["batches"] == 10
+ # Nominal socket timeout windows plus maximal backoff leave setup/upload margin.
+ nominal_seconds = p["batch_size"] * (2 * calls[0] + 30) + p["batch_size"] - 1
+ job_seconds = int(workflow()["jobs"]["review"]["timeout-minutes"]) * 60
+ assert nominal_seconds <= job_seconds - 300
+
+
+@pytest.mark.parametrize("first", [(429, "3"), (503, ""), ConnectionResetError("private API key/body")])
+def test_each_request_records_sanitized_duration_and_outcome(first):
+ ticks = iter((0.0, 10.0, 12.25, 30.0, 33.5))
+ calls = []
+ def request(*_):
+ calls.append(1)
+ if len(calls) == 1:
+ if isinstance(first, Exception):
+ raise first
+ return "private API body", *first
+ return ok()
+ record = review.review_file(file(), "private prompt", "private-key", request=request,
+ sleep=lambda _: None, clock=ticks.__next__)
+ outcome = type(first).__name__ if isinstance(first, Exception) else f"HTTP {first[0]}"
+ assert record["request_attempts"] == [
+ {"attempt": 1, "outcome": outcome, "elapsed_seconds": 2.25},
+ {"attempt": 2, "outcome": "HTTP 200", "elapsed_seconds": 3.5},
+ ]
+ assert record["status"] == "reviewed" and "reason" not in record
+ assert record["elapsed"] == 33.5
+ assert "private" not in json.dumps(record)
+
+
+@pytest.mark.parametrize("wrapped", [False, True])
+def test_terminal_timeout_records_one_sanitized_attempt_without_retry(wrapped):
+ ticks = iter((0.0, 0.0, 180.0))
+ def request(*_):
+ error = TimeoutError("private API key/body")
+ raise urllib.error.URLError(error) if wrapped else error
+ record = review.review_file(file(), "prompt", "test-key", request=request,
+ sleep=lambda _: pytest.fail("timeout must not retry"),
+ clock=ticks.__next__)
+ assert record["status"] == "failed" and record["attempts"] == 1
+ assert record["request_attempts"] == [
+ {"attempt": 1, "outcome": "URLError" if wrapped else "TimeoutError", "elapsed_seconds": 180.0},
+ ]
+ assert record["elapsed"] == 180.0
+ assert "private" not in json.dumps(record)
+
+
+def test_saved_ten_file_plan_keeps_run_and_cli_summary_boundaries(tmp_path):
+ old = plan(11)
+ old.update(batch_size=10, batches=2)
+ download = tmp_path / "download"
+ review.write_json(download / "plan.json", old)
+ batches = [review.run_batch(old, number, "prompt", download / f"batch-{number}.json", "test-key",
+ request=ok, sleep=lambda _: None) for number in range(2)]
+ assert [[r["filename"] for r in batch["files"]] for batch in batches] == [
+ [f"{i}.py" for i in range(10)], ["10.py"],
+ ]
+ assert review.summarize(old, batches)["complete"]
+ out = tmp_path / "summary"
+ assert review.main(["summarize", "--directory", str(download), "--output", str(out),
+ "--run-url", "https://example.test/run"]) == 0
+ result = json.loads((out / "summary.json").read_text())
+ assert result["complete"] and result["counts"] == {"reviewed": 11, "failed": 0, "unreviewable": 0}
+ assert review.main(["gate", "--summary", str(out / "summary.json")]) == 0
+
+
+@pytest.mark.parametrize("invalid", [0, -1, 6, 11, True, "10", None])
+def test_invalid_saved_batch_size_fails_closed(tmp_path, invalid):
+ saved = plan(1)
+ saved["batch_size"] = invalid
+ with pytest.raises(ValueError, match="batch size"):
+ review.run_batch(saved, 0, "prompt", tmp_path / "batch.json", "test-key", request=ok)
+ summary = review.summarize(saved, [reviewed_batch()])
+ assert not summary["complete"] and summary["counts"]["failed"] == 1
+ rebuilt = review.make_plan({**saved, "head_after": saved["head"]})
+ assert rebuilt["errors"] and not review.summarize(rebuilt, [reviewed_batch()])["complete"]
+
+
+@pytest.mark.parametrize("batch_size,timeout_seconds", [(5, 180), (10, 90)])
+def test_real_adapter_keeps_each_plan_timeout_budget(tmp_path, monkeypatch, batch_size, timeout_seconds):
+ calls = []
+ class Response:
+ status = 200
+ def __enter__(self): return self
+ def __exit__(self, *_): return None
+ def read(self, _):
+ return b'{"choices":[{"finish_reason":"stop","message":{"content":"No issue."}}]}'
+ def urlopen(request, *, timeout):
+ calls.append(timeout)
+ return Response()
+ monkeypatch.setattr(review.urllib.request, "urlopen", urlopen)
+ saved = plan(1)
+ saved["batch_size"] = batch_size
+ result = review.run_batch(saved, 0, "prompt", tmp_path / "batch.json", "test-key")
+ assert result["files"][0]["status"] == "reviewed"
+ assert calls == [timeout_seconds]
+
+
def write_review_inputs(tmp_path, p, prompt="prompt"):
plan_path = tmp_path / "plan.json"
prompt_path = tmp_path / "prompt.md"
@@ -357,7 +471,7 @@ def request(*_):
def test_elapsed_is_recorded_with_injected_clock():
result = review.review_file(file(), "prompt", "test-key", request=ok, sleep=lambda _: None,
- clock=iter([0.0, 12.5]).__next__)
+ clock=iter([0.0, 0.0, 12.5]).__next__)
assert result["status"] == "reviewed" and result["elapsed"] == 12.5
skipped = review.review_file(file(patch=None, additions=0, deletions=0), "prompt", "test-key",
request=ok, sleep=lambda _: None, clock=iter([5.0, 5.0]).__next__)
@@ -405,29 +519,34 @@ def failing(item, *_args, **_kwargs):
assert out.count("::warning title=AI review incomplete::") == 2
-def test_slow_successful_file_warns(tmp_path, monkeypatch, capsys):
+@pytest.mark.parametrize("batch_size,timeout_seconds", [(5, 180), (10, 90)])
+def test_slow_successful_file_warns_with_plan_budget(tmp_path, monkeypatch, capsys, batch_size, timeout_seconds):
monkeypatch.setattr(review, "review_file", lambda item, *_a, **_k: {
"filename": item["filename"], "status": "reviewed", "attempts": 1,
"review": "ok", "elapsed": 61.0})
- plan_path, prompt_path = write_review_inputs(tmp_path, plan(1))
+ saved = plan(1)
+ saved["batch_size"] = batch_size
+ plan_path, prompt_path = write_review_inputs(tmp_path, saved)
assert review.main(["review", "--plan", str(plan_path), "--batch", "0",
"--prompt", str(prompt_path), "--output", str(tmp_path / "batch.json")]) == 0
- assert "::warning title=AI review slow::" in capsys.readouterr().out
+ warning = capsys.readouterr().out
+ assert "::warning title=AI review slow::" in warning
+ assert f"{timeout_seconds}s budget" in warning
def test_summarize_tracks_batch_visibility_and_slowest():
- p = plan(11)
- batch = reviewed_batch(names=tuple(f"{i}.py" for i in range(10)))
+ p = plan(6)
+ batch = reviewed_batch(names=tuple(f"{i}.py" for i in range(5)))
for index, record in enumerate(batch["files"]):
record["elapsed"] = float(index)
summary = review.summarize(p, [batch])
assert summary["batches_planned"] == 2 and summary["batches_seen"] == 1
- assert summary["slowest"] == 9.0
+ assert summary["slowest"] == 4.0
def test_gate_warns_when_batch_artifacts_are_missing(tmp_path, capsys):
- p = plan(11)
- summary = review.summarize(p, [reviewed_batch(names=tuple(f"{i}.py" for i in range(10)))])
+ p = plan(6)
+ summary = review.summarize(p, [reviewed_batch(names=tuple(f"{i}.py" for i in range(5)))])
path = tmp_path / "summary.json"
review.write_json(path, summary)
assert review.main(["gate", "--summary", str(path)]) == 0
diff --git a/tests/test_ci_routing.py b/tests/test_ci_routing.py
index 835ca0b8..b3ce28db 100644
--- a/tests/test_ci_routing.py
+++ b/tests/test_ci_routing.py
@@ -327,3 +327,14 @@ def test_summary_cli_returns_success_only_for_complete_result():
result = subprocess.run([sys.executable, "-B", "-X", "utf8", str(ROOT / "scripts/summarize_ci.py")],
env=env, capture_output=True, text=True, encoding="utf-8", timeout=10)
assert result.returncode == 0 and "**PASS**" in result.stdout
+
+
+@pytest.mark.parametrize("path", [
+ "scripts/run_inference.py", "scripts/deploy_w5.sh", "scripts/check_w5_ci.py",
+ "deployment/w5/Dockerfile", "probes/w5_generation/references.json",
+ "tests/test_generation.py", "tests/test_conversation.py", "tests/test_qemu_generation.py",
+ "tests/test_w5_ci.py", "tests/test_riscv_mean_policy.py", "tests/test_riscv_nonlinear_policy.py",
+])
+def test_w5_paths_route_to_existing_runtime_and_tokenizer_gates(path):
+ config = scope.read_config(ROOT / "scripts/ci_scopes.json")
+ assert scope.select_scopes(config, [path]) == {"llm": True, "w4": True}
diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py
index 256a0097..cec20fe3 100644
--- a/tests/test_ci_workflows.py
+++ b/tests/test_ci_workflows.py
@@ -34,7 +34,7 @@ def test_reusable_jobs_share_the_callers_run_and_same_commit():
assert not ({'pull_request', 'push'} & set(workflow(filename)['on']))
assert jobs['llm']['needs'] == 'scope'
assert jobs['llm']['if'] == "github.event_name == 'pull_request' && needs.scope.outputs.llm == 'true'"
- assert workflow('llm-deploy.yml')['jobs']['full-qwen3-onnx']['if'] == "github.event_name != 'pull_request'"
+ assert workflow('llm-deploy.yml')['jobs']['full-qwen3-onnx']['if'] == "github.event_name != 'pull_request' && !inputs.run_w5_validation"
assert 'workflow_dispatch' in workflow('llm-deploy.yml')['on']
@@ -96,7 +96,7 @@ def test_the_w1_w2_job_writes_no_step_summary():
data = workflow('llm-deploy.yml')
writers = {name for name, job in data['jobs'].items()
for step in job.get('steps', []) if 'GITHUB_STEP_SUMMARY' in step.get('run', '')}
- assert writers == {'full-qwen3-onnx', 'w3-execution-summary', 'w4-execution-summary'}
+ assert writers == {'full-qwen3-onnx', 'w3-execution-summary', 'w4-execution-summary', 'w5-execution-summary'}
def test_linux_reproduction_check_runs_without_a_path_filter():
@@ -224,3 +224,33 @@ def test_delegated_rv64_junit_check_requires_one_real_success(tmp_path, outcome)
result = subprocess.run([sys.executable, '-c', checker], cwd=tmp_path,
capture_output=True, text=True)
assert (result.returncode == 0) is (outcome == 'success')
+
+
+@pytest.mark.parametrize("suffix", ["None", "Give brief answers."])
+@pytest.mark.parametrize("outcome", ["pass", "skipped", "failure", "error", "missing", "duplicate"])
+def test_w5_official_three_round_template_must_execute(tmp_path, suffix, outcome):
+ import subprocess
+ import sys
+ import xml.etree.ElementTree as ET
+
+ steps = workflow("llm-deploy.yml")["jobs"]["llm-deploy"]["steps"]
+ actual = next(step for step in steps if step.get("name") == "W5 official three-round conversation template")
+ assert "tests/test_conversation.py::test_official_three_round_rendering_matches_transformers" in actual["run"]
+ assert "if" not in actual and "continue-on-error" not in actual
+ checker = actual["run"].split("python3 - <<'PY'\n", 1)[1].rsplit("\nPY", 1)[0]
+ suite = ET.Element("testsuite")
+ for current in ("None", "Give brief answers."):
+ if current == suffix and outcome == "missing":
+ continue
+ name = f"test_official_three_round_rendering_matches_transformers[{current}]"
+ row = ET.SubElement(suite, "testcase", name=name)
+ if current == suffix:
+ if outcome in ("skipped", "failure", "error"):
+ ET.SubElement(row, outcome)
+ elif outcome == "duplicate":
+ ET.SubElement(suite, "testcase", name=name)
+ directory = tmp_path / "output/w5-multiturn"
+ directory.mkdir(parents=True)
+ ET.ElementTree(suite).write(directory / "tests.xml")
+ result = subprocess.run([sys.executable, "-c", checker], cwd=tmp_path, capture_output=True, text=True)
+ assert (result.returncode == 0) is (outcome == "pass"), result.stderr
diff --git a/tests/test_conversation.py b/tests/test_conversation.py
new file mode 100644
index 00000000..88e9f229
--- /dev/null
+++ b/tests/test_conversation.py
@@ -0,0 +1,302 @@
+"""Host text-history tests; optional official parity loads tokenizers only."""
+
+from dataclasses import FrozenInstanceError
+import json
+import os
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+pytest.importorskip("jinja2")
+from scratchv.runtime import conversation
+from scratchv.runtime.conversation import ChatMessage, Conversation
+from scratchv.runtime.generation import GenerationCancelledError, GenerationResult
+
+
+TEMPLATE = "{%- for m in messages %}{{ m.role }}:{{ m.content }}|{% endfor -%}{{ 'assistant:' if add_generation_prompt else '' }}{{ 'nonthinking:' if enable_thinking is false else 'thinking:' }}"
+
+
+class Tokenizer:
+ model_vocab_size = 258
+ pad_token_id = 256
+ generation_eos_token_ids = (257,)
+
+ def encode(self, text, *, add_special_tokens):
+ assert add_special_tokens is False
+ return list(text.encode("utf-8"))
+
+ def validate_token_ids(self, tokens):
+ if any(type(x) is not int or not 0 <= x < 258 for x in tokens):
+ raise ValueError("invalid token")
+ return list(tokens)
+
+ def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces):
+ assert skip_special_tokens is True
+ assert clean_up_tokenization_spaces is False
+ return bytes(x for x in tokens if x < 256).decode("utf-8")
+
+
+def logits(prepared, token=97):
+ value = np.zeros((1, 256, 258), dtype=np.float32)
+ value[0, prepared.valid_length - 1, token] = 1
+ return value
+
+
+def no_forward(_prepared):
+ raise AssertionError("backend must not run")
+
+
+def test_three_rounds_preserve_exact_order_and_nonthinking_prompt():
+ chat = Conversation(Tokenizer(), TEMPLATE, system_prompt="brief")
+ inputs = []
+ for index, question in enumerate(("first", "second", "third")):
+ expected = "system:brief|" + "".join(
+ "user:" + q + "|assistant:a|" for q in ("first", "second", "third")[:index]
+ ) + "user:" + question + "|assistant:nonthinking:"
+ assert chat.render_prompt(question) == expected
+
+ def forward(prepared):
+ inputs.append(bytes(prepared.input_ids[0, :prepared.valid_length].tolist()).decode("utf-8"))
+ return logits(prepared)
+
+ result = chat.turn(question, forward=forward, max_new_tokens=1)
+ assert result.generated_text == "a"
+ assert result.stop_reason == "max_new_tokens"
+ assert inputs[-1] == expected
+ assert chat.history == (
+ ChatMessage("system", "brief"), ChatMessage("user", "first"), ChatMessage("assistant", "a"),
+ ChatMessage("user", "second"), ChatMessage("assistant", "a"),
+ ChatMessage("user", "third"), ChatMessage("assistant", "a"),
+ )
+
+
+def test_history_snapshot_is_immutable_and_sessions_reset_independently():
+ first = Conversation(Tokenizer(), TEMPLATE, system_prompt="one")
+ second = Conversation(Tokenizer(), TEMPLATE)
+ before = first.history
+ first.turn("first", forward=logits, max_new_tokens=1)
+ assert before == (ChatMessage("system", "one"),)
+ assert second.history == ()
+ with pytest.raises(FrozenInstanceError):
+ first.history[0].content = "changed"
+ first.reset()
+ assert first.history == before
+ second.turn("second", forward=logits, max_new_tokens=1)
+ assert first.history == before
+ assert len(second.history) == 2
+
+
+@pytest.mark.parametrize("role", ["tool", "developer", "function", "", None, 1])
+def test_rejects_unsupported_roles(role):
+ with pytest.raises(ValueError, match="role"):
+ ChatMessage(role, "text")
+
+
+@pytest.mark.parametrize("content", [None, b"text", [], {"type": "image"}, 1, True])
+def test_rejects_nontext_content_before_backend(content):
+ chat = Conversation(Tokenizer(), TEMPLATE)
+ with pytest.raises(TypeError, match="content"):
+ chat.turn(content, forward=no_forward, max_new_tokens=1)
+ assert chat.history == ()
+
+
+def test_rejects_invalid_unicode():
+ with pytest.raises(UnicodeEncodeError):
+ ChatMessage("user", "\ud800")
+
+
+@pytest.mark.parametrize("template", [None, [], "", " "])
+def test_requires_explicit_nonempty_template(template):
+ with pytest.raises((TypeError, ValueError), match="chat_template"):
+ Conversation(Tokenizer(), template)
+
+
+def test_invalid_system_prompt():
+ with pytest.raises(TypeError, match="content"):
+ Conversation(Tokenizer(), TEMPLATE, system_prompt=[])
+
+
+@pytest.mark.parametrize("failure", [RuntimeError("backend failed"), KeyboardInterrupt(), SystemExit(2)])
+def test_error_after_partial_generation_rolls_back_and_preserves_original_exception(failure):
+ chat = Conversation(Tokenizer(), TEMPLATE)
+ chat.turn("committed", forward=logits, max_new_tokens=1)
+ before = chat.history
+ calls = 0
+
+ def forward(prepared):
+ nonlocal calls
+ calls += 1
+ if calls == 2:
+ raise failure
+ return logits(prepared)
+
+ with pytest.raises(type(failure)) as caught:
+ chat.turn("failed", forward=forward, max_new_tokens=3)
+ assert caught.value is failure
+ assert chat.history == before
+ chat.turn("retry", forward=logits, max_new_tokens=1)
+ assert len(chat.history) == len(before) + 2
+
+
+def test_cancel_during_forward_rolls_back():
+ chat = Conversation(Tokenizer(), TEMPLATE)
+ aborted = False
+
+ def forward(prepared):
+ nonlocal aborted
+ aborted = True
+ return logits(prepared)
+
+ with pytest.raises(GenerationCancelledError):
+ chat.turn("cancel", forward=forward, max_new_tokens=1, cancelled=lambda: aborted)
+ assert chat.history == ()
+
+
+def test_timeout_after_forward_rolls_back(monkeypatch):
+ from scratchv.runtime import generation
+ moments = iter((0.0, 0.0, 2.0))
+ monkeypatch.setattr(generation.time, "monotonic", lambda: next(moments))
+ chat = Conversation(Tokenizer(), TEMPLATE)
+ with pytest.raises(TimeoutError):
+ chat.turn("timeout", forward=logits, max_new_tokens=1, timeout_seconds=1)
+ assert chat.history == ()
+
+
+def test_nonfinite_output_rolls_back():
+ chat = Conversation(Tokenizer(), TEMPLATE)
+
+ def forward(prepared):
+ value = logits(prepared)
+ value[0, -1, 0] = np.nan
+ return value
+
+ with pytest.raises(ValueError):
+ chat.turn("bad", forward=forward, max_new_tokens=1)
+ assert chat.history == ()
+
+
+@pytest.mark.parametrize("budget", [0, False, -1])
+def test_zero_budget_no_fake_turn_and_bad_budget_no_mutation(budget):
+ chat = Conversation(Tokenizer(), TEMPLATE)
+ if type(budget) is int and budget == 0:
+ result = chat.turn("valid", forward=no_forward, max_new_tokens=budget)
+ assert result.generated_token_ids == ()
+ else:
+ with pytest.raises((TypeError, ValueError)):
+ chat.turn("valid", forward=no_forward, max_new_tokens=budget)
+ assert chat.history == ()
+
+
+def test_eos_only_reply_commits_legitimate_empty_assistant():
+ chat = Conversation(Tokenizer(), TEMPLATE)
+ result = chat.turn("hello", forward=lambda p: logits(p, 257), max_new_tokens=2)
+ assert result.stop_reason == "eos"
+ assert chat.history == (ChatMessage("user", "hello"), ChatMessage("assistant", ""))
+
+
+def test_capacity_checks_full_rendered_history_without_truncation():
+ chat = Conversation(Tokenizer(), TEMPLATE)
+ chat.turn("prior", forward=logits, max_new_tokens=1)
+ before = chat.history
+ with pytest.raises(ValueError):
+ chat.turn("x" * 256, forward=no_forward, max_new_tokens=1)
+ assert chat.history == before
+ assert "prior" in chat.render_prompt("next")
+
+
+def test_exact_capacity_returns_no_fake_turn_and_last_slot_commits():
+ chat = Conversation(Tokenizer(), "{{ messages[-1].content }}")
+ result = chat.turn("x" * 256, forward=no_forward, max_new_tokens=1)
+ assert result.stop_reason == "capacity"
+ assert chat.history == ()
+ result = chat.turn("x" * 255, forward=logits, max_new_tokens=2)
+ assert result.stop_reason == "capacity"
+ assert chat.history[-1] == ChatMessage("assistant", "a")
+
+
+@pytest.mark.parametrize("action", ["turn", "reset"])
+def test_reentrant_state_mutation_rejected(action):
+ chat = Conversation(Tokenizer(), TEMPLATE)
+
+ def forward(prepared):
+ if action == "turn":
+ chat.turn("nested", forward=no_forward, max_new_tokens=1)
+ else:
+ chat.reset()
+ return logits(prepared)
+
+ with pytest.raises(RuntimeError, match="active turn"):
+ chat.turn("outer", forward=forward, max_new_tokens=1)
+ assert chat.history == ()
+ chat.reset()
+
+
+@pytest.mark.parametrize("config", [{}, {"chat_template": []}, [], {"chat_template": ""}])
+def test_from_directory_rejects_missing_or_ambiguous_template(tmp_path, config):
+ (tmp_path / "tokenizer_config.json").write_text(json.dumps(config), encoding="utf-8")
+ with pytest.raises(ValueError):
+ Conversation.from_directory(tmp_path)
+
+
+def _require_official_tokenizer(root, *, required):
+ source = root / "tokenizer.json"
+ unavailable = not source.is_file()
+ if not unavailable:
+ with source.open("rb") as stream:
+ unavailable = stream.read(128).startswith(b"version https://git-lfs.github.com/spec/")
+ if unavailable:
+ message = "official local Qwen3 tokenizer assets are missing or still Git LFS pointers"
+ if required:
+ pytest.fail(message)
+ pytest.skip(message)
+
+
+@pytest.mark.parametrize("required", [False, True])
+@pytest.mark.parametrize("content", [None, b"version https://git-lfs.github.com/spec/v1\n"])
+def test_official_tokenizer_missing_assets_skip_only_when_optional(tmp_path, required, content):
+ if content is not None:
+ (tmp_path / "tokenizer.json").write_bytes(content)
+ expected = pytest.fail.Exception if required else pytest.skip.Exception
+ with pytest.raises(expected, match="missing or still Git LFS"):
+ _require_official_tokenizer(tmp_path, required=required)
+
+
+@pytest.mark.parametrize("content", [b'{"version": "1.0"}', b"corrupt json"])
+def test_official_tokenizer_nonpointer_contents_are_left_to_parser(tmp_path, content):
+ source = tmp_path / "tokenizer.json"
+ source.write_bytes(content)
+ _require_official_tokenizer(tmp_path, required=False)
+ assert source.read_bytes() == content
+
+
+@pytest.mark.parametrize("system_prompt", [None, "Give brief answers."])
+def test_official_three_round_rendering_matches_transformers(monkeypatch, system_prompt):
+ """Optional in generic tests; explicit assets and dedicated CI remain mandatory."""
+ pytest.importorskip("tokenizers")
+ transformers = pytest.importorskip("transformers")
+ root = Path(os.environ.get("SCRATCHV_QWEN3_TOKENIZER_DIR", Path(__file__).resolve().parents[1] / "assets/qwen3-0.6b/tokenizer"))
+ _require_official_tokenizer(root, required="SCRATCHV_QWEN3_TOKENIZER_DIR" in os.environ)
+ official = transformers.AutoTokenizer.from_pretrained(str(root), local_files_only=True)
+ chat = Conversation.from_directory(root, system_prompt=system_prompt)
+ replies = iter(("Paris", "Berlin", "Tokyo"))
+
+ def fake_generate(prompt, *, tokenizer, **kwargs):
+ reply = next(replies)
+ return GenerationResult(tuple(tokenizer.encode(prompt, add_special_tokens=False)),
+ tuple(tokenizer.encode(reply, add_special_tokens=False)), reply, "max_new_tokens")
+
+ monkeypatch.setattr(conversation, "generate", fake_generate)
+ for text in ("What is France's capital?", "And Germany's?", "What about Japan?"):
+ messages = [dict(role=m.role, content=m.content) for m in chat.history]
+ messages.append(dict(role="user", content=text))
+ expected = official.apply_chat_template(messages, tokenize=False,
+ add_generation_prompt=True, enable_thinking=False)
+ rendered = chat.render_prompt(text)
+ assert rendered == expected
+ assert rendered.endswith("<|im_start|>assistant\n\n\n\n\n")
+ assert chat.tokenizer.encode(rendered, add_special_tokens=False) == official.apply_chat_template(
+ messages, tokenize=True, add_generation_prompt=True, enable_thinking=False)
+ assert len(chat.tokenizer.encode(rendered, add_special_tokens=False)) < 256
+ chat.turn(text, forward=no_forward, max_new_tokens=1)
+ assert len(chat.history) == (7 if system_prompt is not None else 6)
diff --git a/tests/test_generation.py b/tests/test_generation.py
new file mode 100644
index 00000000..a55c6631
--- /dev/null
+++ b/tests/test_generation.py
@@ -0,0 +1,431 @@
+"""Host generation contract boundaries without a model, compiler or QEMU."""
+
+from dataclasses import FrozenInstanceError
+from numbers import Integral
+
+import numpy as np
+import pytest
+
+from scratchv.runtime import generation
+from scratchv.runtime.generation import GenerationCancelledError, generate
+from scratchv.runtime.llm_inputs import SEQ_LENGTH
+
+
+class Tokenizer:
+ """Small tokenizer with one undecodable model-vocabulary tail entry."""
+
+ model_vocab_size = 8
+ pad_token_id = 0
+ generation_eos_token_ids = (5, 6)
+
+ def __init__(self, tokens=(1, 2)):
+ self.tokens = list(tokens)
+ self.encode_calls = []
+ self.decode_calls = []
+
+ def encode(self, text, *, add_special_tokens):
+ self.encode_calls.append((text, add_special_tokens))
+ return self.tokens.copy()
+
+ def validate_token_ids(self, tokens):
+ checked = []
+ for token in tokens:
+ if isinstance(token, (bool, np.bool_)) or not isinstance(token, Integral):
+ raise TypeError("token ID must be an integer")
+ if not 0 <= token < 7:
+ raise ValueError("token ID is absent from tokenizer vocabulary")
+ checked.append(int(token))
+ return checked
+
+ def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces):
+ tokens = self.validate_token_ids(tokens)
+ self.decode_calls.append((tuple(tokens), skip_special_tokens,
+ clean_up_tokenization_spaces))
+ return " ".join(str(token) for token in tokens if token not in (0, 5, 6))
+
+
+def logits_for(prepared, token=3):
+ logits = np.zeros((1, SEQ_LENGTH, 8), dtype=np.float32)
+ # Padding rows deliberately prefer an undecodable model-vocabulary tail ID.
+ logits[0, :, 7] = 10
+ logits[0, prepared.valid_length - 1, :] = 0
+ logits[0, prepared.valid_length - 1, token] = 20
+ return logits
+
+
+def no_forward(_prepared):
+ raise AssertionError("forward must not be called")
+
+
+def test_raw_prompt_last_valid_row_and_owned_result():
+ tokenizer = Tokenizer()
+ inputs = []
+
+ def forward(prepared):
+ inputs.append(prepared)
+ return logits_for(prepared, 3 if len(inputs) == 1 else 4)
+
+ result = generate(" raw\nCafe\u0301 ", tokenizer=tokenizer, forward=forward,
+ max_new_tokens=2)
+ assert tokenizer.encode_calls == [(" raw\nCafe\u0301 ", False)]
+ assert result.prompt_token_ids == (1, 2)
+ assert result.generated_token_ids == (3, 4)
+ assert result.generated_text == "3 4"
+ assert result.stop_reason == "max_new_tokens"
+ assert tokenizer.decode_calls == [((3, 4), True, False)]
+ assert [item.valid_length for item in inputs] == [2, 3]
+ np.testing.assert_array_equal(inputs[1].input_ids[0, :3], [1, 2, 3])
+ assert inputs[1].attention_mask[0, 0, 2, 2] == 0
+ assert inputs[1].attention_mask[0, 0, 2, 3] < 0
+ tokenizer.tokens[:] = [4]
+ assert result.prompt_token_ids == (1, 2)
+ with pytest.raises((FrozenInstanceError, AttributeError)):
+ result.stop_reason = "eos"
+
+
+def test_zero_budget_has_no_forward_or_generated_tokens():
+ result = generate("prompt", tokenizer=Tokenizer(), forward=no_forward,
+ max_new_tokens=0)
+ assert result.generated_token_ids == ()
+ assert result.generated_text == ""
+ assert result.stop_reason == "max_new_tokens"
+
+
+@pytest.mark.parametrize("budget", [0, 1, 1000])
+def test_full_prompt_stops_at_capacity_without_forward(budget):
+ result = generate("prompt", tokenizer=Tokenizer([1] * SEQ_LENGTH),
+ forward=no_forward, max_new_tokens=budget)
+ assert result.prompt_token_ids == (1,) * SEQ_LENGTH
+ assert result.generated_token_ids == ()
+ assert result.stop_reason == "capacity"
+
+
+@pytest.mark.parametrize("tokens,exception", [
+ ([], ValueError), ([1] * 257, ValueError), ([-1], ValueError),
+ ([7], ValueError), ([True], TypeError), ([1.5], TypeError),
+])
+def test_invalid_prompt_rejected_even_with_zero_budget(tokens, exception):
+ with pytest.raises(exception):
+ generate("prompt", tokenizer=Tokenizer(tokens), forward=no_forward,
+ max_new_tokens=0)
+
+
+@pytest.mark.parametrize("prompt", [None, b"prompt", 123, ["prompt"]])
+def test_prompt_must_be_text(prompt):
+ with pytest.raises(TypeError):
+ generate(prompt, tokenizer=Tokenizer(), forward=no_forward,
+ max_new_tokens=0)
+
+
+@pytest.mark.parametrize("budget,exception", [
+ (-1, ValueError), (True, TypeError), (np.bool_(False), TypeError),
+ (1.5, TypeError), (None, TypeError),
+])
+def test_invalid_budget_rejected_before_forward(budget, exception):
+ with pytest.raises(exception):
+ generate("prompt", tokenizer=Tokenizer(), forward=no_forward,
+ max_new_tokens=budget)
+
+
+@pytest.mark.parametrize("eos", [5, 6])
+def test_each_generation_eos_stops_and_is_retained_in_ids(eos):
+ calls = []
+
+ def forward(prepared):
+ calls.append(prepared.valid_length)
+ return logits_for(prepared, eos)
+
+ result = generate("prompt", tokenizer=Tokenizer(), forward=forward,
+ max_new_tokens=5)
+ assert calls == [2]
+ assert result.generated_token_ids == (eos,)
+ assert result.generated_text == ""
+ assert result.stop_reason == "eos"
+
+
+def test_eos_in_prompt_does_not_stop_generation():
+ result = generate("prompt", tokenizer=Tokenizer([1, 5]), forward=logits_for,
+ max_new_tokens=1)
+ assert result.prompt_token_ids == (1, 5)
+ assert result.generated_token_ids == (3,)
+ assert result.stop_reason == "max_new_tokens"
+
+
+@pytest.mark.parametrize("token,budget,reason", [
+ (5, 1, "eos"), (5, 3, "eos"), (3, 1, "capacity"), (3, 3, "capacity"),
+])
+def test_last_slot_stop_priority_and_no_extra_forward(token, budget, reason):
+ calls = []
+
+ def forward(prepared):
+ calls.append(prepared.valid_length)
+ return logits_for(prepared, token)
+
+ result = generate("prompt", tokenizer=Tokenizer([1] * 255), forward=forward,
+ max_new_tokens=budget)
+ assert calls == [255]
+ assert result.generated_token_ids == (token,)
+ assert result.stop_reason == reason
+
+
+def test_greedy_tie_uses_lowest_id_without_sampling():
+ def forward(prepared):
+ logits = logits_for(prepared, 4)
+ logits[0, prepared.valid_length - 1, 3] = 20
+ return logits
+
+ result = generate("prompt", tokenizer=Tokenizer(), forward=forward,
+ max_new_tokens=1)
+ assert result.generated_token_ids == (3,)
+
+
+@pytest.mark.parametrize("kind,exception", [
+ ("shape", ValueError), ("dtype", TypeError),
+ ("valid_nan", ValueError), ("padding_inf", ValueError),
+])
+def test_malformed_or_nonfinite_output_is_rejected(kind, exception):
+ tokenizer = Tokenizer()
+
+ def forward(prepared):
+ logits = logits_for(prepared)
+ if kind == "shape":
+ return logits[:, :-1, :]
+ if kind == "dtype":
+ return logits.astype(np.float64)
+ if kind == "valid_nan":
+ logits[0, prepared.valid_length - 1, 3] = np.nan
+ else:
+ logits[0, -1, 0] = np.inf
+ return logits
+
+ with pytest.raises(exception):
+ generate("prompt", tokenizer=tokenizer, forward=forward, max_new_tokens=1)
+ assert tokenizer.decode_calls == []
+
+
+def test_undecodable_model_tail_argmax_is_error_not_filtered_or_resampled():
+ tokenizer = Tokenizer()
+ with pytest.raises(ValueError, match="absent"):
+ generate("prompt", tokenizer=tokenizer,
+ forward=lambda prepared: logits_for(prepared, 7), max_new_tokens=1)
+ assert tokenizer.decode_calls == []
+
+
+@pytest.mark.parametrize("error", [RuntimeError("backend failed"), TimeoutError("backend timeout")])
+def test_backend_exception_identity_is_preserved(error):
+ tokenizer = Tokenizer()
+
+ def forward(_prepared):
+ raise error
+
+ with pytest.raises(type(error)) as caught:
+ generate("prompt", tokenizer=tokenizer, forward=forward, max_new_tokens=2)
+ assert caught.value is error
+ assert tokenizer.decode_calls == []
+
+
+def test_cancellation_before_first_forward_and_after_forward():
+ with pytest.raises(GenerationCancelledError):
+ generate("prompt", tokenizer=Tokenizer(), forward=no_forward,
+ max_new_tokens=1, cancelled=lambda: True)
+
+ cancelled = False
+ tokenizer = Tokenizer()
+ calls = []
+
+ def forward(prepared):
+ nonlocal cancelled
+ calls.append(prepared.valid_length)
+ cancelled = True
+ return logits_for(prepared)
+
+ with pytest.raises(GenerationCancelledError):
+ generate("prompt", tokenizer=tokenizer, forward=forward,
+ max_new_tokens=2, cancelled=lambda: cancelled)
+ assert calls == [2]
+ assert tokenizer.decode_calls == []
+
+
+@pytest.mark.parametrize("tokens,budget", [((1, 2), 0), ((1,) * 256, 2)])
+def test_cancellation_applies_to_no_forward_completion(tokens, budget):
+ with pytest.raises(GenerationCancelledError):
+ generate("prompt", tokenizer=Tokenizer(tokens), forward=no_forward,
+ max_new_tokens=budget, cancelled=lambda: True)
+
+
+def test_cancellation_after_decode_does_not_return_success():
+ tokenizer = Tokenizer()
+ cancelled = False
+ original_decode = tokenizer.decode
+
+ def decode(*args, **kwargs):
+ nonlocal cancelled
+ text = original_decode(*args, **kwargs)
+ cancelled = True
+ return text
+
+ tokenizer.decode = decode
+ with pytest.raises(GenerationCancelledError):
+ generate("prompt", tokenizer=tokenizer, forward=logits_for,
+ max_new_tokens=1, cancelled=lambda: cancelled)
+
+
+@pytest.mark.parametrize("timeout,exception", [
+ (-1, ValueError), (float("nan"), ValueError), (float("inf"), ValueError),
+ (True, TypeError), ("1", TypeError),
+])
+def test_invalid_timeout_rejected_before_forward(timeout, exception):
+ with pytest.raises(exception):
+ generate("prompt", tokenizer=Tokenizer(), forward=no_forward,
+ max_new_tokens=0, timeout_seconds=timeout)
+
+
+@pytest.mark.parametrize("budget", [0, 1])
+def test_zero_timeout_fails_before_forward_even_at_zero_budget(monkeypatch, budget):
+ monkeypatch.setattr(generation.time, "monotonic", lambda: 10.0)
+ with pytest.raises(TimeoutError):
+ generate("prompt", tokenizer=Tokenizer(), forward=no_forward,
+ max_new_tokens=budget, timeout_seconds=0)
+
+
+def test_elapsed_forward_is_rejected_without_exposing_partial_success(monkeypatch):
+ now = [10.0]
+ monkeypatch.setattr(generation.time, "monotonic", lambda: now[0])
+ tokenizer = Tokenizer()
+
+ def forward(prepared):
+ now[0] = 12.0
+ return logits_for(prepared, 5)
+
+ with pytest.raises(TimeoutError):
+ generate("prompt", tokenizer=tokenizer, forward=forward,
+ max_new_tokens=2, timeout_seconds=1)
+ assert tokenizer.decode_calls == []
+
+
+def test_timeout_after_decode_does_not_return_success(monkeypatch):
+ now = [10.0]
+ monkeypatch.setattr(generation.time, "monotonic", lambda: now[0])
+ tokenizer = Tokenizer()
+ original_decode = tokenizer.decode
+
+ def decode(*args, **kwargs):
+ text = original_decode(*args, **kwargs)
+ now[0] = 11.0
+ return text
+
+ tokenizer.decode = decode
+ with pytest.raises(TimeoutError):
+ generate("prompt", tokenizer=tokenizer, forward=logits_for,
+ max_new_tokens=1, timeout_seconds=1)
+
+
+def test_workspace_reuse_and_backend_input_mutation_do_not_change_history():
+ tokenizer = Tokenizer()
+ buffer = np.empty((1, SEQ_LENGTH, 8), dtype=np.float32)
+ seen = []
+ inputs = []
+
+ def forward(prepared):
+ seen.append(tuple(prepared.input_ids[0, :prepared.valid_length]))
+ inputs.append(prepared)
+ buffer[:] = logits_for(prepared, 3 if len(seen) == 1 else 4)
+ prepared.input_ids[:] = 0
+ prepared.attention_mask[:] = 0
+ return buffer
+
+ result = generate("prompt", tokenizer=tokenizer, forward=forward,
+ max_new_tokens=2)
+ assert seen == [(1, 2), (1, 2, 3)]
+ assert inputs[0] is not inputs[1]
+ assert not np.shares_memory(inputs[0].input_ids, inputs[1].input_ids)
+ assert not np.shares_memory(inputs[0].attention_mask, inputs[1].attention_mask)
+ buffer[:] = np.nan
+ assert result.generated_token_ids == (3, 4)
+ assert result.prompt_token_ids == (1, 2)
+
+
+def test_independent_calls_do_not_share_context_after_success_or_error():
+ tokenizer = Tokenizer([1])
+ seen = []
+
+ def forward(prepared):
+ seen.append(tuple(prepared.input_ids[0, :prepared.valid_length]))
+ return logits_for(prepared)
+
+ first = generate("first", tokenizer=tokenizer, forward=forward, max_new_tokens=1)
+ tokenizer.tokens[:] = [2, 4]
+ with pytest.raises(RuntimeError, match="failed"):
+ generate("failed", tokenizer=tokenizer,
+ forward=lambda _prepared: (_ for _ in ()).throw(RuntimeError("failed")),
+ max_new_tokens=1)
+ second = generate("second", tokenizer=tokenizer, forward=forward, max_new_tokens=1)
+ assert seen == [(1,), (2, 4)]
+ assert first.prompt_token_ids == (1,)
+ assert second.prompt_token_ids == (2, 4)
+ assert first.generated_token_ids == second.generated_token_ids == (3,)
+
+
+def test_decode_error_propagates_without_conversion_to_stop_reason():
+ tokenizer = Tokenizer()
+ error = ValueError("decode failed")
+
+ def decode(*_args, **_kwargs):
+ raise error
+
+ tokenizer.decode = decode
+ with pytest.raises(ValueError) as caught:
+ generate("prompt", tokenizer=tokenizer, forward=logits_for, max_new_tokens=1)
+ assert caught.value is error
+
+
+@pytest.mark.parametrize("error", [
+ RuntimeError("second step failed"), TimeoutError("second step timeout"),
+ KeyboardInterrupt("interrupted second step"),
+])
+def test_second_step_error_keeps_identity_and_reusable_backend_has_no_history_leak(error):
+ tokenizer = Tokenizer()
+ seen = []
+
+ def forward(prepared):
+ seen.append(tuple(prepared.input_ids[0, :prepared.valid_length]))
+ if len(seen) == 2:
+ raise error
+ return logits_for(prepared, 3 if len(seen) == 1 else 4)
+
+ with pytest.raises(type(error)) as caught:
+ generate("first", tokenizer=tokenizer, forward=forward, max_new_tokens=3)
+ assert caught.value is error
+ assert seen == [(1, 2), (1, 2, 3)]
+ assert tokenizer.decode_calls == []
+
+ retry = generate("retry", tokenizer=tokenizer, forward=forward, max_new_tokens=1)
+ assert seen[-1] == (1, 2)
+ assert retry.generated_token_ids == (4,)
+ assert tokenizer.decode_calls == [((4,), True, False)]
+
+
+def test_second_step_cancellation_discards_partial_result_and_allows_backend_reuse():
+ tokenizer = Tokenizer()
+ cancelled = False
+ seen = []
+
+ def forward(prepared):
+ nonlocal cancelled
+ seen.append(tuple(prepared.input_ids[0, :prepared.valid_length]))
+ if len(seen) == 2:
+ cancelled = True
+ return logits_for(prepared, 3 if len(seen) == 1 else 4)
+
+ with pytest.raises(GenerationCancelledError):
+ generate("first", tokenizer=tokenizer, forward=forward,
+ max_new_tokens=3, cancelled=lambda: cancelled)
+ assert seen == [(1, 2), (1, 2, 3)]
+ assert tokenizer.decode_calls == []
+
+ cancelled = False
+ retry = generate("retry", tokenizer=tokenizer, forward=forward,
+ max_new_tokens=1, cancelled=lambda: cancelled)
+ assert seen[-1] == (1, 2)
+ assert retry.generated_token_ids == (4,)
+ assert tokenizer.decode_calls == [((4,), True, False)]
diff --git a/tests/test_qemu_generation.py b/tests/test_qemu_generation.py
new file mode 100644
index 00000000..c327145c
--- /dev/null
+++ b/tests/test_qemu_generation.py
@@ -0,0 +1,422 @@
+"""Generation ABI, live guest cancellation, and reusable 1/4-core small graphs."""
+import json
+import os
+from pathlib import Path
+import socket
+import threading
+import time
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+
+from scratchv.backend.tensor_c_codegen import TensorCCodegen, TensorSpec
+from scratchv.frontend.onnx_parser import ONNXParser
+from scratchv.ir.types import DataType
+from scratchv.runtime import qemu_generation as adapter
+from scratchv.runtime import riscv_external as runtime
+from scratchv.runtime.generation import GenerationCancelledError, generate
+from scratchv.runtime.llm_inputs import prepare_inputs
+from scratchv.runtime.qemu_generation import QemuForward
+from scratchv.runtime.riscv_tensor import discover_toolchain
+from scratchv.runtime.weight_bundle import write_weight_bundle
+
+
+class SmallTokenizer:
+ model_vocab_size = 8
+ pad_token_id = 0
+ generation_eos_token_ids = (6,)
+
+ def encode(self, text, *, add_special_tokens=False):
+ assert not add_special_tokens
+ return [int(part) for part in text.split()]
+
+ def validate_token_ids(self, tokens):
+ result = list(tokens)
+ if any(isinstance(x, (bool, np.bool_)) or not isinstance(x, (int, np.integer))
+ or not 0 <= x < 8 for x in result):
+ raise ValueError("invalid small token")
+ return result
+
+ def decode(self, tokens, *, skip_special_tokens=True, clean_up_tokenization_spaces=False):
+ return " ".join(str(x) for x in tokens if not skip_special_tokens or x not in (0, 6))
+
+
+def make_small_onnx(path):
+ """A real input-dependent 256-row graph, with a small vocabulary/weights."""
+ import onnx
+ from onnx import helper, numpy_helper, TensorProto
+ embedding = np.zeros((8, 128), np.float32)
+ embedding[np.arange(8), np.arange(8)] = 1
+ projection = np.zeros((128, 8), np.float32)
+ projection[np.arange(8), [1, 2, 3, 4, 6, 6, 6, 6]] = 8
+ graph = helper.make_graph([
+ helper.make_node("Gather", ["embedding", "input_ids"], ["hidden"], axis=0),
+ helper.make_node("MatMul", ["hidden", "projection"], ["logits"]),
+ ], "w5-small-generation", [
+ helper.make_tensor_value_info("input_ids", TensorProto.INT64, [1, 256]),
+ helper.make_tensor_value_info("attention_mask", TensorProto.FLOAT, [1, 1, 256, 256]),
+ ], [helper.make_tensor_value_info("logits", TensorProto.FLOAT, [1, 256, 8])], [
+ numpy_helper.from_array(embedding, "embedding"),
+ numpy_helper.from_array(projection, "projection"),
+ ])
+ model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)], ir_version=10)
+ onnx.checker.check_model(model)
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ onnx.save(model, path)
+ return path
+
+
+def abi_executable(**changes):
+ base = dict(inputs=(TensorSpec("input_ids", DataType.INT64, (1, 256)),
+ TensorSpec("attention_mask", DataType.FLOAT32, (1, 1, 256, 256))),
+ output=TensorSpec("logits", DataType.FLOAT32, (1, 256, 8)),
+ evidence={"qemu_workspace_layout": "none"}, harts=1)
+ base.update(changes)
+ return SimpleNamespace(**base)
+
+
+@pytest.mark.parametrize("changes,error", [
+ ({"evidence": {"qemu_workspace_layout": "two-buffer-pad-4k"}}, "layout=none"),
+ ({"inputs": ()}, "exactly"),
+ ({"inputs": (TensorSpec("input_ids", DataType.INT32, (1,256)),
+ TensorSpec("attention_mask", DataType.FLOAT32, (1,1,256,256)))}, "shape/dtype"),
+ ({"output": TensorSpec("logits", DataType.FLOAT32, (1,8))}, "full FP32"),
+ ({"output": TensorSpec("logits", DataType.INT64, (1,256,8))}, "full FP32"),
+])
+def test_rejects_incompatible_abi_before_creating_directory(tmp_path, changes, error):
+ with pytest.raises(ValueError, match=error):
+ QemuForward(abi_executable(**changes), tmp_path / "runs")
+ assert not (tmp_path / "runs").exists()
+
+
+@pytest.mark.parametrize("kwargs,exception", [
+ ({"timeout": 0}, ValueError), ({"timeout": float("nan")}, ValueError),
+ ({"timeout": True}, TypeError), ({"deadline": float("inf")}, ValueError),
+ ({"cancelled": 1}, TypeError),
+])
+def test_invalid_options(tmp_path, kwargs, exception):
+ with pytest.raises(exception):
+ QemuForward(abi_executable(), tmp_path / "runs", **kwargs)
+
+
+def test_serial_adapter_reuses_executable_and_keeps_full_output_contract(tmp_path, monkeypatch):
+ exe = abi_executable()
+ backend = QemuForward(exe, tmp_path / "runs")
+ calls = []
+ def execute(executable, inputs, run_dir, **kwargs):
+ assert executable is exe
+ assert not backend._lock.acquire(blocking=False)
+ calls.append((inputs["input_ids"].copy(), run_dir, kwargs))
+ result = np.zeros((1, 256, 8), np.float32)
+ if len(calls) == 2:
+ result = result[:, :1, :]
+ return result, {"passed": True}
+ monkeypatch.setattr(adapter, "run_external", execute)
+ prepared = prepare_inputs([1], pad_token_id=0, vocab_size=8)
+ backend(prepared)
+ with pytest.raises(ValueError, match="fixed ABI"):
+ backend(prepared)
+ backend(prepared)
+ assert [x[1].name for x in calls] == ["step-0000", "step-0001", "step-0002"]
+ assert [r["passed"] for r in backend.runs] == [True, False, True]
+ assert all(x[2]["cancelled"] is None for x in calls)
+ with pytest.raises(FileExistsError):
+ QemuForward(exe, tmp_path / "runs")
+
+
+def test_adapter_nonoverlap_cancel_deadline_and_original_exception(tmp_path, monkeypatch):
+ backend = QemuForward(abi_executable(), tmp_path / "runs")
+ prepared = prepare_inputs([1], pad_token_id=0, vocab_size=8)
+ backend._lock.acquire()
+ try:
+ with pytest.raises(RuntimeError, match="overlapping"):
+ backend(prepared)
+ finally:
+ backend._lock.release()
+ backend.cancelled = lambda: True
+ with pytest.raises(GenerationCancelledError):
+ backend(prepared)
+ backend.cancelled = None
+ backend.deadline = time.monotonic() - 1
+ with pytest.raises(TimeoutError, match="generation"):
+ backend(prepared)
+ assert backend.runs == []
+ backend.deadline = None
+ error = RuntimeError("backend error")
+ def fail(*args, **kwargs):
+ raise error
+ monkeypatch.setattr(adapter, "run_external", fail)
+ with pytest.raises(RuntimeError) as actual:
+ backend(prepared)
+ assert actual.value is error
+ assert backend.runs[0]["status"] == "FAIL"
+
+
+@pytest.mark.parametrize("phase", ["greeting", "partial", "reply"])
+def test_qmp_cancellation_interrupts_missing_or_partial_response(phase):
+ listener = socket.socket()
+ listener.bind(("127.0.0.1", 0))
+ listener.listen(1)
+ listener.settimeout(3)
+ ready, stop = threading.Event(), threading.Event()
+ errors = []
+ def serve():
+ try:
+ with listener.accept()[0] as connection:
+ connection.settimeout(3)
+ with connection.makefile("rwb", buffering=0) as stream:
+ if phase != "greeting":
+ stream.write(b'{"QMP":{}}\n')
+ request = json.loads(stream.readline())
+ stream.write(json.dumps({"return":{}, "id":request["id"]}).encode()+b'\n')
+ request = json.loads(stream.readline())
+ assert request["execute"] == "stop"
+ if phase == "partial":
+ stream.write(b'{"return":')
+ ready.set()
+ assert stop.wait(3)
+ except BaseException as exc:
+ errors.append(exc)
+ worker = threading.Thread(target=serve)
+ worker.start()
+ client = None
+ began = time.monotonic()
+ try:
+ with pytest.raises(GenerationCancelledError):
+ client = runtime.QMP(listener.getsockname()[1], began + 5, cancelled=ready.is_set)
+ client.command("stop")
+ assert time.monotonic() - began < 1
+ finally:
+ if client is not None:
+ client.close()
+ stop.set()
+ worker.join(4)
+ listener.close()
+ assert not worker.is_alive()
+ assert not errors
+
+
+def _assert_reaped(report):
+ assert report["qemu_reaped"]
+ with pytest.raises(ProcessLookupError):
+ os.kill(report["qemu_pid"], 0)
+ with pytest.raises(ProcessLookupError):
+ os.killpg(report["qemu_pid"], 0)
+ assert not report.get("cleanup_errors")
+
+
+@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="requires explicit SCRATCHV_QEMU/CC")
+@pytest.mark.parametrize("harts", [1, 4])
+def test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel(tmp_path, monkeypatch, harts):
+ import onnxruntime as ort
+ path = make_small_onnx(tmp_path / "model.onnx")
+ parser = ONNXParser()
+ program = parser.parse(str(path))
+ model = TensorCCodegen(program, initializers=parser.initializers,
+ constant_storage="external").generate()
+ bundle = tmp_path / "weights"
+ write_weight_bundle(model.external_weights, model.external_initializers, bundle)
+ original_harness = runtime.guest_harness
+ def waiting_fixture(*args):
+ source = original_harness(*args)
+ # Token 7 deliberately parks this test guest, so timeout/cancel happen
+ # while a real QEMU process is alive instead of before it is launched.
+ return source.replace(" int status=", ' if (((const long *)inputs[0])[0] == 7) for (;;) __asm__ volatile("wfi");\n int status=')
+ monkeypatch.setattr(runtime, "guest_harness", waiting_fixture)
+ exe = runtime.build_external(model, bundle, tmp_path / "build", discover_toolchain(), harts=harts)
+ backend = QemuForward(exe, tmp_path / "runs", timeout=10)
+ options = ort.SessionOptions()
+ options.intra_op_num_threads = options.inter_op_num_threads = 1
+ session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"])
+ comparisons = []
+ def checked(prepared):
+ actual = backend(prepared)
+ expected = session.run(None, prepared.as_feed())[0]
+ np.testing.assert_array_equal(actual, expected)
+ comparisons.append({"valid_length": prepared.valid_length, "elements": actual.size,
+ "max_abs": 0.0})
+ return actual
+ first = generate("1", tokenizer=SmallTokenizer(), forward=checked, max_new_tokens=8)
+ assert first.generated_token_ids == (2, 3, 4, 6) and first.stop_reason == "eos"
+ second = generate("3", tokenizer=SmallTokenizer(), forward=checked, max_new_tokens=1)
+ assert second.generated_token_ids == (4,) and second.stop_reason == "max_new_tokens"
+ assert len(backend.runs) == 5
+ for report in backend.runs:
+ _assert_reaped(report)
+ assert report["harts"] == harts and report["qemu_workspace_layout"] == "none"
+ assert report["weight_transport"]["retained_bytes"] == 0
+ waiting = prepare_inputs([7], pad_token_id=0, vocab_size=8)
+ backend.timeout = 0.2
+ with pytest.raises(TimeoutError, match="forward exceeded"):
+ backend(waiting)
+ _assert_reaped(backend.runs[-1])
+ backend.timeout = 10
+ original_wait = runtime._UartCompletionWaiter.wait
+ started = []
+ cancel_at = [None]
+ def record_wait(self, process, *args, **kwargs):
+ assert process.poll() is None
+ started.append(process.pid)
+ cancel_at[0] = time.monotonic() + 0.05
+ return original_wait(self, process, *args, **kwargs)
+ monkeypatch.setattr(runtime._UartCompletionWaiter, "wait", record_wait)
+ backend.cancelled = lambda: cancel_at[0] is not None and time.monotonic() >= cancel_at[0]
+ before = time.monotonic()
+ with pytest.raises(GenerationCancelledError):
+ backend(waiting)
+ assert time.monotonic() - before < 2
+ _assert_reaped(backend.runs[-1])
+ assert started == [backend.runs[-1]["qemu_pid"]]
+ backend.cancelled = None
+ third = generate("4", tokenizer=SmallTokenizer(), forward=checked, max_new_tokens=2)
+ assert third.generated_token_ids == (6,) and third.stop_reason == "eos"
+ _assert_reaped(backend.runs[-1])
+ assert all(run["command"][run["command"].index("-kernel")+1] == str(exe.elf_path)
+ for run in backend.runs)
+ (tmp_path / "generation-check.json").write_text(json.dumps({
+ "harts": harts, "full_model": False, "qemu_workspace_layout": "none",
+ "first_tokens": list(first.generated_token_ids), "different_context": list(second.generated_token_ids),
+ "after_abort_tokens": list(third.generated_token_ids), "numeric_comparisons": comparisons,
+ "one_elf_reused": True, "one_bundle_reused": True, "runs": backend.runs,
+ }, indent=2)+"\n")
+
+
+def test_cancellable_qmp_preserves_fragmented_events_and_matching_id():
+ listener = socket.socket()
+ listener.bind(("127.0.0.1", 0))
+ listener.listen(1)
+ listener.settimeout(3)
+ errors = []
+ def serve():
+ try:
+ with listener.accept()[0] as connection:
+ connection.settimeout(3)
+ with connection.makefile("rwb", buffering=0) as stream:
+ stream.write(b'{"QMP":{}}\n')
+ capability = json.loads(stream.readline())
+ stream.write(json.dumps({"return":{}, "id":capability["id"]}).encode()+b'\n')
+ request = json.loads(stream.readline())
+ payload = (json.dumps({"event":"STOP"})+'\n'+
+ json.dumps({"return":{}, "id":request["id"]-1})+'\n'+
+ json.dumps({"return":{"status":"paused"}, "id":request["id"]})+'\n').encode()
+ stream.write(payload[:7])
+ time.sleep(0.01)
+ stream.write(payload[7:])
+ except BaseException as exc:
+ errors.append(exc)
+ worker = threading.Thread(target=serve)
+ worker.start()
+ client = None
+ try:
+ client = runtime.QMP(listener.getsockname()[1], time.monotonic()+3, cancelled=lambda: False)
+ assert client.command("stop") == {"status":"paused"}
+ finally:
+ if client is not None:
+ client.close()
+ worker.join(4)
+ listener.close()
+ assert not worker.is_alive()
+ assert not errors
+
+
+@pytest.mark.parametrize("contents,error", [
+ ("null", TimeoutError("original timeout")),
+ ("[1]", GenerationCancelledError("original cancellation")),
+ ('"not an object"', KeyboardInterrupt("original interruption")),
+ ("{", RuntimeError("original backend error")),
+ ("[" * 2000 + "0" + "]" * 2000, TimeoutError("timeout with deeply nested report")),
+ ('{"elapsed_s": NaN}', TimeoutError("timeout with nonfinite report")),
+ ('{"samples": [Infinity, -Infinity]}', GenerationCancelledError("cancel with nonfinite report")),
+ (None, TimeoutError("original timeout with unreadable report")),
+])
+def test_report_read_failure_preserves_backend_error_and_adapter_reuse(
+ tmp_path, monkeypatch, contents, error,
+):
+ backend = QemuForward(abi_executable(), tmp_path / "runs")
+ prepared = prepare_inputs([1], pad_token_id=0, vocab_size=8)
+ calls = []
+ original_read = Path.read_text
+
+ def read(path, *args, **kwargs):
+ if contents is None and path.name == "run.json":
+ raise PermissionError("diagnostic report unreadable")
+ return original_read(path, *args, **kwargs)
+
+ def execute(executable, inputs, run_dir, **kwargs):
+ calls.append((run_dir.name, inputs["input_ids"].copy()))
+ if len(calls) == 1:
+ run_dir.mkdir()
+ (run_dir / "run.json").write_text("{}" if contents is None else contents)
+ raise error
+ result = np.zeros((1, 256, 8), dtype=np.float32)
+ return result, {"passed": True}
+
+ monkeypatch.setattr(Path, "read_text", read)
+ monkeypatch.setattr(adapter, "run_external", execute)
+ with pytest.raises(type(error)) as caught:
+ backend(prepared)
+ assert caught.value is error
+ assert len(backend.runs) == 1
+ assert backend.runs[0]["passed"] is False
+ assert backend.runs[0]["error"] == f"{type(error).__name__}: {error}"
+ assert backend.runs[0]["report_read_error"]
+ json.dumps(backend.runs, allow_nan=False) # The CLI's finally report must still serialize.
+ np.testing.assert_array_equal(backend(prepared), np.zeros((1, 256, 8), np.float32))
+ assert [run["passed"] for run in backend.runs] == [False, True]
+ assert [call[0] for call in calls] == ["step-0000", "step-0001"]
+ np.testing.assert_array_equal(calls[0][1], calls[1][1])
+
+
+@pytest.mark.parametrize("error", [
+ GenerationCancelledError("cancelled during second token"),
+ TimeoutError("timeout during second token"),
+ RuntimeError("backend failed during second token"),
+])
+def test_conversation_and_reused_backend_rollback_together_after_partial_turn(
+ tmp_path, monkeypatch, error,
+):
+ pytest.importorskip("jinja2")
+ from scratchv.runtime.conversation import ChatMessage, Conversation
+
+ backend = QemuForward(abi_executable(), tmp_path / "runs")
+ chat = Conversation(SmallTokenizer(),
+ "{% for m in messages %}{{ m.content }} {% endfor %}")
+ buffer = np.empty((1, 256, 8), dtype=np.float32)
+ seen = []
+
+ def execute(executable, inputs, run_dir, **kwargs):
+ run_dir.mkdir()
+ # The test's nonzero prefix denotes valid token IDs; padding is zero.
+ prefix = tuple(int(x) for x in inputs["input_ids"][0] if x != 0)
+ seen.append(prefix)
+ if len(seen) == 3:
+ (run_dir / "run.json").write_text("null")
+ raise error
+ buffer.fill(0)
+ token = {1: 2, 2: 4, 4: 6}[len(seen)]
+ buffer[0, len(prefix) - 1, token] = 10
+ return buffer, {"passed": True}
+
+ monkeypatch.setattr(adapter, "run_external", execute)
+ first = chat.turn("1", forward=backend, max_new_tokens=1)
+ committed = chat.history
+ assert committed == (ChatMessage("user", "1"), ChatMessage("assistant", "2"))
+ with pytest.raises(type(error)) as caught:
+ chat.turn("3", forward=backend, max_new_tokens=3)
+ assert caught.value is error
+ assert chat.history == committed
+ assert seen == [(1,), (1, 2, 3), (1, 2, 3, 4)]
+
+ retry = chat.turn("5", forward=backend, max_new_tokens=2)
+ assert seen[-1] == (1, 2, 5) # No failed user 3 or partial assistant 4.
+ assert retry.generated_token_ids == (6,) and retry.stop_reason == "eos"
+ assert chat.history == committed + (ChatMessage("user", "5"), ChatMessage("assistant", ""))
+ assert [run["passed"] for run in backend.runs] == [True, True, False, True]
+ assert [Path(run["run_dir"]).name for run in backend.runs] == [
+ "step-0000", "step-0001", "step-0002", "step-0003",
+ ]
+ buffer.fill(np.nan)
+ assert first.generated_token_ids == (2,) and first.generated_text == "2"
+ assert retry.generated_token_ids == (6,) and retry.generated_text == ""
diff --git a/tests/test_riscv_external_contract.py b/tests/test_riscv_external_contract.py
index 81312b9e..74244d4f 100644
--- a/tests/test_riscv_external_contract.py
+++ b/tests/test_riscv_external_contract.py
@@ -578,3 +578,127 @@ def inaccessible_report(path, *args, **kwargs):
# Transport observation/reporting is attempted even after monitor failure.
assert report["weight_transport"]["retained_observation"] == "complete"
assert report["weight_transport"]["retained_bytes"] == 0
+
+
+@pytest.mark.parametrize("policy", ["sequential", "two-four-lane-last-axis"])
+def test_external_build_records_explicit_mean_policy_without_compiler(tmp_path, monkeypatch, policy):
+ import subprocess
+ builder = IRBuilder()
+ x = Value("x", shape=(2, 4))
+ builder.new_function("main", [x]); builder.new_block("entry"); builder.ret(builder.neg(x))
+ artifact = TensorCCodegen(builder.program, constant_storage="external", mean_policy=policy).generate()
+ bundle = tmp_path / "weights"
+ manifest = write_weight_bundle(artifact.external_weights, artifact.external_initializers, bundle)
+ _, input_bytes = runtime.input_layout(artifact.inputs)
+ layout = plan_guest_memory(workspace_bytes=artifact.workspace_bytes,
+ input_bytes=input_bytes, output_bytes=artifact.output.nbytes, weight_bytes=manifest["total_bytes"])
+
+ def compile_fixture(command, *, cwd, **kwargs):
+ _elf(Path(cwd) / "model.elf", layout)
+ return subprocess.CompletedProcess(command, 0, b"", b"")
+
+ monkeypatch.setattr(runtime, "_run_process", compile_fixture)
+ monkeypatch.setattr(runtime, "toolchain_versions", lambda *_: {})
+ executable = runtime.build_external(artifact, bundle, tmp_path / "build", RiscVToolchain(("mock-cc",), "mock-qemu"))
+ assert executable.evidence["mean_policy"] == policy
+ assert json.loads((tmp_path / "build/build.json").read_text())["mean_policy"] == policy
+ assert executable.harts == 1 and executable.evidence["qemu_workspace_layout"] == "none"
+
+
+@pytest.mark.parametrize("policy", [None, True, "unsupported"])
+def test_external_build_rejects_unknown_mean_policy_before_creating_artifacts(tmp_path, policy):
+ artifact = SimpleNamespace(constant_storage="external", mean_policy=policy)
+ with pytest.raises(ValueError, match="Mean policy"):
+ runtime.build_external(artifact, tmp_path / "weights", tmp_path / "build", None)
+ assert not (tmp_path / "build").exists()
+
+
+@pytest.mark.parametrize("policy", [None, "sequential", "two-four-lane-last-axis"])
+def test_external_run_reports_mean_policy_including_legacy_default(tmp_path, monkeypatch, policy):
+ import numpy as np
+ from dataclasses import replace
+ executable = _unlaunchable_executable(tmp_path)
+ if policy is not None:
+ executable = replace(executable, evidence={**executable.evidence, "mean_policy": policy})
+ run_dir = tmp_path / "run"
+ # A missing QEMU executable stops before any process/model execution, but
+ # still exercises the runner's real failure-report publication.
+ with pytest.raises(OSError):
+ runtime.run_external(executable, {"x": np.ones(1, np.float32)}, run_dir)
+ report = json.loads((run_dir / "run.json").read_text())
+ assert report["mean_policy"] == (policy or "sequential")
+ assert report["status"] == "FAIL" and report["guest_completed"] is False
+
+
+@pytest.mark.parametrize("policy", [None, True, "unsupported"])
+def test_external_run_rejects_invalid_mean_policy_before_launch(tmp_path, policy):
+ import numpy as np
+ from dataclasses import replace
+ executable = _unlaunchable_executable(tmp_path)
+ executable = replace(executable, evidence={**executable.evidence, "mean_policy": policy})
+ run_dir = tmp_path / "run"
+ with pytest.raises(ValueError, match="Mean policy"):
+ runtime.run_external(executable, {"x": np.ones(1, np.float32)}, run_dir)
+ assert not run_dir.exists()
+
+
+@pytest.mark.parametrize("policy", ["libm", "fp32-reference"])
+def test_external_build_records_explicit_nonlinear_policy_without_compiler(tmp_path, monkeypatch, policy):
+ import subprocess
+ builder = IRBuilder()
+ x = Value("x", shape=(2, 4))
+ builder.new_function("main", [x]); builder.new_block("entry"); builder.ret(builder.neg(x))
+ artifact = TensorCCodegen(builder.program, constant_storage="external", nonlinear_policy=policy).generate()
+ bundle = tmp_path / "weights"
+ manifest = write_weight_bundle(artifact.external_weights, artifact.external_initializers, bundle)
+ _, input_bytes = runtime.input_layout(artifact.inputs)
+ layout = plan_guest_memory(workspace_bytes=artifact.workspace_bytes,
+ input_bytes=input_bytes, output_bytes=artifact.output.nbytes, weight_bytes=manifest["total_bytes"])
+
+ def compile_fixture(command, *, cwd, **kwargs):
+ _elf(Path(cwd) / "model.elf", layout)
+ return subprocess.CompletedProcess(command, 0, b"", b"")
+
+ monkeypatch.setattr(runtime, "_run_process", compile_fixture)
+ monkeypatch.setattr(runtime, "toolchain_versions", lambda *_: {})
+ executable = runtime.build_external(artifact, bundle, tmp_path / "build", RiscVToolchain(("mock-cc",), "mock-qemu"))
+ assert executable.evidence["nonlinear_policy"] == policy
+ assert json.loads((tmp_path / "build/build.json").read_text())["nonlinear_policy"] == policy
+ assert executable.harts == 1 and executable.evidence["qemu_workspace_layout"] == "none"
+
+
+@pytest.mark.parametrize("policy", [None, True, "unsupported"])
+def test_external_build_rejects_unknown_nonlinear_policy_before_creating_artifacts(tmp_path, policy):
+ artifact = SimpleNamespace(constant_storage="external", nonlinear_policy=policy)
+ with pytest.raises(ValueError, match="nonlinear policy"):
+ runtime.build_external(artifact, tmp_path / "weights", tmp_path / "build", None)
+ assert not (tmp_path / "build").exists()
+
+
+@pytest.mark.parametrize("policy", [None, "libm", "fp32-reference"])
+def test_external_run_reports_nonlinear_policy_including_legacy_default(tmp_path, monkeypatch, policy):
+ import numpy as np
+ from dataclasses import replace
+ executable = _unlaunchable_executable(tmp_path)
+ if policy is not None:
+ executable = replace(executable, evidence={**executable.evidence, "nonlinear_policy": policy})
+ run_dir = tmp_path / "run"
+ # A missing QEMU executable stops before any process/model execution, but
+ # still exercises the runner's real failure-report publication.
+ with pytest.raises(OSError):
+ runtime.run_external(executable, {"x": np.ones(1, np.float32)}, run_dir)
+ report = json.loads((run_dir / "run.json").read_text())
+ assert report["nonlinear_policy"] == (policy or "libm")
+ assert report["status"] == "FAIL" and report["guest_completed"] is False
+
+
+@pytest.mark.parametrize("policy", [None, True, "unsupported"])
+def test_external_run_rejects_invalid_nonlinear_policy_before_launch(tmp_path, policy):
+ import numpy as np
+ from dataclasses import replace
+ executable = _unlaunchable_executable(tmp_path)
+ executable = replace(executable, evidence={**executable.evidence, "nonlinear_policy": policy})
+ run_dir = tmp_path / "run"
+ with pytest.raises(ValueError, match="nonlinear policy"):
+ runtime.run_external(executable, {"x": np.ones(1, np.float32)}, run_dir)
+ assert not run_dir.exists()
diff --git a/tests/test_riscv_external_harts.py b/tests/test_riscv_external_harts.py
index a8ea458d..59cbe58a 100644
--- a/tests/test_riscv_external_harts.py
+++ b/tests/test_riscv_external_harts.py
@@ -1,5 +1,5 @@
"""Real small-guest checks for external-runtime hart ownership and barriers."""
-from dataclasses import replace
+from dataclasses import FrozenInstanceError, asdict, fields, replace
import os
import shutil
import subprocess
@@ -10,8 +10,8 @@
from scratchv.backend.tensor_c_codegen import TensorCArtifact, TensorCCodegen, TensorSpec
from scratchv.ir.builder import IRBuilder
from scratchv.ir.types import DataType, Value
-from scratchv.runtime.riscv_external import build_external, run_external
-from scratchv.runtime.riscv_harts import resolve_harts
+from scratchv.runtime.riscv_external import ExternalExecutable, build_external, run_external
+from scratchv.runtime.riscv_harts import UNSET, read_cores, resolve_cores, resolve_harts, select_cores
from scratchv.runtime.riscv_tensor import RiscVToolchain, discover_toolchain
from scratchv.runtime.weight_bundle import MIB, plan_guest_memory, write_weight_bundle
@@ -367,3 +367,183 @@ def test_doorbells_before_wait_coalesce_and_spurious_wakes_do_not_repeat_tasks(t
np.testing.assert_array_equal(output[:65], expected)
assert report["passed"] and report["harts"] == harts
np.testing.assert_array_equal(output[73:73+harts], 0)
+
+
+@pytest.mark.parametrize("cores", [1, 2, 4])
+def test_cores_and_legacy_harts_select_the_same_layout(cores):
+ assert select_cores(cores) == select_cores(harts=cores) == select_cores(cores, harts=cores)
+ assert resolve_cores(cores) == resolve_harts(cores)
+ assert plan_guest_memory(cores=cores) == plan_guest_memory(harts=cores)
+ assert select_cores() == 1
+
+
+@pytest.mark.parametrize("bad", [None, True, False, 0, 3, 1.0, "2"])
+def test_invalid_cores_and_alias_conflicts_fail_before_build(tmp_path, bad):
+ for kwargs in ({"cores": bad}, {"cores": 4, "harts": bad}):
+ with pytest.raises(ValueError, match="cores"):
+ plan_guest_memory(**kwargs)
+ with pytest.raises(ValueError, match="cores"):
+ build_external(artifact(), tmp_path / "missing", tmp_path / "build",
+ RiscVToolchain(("unused",), "unused"), **kwargs)
+ assert not (tmp_path / "build").exists()
+ with pytest.raises(ValueError, match="disagree"):
+ select_cores(4, harts=1)
+ with pytest.raises(ValueError, match="disagree"):
+ select_cores("auto", harts=4, allow_auto=True)
+
+
+@pytest.mark.parametrize("record,expected", [
+ ({}, 1), ({"cores": 4}, 4), ({"harts": 4}, 4),
+ ({"cores": 2, "harts": 2}, 2),
+])
+def test_core_metadata_reads_both_generations(record, expected):
+ assert read_cores(record) == expected
+
+
+@pytest.mark.parametrize("record", [
+ {"cores": 4, "harts": 1}, {"cores": 1, "harts": True},
+ {"cores": False, "harts": 1}, {"cores": None},
+ {"cores": "4"}, {"harts": 1.0}, {"cores": "auto"},
+])
+def test_core_metadata_rejects_invalid_or_conflicting_aliases(record):
+ with pytest.raises(ValueError, match="cores"):
+ read_cores(record)
+
+
+def test_required_core_metadata_does_not_invent_a_single_core():
+ with pytest.raises(ValueError, match="Missing cores"):
+ read_cores({}, default=UNSET)
+
+
+def test_executable_constructor_and_dataclass_replace_accept_both_spellings(tmp_path):
+ positional = (tmp_path / "model.elf", tmp_path, {}, {}, (), None, (),
+ RiscVToolchain(("unused",), "unused"), {})
+ original = ExternalExecutable(*positional, cores=4)
+ assert original.cores == original.harts == 4
+ assert ExternalExecutable(*positional, harts=4) == original
+ assert ExternalExecutable(*positional, cores=4, harts=4) == original
+ assert ExternalExecutable(*positional).cores == 1
+ assert replace(original, evidence={"changed": True}).cores == 4
+ assert replace(original, cores=2).cores == replace(original, harts=2).cores == 2
+ with pytest.raises(ValueError, match="disagree"):
+ ExternalExecutable(*positional, cores=4, harts=1)
+ with pytest.raises(ValueError, match="disagree"):
+ replace(original, cores=2, harts=1)
+
+
+@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="explicit RV64 tools required")
+@pytest.mark.parametrize("cores", [1, 4])
+def test_new_core_api_runs_small_graph_and_reuses_legacy_build_evidence(tmp_path, cores):
+ model = artifact()
+ write_weight_bundle((), (), tmp_path / "weights")
+ executable = build_external(model, tmp_path / "weights", tmp_path / "build",
+ discover_toolchain(), cores=cores)
+ assert executable.cores == executable.harts == cores
+ assert executable.evidence["cores"] == executable.evidence["harts"] == cores
+ first, report = run_external(executable, {}, tmp_path / "current", timeout=15)
+ assert report["passed"] and report["cores"] == report["harts"] == cores
+ evidence = dict(executable.evidence)
+ del evidence["cores"]
+ legacy, old_report = run_external(replace(executable, evidence=evidence), {},
+ tmp_path / "legacy", timeout=15)
+ assert old_report["passed"] and old_report["cores"] == cores
+ np.testing.assert_array_equal(first, legacy)
+ expected = np.full(65, sum(range(1, 33)), dtype=np.int64)
+ expected[0] += 1000
+ expected[:3] += 2000
+ np.testing.assert_array_equal(first[:65], expected)
+ bad = replace(executable, evidence={**evidence, "cores": 2 if cores == 1 else 1})
+ with pytest.raises(ValueError, match="disagree"):
+ run_external(bad, {}, tmp_path / "conflicting", timeout=1)
+ assert not (tmp_path / "conflicting").exists()
+
+
+@pytest.mark.parametrize("definitions", [
+ [], ["-DSCRATCHV_CORES=4"], ["-DSCRATCHV_HARTS=4"],
+ ["-DSCRATCHV_CORES=4", "-DSCRATCHV_HARTS=4"],
+])
+def test_runtime_macro_aliases_support_existing_c_and_assembly(definitions):
+ from scratchv.runtime.riscv_harts import STARTUP_ASSEMBLY, WORKER_SOURCE
+ from scratchv.backend.tensor_c_codegen import _EXTERNAL_PRELUDE
+ compiler = shutil.which("cc")
+ if not compiler:
+ pytest.skip("host C preprocessor required")
+ expected = 4 if definitions else 1
+ for source, language in ((STARTUP_ASSEMBLY, "assembler-with-cpp"),
+ (WORKER_SOURCE, "c"), (_EXTERNAL_PRELUDE, "c")):
+ completed = subprocess.run([compiler, "-E", "-P", "-x", language, *definitions, "-"],
+ input=source + "\nSCRATCHV_CORES SCRATCHV_HARTS\n",
+ text=True, capture_output=True, check=True)
+ assert completed.stdout.rstrip().endswith(f"{expected} {expected}")
+
+
+def test_runtime_macro_alias_conflicts_are_rejected():
+ from scratchv.runtime.riscv_harts import STARTUP_ASSEMBLY, WORKER_SOURCE
+ from scratchv.backend.tensor_c_codegen import _EXTERNAL_PRELUDE
+ compiler = shutil.which("cc")
+ if not compiler:
+ pytest.skip("host C preprocessor required")
+ for source, language in ((STARTUP_ASSEMBLY, "assembler-with-cpp"),
+ (WORKER_SOURCE, "c"), (_EXTERNAL_PRELUDE, "c")):
+ completed = subprocess.run([compiler, "-E", "-P", "-x", language,
+ "-DSCRATCHV_CORES=4", "-DSCRATCHV_HARTS=1", "-"],
+ input=source, text=True, capture_output=True)
+ assert completed.returncode != 0 and "disagree" in completed.stderr
+
+
+@pytest.mark.parametrize("kwargs,expected", [
+ ({}, 1), ({"cores": 1}, 1), ({"cores": 2}, 2), ({"cores": 4}, 4),
+ ({"harts": 4}, 4), ({"cores": 4, "harts": 4}, 4),
+])
+def test_executable_asdict_preserves_public_core_and_legacy_fields(tmp_path, kwargs, expected):
+ original = ExternalExecutable(
+ tmp_path / "model.elf", tmp_path, {}, {}, (), None, (),
+ RiscVToolchain(("unused",), "unused"), {"labels": ["original"]}, **kwargs)
+ record = asdict(original)
+ assert record["cores"] == record["harts"] == expected
+ assert type(record["cores"]) is type(record["harts"]) is int
+ assert set(record) == {
+ "elf_path", "bundle_dir", "manifest", "layout", "inputs",
+ "output", "weights", "toolchain", "evidence", "cores", "harts",
+ }
+ record["evidence"]["labels"].append("copy")
+ assert original.evidence == {"labels": ["original"]}
+ shallow = {field.name: getattr(original, field.name) for field in fields(original)}
+ assert ExternalExecutable(**shallow) == original
+ shallow.pop("cores")
+ assert ExternalExecutable(**shallow) == original
+ with pytest.raises(FrozenInstanceError):
+ original.cores = 2
+ with pytest.raises(FrozenInstanceError):
+ original.harts = 2
+
+
+@pytest.mark.parametrize("initial", [1, 2, 4])
+@pytest.mark.parametrize("updated", [1, 2, 4])
+def test_executable_replace_keeps_serialized_core_aliases_equal(tmp_path, initial, updated):
+ original = ExternalExecutable(
+ tmp_path / "model.elf", tmp_path, {}, {}, (), None, (),
+ RiscVToolchain(("unused",), "unused"), {}, cores=initial)
+ for change in ({"cores": updated}, {"harts": updated},
+ {"cores": updated, "harts": updated},
+ {"cores": initial, "harts": updated},
+ {"cores": updated, "harts": initial}):
+ changed = replace(original, **change)
+ assert changed.cores == changed.harts == updated
+ copied = replace(changed, evidence={"modified": True})
+ record = asdict(copied)
+ assert record["cores"] == record["harts"] == updated
+ assert original.cores == original.harts == initial
+ others = [value for value in (1, 2, 4) if value != initial]
+ with pytest.raises(ValueError, match="disagree"):
+ replace(original, cores=others[0], harts=others[1])
+
+
+@pytest.mark.parametrize("field", ["cores", "harts"])
+@pytest.mark.parametrize("bad", [None, True, False, 1.0, "4", 0, 3, "auto"])
+def test_executable_replace_rejects_invalid_count_before_alias_sync(tmp_path, field, bad):
+ original = ExternalExecutable(
+ tmp_path / "model.elf", tmp_path, {}, {}, (), None, (),
+ RiscVToolchain(("unused",), "unused"), {}, cores=1)
+ with pytest.raises(ValueError, match="cores"):
+ replace(original, **{field: bad})
diff --git a/tests/test_riscv_mean_policy.py b/tests/test_riscv_mean_policy.py
new file mode 100644
index 00000000..5947e0ce
--- /dev/null
+++ b/tests/test_riscv_mean_policy.py
@@ -0,0 +1,72 @@
+"""Small real RV64 checks for the explicitly selected last-axis mean order."""
+import os
+import json
+
+import numpy as np
+import pytest
+
+from scratchv.backend.tensor_c_codegen import TensorCCodegen
+from scratchv.ir.builder import IRBuilder
+from scratchv.ir.types import Value
+from scratchv.runtime.riscv_external import build_external, run_external
+from scratchv.runtime.riscv_tensor import discover_toolchain
+from scratchv.runtime.weight_bundle import write_weight_bundle
+from scratchv.verification.fp32_eigen import mean_last
+
+
+@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="explicit RV64 tools required")
+@pytest.mark.parametrize("harts", [1, 4])
+@pytest.mark.parametrize("width", [13, 128, 1024])
+def test_explicit_mean_order_rv64_and_context_reuse(tmp_path, harts, width):
+ builder = IRBuilder()
+ x = Value("x", shape=(7, width))
+ builder.new_function("main", [x])
+ builder.new_block("entry")
+ builder.ret(builder.reduce_mean(x, axes=(-1,), keepdims=True))
+ artifact = TensorCCodegen(builder.program, constant_storage="external", kernel_calls=True,
+ mean_policy="two-four-lane-last-axis").generate()
+ write_weight_bundle(artifact.external_weights, artifact.external_initializers, tmp_path / "weights")
+ executable = build_external(artifact, tmp_path / "weights", tmp_path / "build",
+ discover_toolchain(), harts=harts)
+ assert executable.evidence["mean_policy"] == "two-four-lane-last-axis"
+ assert executable.evidence["qemu_workspace_layout"] == "none"
+ rng = np.random.default_rng(1200 + width)
+ for index in range(2):
+ data = (rng.normal(size=(7, width)) * (1 if index == 0 else 256)).astype(np.float32)
+ # Exact signs and cancellation exercise more than a positive-only mean.
+ data[0] = np.float32(-0.0)
+ data[1, ::2] *= np.float32(-1)
+ expected = mean_last(data)
+ before = data.copy()
+ actual, report = run_external(executable, {"x": data}, tmp_path / f"run{index}", timeout=30)
+ np.testing.assert_array_equal(actual.view(np.uint32), expected.view(np.uint32))
+ np.testing.assert_array_equal(data.view(np.uint32), before.view(np.uint32))
+ assert report["passed"] and report["qemu_reaped"] and report["harts"] == harts
+ assert report["mean_policy"] == "two-four-lane-last-axis"
+
+
+@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="explicit RV64 tools required")
+@pytest.mark.parametrize("harts", [1, 4])
+def test_explicit_mean_overflow_propagates_and_build_remains_reusable(tmp_path, harts):
+ builder = IRBuilder()
+ x = Value("x", shape=(2, 16))
+ builder.new_function("main", [x])
+ builder.new_block("entry")
+ builder.ret(builder.reduce_mean(x, axes=(-1,), keepdims=False))
+ artifact = TensorCCodegen(builder.program, constant_storage="external", kernel_calls=True,
+ mean_policy="two-four-lane-last-axis").generate()
+ write_weight_bundle(artifact.external_weights, artifact.external_initializers, tmp_path / "weights")
+ executable = build_external(artifact, tmp_path / "weights", tmp_path / "build",
+ discover_toolchain(), harts=harts)
+ with pytest.raises(RuntimeError, match="status 0x00000002"):
+ run_external(executable, {"x": np.full((2, 16), np.finfo(np.float32).max, np.float32)},
+ tmp_path / "overflow", timeout=30)
+ failure = json.loads((tmp_path / "overflow/run.json").read_text())
+ assert not failure["passed"] and failure["qemu_reaped"]
+ with pytest.raises(ProcessLookupError):
+ os.killpg(failure["qemu_pid"], 0)
+ # A valid new context uses the exact same executable after a guest error.
+ data = np.arange(32, dtype=np.float32).reshape(2, 16) / np.float32(8)
+ actual, report = run_external(executable, {"x": data}, tmp_path / "reused", timeout=30)
+ np.testing.assert_array_equal(actual.view(np.uint32), mean_last(data, False).view(np.uint32))
+ assert report["passed"] and report["qemu_reaped"]
diff --git a/tests/test_riscv_nonlinear_policy.py b/tests/test_riscv_nonlinear_policy.py
new file mode 100644
index 00000000..6e3ecf50
--- /dev/null
+++ b/tests/test_riscv_nonlinear_policy.py
@@ -0,0 +1,67 @@
+"""Real RV64 integration of the explicit portable nonlinear/Mean policies."""
+import json
+import os
+
+import numpy as np
+import pytest
+
+from scratchv.backend.tensor_c_codegen import TensorCCodegen
+from scratchv.ir.builder import IRBuilder
+from scratchv.ir.types import Value
+from scratchv.runtime.riscv_external import build_external, run_external
+from scratchv.runtime.riscv_tensor import discover_toolchain
+from scratchv.runtime.weight_bundle import write_weight_bundle
+from scratchv.verification.fp32_eigen import mean_last, trig
+from scratchv.verification.fp32_reference import sigmoid, softmax
+
+
+@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="explicit RV64 tools required")
+@pytest.mark.parametrize("harts", [1, 4])
+def test_reference_nonlinear_rv64_contexts_and_error_recovery(tmp_path, monkeypatch, harts):
+ # Select the saved arithmetic order, independent of the host's SIMD support.
+ monkeypatch.setenv("SCRATCHV_FP32_REFERENCE_CPU", "avx512")
+ graph = IRBuilder()
+ x = Value("x", shape=(128, 256))
+ graph.new_function("main", [x])
+ graph.new_block("entry")
+ graph.ret(graph.concat([
+ graph.sin(x), graph.cos(x), graph.sigmoid(x),
+ graph.softmax(x, axis=-1), graph.softmax(x, axis=0),
+ graph.reduce_mean(x, axes=(-1,), keepdims=True),
+ ], axis=1))
+ artifact = TensorCCodegen(
+ graph.program, constant_storage="external", kernel_calls=True,
+ mean_policy="two-four-lane-last-axis", nonlinear_policy="fp32-reference",
+ ).generate()
+ write_weight_bundle(artifact.external_weights, artifact.external_initializers, tmp_path / "weights")
+ executable = build_external(artifact, tmp_path / "weights", tmp_path / "build",
+ discover_toolchain(), harts=harts)
+ assert executable.evidence["nonlinear_policy"] == "fp32-reference"
+ assert executable.evidence["mean_policy"] == "two-four-lane-last-axis"
+ assert executable.evidence["qemu_workspace_layout"] == "none"
+ rng = np.random.default_rng(2718)
+ for context in range(2):
+ data = rng.uniform(-255, 255, size=x.shape).astype(np.float32)
+ data[0, :8] = np.array([0., -0., 1e-44, -1e-44, -18., 18., -104., 104.], np.float32)
+ wanted = np.concatenate([
+ trig(data, sine=True), trig(data, sine=False), sigmoid([data], {}, None),
+ softmax([data], {"axis": -1}, None), softmax([data], {"axis": 0}, None),
+ mean_last(data),
+ ], axis=1)
+ saved = data.copy()
+ actual, record = run_external(executable, {"x": data}, tmp_path / f"context-{context}", timeout=60)
+ np.testing.assert_array_equal(actual.view(np.uint32), wanted.view(np.uint32))
+ np.testing.assert_array_equal(data.view(np.uint32), saved.view(np.uint32))
+ assert record["passed"] and record["qemu_reaped"] and record["harts"] == harts
+ assert record["nonlinear_policy"] == "fp32-reference"
+ if context == 0:
+ # Finite inputs whose stable Softmax subtraction overflows must
+ # still fail, then the same ELF/bundle must accept a fresh context.
+ invalid = np.zeros(x.shape, np.float32)
+ invalid[-1, :2] = [np.finfo(np.float32).max, -np.finfo(np.float32).max]
+ with pytest.raises(RuntimeError, match="status 0x00000002"):
+ run_external(executable, {"x": invalid}, tmp_path / "overflow", timeout=60)
+ failure = json.loads((tmp_path / "overflow/run.json").read_text())
+ assert not failure["passed"] and failure["qemu_reaped"]
+ with pytest.raises(ProcessLookupError):
+ os.killpg(failure["qemu_pid"], 0)
diff --git a/tests/test_tensor_c_external.py b/tests/test_tensor_c_external.py
index 3559ed04..1d6e2225 100644
--- a/tests/test_tensor_c_external.py
+++ b/tests/test_tensor_c_external.py
@@ -1365,7 +1365,7 @@ def test_column_panel_rv64_errors_recover_in_same_guest_workspace(tmp_path):
from scratchv.runtime.riscv_tensor import discover_toolchain
from scratchv.runtime.weight_bundle import write_weight_bundle
- # Only the final row is poisoned, owned by a worker in the 4-hart 2x4 partition.
+ # Only the final row is poisoned, owned by a worker in the 4-core 2x4 partition.
program, bindings, data = _panel_graph(k=129, n=17, m=17, transpose=True)
bindings["panel_weight"].fill(2.0)
data.fill(0.25)
diff --git a/tests/test_tensor_c_mean_policy.py b/tests/test_tensor_c_mean_policy.py
new file mode 100644
index 00000000..f88f6628
--- /dev/null
+++ b/tests/test_tensor_c_mean_policy.py
@@ -0,0 +1,175 @@
+"""Explicit FP32 Mean order; use the existing oracle and Host ABI helpers."""
+
+import numpy as np
+import pytest
+
+from scratchv.backend.tensor_c_codegen import TensorCCodegen, TensorCCodegenError
+from scratchv.ir.types import DataType, Value
+from scratchv.verification.fp32_eigen import mean_last
+from tests.test_tensor_c_codegen import builder
+from tests.test_tensor_c_external import arguments, host_cc, library, pointers
+
+
+POLICY = "two-four-lane-last-axis"
+
+
+def make_mean(shape, *, axes=(-1,), keepdims=True, **options):
+ x = Value("x", shape=shape)
+ graph = builder(x)
+ graph.ret(graph.reduce_mean(x, axes, keepdims))
+ return TensorCCodegen(graph.program, **options).generate()
+
+
+@pytest.mark.parametrize("invalid", [None, "", "eigen", True, 1, []])
+def test_mean_policy_rejects_unknown_values(invalid):
+ with pytest.raises(TensorCCodegenError, match="mean_policy"):
+ make_mean((2, 3), mean_policy=invalid)
+
+
+@pytest.mark.parametrize("storage,kernels", [
+ ("inline", False), ("external", False), ("external", True),
+])
+def test_mean_policy_default_and_non_last_axes_are_unchanged(storage, kernels):
+ options = dict(constant_storage=storage, kernel_calls=kernels)
+ implicit = make_mean((2, 3, 5), **options)
+ explicit = make_mean((2, 3, 5), mean_policy="sequential", **options)
+ assert implicit == explicit
+ assert implicit.mean_policy == "sequential"
+ assert "sv_two_four_lane_mean_last" not in implicit.source
+ for axes in ((0,), (1,), (0, 2), None):
+ old = make_mean((2, 3, 5), axes=axes, **options)
+ selected = make_mean((2, 3, 5), axes=axes, mean_policy=POLICY, **options)
+ assert selected.mean_policy == POLICY
+ assert selected.source == old.source
+ assert selected.workspace_bytes == old.workspace_bytes
+ selected = make_mean((2, 3, 5), mean_policy=POLICY, **options)
+ assert "sv_two_four_lane_mean_last" in selected.source
+ assert "SPDX-License-Identifier: MPL-2.0" in selected.source
+ assert selected.workspace_bytes == implicit.workspace_bytes
+ # Canonical positive/negative axis spellings choose identical code.
+ assert selected.source == make_mean((2, 3, 5), axes=(2,),
+ mean_policy=POLICY, **options).source
+
+
+@pytest.mark.parametrize("storage,kernels", [
+ ("inline", False), ("external", False), ("external", True),
+])
+def test_mean_policy_matches_existing_fp32_oracle_at_prefix_tail_boundaries(
+ tmp_path, host_cc, storage, kernels):
+ widths = (1, 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 31, 127, 128, 1024)
+ params = [Value(f"x{width}", shape=(5, width)) for width in widths]
+ graph = builder(*params)
+ outputs = []
+ for x in params:
+ for keepdims in (False, True):
+ y = graph.reduce_mean(x, (-1,), keepdims)
+ outputs.append(graph.reshape(y, (5,)))
+ graph.ret(graph.concat(outputs, axis=0))
+ codegen = TensorCCodegen(graph.program, constant_storage=storage,
+ kernel_calls=kernels, mean_policy=POLICY)
+ artifact = codegen.generate()
+ assert artifact == codegen.generate() # Helper state resets on regeneration.
+ assert artifact.source.count("static float sv_two_four_lane_mean_last(") == 1
+ rng = np.random.default_rng(53)
+ with library(tmp_path, artifact, host_cc) as run:
+ for kind in ("random", "cancellation", "negative-zero", "subnormal"):
+ feed = {}
+ for x in params:
+ # Legal 4-byte ABI alignment, deliberately shifted from 16-byte
+ # alignment; canonical row order must not depend on its address.
+ backing = np.empty(5 * x.shape[-1] + 1, np.float32)
+ values = backing[1:].reshape(x.shape)
+ if kind == "random":
+ values[:] = rng.normal(size=x.shape).astype(np.float32)
+ elif kind == "cancellation":
+ values[:] = np.resize(np.array([2**24, 1, -2**24, 1, -3, 2, 5],
+ np.float32), x.shape)
+ elif kind == "negative-zero":
+ values[:] = np.float32(-0.0)
+ else:
+ values[:] = np.resize(np.array([1, -2, 3, -4], np.float32)
+ * np.nextafter(np.float32(0), np.float32(1)),
+ x.shape)
+ values.flags.writeable = False
+ feed[x.name] = values
+ expected = np.concatenate([mean_last(feed[x.name], keepdims=False)
+ for x in params for _ in range(2)])
+ saved = [values.copy() for values in feed.values()]
+ if storage == "external":
+ weights, workspace, actual, args = arguments(artifact, feed)
+ else:
+ actual = np.empty(artifact.output.shape, np.float32)
+ args = [pointers(list(feed.values())), actual.ctypes.data]
+ assert run(*args) == 0
+ np.testing.assert_array_equal(actual.view(np.uint32), expected.view(np.uint32))
+ for original, values in zip(saved, feed.values()):
+ np.testing.assert_array_equal(values.view(np.uint32), original.view(np.uint32))
+
+
+@pytest.mark.parametrize("shape,axes", [((2, 0), (-1,)), ((0,), None)])
+def test_mean_policy_empty_reduction_keeps_existing_rejection(shape, axes):
+ with pytest.raises(TensorCCodegenError, match="empty axis"):
+ make_mean(shape, axes=axes, mean_policy=POLICY)
+
+
+@pytest.mark.parametrize("storage,kernels", [
+ ("inline", False), ("external", False), ("external", True),
+])
+def test_mean_policy_empty_leading_dimensions(tmp_path, host_cc, storage, kernels):
+ model = make_mean((2, 0, 7), keepdims=False, constant_storage=storage,
+ kernel_calls=kernels, mean_policy=POLICY)
+ x = np.empty((2, 0, 7), np.float32)
+ with library(tmp_path, model, host_cc) as run:
+ if storage == "external":
+ weights, workspace, output, args = arguments(model, x)
+ else:
+ output = np.empty(model.output.shape, np.float32)
+ args = [pointers([x]), output.ctypes.data]
+ assert run(*args) == 0
+ assert output.shape == (2, 0)
+ np.testing.assert_array_equal(output, mean_last(x, keepdims=False))
+
+
+@pytest.mark.parametrize("kernels", [False, True])
+def test_mean_policy_numeric_error_propagation_and_workspace_recovery(tmp_path, host_cc, kernels):
+ x = Value("x", shape=(5, 13))
+ graph = builder(x)
+ first = graph.reduce_mean(x, (-1,), True)
+ shifted = graph.add(x, first)
+ second = graph.reduce_mean(shifted, (-1,), True)
+ graph.ret(graph.add(second, second))
+ model = TensorCCodegen(graph.program, constant_storage="external",
+ kernel_calls=kernels, mean_policy=POLICY).generate()
+ assert model.workspace_bytes < (5 * 13 + 5 * 3) * 4
+ x0 = np.arange(65, dtype=np.float32).reshape(5, 13) / np.float32(10)
+ weights, workspace, output, args = arguments(model, x0)
+ with library(tmp_path, model, host_cc) as run:
+ for bad in (np.nan, np.inf, np.finfo(np.float32).max):
+ x0[:] = bad
+ output[:] = -123
+ assert run(*args) == 2
+ assert np.all(output == -123) # No final result copied on failure.
+ # New input pointer/context, exact same caller workspace.
+ x1 = np.linspace(-2, 3, 65, dtype=np.float32).reshape(5, 13)
+ args[0] = pointers([x1])
+ assert run(*args) == 0
+ expected = mean_last(x1 + mean_last(x1))
+ expected = expected + expected
+ np.testing.assert_array_equal(output.view(np.uint32), expected.view(np.uint32))
+ args[0] = pointers([x0])
+ # Capacity and overlap validation must still precede touching workspace.
+ workspace[:] = np.uint64(0x1122334455667788)
+ output[:] = -123
+ args[0] = pointers([x1])
+ args[3] = model.workspace_bytes - 1
+ assert run(*args) == 1
+ assert np.all(workspace == np.uint64(0x1122334455667788))
+ assert np.all(output == -123)
+
+
+def test_mean_policy_does_not_bypass_fp32_type_contract():
+ x = Value("x", DataType.INT32, shape=(2, 4))
+ graph = builder(x)
+ graph.ret(graph.reduce_mean(x, (-1,), True))
+ with pytest.raises(TensorCCodegenError):
+ TensorCCodegen(graph.program, mean_policy=POLICY).generate()
diff --git a/tests/test_tensor_c_nonlinear_policy.py b/tests/test_tensor_c_nonlinear_policy.py
new file mode 100644
index 00000000..f422d00e
--- /dev/null
+++ b/tests/test_tensor_c_nonlinear_policy.py
@@ -0,0 +1,239 @@
+"""Explicit portable nonlinear arithmetic, using the existing FP32 oracle/ABI."""
+from dataclasses import replace
+import ctypes
+import re
+
+import numpy as np
+import pytest
+
+from scratchv.backend.tensor_c_codegen import TensorCCodegen, TensorCCodegenError
+from scratchv.ir.types import Value
+from scratchv.verification import fp32_reference
+from scratchv.verification.fp32_eigen import trig
+from tests.test_tensor_c_codegen import builder
+from tests.test_tensor_c_external import (
+ arguments, host_cc, library, pointers, _PARALLEL_HOST_RUNTIME,
+ _softmax_pair_experiment, _softmax_experiment,
+)
+
+POLICY = "fp32-reference"
+MODES = [("inline", False), ("external", False), ("external", True)]
+
+
+def oracle_softmax(x, axis=-1):
+ # Explicit reference arithmetic, independent of local/hidden CPU features.
+ moved = np.moveaxis(x, axis, -1)
+ weights = fp32_reference._exp_nonpositive(
+ moved - moved.max(axis=-1, keepdims=True), cpu_strategy="avx512")
+ result = weights * (np.float32(1) / fp32_reference._sum_lanes(weights, 16))
+ return np.moveaxis(result, -1, axis)
+
+
+def abi(model, feed):
+ if model.constant_storage == "external":
+ weights, workspace, output, args = arguments(model, feed)
+ return output, args, (weights, workspace)
+ output = np.full(model.output.shape, -999, np.float32)
+ return output, [pointers([feed[x.name] for x in model.inputs]), output.ctypes.data], ()
+
+
+@pytest.mark.parametrize("invalid", [None, "", "auto", "avx512", True, 1, []])
+def test_nonlinear_policy_rejects_invalid_values(invalid):
+ x = Value("x", shape=(4, 8))
+ graph = builder(x)
+ graph.ret(graph.softmax(x))
+ with pytest.raises(TensorCCodegenError, match="nonlinear_policy"):
+ TensorCCodegen(graph.program, nonlinear_policy=invalid).generate()
+
+
+@pytest.mark.parametrize("storage,kernels", MODES)
+def test_default_source_and_unused_operations_stay_unchanged(storage, kernels):
+ x = Value("x", shape=(4, 17))
+ graph = builder(x)
+ graph.ret(graph.softmax(graph.sigmoid(graph.add(graph.sin(x), graph.cos(x)))))
+ options = dict(constant_storage=storage, kernel_calls=kernels)
+ old = TensorCCodegen(graph.program, **options).generate()
+ assert old == TensorCCodegen(graph.program, nonlinear_policy="libm", **options).generate()
+ assert old.nonlinear_policy == "libm" and old.mean_policy == "sequential"
+ assert "sv_fp32_reference_" not in old.source
+ generator = TensorCCodegen(graph.program, nonlinear_policy=POLICY, **options)
+ selected = generator.generate()
+ assert selected == generator.generate()
+ assert selected.workspace_bytes == old.workspace_bytes
+ for name in ("trig", "sigmoid", "exp_nonpositive", "sum_sixteen"):
+ assert selected.source.count(f"static float sv_fp32_reference_{name}(") == 1
+ assert "SPDX-License-Identifier: MPL-2.0" in selected.source
+ assert "Copyright (c) Microsoft Corporation" in selected.source
+ generator.nonlinear_policy = "libm"
+ assert generator.generate() == old # A prior reference build leaves no state.
+ plain = builder(x)
+ plain.ret(plain.add(plain.exp(x), plain.pow(x, plain.make_const(2.0))))
+ implicit = TensorCCodegen(plain.program, **options).generate()
+ explicit = TensorCCodegen(plain.program, nonlinear_policy=POLICY, **options).generate()
+ assert explicit.source == implicit.source # Exp/Pow are outside this policy.
+ assert explicit.mean_policy == "sequential"
+
+
+@pytest.mark.parametrize("storage,kernels", MODES)
+def test_trig_and_sigmoid_match_fixed_reference_and_preserve_libm_fallback(
+ tmp_path, host_cc, storage, kernels):
+ threshold = np.float32(18000)
+ small = np.nextafter(np.float32(0), np.float32(1))
+ values = np.concatenate([
+ np.linspace(-17999, 17999, 513, dtype=np.float32),
+ np.array([0, -0.0, small, -small, 18, -18,
+ np.nextafter(threshold, np.float32(0)), threshold,
+ np.nextafter(threshold, np.float32(np.inf)), 1e20,
+ np.finfo(np.float32).max], np.float32),
+ np.array([-threshold, -np.nextafter(threshold, np.float32(0)),
+ -np.nextafter(threshold, np.float32(np.inf)), -1e20,
+ -np.finfo(np.float32).max], np.float32),
+ ])
+ x = Value("x", shape=values.shape)
+ graph = builder(x)
+ graph.ret(graph.concat([graph.sin(x), graph.cos(x), graph.sigmoid(x)], axis=0))
+ opts = dict(constant_storage=storage, kernel_calls=kernels)
+ baseline = TensorCCodegen(graph.program, **opts).generate()
+ selected = TensorCCodegen(graph.program, nonlinear_policy=POLICY, **opts).generate()
+ feed = {"x": values}
+ with library(tmp_path / "libm", baseline, host_cc) as run:
+ original, args, keep = abi(baseline, feed)
+ assert run(*args) == 0
+ with library(tmp_path / "reference", selected, host_cc) as run:
+ actual, args, keep = abi(selected, feed)
+ assert run(*args) == 0
+ expected = np.concatenate([trig(values, sine=True), trig(values, sine=False),
+ fp32_reference.sigmoid([values], {}, None)])
+ fallback = np.abs(values) >= threshold
+ for block in (0, 1):
+ # The explicit fallback is the original platform libm, not NumPy.
+ expected[block*values.size:(block+1)*values.size][fallback] = (
+ original[block*values.size:(block+1)*values.size][fallback])
+ np.testing.assert_array_equal(actual.view("u4"), expected.view("u4"))
+ assert np.signbit(actual[np.flatnonzero(np.signbit(values) & (values == 0))]).all()
+ for invalid in (np.nan, np.inf, -np.inf):
+ bad = values.copy()
+ bad[0] = invalid
+ args[0] = pointers([bad])
+ actual.fill(-999)
+ assert run(*args) == 2 and np.all(actual == -999)
+ args[0] = pointers([values])
+ assert run(*args) == 0
+ np.testing.assert_array_equal(actual.view("u4"), expected.view("u4"))
+
+
+@pytest.mark.parametrize("storage,kernels", MODES)
+def test_softmax_axes_tails_empty_rows_and_fixed_cpu_independent_order(
+ tmp_path, host_cc, monkeypatch, storage, kernels):
+ specifications = [((3, n), -1) for n in (1, 7, 16, 17, 257)]
+ specifications += [((2, 17, 3), 1), ((2, 17, 3), 0), ((0, 17), -1)]
+ params = [Value(f"x{i}", shape=shape) for i, (shape, _) in enumerate(specifications)]
+ graph = builder(*params)
+ graph.ret(graph.concat([graph.reshape(graph.softmax(x, axis=axis), (int(np.prod(shape)),))
+ for x, (shape, axis) in zip(params, specifications)], axis=0))
+ feed = {}
+ rng = np.random.default_rng(217)
+ for x in params:
+ feed[x.name] = rng.normal(size=x.shape).astype("f4") * np.float32(20)
+ expected = np.concatenate([oracle_softmax(feed[x.name], axis).ravel()
+ for x, (_, axis) in zip(params, specifications)])
+ # Backend selection must not consult host CPU dispatch or reference env.
+ monkeypatch.setenv("SCRATCHV_FP32_REFERENCE_CPU", "avx2-fma3")
+ model = TensorCCodegen(graph.program, constant_storage=storage, kernel_calls=kernels,
+ nonlinear_policy=POLICY).generate()
+ assert "ldexpf" in model.source and "sv_reference_lanes[16]" in model.source
+ actual, args, keep = abi(model, feed)
+ with library(tmp_path, model, host_cc) as run:
+ assert run(*args) == 0
+ np.testing.assert_array_equal(actual.view("u4"), expected.view("u4"))
+
+
+@pytest.mark.parametrize("kernels", [False, True])
+def test_reference_softmax_reuses_callback_with_separate_contexts_and_recovers(
+ tmp_path, host_cc, kernels):
+ program, feed = _softmax_pair_experiment()
+ model = TensorCCodegen(program, constant_storage="external", kernel_calls=kernels,
+ nonlinear_policy=POLICY).generate()
+ assert len(re.findall(r"static void sv_softmax_\w+_range\(", model.source)) == 1
+ expected = np.concatenate([oracle_softmax(feed["x"]), oracle_softmax(feed["y"])], axis=0)
+ parallel = replace(model, source=model.source + _PARALLEL_HOST_RUNTIME)
+ for harts in (1, 4):
+ actual, args, keep = abi(model, feed)
+ with library(tmp_path / str(harts), parallel,
+ [*host_cc, f"-DSCRATCHV_HARTS={harts}", "-pthread"]) as run:
+ for inputs, wanted in [(feed, expected),
+ ({"x": feed["y"], "y": feed["x"]},
+ np.concatenate(np.split(expected, 2)[::-1], axis=0)),
+ (feed, expected)]:
+ args[0] = pointers([inputs["x"], inputs["y"]])
+ assert run(*args) == 0
+ np.testing.assert_array_equal(actual.view("u4"), wanted.view("u4"))
+ bad = feed["y"].copy()
+ bad[-1, :2] = [np.finfo("f4").max, -np.finfo("f4").max]
+ args[0] = pointers([feed["x"], bad])
+ actual.fill(-999)
+ assert run(*args) == 2 and np.all(actual == -999)
+ args[0] = pointers([feed["x"], feed["y"]])
+ assert run(*args) == 0
+ np.testing.assert_array_equal(actual.view("u4"), expected.view("u4"))
+
+
+def test_reference_softmax_chain_reuses_workspace(tmp_path, host_cc):
+ program, x = _softmax_experiment((32, 1024), passes=4)
+ model = TensorCCodegen(program, constant_storage="external", kernel_calls=True,
+ nonlinear_policy=POLICY).generate()
+ assert model.workspace_bytes < 4*x.nbytes
+ expected = x
+ for _ in range(4):
+ expected = oracle_softmax(expected)
+ parallel = replace(model, source=model.source + _PARALLEL_HOST_RUNTIME)
+ actual, args, keep = abi(model, {"x": x})
+ with library(tmp_path, parallel, [*host_cc, "-DSCRATCHV_HARTS=4", "-pthread"]) as run:
+ assert run(*args) == 0
+ np.testing.assert_array_equal(actual.view("u4"), expected.view("u4"))
+
+
+@pytest.mark.parametrize("kernels", [False, True])
+def test_reference_softmax_output_finite_checks_before_workspace_recovery(
+ tmp_path, host_cc, kernels):
+ shape = (17, 2049)
+ x, y = Value("x", shape=shape), Value("y", shape=shape)
+ graph = builder(x, y)
+ graph.softmax(x)
+ graph.ret(graph.softmax(y)) # No downstream finite check can hide failure.
+ model = TensorCCodegen(graph.program, constant_storage="external", kernel_calls=kernels,
+ nonlinear_policy=POLICY).generate()
+ feed = {"x": np.zeros(shape, "f4"), "y": np.zeros(shape, "f4")}
+ feed["y"][-1, 0] = -13
+ expected = oracle_softmax(feed["y"])
+ source = model.source.replace("float v=sv_fp32_reference_exp_nonpositive(shifted);",
+ "float v=sv_test_reference_exp(shifted);")
+ fault = r"""
+int sv_test_fault, sv_test_hits;
+float sv_test_reference_exp(float x) {
+ if (sv_test_fault && x == -13.0f) {
+ __atomic_fetch_add(&sv_test_hits,1,__ATOMIC_RELAXED);
+ if (sv_test_fault == 1) return -2048.0f;
+ union {uint32_t u; float f;} bad={0x7fc00000U}; return bad.f;
+ }
+ return sv_fp32_reference_exp_nonpositive(x);
+}
+"""
+ injected = replace(model, source="extern float sv_test_reference_exp(float);\n"
+ + source + _PARALLEL_HOST_RUNTIME + fault)
+ for harts in (1, 4):
+ actual, args, keep = abi(model, feed)
+ directory = tmp_path / str(harts)
+ with library(directory, injected,
+ [*host_cc, f"-DSCRATCHV_HARTS={harts}", "-pthread"]) as run:
+ loaded = ctypes.CDLL(str(directory / "model.so"))
+ mode = ctypes.c_int.in_dll(loaded, "sv_test_fault")
+ hits = ctypes.c_int.in_dll(loaded, "sv_test_hits")
+ for number in (1, 2):
+ mode.value, hits.value = number, 0
+ actual.fill(-999)
+ assert run(*args) == 2 and hits.value == 1
+ assert np.all(actual == -999)
+ mode.value, hits.value = 0, 0
+ assert run(*args) == 0 and hits.value == 0
+ np.testing.assert_array_equal(actual.view("u4"), expected.view("u4"))
diff --git a/tests/test_w3_workflows.py b/tests/test_w3_workflows.py
index b27bdb33..b8306a8d 100644
--- a/tests/test_w3_workflows.py
+++ b/tests/test_w3_workflows.py
@@ -6,6 +6,7 @@
import ast
import os
from pathlib import Path
+import re
import shutil
import subprocess
@@ -46,17 +47,21 @@ def run_script(bash, script, tmp_path, **values):
cwd=tmp_path, env=env, capture_output=True, text=True)
-def evaluate_guard(expression, event, enabled, w3_validation=False):
+def evaluate_guard(expression, event, enabled, w3_validation=False, w5_validation=False):
"""Evaluate only the tiny boolean subset used by the real job guards."""
value = expression.removeprefix("${{").removesuffix("}}").strip()
value = value.replace("github.event_name", repr(event))
value = value.replace("vars.W3_NIGHTLY_ENABLED", repr(enabled))
value = value.replace("inputs.run_w3_validation", repr(w3_validation))
+ value = value.replace("inputs.run_w5_validation", repr(w5_validation))
value = value.replace("always()", "True").replace("||", "or").replace("&&", "and")
+ value = re.sub(r"!(?!=)", "not ", value).strip()
def visit(node):
if isinstance(node, ast.Constant):
return node.value
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not):
+ return not bool(visit(node.operand))
if isinstance(node, ast.BoolOp):
values = [bool(visit(item)) for item in node.values]
return all(values) if isinstance(node.op, ast.And) else any(values)
@@ -121,17 +126,18 @@ def test_nightly_expensive_jobs_require_opt_in_or_explicit_dispatch(event, enabl
@pytest.mark.parametrize("event", ["pull_request", "schedule", "workflow_dispatch"])
@pytest.mark.parametrize("selected", [False, True])
-def test_existing_manual_entry_runs_w3_only_when_explicitly_requested(event, selected):
+@pytest.mark.parametrize("w5_selected", [False, True])
+def test_existing_manual_entry_runs_w3_only_when_explicitly_requested(event, selected, w5_selected):
data = workflow("llm-deploy.yml")
option = data["on"]["workflow_dispatch"]["inputs"]["run_w3_validation"]
assert option["type"] == "boolean" and option["default"] == "false"
job = data["jobs"]["w3-validation"]
assert job["uses"] == "./.github/workflows/w3-nightly.yml"
- assert evaluate_guard(job["if"], event, "", selected) is (
- event == "workflow_dispatch" and selected)
+ expected = event == "workflow_dispatch" and selected and not w5_selected
+ assert evaluate_guard(job["if"], event, "", selected, w5_selected) is expected
# The called workflow inherits the caller event. A manual call must run
# both complete gates even while the scheduled nightly opt-in is disabled.
- if event == "workflow_dispatch" and selected:
+ if expected:
assert all(evaluate_guard(item["if"], event, "")
for item in workflow("w3-nightly.yml")["jobs"].values())
diff --git a/tests/test_w4_ci_summary.py b/tests/test_w4_ci_summary.py
index 7a15cd29..4da16198 100644
--- a/tests/test_w4_ci_summary.py
+++ b/tests/test_w4_ci_summary.py
@@ -168,7 +168,7 @@ def test_workflow_publishes_summaries_even_on_failure_and_runs_tests():
assert step["if"] == "always()"
assert "scripts/summarize_w4_ci.py" in step["run"]
assert all(".outcome" in value for value in step["env"].values())
- assert set(workflow["on"]) == {"workflow_call"}
+ assert set(workflow["on"]) == {"workflow_call", "workflow_dispatch"}
ci = yaml.load((ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8"), Loader=yaml.BaseLoader)
assert "paths" not in ci["on"]["pull_request"]
assert any(job.get("uses") == "./.github/workflows/w4-full-numeric.yml"
diff --git a/tests/test_w4_diagnostic.py b/tests/test_w4_diagnostic.py
index 5a2d5c04..60521364 100644
--- a/tests/test_w4_diagnostic.py
+++ b/tests/test_w4_diagnostic.py
@@ -191,19 +191,19 @@ def test_diagnostic_cli_rejects_unbounded_timeout_before_creating_output(tmp_pat
@pytest.mark.parametrize("policy,kernel_calls,flags", [
("sequential", True, []),
("sequential", False, ["--no-kernel-calls"]),
- ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--kernel-calls", "--harts", "4"]),
+ ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--kernel-calls", "--cores", "4"]),
("blocked_fma", False, ["--matmul-policy", "blocked_fma", "--no-kernel-calls"]),
- ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--harts", "auto"]),
+ ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--cores", "auto"]),
("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--matmul-tile", "1x4"]),
- ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--matmul-tile", "2x4", "--harts", "4"]),
+ ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--matmul-tile", "2x4", "--cores", "4"]),
("blocked_fma", False, ["--matmul-policy", "blocked_fma", "--matmul-tile", "1x8",
- "--no-kernel-calls", "--harts", "auto"]),
+ "--no-kernel-calls", "--cores", "auto"]),
("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--matmul-rhs-layout", "column-panel",
- "--matmul-tile", "2x4", "--harts", "4"]),
+ "--matmul-tile", "2x4", "--cores", "4"]),
("blocked_fma", False, ["--matmul-policy", "blocked_fma", "--matmul-rhs-layout", "column-panel",
- "--no-kernel-calls", "--harts", "auto"]),
+ "--no-kernel-calls", "--cores", "auto"]),
("sequential", True, ["--qemu-workspace-layout", "two-buffer-pad-4k"]),
- ("sequential", True, ["--qemu-workspace-layout", "two-buffer-pad-4k", "--harts", "4"]),
+ ("sequential", True, ["--qemu-workspace-layout", "two-buffer-pad-4k", "--cores", "4"]),
])
def test_diagnostic_codegen_policy_reaches_build_and_failed_build_never_accepts_gate(
tmp_path, monkeypatch, policy, kernel_calls, flags):
@@ -240,7 +240,7 @@ def test_diagnostic_codegen_policy_reaches_build_and_failed_build_never_accepts_
monkeypatch.setattr(onnx_parser, "ONNXParser", lambda: parser)
monkeypatch.setattr(diagnostic, "discover_toolchain", lambda *a: object())
generated = []
- requested_harts = flags[flags.index("--harts") + 1] if "--harts" in flags else "1"
+ requested_harts = flags[flags.index("--cores") + 1] if "--cores" in flags else "1"
requested_harts = "auto" if requested_harts == "auto" else int(requested_harts)
expected_harts = 2 if requested_harts == "auto" else requested_harts
probes = []
@@ -251,7 +251,7 @@ def available_cpus(pid):
monkeypatch.setattr(os, "sched_getaffinity", available_cpus, raising=False)
def stopped_build(artifact, *a, **kw):
- assert type(kw["harts"]) is int and kw["harts"] == expected_harts
+ assert type(kw["cores"]) is int and kw["cores"] == expected_harts
generated.append(artifact)
raise RuntimeError("intentional unit-test build stop")
@@ -276,7 +276,7 @@ def stopped_build(artifact, *a, **kw):
assert report["matmul_rhs_layout"] == report["codegen"]["matmul_rhs_layout"] == requested_rhs_layout
assert report["codegen"]["matmul_rhs_panels"] == list(artifact.matmul_rhs_panels)
assert report["harts_requested"] == requested_harts
- assert report["harts"] == expected_harts
+ assert report["cores"] == expected_harts
assert report["qemu_workspace_layout_requested"] == requested_layout
assert report["codegen"]["qemu_workspace_layout"] == requested_layout
assert probes == ([0] if requested_harts == "auto" else [])
diff --git a/tests/test_w4_full_probe_contract.py b/tests/test_w4_full_probe_contract.py
index 9e6d1929..deff5236 100644
--- a/tests/test_w4_full_probe_contract.py
+++ b/tests/test_w4_full_probe_contract.py
@@ -370,7 +370,7 @@ def test_reused_executable_rejects_invalid_harts(tmp_path, monkeypatch, harts):
path, record = _saved_executable(tmp_path, monkeypatch)
record["harts"] = record["evidence"]["harts"] = harts
path.write_text(json.dumps(record), encoding="utf-8")
- with pytest.raises(ValueError, match="hart"):
+ with pytest.raises(ValueError, match="core|hart"):
run.load_executable(path)
@@ -378,13 +378,17 @@ def test_reused_executable_rejects_hart_count_changed_without_rebuild(tmp_path,
path, record = _saved_executable(tmp_path, monkeypatch)
record["harts"] = 4
path.write_text(json.dumps(record), encoding="utf-8")
- with pytest.raises(ValueError, match="hart"):
+ with pytest.raises(ValueError, match="core|hart"):
run.load_executable(path)
@pytest.mark.parametrize("requested", [None, 2, 4, "auto"])
@pytest.mark.parametrize("layout", ["none", "two-buffer-pad-4k"])
-def test_full_cli_resolves_harts_once_before_tiny_graph_build(tmp_path, monkeypatch, requested, layout):
+@pytest.mark.parametrize("mean_policy,nonlinear_policy", [
+ ("sequential", "libm"), ("two-four-lane-last-axis", "libm"),
+ ("sequential", "fp32-reference"), ("two-four-lane-last-axis", "fp32-reference"),
+])
+def test_full_cli_resolves_harts_once_before_tiny_graph_build(tmp_path, monkeypatch, requested, layout, mean_policy, nonlinear_policy):
from scratchv.frontend import onnx_parser
from scratchv.ir.builder import IRBuilder
from scratchv.ir.types import Value
@@ -412,22 +416,28 @@ def available_cpus(pid):
expected = 2 if requested == "auto" else requested or 1
built = []
def stopped_build(artifact, *a, **kw):
- built.append(kw["harts"])
- assert type(kw["harts"]) is int and kw["harts"] == expected
+ built.append(kw["cores"])
+ assert type(kw["cores"]) is int and kw["cores"] == expected
assert artifact.qemu_workspace_layout == layout
+ assert artifact.mean_policy == mean_policy
+ assert artifact.nonlinear_policy == nonlinear_policy
raise RuntimeError("intentional tiny-graph build stop")
monkeypatch.setattr(run, "build_external", stopped_build)
if requested is not None:
- arguments += ["--harts", str(requested)]
+ arguments += ["--cores" if layout == "none" else "--harts", str(requested)]
if layout != "none":
arguments += ["--qemu-workspace-layout", layout]
+ arguments += ["--mean-policy", mean_policy, "--nonlinear-policy", nonlinear_policy]
assert run.main(arguments + ["--build-only"]) == 1
saved = json.loads((output / "report.json").read_text())
assert built == [expected]
assert probes == ([0] if requested == "auto" else [])
assert saved["harts_requested"] == (requested if requested is not None else 1)
- assert saved["harts"] == expected
+ assert saved["cores"] == saved["harts"] == expected
+ assert saved["cores_requested"] == saved["harts_requested"]
assert saved["qemu_workspace_layout_requested"] == saved["codegen"]["qemu_workspace_layout"] == layout
+ assert saved["mean_policy"] == saved["codegen"]["mean_policy"] == mean_policy
+ assert saved["nonlinear_policy"] == saved["codegen"]["nonlinear_policy"] == nonlinear_policy
assert "intentional tiny-graph build stop" in saved["error"]
assert not saved["full_qemu_forward_executed"]
@@ -448,7 +458,7 @@ def test_full_cli_auto_reuse_requires_resolved_harts_to_match_build(tmp_path, mo
if built_harts == 4:
assert result == 0 and saved["status"] == "BUILD_ONLY"
else:
- assert result == 1 and "hart count" in saved["error"]
+ assert result == 1 and "core count" in saved["error"]
assert saved["stage"] == "load_existing_build"
@@ -501,7 +511,7 @@ def test_full_cli_matmul_tile_reaches_tiny_graph_build(tmp_path, monkeypatch, ti
def stopped_build(artifact, *a, **kw):
generated.append(artifact)
- assert kw["harts"] == 1 and artifact.qemu_workspace_layout == "none"
+ assert kw["cores"] == 1 and artifact.qemu_workspace_layout == "none"
raise RuntimeError("intentional tile build stop")
monkeypatch.setattr(run, "build_external", stopped_build)
@@ -597,7 +607,7 @@ def test_full_cli_rhs_layout_reaches_tiny_graph_build(tmp_path, monkeypatch, rhs
def stopped_build(artifact, *a, **kw):
generated.append(artifact)
- assert kw["harts"] == 1 and artifact.qemu_workspace_layout == "none"
+ assert kw["cores"] == 1 and artifact.qemu_workspace_layout == "none"
raise RuntimeError("intentional RHS layout build stop")
monkeypatch.setattr(run, "build_external", stopped_build)
@@ -705,3 +715,179 @@ def test_full_cli_rejects_rhs_layout_before_output_creation(tmp_path, layout):
"--matmul-rhs-layout", layout])
assert exc.value.code == 2
assert not target.exists()
+
+
+@pytest.mark.parametrize("policy", [None, "sequential", "two-four-lane-last-axis"])
+def test_load_retains_mean_policy_and_legacy_default(tmp_path, monkeypatch, policy):
+ path, record = _saved_executable(tmp_path, monkeypatch)
+ if policy is not None:
+ record["evidence"]["mean_policy"] = policy
+ path.write_text(json.dumps(record))
+ exe = run.load_executable(path)
+ assert exe.evidence.get("mean_policy", "sequential") == (policy or "sequential")
+ if policy is None:
+ assert "mean_policy" not in exe.evidence # Do not relabel the legacy artifact.
+
+
+@pytest.mark.parametrize("policy", [None, True, "reference-v4", 1])
+def test_load_rejects_invalid_explicit_mean_policy(tmp_path, monkeypatch, policy):
+ path, record = _saved_executable(tmp_path, monkeypatch)
+ record["evidence"]["mean_policy"] = policy
+ path.write_text(json.dumps(record))
+ with pytest.raises(ValueError, match="Mean policy"):
+ run.load_executable(path)
+
+
+@pytest.mark.parametrize("built", [None, "sequential", "two-four-lane-last-axis"])
+@pytest.mark.parametrize("requested", ["sequential", "two-four-lane-last-axis"])
+def test_w4_mean_policy_reuse_requires_explicit_matching_build(tmp_path, monkeypatch, built, requested):
+ arguments, output = _mock_preflight_and_reuse(tmp_path, monkeypatch)
+ existing = run.load_executable(None)
+ evidence = {} if built is None else {"mean_policy": built}
+ monkeypatch.setattr(run, "load_executable", lambda *_: replace(existing, evidence=evidence))
+ previous = tmp_path / "previous/report.json"
+ record = json.loads(previous.read_text())
+ if built is not None:
+ record["mean_policy"] = built
+ previous.write_text(json.dumps(record))
+ result = run.main(arguments + ["--mean-policy", requested, "--build-only"])
+ saved = json.loads((output / "report.json").read_text())
+ assert saved["mean_policy"] == requested and not saved["full_qemu_forward_executed"]
+ if (built or "sequential") == requested:
+ assert result == 0 and saved["status"] == "BUILD_ONLY"
+ else:
+ assert result == 1 and "Mean policy" in saved["error"]
+
+
+def test_w4_mean_policy_report_disagreement_rejects_reuse(tmp_path, monkeypatch):
+ arguments, output = _mock_preflight_and_reuse(tmp_path, monkeypatch)
+ existing = run.load_executable(None)
+ monkeypatch.setattr(run, "load_executable", lambda *_: replace(existing,
+ evidence={"mean_policy": "two-four-lane-last-axis"}))
+ assert run.main(arguments + ["--mean-policy", "two-four-lane-last-axis", "--build-only"]) == 1
+ saved = json.loads((output / "report.json").read_text())
+ assert "Mean policy" in saved["error"] and not saved["full_qemu_forward_executed"]
+
+
+@pytest.mark.parametrize("policy", [None, "libm", "fp32-reference"])
+def test_load_retains_nonlinear_policy_and_legacy_default(tmp_path, monkeypatch, policy):
+ path, record = _saved_executable(tmp_path, monkeypatch)
+ if policy is not None:
+ record["evidence"]["nonlinear_policy"] = policy
+ path.write_text(json.dumps(record))
+ exe = run.load_executable(path)
+ assert exe.evidence.get("nonlinear_policy", "libm") == (policy or "libm")
+ if policy is None:
+ assert "nonlinear_policy" not in exe.evidence # Do not relabel the legacy artifact.
+
+
+@pytest.mark.parametrize("policy", [None, True, "reference-v4", 1])
+def test_load_rejects_invalid_explicit_nonlinear_policy(tmp_path, monkeypatch, policy):
+ path, record = _saved_executable(tmp_path, monkeypatch)
+ record["evidence"]["nonlinear_policy"] = policy
+ path.write_text(json.dumps(record))
+ with pytest.raises(ValueError, match="nonlinear policy"):
+ run.load_executable(path)
+
+
+@pytest.mark.parametrize("built", [None, "libm", "fp32-reference"])
+@pytest.mark.parametrize("requested", ["libm", "fp32-reference"])
+def test_w4_nonlinear_policy_reuse_requires_explicit_matching_build(tmp_path, monkeypatch, built, requested):
+ arguments, output = _mock_preflight_and_reuse(tmp_path, monkeypatch)
+ existing = run.load_executable(None)
+ evidence = {} if built is None else {"nonlinear_policy": built}
+ monkeypatch.setattr(run, "load_executable", lambda *_: replace(existing, evidence=evidence))
+ previous = tmp_path / "previous/report.json"
+ record = json.loads(previous.read_text())
+ if built is not None:
+ record["nonlinear_policy"] = built
+ previous.write_text(json.dumps(record))
+ result = run.main(arguments + ["--nonlinear-policy", requested, "--build-only"])
+ saved = json.loads((output / "report.json").read_text())
+ assert saved["nonlinear_policy"] == requested and not saved["full_qemu_forward_executed"]
+ if (built or "libm") == requested:
+ assert result == 0 and saved["status"] == "BUILD_ONLY"
+ else:
+ assert result == 1 and "nonlinear policy" in saved["error"]
+
+
+def test_w4_nonlinear_policy_report_disagreement_rejects_reuse(tmp_path, monkeypatch):
+ arguments, output = _mock_preflight_and_reuse(tmp_path, monkeypatch)
+ existing = run.load_executable(None)
+ monkeypatch.setattr(run, "load_executable", lambda *_: replace(existing,
+ evidence={"nonlinear_policy": "fp32-reference"}))
+ assert run.main(arguments + ["--nonlinear-policy", "fp32-reference", "--build-only"]) == 1
+ saved = json.loads((output / "report.json").read_text())
+ assert "nonlinear policy" in saved["error"] and not saved["full_qemu_forward_executed"]
+
+
+@pytest.mark.parametrize("policy", ["unknown", "sequential", ""])
+def test_w4_unknown_nonlinear_policy_fails_before_output(tmp_path, policy):
+ target = tmp_path / "report"
+ with pytest.raises(SystemExit) as exc:
+ run.main(["--model-dir", str(tmp_path), "--output-dir", str(target),
+ "--nonlinear-policy", policy])
+ assert exc.value.code == 2 and not target.exists()
+
+
+@pytest.mark.parametrize("cores", [1, 4])
+@pytest.mark.parametrize("top_names,evidence_names", [
+ (("cores",), ("cores",)), (("harts",), ("harts",)),
+ (("cores",), ("harts",)), (("harts",), ("cores",)),
+ (("cores", "harts"), ("cores", "harts")),
+])
+def test_core_metadata_aliases_roundtrip_without_rewriting_source(
+ tmp_path, monkeypatch, cores, top_names, evidence_names):
+ path, record = _saved_executable(tmp_path, monkeypatch)
+ record.update({name: cores for name in top_names})
+ record["evidence"].update({name: cores for name in evidence_names})
+ record["layout"] = plan_guest_memory(input_bytes=2048, output_bytes=256 * 151936 * 4,
+ workspace_bytes=64, cores=cores)
+ original = json.dumps(record)
+ path.write_text(original)
+ executable = run.load_executable(path)
+ assert executable.cores == executable.harts == cores
+ assert path.read_text() == original
+ out = tmp_path / "resaved"
+ out.mkdir()
+ run.save_executable(executable, out)
+ saved = json.loads((out / "executable.json").read_text())
+ assert saved["cores"] == saved["harts"] == cores
+ assert saved["evidence"]["cores"] == saved["evidence"]["harts"] == cores
+ reloaded = run.load_executable(out / "executable.json")
+ assert reloaded.cores == cores and reloaded.elf_path == executable.elf_path
+
+
+@pytest.mark.parametrize("container", ["top", "evidence"])
+@pytest.mark.parametrize("fields", [
+ {"cores": 4, "harts": 1}, {"cores": 1, "harts": True},
+ {"cores": None}, {"cores": True}, {"cores": "1"}, {"cores": "auto"}, {"cores": 3},
+])
+def test_core_metadata_rejects_conflicts_and_invalid_values(tmp_path, monkeypatch, container, fields):
+ path, record = _saved_executable(tmp_path, monkeypatch)
+ (record if container == "top" else record["evidence"]).update(fields)
+ path.write_text(json.dumps(record))
+ with pytest.raises(ValueError, match="core|hart"):
+ run.load_executable(path)
+
+
+def test_core_metadata_without_either_field_stays_single_core(tmp_path, monkeypatch):
+ path, _ = _saved_executable(tmp_path, monkeypatch)
+ executable = run.load_executable(path)
+ assert executable.cores == executable.harts == 1
+ assert executable.evidence["cores"] == executable.evidence["harts"] == 1
+
+
+@pytest.mark.parametrize("entry", ["full", "diagnostic"])
+@pytest.mark.parametrize("flags", [
+ ["--cores", "1", "--harts", "4"], ["--harts", "4", "--cores", "1"],
+ ["--cores", "auto", "--harts", "4"], ["--cores", "3"],
+])
+def test_w4_core_alias_conflict_stops_before_output(tmp_path, entry, flags):
+ from probes.w4_qwen3_full import diagnostic
+ module = run if entry == "full" else diagnostic
+ output = tmp_path / "output"
+ with pytest.raises(SystemExit) as exc:
+ module.main(["--model-dir", str(tmp_path / "model"), "--output-dir", str(output), *flags])
+ assert exc.value.code == 2
+ assert not output.exists()
diff --git a/tests/test_w4_workflows.py b/tests/test_w4_workflows.py
index 127d9b4f..18f2db58 100644
--- a/tests/test_w4_workflows.py
+++ b/tests/test_w4_workflows.py
@@ -17,6 +17,14 @@
ROOT = Path(__file__).resolve().parents[1]
+W5_REQUIRED = (
+ [f"test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel[{h}]" for h in (1, 4)]
+ + [f"test_explicit_mean_order_rv64_and_context_reuse[{w}-{h}]"
+ for w in (13, 128, 1024) for h in (1, 4)]
+ + [f"test_explicit_mean_overflow_propagates_and_build_remains_reusable[{h}]" for h in (1, 4)]
+ + [f"test_reference_nonlinear_rv64_contexts_and_error_recovery[{h}]" for h in (1, 4)]
+)
+
PATH = ROOT / ".github/workflows/w4-full-numeric.yml"
NIGHTLY_PATH = ROOT / ".github/workflows/w4-nightly.yml"
LLM_PATH = ROOT / ".github/workflows/llm-deploy.yml"
@@ -51,14 +59,16 @@ def _guard(expression, event):
def test_expensive_forward_is_not_automatic_on_pr_or_enabled_as_a_schedule():
data = workflow()
- assert set(data["on"]) == {"workflow_call"}
+ assert set(data["on"]) == {"workflow_call", "workflow_dispatch"}
run_full = data["on"]["workflow_call"]["inputs"]["run_full"]
assert run_full["type"] == "boolean" and run_full["default"] == "false"
assert data["permissions"] == {"contents": "read"}
assert data["concurrency"]["cancel-in-progress"] == "false"
full = data["jobs"]["full-numeric"]
assert full["needs"] == "unit"
- assert full["if"] == "${{ inputs.run_full }}"
+ assert full["if"] == ("${{ !inputs.run_w5 && inputs.mode != 'w5-generation' && "
+ "(inputs.run_full || (github.event_name == 'workflow_dispatch' "
+ "&& inputs.mode == 'w4-forward')) }}")
assert "if" not in data["jobs"]["unit"]
@@ -86,7 +96,8 @@ def test_same_ref_unit_and_full_calls_do_not_share_a_concurrency_group():
assert "${{ inputs.run_full && 'full' || 'unit' }}" in group
assert concurrency["cancel-in-progress"] == "false"
fixed = group.replace("${{ github.ref }}", "refs/heads/candidate").replace(
- "${{ inputs.case || 'full_seed_0' }}", "full_seed_0")
+ "${{ inputs.case || 'full_seed_0' }}", "full_seed_0").replace(
+ "${{ inputs.run_w5 && 'w5' || inputs.mode || 'call' }}", "call")
unit = fixed.replace("${{ inputs.run_full && 'full' || 'unit' }}", "unit")
full = fixed.replace("${{ inputs.run_full && 'full' || 'unit' }}", "full")
assert unit != full and "${{" not in unit + full
@@ -104,20 +115,21 @@ def test_registered_llm_entry_calls_same_commit_w4_without_changing_w3_or_pr_def
assert inputs["w4_case"]["options"] == ["full_seed_0", "full_seed_42", "one_token",
"short_17", "short_255", "changed_future", "changed_padding"]
w4 = data["jobs"]["w4-validation"]
- assert w4["if"] == "${{ github.event_name == 'workflow_dispatch' && inputs.run_w4_validation }}"
+ assert w4["if"] == "${{ github.event_name == 'workflow_dispatch' && inputs.run_w4_validation && !inputs.run_w5_validation }}"
assert w4["uses"] == "./.github/workflows/w4-full-numeric.yml"
assert w4["with"] == {"run_full": "true", "case": "${{ inputs.w4_case || 'full_seed_0' }}"}
assert "continue-on-error" not in w4
for event in ("pull_request", "schedule", "workflow_dispatch"):
for enabled in (False, True):
guard = w4["if"].removeprefix("${{").removesuffix("}}")
- event_guard, flag = guard.split("&&")
+ event_guard, flag, w5_guard = guard.split("&&")
allowed = _guard(event_guard, event) and enabled
assert flag.strip() == "inputs.run_w4_validation"
+ assert w5_guard.strip() == "!inputs.run_w5_validation"
assert allowed is (event == "workflow_dispatch" and enabled)
assert inputs["run_w3_validation"]["default"] == "false"
w3 = data["jobs"]["w3-validation"]
- assert w3["if"] == "${{ github.event_name == 'workflow_dispatch' && inputs.run_w3_validation }}"
+ assert w3["if"] == "${{ github.event_name == 'workflow_dispatch' && inputs.run_w3_validation && !inputs.run_w5_validation }}"
assert w3["uses"] == "./.github/workflows/w3-nightly.yml"
assert data["jobs"]["w2-acceptance"]["needs"] == ["llm-deploy", "full-qwen3-frontend"]
assert data["permissions"] == {"contents": "read"}
@@ -309,7 +321,7 @@ def test_actual_ci_report_gate_rejects_false_or_incomplete_acceptance(tmp_path,
"test_post_execution_transport_corruption_fails_and_retains_copies",
"test_success_copy_cleanup_failure_is_not_a_pass",
"test_actual_shared_library_executes_kernel_calls_and_unloads[sequential]",
- "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]"])
+ "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]"] + W5_REQUIRED)
def test_actual_junit_gate_requires_real_qemu_execution(tmp_path, result, required):
folder = tmp_path / "output/w4-unit"
folder.mkdir(parents=True)
@@ -330,7 +342,7 @@ def test_actual_junit_gate_requires_real_qemu_execution(tmp_path, result, requir
"test_post_execution_transport_corruption_fails_and_retains_copies",
"test_success_copy_cleanup_failure_is_not_a_pass",
"test_actual_shared_library_executes_kernel_calls_and_unloads[sequential]",
- "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]"):
+ "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]") + tuple(W5_REQUIRED):
if name != required:
case += f''
(folder / "tests.xml").write_text(f"{case}", encoding="utf-8")
@@ -556,7 +568,7 @@ def test_ci_summary_actual_bash_failure_matrix(bash, tmp_path, event, requested,
enabled = requested == "true"
expected = unit == "success" and full == ("success" if enabled else "skipped")
env = dict(os.environ, GITHUB_EVENT_NAME=event, UNIT_RESULT=unit, FULL_NUMERIC_RESULT=full,
- FULL_REQUESTED=requested,
+ FULL_REQUESTED=requested, W5_REQUESTED="false", W5_GENERATION_RESULT="skipped",
SELECTED_CASE="full_seed_0", GITHUB_SHA="a" * 40, GITHUB_RUN_ID="123", GITHUB_RUN_ATTEMPT="1",
GITHUB_STEP_SUMMARY=(tmp_path / "summary.md").as_posix())
script = step("Record W4 CI result without promoting skipped gates")["run"].replace("python3 -", '"' + sys.executable.replace("\\", "/") + '" -')
@@ -593,7 +605,7 @@ def test_nightly_summary_preserves_disabled_and_rejects_every_missing_gate(bash,
def test_failure_summaries_keep_json_artifacts_and_all_prerequisites():
- for path, jobname, needs in ((PATH, "ci-status", ["unit", "full-numeric"]),
+ for path, jobname, needs in ((PATH, "ci-status", ["unit", "full-numeric", "w5-generation"]),
(NIGHTLY_PATH, "nightly-status", "full-numeric")):
job = workflow(path)["jobs"][jobname]
assert job["if"] == "${{ always() }}" and job["needs"] == needs
@@ -602,3 +614,294 @@ def test_failure_summaries_keep_json_artifacts_and_all_prerequisites():
assert uploads[0]["with"]["if-no-files-found"] == "error"
assert uploads[0]["with"]["path"].endswith("/status.json")
assert all("continue-on-error" not in s for s in job["steps"])
+
+
+def _full_enabled(expression, event, mode, legacy, run_w5=False):
+ # Evaluate only the fixed, repository-owned workflow boolean expressions.
+ expression = expression.removeprefix("${{").removesuffix("}}").strip()
+ for key, value in (("github.event_name", event), ("inputs.mode", mode), ("inputs.run_full", legacy), ("inputs.run_w5", run_w5)):
+ expression = expression.replace(key, repr(value))
+ return bool(eval(re.sub(r"!(?!=)", " not ", expression.replace("&&", " and ").replace("||", " or ")),
+ {"__builtins__": {}}))
+
+
+@pytest.mark.parametrize("event,mode,legacy,w4,w5", [
+ ("pull_request", "small", False, False, False),
+ ("pull_request", "w5-generation", False, False, False),
+ ("schedule", "w5-generation", False, False, False),
+ ("workflow_dispatch", "small", False, False, False),
+ ("workflow_dispatch", "w4-forward", False, True, False),
+ ("workflow_dispatch", "w5-generation", False, False, True),
+ ("workflow_call", "", True, True, False),
+ ("workflow_dispatch", "", True, True, False), # Existing LLM Deploy caller.
+ ("schedule", "", True, True, False), # Existing opt-in W4 nightly caller.
+ ("workflow_dispatch", "w5-generation", True, False, False),
+])
+def test_w5_manual_selection_is_exclusive_and_old_callers_remain(event, mode, legacy, w4, w5):
+ data = workflow()
+ choice = data["on"]["workflow_dispatch"]["inputs"]["mode"]
+ assert choice == {"description": "Small regressions by default; full modes are mutually exclusive",
+ "type": "choice", "default": "small",
+ "options": ["small", "w4-forward", "w5-generation"]}
+ assert _full_enabled(data["jobs"]["full-numeric"]["if"], event, mode, legacy) is w4
+ assert _full_enabled(data["jobs"]["w5-generation"]["if"], event, mode, legacy) is w5
+ assert not (w4 and w5)
+ assert set(data["on"]["workflow_call"]["inputs"]) == {"run_full", "run_w5", "case"}
+ assert "schedule" not in data["on"] and "pull_request" not in data["on"]
+
+
+def test_w5_manual_job_reuses_deployment_and_preserves_bounded_full_scope():
+ job = workflow()["jobs"]["w5-generation"]
+ assert job["needs"] == "unit"
+ run = step("Run the frozen W5 suite through the deployment script")
+ assert "bash scripts/deploy_w5.sh" in run["run"] and "--reference-suite" in run["run"]
+ assert "--model /assets/onnx/model.onnx" in run["run"]
+ assert "--verify-onnx /assets/onnx/model.onnx" in run["run"]
+ for flag in ("--harts 4", "--matmul-policy blocked_fma", "--matmul-tile 2x4",
+ "--matmul-rhs-layout column-panel", "--mean-policy two-four-lane-last-axis",
+ "--nonlinear-policy fp32-reference"):
+ assert flag in run["run"]
+ assert "probes.w4_qwen3_full.run" not in run["run"] and "--max-new-tokens" not in run["run"]
+ guest = int(re.search(r"--total-timeout (\d+)", run["run"]).group(1))
+ outer = int(re.search(r"--kill-after=30s (\d+)m", run["run"]).group(1))
+ assert 10540 < guest < outer * 60 < int(run["timeout-minutes"]) * 60
+ assert int(run["timeout-minutes"]) + 40 <= int(job["timeout-minutes"]) <= 360
+ bounded_steps = sum(int(item.get("timeout-minutes", 0)) for item in job["steps"])
+ assert bounded_steps + 10 <= int(job["timeout-minutes"]) # setup, diagnostics and artifact upload
+ assert job["steps"].index(step("Build the clean W5 deployment image")) < job["steps"].index(run)
+ assert job["steps"].index(step("Acquire only the fixed W5 ONNX and tokenizer assets")) < job["steps"].index(run)
+ upload = step("Upload W5 reports and failure diagnostics only")
+ assert upload["if"] == "always()" and upload["with"]["retention-days"] == "30"
+ assert not any(suffix in upload["with"]["path"] for suffix in (".bin", ".npy", ".onnx", ".elf", ".zip"))
+ gate = step("Require complete W5 generation evidence")
+ assert gate["if"] == "always()" and "scripts/check_w5_ci.py" in gate["run"]
+ assert "--exit-code-file output/w5-ci/runner-exit-code.txt" in gate["run"]
+ assert "--log-exit-code-file output/w5-ci/log-exit-code.txt" in gate["run"]
+ assert all("continue-on-error" not in item for item in job["steps"])
+
+
+@pytest.mark.parametrize("exit_code", [0, 1, 124, 130, 143])
+@pytest.mark.parametrize("log_failure", [False, True], ids=["log-ok", "log-full"])
+def test_w5_deployment_pipeline_preserves_actual_exit(bash, tmp_path, exit_code, log_failure):
+ if log_failure and not Path("/dev/full").exists():
+ pytest.skip("Real tee write-failure check requires /dev/full")
+ output = tmp_path / "output/w5-ci"
+ output.mkdir(parents=True)
+ if log_failure:
+ (output / "runner.log").symlink_to("/dev/full")
+ script = step("Run the frozen W5 suite through the deployment script")["run"]
+ # Replace only the expensive producer; execute the actual workflow and real tee.
+ wrapper = ('timeout() { printf "%s\\n" "$@" > "$ARG_LOG"; '
+ 'printf "generation stdout\\n"; printf "generation stderr\\n" >&2; '
+ 'return "$MOCK_EXIT"; }\n')
+ capture = tmp_path / "arguments.txt"
+ result = subprocess.run([bash, "--noprofile", "--norc", "-c", wrapper + script], cwd=tmp_path,
+ env=dict(os.environ, ARG_LOG=str(capture), MOCK_EXIT=str(exit_code)),
+ capture_output=True, text=True, timeout=10)
+ assert (result.returncode != 0) is (log_failure or exit_code != 0), result.stderr
+ log_exit = int((output / "log-exit-code.txt").read_text())
+ assert (log_exit != 0) is log_failure
+ assert result.returncode == (exit_code or log_exit), result.stderr
+ assert int((output / "runner-exit-code.txt").read_text()) == exit_code
+ assert "generation stdout" in result.stdout and "generation stderr" in result.stdout
+ if log_failure:
+ assert "runner.log" in result.stderr
+ else:
+ assert (output / "runner.log").read_text() == "generation stdout\ngeneration stderr\n"
+ args = capture.read_text().splitlines()
+ assert "scripts/deploy_w5.sh" in args and "--reference-suite" in args
+ assert args[args.index("--total-timeout") + 1] == "16500"
+
+
+@pytest.mark.parametrize("status_file", ["runner-exit-code.txt", "log-exit-code.txt"])
+def test_w5_deployment_pipeline_rejects_status_write_failure(bash, tmp_path, status_file):
+ if not Path("/dev/full").exists():
+ pytest.skip("Real status write-failure check requires /dev/full")
+ output = tmp_path / "output/w5-ci"
+ output.mkdir(parents=True)
+ (output / status_file).symlink_to("/dev/full")
+ script = step("Run the frozen W5 suite through the deployment script")["run"]
+ wrapper = 'timeout() { printf "generation completed\\n"; return 0; }\n'
+ result = subprocess.run([bash, "--noprofile", "--norc", "-c", wrapper + script],
+ cwd=tmp_path, capture_output=True, text=True, timeout=10)
+ assert result.returncode != 0, result.stderr
+ assert (output / "runner.log").read_text() == "generation completed\n"
+ assert result.stderr # The status write, not generation or tee, must fail.
+ if status_file == "log-exit-code.txt":
+ assert (output / "runner-exit-code.txt").read_text() == "0\n"
+
+
+@pytest.mark.parametrize("cpus,ram,disk,passed", [
+ (4, 10, 10, True), (8, 16, 30, True),
+ (3, 16, 30, False), (4, 9, 30, False), (4, 16, 9, False)])
+def test_w5_resource_shortfall_fails_before_generation(tmp_path, monkeypatch, cpus, ram, disk, passed):
+ (tmp_path / "output/w5-ci").mkdir(parents=True)
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(os, "sched_getaffinity", lambda _: set(range(cpus)))
+ monkeypatch.setattr(shutil, "disk_usage", lambda _: SimpleNamespace(free=disk * 1024**3))
+ real_read = Path.read_text
+ def memory(path, *args, **kwargs):
+ if path.as_posix() == "/proc/meminfo":
+ return f"MemTotal: 16777216 kB\\nMemAvailable: {ram * 1024**2} kB\\n".replace("\\n", "\n")
+ return real_read(path, *args, **kwargs)
+ monkeypatch.setattr(Path, "read_text", memory)
+ code = python_body("Require four CPUs and capacity for 24 retained logits")
+ if passed:
+ exec(compile(code, "w5-resource-preflight", "exec"), {})
+ else:
+ with pytest.raises(SystemExit, match="generation NOT RUN"):
+ exec(compile(code, "w5-resource-preflight", "exec"), {})
+ row = json.loads((tmp_path / "output/w5-ci/resource-preflight.json").read_text())
+ assert row["passed"] is passed and row["available_cpus"] == cpus
+
+
+@pytest.mark.parametrize("unit,full,w5,requested,expected", [
+ ("success", "skipped", "success", "true", True),
+ ("success", "skipped", "skipped", "false", True),
+ ("success", "skipped", "skipped", "true", False),
+ ("success", "skipped", "failure", "true", False),
+ ("success", "skipped", "cancelled", "true", False),
+ ("success", "skipped", "", "true", False),
+ ("failure", "skipped", "success", "true", False),
+ ("cancelled", "skipped", "success", "true", False),
+ ("skipped", "skipped", "success", "true", False),
+ ("success", "success", "success", "true", False),
+ ("success", "skipped", "success", "false", False),
+])
+def test_w5_summary_cannot_promote_skip_or_duplicate_full_execution(
+ tmp_path, unit, full, w5, requested, expected):
+ (tmp_path / "output/w4-status").mkdir(parents=True)
+ env = dict(os.environ, UNIT_RESULT=unit, FULL_NUMERIC_RESULT=full, W5_GENERATION_RESULT=w5,
+ FULL_REQUESTED="false", W5_REQUESTED=requested, SELECTED_CASE="full_seed_0",
+ GITHUB_SHA="a" * 40, GITHUB_RUN_ID="123", GITHUB_RUN_ATTEMPT="1",
+ GITHUB_STEP_SUMMARY=str(tmp_path / "summary.md"))
+ result = subprocess.run([sys.executable, "-c", python_body("Record W4 CI result without promoting skipped gates")],
+ cwd=tmp_path, env=env, capture_output=True, text=True)
+ assert (result.returncode == 0) is expected, result.stderr
+ row = json.loads((tmp_path / "output/w4-status/status.json").read_text())
+ assert row["w5_generation_accepted"] is (expected and requested == "true")
+ assert row["full_numeric_accepted"] is False
+
+
+@pytest.mark.parametrize("event,w5,w3,w4,passed", [
+ ("workflow_dispatch", True, False, False, True),
+ ("workflow_dispatch", True, True, False, False),
+ ("workflow_dispatch", True, False, True, False),
+ ("workflow_dispatch", True, True, True, False),
+ ("workflow_dispatch", False, True, True, True),
+ ("pull_request", False, False, False, True),
+ ("schedule", False, False, False, True),
+ ("schedule", True, False, False, False),
+])
+def test_registered_entry_rejects_conflicts_before_any_expensive_job(tmp_path, event, w5, w3, w4, passed):
+ data = workflow(LLM_PATH)
+ choice = data["on"]["workflow_dispatch"]["inputs"]["run_w5_validation"]
+ assert choice["type"] == "boolean" and choice["default"] == "false"
+ caller = data["jobs"]["w5-validation"]
+ assert caller["uses"] == "./.github/workflows/w4-full-numeric.yml"
+ assert caller["with"] == {"run_w5": "true"}
+ for name in ("w5-validation", "w3-validation", "w4-validation", "llm-deploy",
+ "full-qwen3-onnx", "full-qwen3-frontend"):
+ assert data["jobs"][name]["needs"] == "validation-selection"
+ for name in ("llm-deploy", "full-qwen3-onnx", "full-qwen3-frontend", "w2-acceptance"):
+ assert "!inputs.run_w5_validation" in data["jobs"][name]["if"]
+ result = subprocess.run([sys.executable, "-c", python_body("Reject conflicting W5 and legacy full selections", LLM_PATH)],
+ cwd=tmp_path, env=dict(os.environ, GITHUB_EVENT_NAME=event, W5_REQUESTED=str(w5).lower(),
+ W3_REQUESTED=str(w3).lower(), W4_REQUESTED=str(w4).lower()),
+ capture_output=True, text=True)
+ assert (result.returncode == 0) is passed, result.stderr
+
+
+@pytest.mark.parametrize("event,mode,legacy,w5,allowed", [
+ ("workflow_dispatch", "", False, True, True),
+ ("workflow_dispatch", "small", False, False, True),
+ ("workflow_dispatch", "w5-generation", False, False, True),
+ ("workflow_dispatch", "", True, False, True),
+ ("schedule", "", True, False, True),
+ ("workflow_dispatch", "", True, True, False),
+ ("workflow_dispatch", "w4-forward", False, True, False),
+ ("workflow_dispatch", "w5-generation", True, False, False),
+ ("pull_request", "", False, True, False),
+ ("schedule", "", False, True, False),
+])
+def test_reusable_w5_input_is_manual_and_mutually_exclusive(tmp_path, event, mode, legacy, w5, allowed):
+ data = workflow()
+ option = data["on"]["workflow_call"]["inputs"]["run_w5"]
+ assert option["type"] == "boolean" and option["default"] == "false"
+ unit = data["jobs"]["unit"]
+ first = unit["steps"][0]
+ assert first["name"] == "Reject conflicting or nonmanual full-generation selections"
+ result = subprocess.run([sys.executable, "-c", python_body(first["name"])], cwd=tmp_path,
+ env=dict(os.environ, GITHUB_EVENT_NAME=event, RUN_MODE=mode,
+ W5_REQUESTED=str(w5).lower(), W4_REQUESTED=str(legacy).lower()),
+ capture_output=True, text=True)
+ assert (result.returncode == 0) is allowed, result.stderr
+ w4_enabled = _full_enabled(data["jobs"]["full-numeric"]["if"], event, mode, legacy, w5)
+ w5_enabled = _full_enabled(data["jobs"]["w5-generation"]["if"], event, mode, legacy, w5)
+ assert not (w4_enabled and w5_enabled)
+ if w5 and allowed:
+ assert w5_enabled and not w4_enabled
+ if not allowed:
+ assert not w5_enabled and not w4_enabled
+
+
+@pytest.mark.parametrize("selection,result,legacy,requested,expected", [
+ ("success", "success", "skipped", "true", "PASS"),
+ ("success", "skipped", "skipped", "true", "FAIL"),
+ ("success", "cancelled", "skipped", "true", "FAIL"),
+ ("success", "failure", "skipped", "true", "FAIL"),
+ ("failure", "skipped", "skipped", "true", "FAIL"),
+ ("cancelled", "skipped", "skipped", "true", "FAIL"),
+ ("success", "success", "success", "true", "FAIL"),
+ ("success", "success", "failure", "true", "FAIL"),
+ ("success", "success", "cancelled", "true", "FAIL"),
+ ("success", "skipped", "success", "false", "NOT RUN"),
+ ("success", "success", "skipped", "false", "FAIL"),
+])
+def test_w5_entry_summary_requires_success_and_no_legacy_execution(
+ tmp_path, selection, result, legacy, requested, expected):
+ job = workflow(LLM_PATH)["jobs"]["w5-execution-summary"]
+ assert job["if"] == "always()"
+ results = {name: {"result": legacy} for name in job["needs"]}
+ results["validation-selection"] = {"result": selection}
+ results["w5-validation"] = {"result": result}
+ summary = tmp_path / "summary.md"
+ completed = subprocess.run([sys.executable, "-c", python_body("Require the requested W5-only execution", LLM_PATH)],
+ cwd=tmp_path, env=dict(os.environ, GITHUB_EVENT_NAME="workflow_dispatch", W5_REQUESTED=requested,
+ W5_JOB_RESULTS=json.dumps(results), GITHUB_STEP_SUMMARY=str(summary)),
+ capture_output=True, text=True)
+ assert (completed.returncode == 0) is (expected != "FAIL"), completed.stderr
+ assert f"**{expected}**" in summary.read_text()
+
+
+@pytest.mark.parametrize("suffix", ["None", "Give brief answers."])
+@pytest.mark.parametrize("outcome", ["pass", "skipped", "failure", "error", "missing", "duplicate"])
+def test_w5_only_template_gate_uses_existing_image_and_rejects_missing_execution(tmp_path, suffix, outcome):
+ import xml.etree.ElementTree as ET
+ item = step("Require the official three-round template in the clean W5 image")
+ command = item["run"]
+ assert "docker run --rm --init" in command and '"$SCRATCHV_W5_IMAGE"' in command
+ assert "SCRATCHV_QWEN3_TOKENIZER_DIR=/assets/tokenizer" in command
+ assert "dst=/tmp/test_conversation.py,readonly" in command
+ assert "--reference-suite" not in command and "scripts/run_inference.py" not in command
+ assert "pip install" not in command and "docker build" not in command
+ job = workflow()["jobs"]["w5-generation"]
+ assert job["steps"].index(step("Acquire only the fixed W5 ONNX and tokenizer assets")) < job["steps"].index(item)
+ assert job["steps"].index(item) < job["steps"].index(step("Run the frozen W5 suite through the deployment script"))
+ suite = ET.Element("testsuite")
+ for current in ("None", "Give brief answers."):
+ if current == suffix and outcome == "missing":
+ continue
+ name = f"test_official_three_round_rendering_matches_transformers[{current}]"
+ row = ET.SubElement(suite, "testcase", name=name)
+ if current == suffix:
+ if outcome in ("skipped", "failure", "error"):
+ ET.SubElement(row, outcome)
+ elif outcome == "duplicate":
+ ET.SubElement(suite, "testcase", name=name)
+ target = tmp_path / "output/w5-ci/multiturn-tests.xml"
+ target.parent.mkdir(parents=True)
+ ET.ElementTree(suite).write(target)
+ result = subprocess.run([sys.executable, "-c", python_body(item["name"])], cwd=tmp_path,
+ capture_output=True, text=True)
+ assert (result.returncode == 0) is (outcome == "pass"), result.stderr
diff --git a/tests/test_w5_ci.py b/tests/test_w5_ci.py
new file mode 100644
index 00000000..ff27baa3
--- /dev/null
+++ b/tests/test_w5_ci.py
@@ -0,0 +1,273 @@
+"""Model-free W5 CI evidence checks using the frozen generation contract."""
+
+from copy import deepcopy
+from importlib.util import module_from_spec, spec_from_file_location
+import json
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+POLICIES = {
+ "harts": 4,
+ "qemu_workspace_layout": "none",
+ "matmul_policy": "blocked_fma",
+ "mean_policy": "two-four-lane-last-axis",
+ "nonlinear_policy": "fp32-reference",
+ "matmul_tile": "2x4",
+ "matmul_rhs_layout": "column-panel",
+}
+
+
+@pytest.fixture
+def gate():
+ spec = spec_from_file_location("w5_ci_gate_tests", ROOT / "scripts/check_w5_ci.py")
+ module = module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+@pytest.fixture
+def reference():
+ return json.loads((ROOT / "probes/w5_generation/references.json").read_text())
+
+
+@pytest.fixture
+def report(reference):
+ cases, numeric_steps, runs = [], [], []
+ for expected in reference["cases"]:
+ cases.append({
+ key: deepcopy(expected[key])
+ for key in ("name", "prompt", "max_new_tokens", "prompt_token_ids",
+ "generated_token_ids", "generated_text", "stop_reason")
+ })
+ cases[-1].update(numeric_step_count=8, forward_count=8, reference_passed=True)
+ for step in expected["steps"]:
+ numeric_steps.append({
+ "passed": True, "selected_row_passed": True, "max_abs": 0.0002,
+ "atol": 0.001, "rtol": 0.0, "elements_compared": 38895616,
+ "case": expected["name"], "step": step["step"],
+ "valid_length": step["valid_length"],
+ "qemu_token_id": step["token_id"], "ort_token_id": step["token_id"],
+ "reference_token_id": step["token_id"], "reference_token_passed": True,
+ })
+ runs.append({
+ **POLICIES, "step": len(runs), "valid_length": step["valid_length"],
+ "passed": True, "status": "PASS", "guest_completed": True,
+ "qemu_reaped": True, "exit_code": 0,
+ "elf_sha256": "a" * 64, "weight_sha256": "b" * 64,
+ })
+ rows = []
+ for case, bits, inexact in (
+ ("rounding_sensitive", "0xbf1825f2", 1),
+ ("fused_cancellation", "0xa8800000", 0),
+ ):
+ for repeat in range(6):
+ requested = int(repeat in (1, 4))
+ rows.append({
+ "case": case, "repeat": repeat, "instruction_rm": 0 if repeat < 3 else 7,
+ "requested_fflags": requested, "fcsr_before": requested,
+ "fcsr_after": requested | inexact, "expected_fcsr_after": requested | inexact,
+ "actual_bits": bits, "expected_bits": bits,
+ "bits_match": True, "csr_match": True, "passed": True,
+ })
+ return {
+ **POLICIES, "schema_version": 1, "status": "PASS", "passed": True,
+ "stage": "complete", "mode": "raw", "harts_requested": 4,
+ "cases": cases, "numeric_steps": numeric_steps, "runs": runs,
+ "qemu_forwards": 24, "forward_attempts": 24, "full_model_gate": "PASS",
+ "build": {
+ **POLICIES, "sha256": "a" * 64,
+ "w5_graph_audit": {"passed": True, "layer_count": 28, "tied_embeddings": True},
+ },
+ "fma_conformance": {
+ "passed": True, "status": "PASS",
+ "comparison": {"passed": True, "criterion": "exact binary32 bits and FCSR",
+ "rows": rows, "failed_rows": 0},
+ },
+ }
+
+
+def test_complete_three_by_eight_evidence_passes_without_diagnostic_extras(gate, report, reference):
+ assert len(report["cases"]) == 3
+ assert len(report["runs"]) == len(report["numeric_steps"]) == 24
+ # Production reports need no optional frozen_* diagnostics or output arrays.
+ assert not any(key.startswith("frozen_") for key in report["numeric_steps"][0])
+ assert gate.validate_report(report, reference) is None
+
+
+@pytest.mark.parametrize("collection", ["cases", "numeric_steps", "runs"])
+@pytest.mark.parametrize("damage", ["missing", "duplicate"])
+def test_incomplete_or_duplicate_evidence_cannot_be_hidden_by_pass(gate, report, reference, collection, damage):
+ if damage == "missing":
+ report[collection].pop()
+ else:
+ report[collection][-1] = deepcopy(report[collection][0])
+ with pytest.raises(ValueError):
+ gate.validate_report(report, reference)
+
+
+@pytest.mark.parametrize("field,value", [
+ ("generated_token_ids", [12095] * 8),
+ ("generated_text", "Paris"),
+ ("stop_reason", "eos"),
+ ("prompt_token_ids", [1, 2, 3, 4, 5]),
+ ("numeric_step_count", 7),
+])
+def test_frozen_output_and_history_must_match(gate, report, reference, field, value):
+ report["cases"][0][field] = value
+ with pytest.raises(ValueError):
+ gate.validate_report(report, reference)
+
+
+@pytest.mark.parametrize("field,value", [
+ ("max_abs", 0.001),
+ ("max_abs", float("nan")),
+ ("max_abs", float("inf")),
+ ("atol", 0.01),
+ ("rtol", 0.001),
+ ("elements_compared", 151936),
+ ("passed", False),
+ ("qemu_token_id", 0),
+ ("ort_token_id", 0),
+ ("valid_length", 6),
+])
+def test_each_step_needs_strict_full_logits_and_frozen_token_evidence(gate, report, reference, field, value):
+ report["numeric_steps"][0][field] = value
+ with pytest.raises(ValueError):
+ gate.validate_report(report, reference)
+
+
+@pytest.mark.parametrize("field,value", [
+ ("qemu_reaped", False),
+ ("guest_completed", False),
+ ("exit_code", 130),
+ ("status", "CANCELLED"),
+ ("error", "deadline expired"),
+ ("cleanup_errors", ["unable to reap QEMU"]),
+ ("elf_sha256", "c" * 64),
+ ("weight_sha256", "c" * 64),
+])
+def test_cancel_failure_or_different_executable_cannot_pass(gate, report, reference, field, value):
+ report["runs"][-1][field] = value
+ with pytest.raises(ValueError):
+ gate.validate_report(report, reference)
+
+
+@pytest.mark.parametrize("location,field,value", [
+ ("top", "harts", 1),
+ ("top", "qemu_workspace_layout", "two-buffer-pad-4k"),
+ ("top", "matmul_tile", "1x4"),
+ ("build", "mean_policy", "sequential"),
+ ("build", "nonlinear_policy", "libm"),
+ ("run", "mean_policy", "sequential"),
+ ("run", "nonlinear_policy", "libm"),
+])
+def test_claimed_strategy_must_match_build_and_every_run(gate, report, reference, location, field, value):
+ target = report if location == "top" else report["build"] if location == "build" else report["runs"][7]
+ target[field] = value
+ with pytest.raises(ValueError):
+ gate.validate_report(report, reference)
+
+
+def test_small_graph_cannot_claim_full_model_gate(gate, report, reference):
+ report["build"]["w5_graph_audit"]["layer_count"] = 2
+ with pytest.raises(ValueError):
+ gate.validate_report(report, reference)
+
+
+@pytest.mark.parametrize("damage", ["missing", "failed"])
+def test_fma_summary_cannot_mask_incomplete_or_failing_rows(gate, report, reference, damage):
+ rows = report["fma_conformance"]["comparison"]["rows"]
+ if damage == "missing":
+ rows.pop()
+ else:
+ rows[7]["csr_match"] = False
+ with pytest.raises(ValueError):
+ gate.validate_report(report, reference)
+
+
+@pytest.mark.parametrize("exit_text", ["0\n", "130\n", ""])
+@pytest.mark.parametrize("log_text", [
+ pytest.param(None, id="legacy-no-log-check"),
+ pytest.param("0\n", id="log-success"),
+ pytest.param("1\n", id="log-failure"),
+ pytest.param("", id="log-empty"),
+ pytest.param("invalid\n", id="log-invalid"),
+ pytest.param("missing", id="log-missing"),
+ pytest.param("directory", id="log-unreadable"),
+])
+def test_cli_checks_real_exit_code_and_writes_status(gate, report, tmp_path, monkeypatch, exit_text, log_text):
+ report_path = tmp_path / "report.json"
+ report_path.write_text(json.dumps(report))
+ exit_path = tmp_path / "exit-code.txt"
+ exit_path.write_text(exit_text)
+ output = tmp_path / "gate.json"
+ summary = tmp_path / "summary.md"
+ monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary))
+ argv = ["--report", str(report_path), "--exit-code-file", str(exit_path),
+ "--output", str(output)]
+ if log_text is not None:
+ log_path = tmp_path / "log-exit-code.txt"
+ if log_text == "directory":
+ log_path.mkdir()
+ elif log_text != "missing":
+ log_path.write_text(log_text)
+ argv += ["--log-exit-code-file", str(log_path)]
+ expected = 0 if exit_text == "0\n" and log_text in (None, "0\n") else 1
+ code = gate.main(argv)
+ assert code == expected
+ result = json.loads(output.read_text())
+ assert result["status"] == ("PASS" if expected == 0 else "FAIL")
+ assert result["passed"] is (expected == 0)
+ assert result["status"] in summary.read_text()
+ if exit_text == "0\n" and log_text in ("1\n", "", "invalid\n"):
+ assert "Log capture process" in result["error"]
+ if expected:
+ assert "24/24 numeric steps" not in summary.read_text()
+
+
+def test_missing_run_report_produces_failure_artifact(gate, tmp_path):
+ exit_path = tmp_path / "exit-code.txt"
+ exit_path.write_text("0\n")
+ output = tmp_path / "gate.json"
+ assert gate.main(["--report", str(tmp_path / "missing.json"),
+ "--exit-code-file", str(exit_path), "--output", str(output)]) == 1
+ result = json.loads(output.read_text())
+ assert result["status"] == "FAIL" and result["passed"] is False
+
+
+@pytest.mark.parametrize("form", ["cores", "harts", "both", "mixed"])
+def test_core_field_compatibility_at_report_build_and_each_forward(gate, report, reference, form):
+ records = [report, report["build"], *report["runs"]]
+ for index, record in enumerate(records):
+ count = record.pop("harts")
+ selected = ("cores" if index % 2 else "harts") if form == "mixed" else form
+ if selected in ("cores", "both"):
+ record["cores"] = count
+ if selected in ("harts", "both"):
+ record["harts"] = count
+ assert gate.validate_report(report, reference) is None
+
+
+@pytest.mark.parametrize("location", ["top", "build", "run"])
+@pytest.mark.parametrize("fields", [
+ {},
+ {"cores": 1},
+ {"cores": True},
+ {"cores": 4.0},
+ {"cores": "4"},
+ {"cores": None},
+ {"cores": 4, "harts": 1},
+ {"cores": 1, "harts": 4},
+ {"cores": 4, "harts": 4.0},
+ {"cores": 4, "harts": None},
+])
+def test_missing_invalid_or_conflicting_core_fields_cannot_pass(
+ gate, report, reference, location, fields):
+ target = report if location == "top" else report["build"] if location == "build" else report["runs"][7]
+ target.pop("harts")
+ target.update(fields)
+ with pytest.raises(ValueError):
+ gate.validate_report(report, reference)
diff --git a/tests/test_w5_deployment.py b/tests/test_w5_deployment.py
new file mode 100644
index 00000000..09c4464f
--- /dev/null
+++ b/tests/test_w5_deployment.py
@@ -0,0 +1,244 @@
+"""Exercise deployment orchestration without requiring a container engine."""
+import json
+import os
+from pathlib import Path
+import shlex
+import shutil
+import signal
+import subprocess
+import sys
+import time
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPT = ROOT / "scripts/deploy_w5.sh"
+
+
+@pytest.fixture
+def setup(tmp_path):
+ assets = tmp_path / "model assets"
+ assets.mkdir()
+ output = tmp_path / "new output"
+ calls = tmp_path / "calls.jsonl"
+ engine = tmp_path / "container engine"
+ engine.write_text(
+ "#!" + sys.executable + "\n"
+ "import json, os, signal, subprocess, sys, time\n"
+ "from pathlib import Path\n"
+ "with open(os.environ['ENGINE_CALLS'], 'a') as f:\n"
+ " f.write(json.dumps(sys.argv[1:]) + '\\n')\n"
+ "if sys.argv[1] == os.environ.get('ENGINE_BLOCK_STAGE'):\n"
+ " worker_code = 'import time; time.sleep(60)'\n"
+ " if os.environ.get('ENGINE_IGNORE_TERM'):\n"
+ " signal.signal(signal.SIGTERM, signal.SIG_IGN)\n"
+ " worker_code = 'import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); time.sleep(60)'\n"
+ " worker = subprocess.Popen([sys.executable, '-c', worker_code])\n"
+ " Path(os.environ['ENGINE_PROCESS_FILE']).write_text(json.dumps([os.getpid(), worker.pid]))\n"
+ " time.sleep(60)\n"
+ "sys.exit(int(os.environ.get('ENGINE_' + sys.argv[1].upper() + '_EXIT', '0')))\n"
+ )
+ engine.chmod(0o755)
+ env = dict(os.environ, SCRATCHV_CONTAINER_ENGINE=str(engine),
+ SCRATCHV_W5_ASSETS=str(assets), SCRATCHV_W5_OUTPUT=str(output),
+ ENGINE_CALLS=str(calls))
+ env.pop("SCRATCHV_W5_BASE_IMAGE", None)
+ return env, calls, output
+
+
+def run(env, *args):
+ return subprocess.run(["bash", str(SCRIPT), *args], env=env,
+ text=True, capture_output=True, timeout=10)
+
+
+def recorded(path):
+ return [json.loads(line) for line in path.read_text().splitlines()]
+
+
+def test_unavailable_engine_does_not_create_output(setup):
+ env, calls, output = setup
+ env["SCRATCHV_CONTAINER_ENGINE"] += "-absent"
+ result = run(env, "--help")
+ assert result.returncode == 127
+ assert "unavailable" in result.stderr
+ assert not calls.exists()
+ assert not output.exists()
+
+
+def test_unready_engine_does_not_build_or_create_output(setup):
+ env, calls, output = setup
+ env["ENGINE_INFO_EXIT"] = "1"
+ result = run(env)
+ assert result.returncode == 1
+ assert "not ready" in result.stderr
+ assert recorded(calls) == [["info"]]
+ assert not output.exists()
+
+
+def test_missing_assets_do_not_build_or_run(setup):
+ env, calls, output = setup
+ env["SCRATCHV_W5_ASSETS"] += "-absent"
+ result = run(env)
+ assert result.returncode == 2
+ assert "Assets directory does not exist" in result.stderr
+ assert recorded(calls) == [["info"]]
+ assert not output.exists()
+
+
+def test_build_failure_never_starts_generation(setup):
+ env, calls, output = setup
+ env["ENGINE_BUILD_EXIT"] = "7"
+ result = run(env, "--prompt", "The capital of France is")
+ assert result.returncode == 7
+ assert [call[0] for call in recorded(calls)] == ["info", "build"]
+
+
+@pytest.mark.parametrize("option", ["--cores", "--harts"])
+def test_arguments_mounts_and_invoking_user_are_preserved(setup, option):
+ env, calls, output = setup
+ args = ["--model", "/assets/model.onnx", "--tokenizer-dir", "/assets/tokenizer",
+ "--prompt", "quotes '$HOME'; * stay literal", option, "auto",
+ "--output-dir", "/output/second run"]
+ result = run(env, *args)
+ assert result.returncode == 0, result.stderr
+ invocations = recorded(calls)
+ assert [call[0] for call in invocations] == ["info", "build", "run"]
+ build = invocations[1]
+ assert build[build.index("--build-arg") + 1] == "BASE_IMAGE=ubuntu:24.04"
+ command = invocations[-1]
+ assert "--rm" in command and "--init" in command
+ assert command[command.index("--user") + 1] == f"{os.getuid()}:{os.getgid()}"
+ assert f"type=bind,src={env['SCRATCHV_W5_ASSETS']},dst=/assets,readonly" in command
+ assert f"type=bind,src={output},dst=/output" in command
+ assert command[-len(args):] == args
+ image_index = command.index("scratchv-w5:local")
+ assert command[image_index + 1:image_index + 3] == ["--output-dir", "/output/run"]
+ assert "SCRATCHV_ZIG_CACHE=/output/.zig-cache" in command
+
+
+@pytest.mark.parametrize("value,expected", [
+ ("", "ubuntu:24.04"),
+ ("public.ecr.aws/ubuntu/ubuntu:24.04", "public.ecr.aws/ubuntu/ubuntu:24.04"),
+ ("literal $(printf not-expanded); *", "literal $(printf not-expanded); *"),
+])
+def test_base_image_override_is_one_literal_build_argument(setup, value, expected):
+ env, calls, output = setup
+ env["SCRATCHV_W5_BASE_IMAGE"] = value
+ result = run(env, "--help")
+ assert result.returncode == 0, result.stderr
+ invocations = recorded(calls)
+ assert [call[0] for call in invocations] == ["info", "build", "run"]
+ build = invocations[1]
+ assert build.count("--build-arg") == 1
+ assert build[build.index("--build-arg") + 1] == "BASE_IMAGE=" + expected
+ assert not any(arg.startswith("BASE_IMAGE=") for arg in invocations[-1])
+
+
+def test_model_failure_exit_status_is_propagated(setup):
+ env, calls, output = setup
+ env["ENGINE_RUN_EXIT"] = "9"
+ assert run(env).returncode == 9
+
+
+def test_mount_delimiter_is_rejected_before_build(setup):
+ env, calls, output = setup
+ bad_assets = Path(env["SCRATCHV_W5_ASSETS"]).with_name("bad,assets")
+ bad_assets.mkdir()
+ env["SCRATCHV_W5_ASSETS"] = str(bad_assets)
+ result = run(env)
+ assert result.returncode == 2
+ assert "commas" in result.stderr
+ assert recorded(calls) == [["info"]]
+
+
+def test_container_copy_inputs_exist_and_do_not_include_assets():
+ # Detect a missing packaged entry point without claiming an image was built.
+ dockerfile = ROOT / "deployment/w5/Dockerfile"
+ for line in dockerfile.read_text().splitlines():
+ if not line.startswith("COPY "):
+ continue
+ for source in shlex.split(line)[1:-1]:
+ assert (ROOT / source).exists(), source
+ assert source not in {".", "assets", "output", ".venv", ".git"}
+
+
+def test_packaged_cli_help_resolves_transitive_modules(tmp_path):
+ # Use only the image COPY inputs; no host model or generated ELF is copied.
+ # This catches missing probe/manifest dependencies before a Docker build.
+ dockerfile = ROOT / "deployment/w5/Dockerfile"
+ for line in dockerfile.read_text().splitlines():
+ if not line.startswith("COPY "):
+ continue
+ parts = shlex.split(line)
+ for source in parts[1:-1]:
+ src = ROOT / source
+ target = tmp_path / source
+ target.parent.mkdir(parents=True, exist_ok=True)
+ if src.is_file():
+ shutil.copyfile(src, target)
+ elif source == "probes":
+ for item in src.rglob("*"):
+ if item.is_file() and item.suffix in {".py", ".json"}:
+ dest = target / item.relative_to(src)
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copyfile(item, dest)
+ else:
+ shutil.copytree(src, target,
+ ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
+ env = dict(os.environ, PYTHONPATH=str(tmp_path), PYTHONDONTWRITEBYTECODE="1")
+ result = subprocess.run([sys.executable, str(tmp_path / "scripts/run_inference.py"), "--help"],
+ cwd=tmp_path, env=env, text=True, capture_output=True, timeout=30)
+ assert result.returncode == 0, result.stderr
+ assert "--reuse-build" in result.stdout
+ assert "--reference-suite" in result.stdout
+
+
+@pytest.mark.parametrize("stage,signum,ignore_term", [
+ ("info", signal.SIGTERM, False),
+ ("info", signal.SIGINT, False),
+ ("build", signal.SIGTERM, False),
+ ("build", signal.SIGINT, False),
+ ("build", signal.SIGTERM, True),
+])
+def test_cancel_stage_reaps_client_and_stops_helpers_without_starting_model(
+ setup, tmp_path, stage, signum, ignore_term,
+):
+ """A signal to Bash alone must reach the owned container-client group."""
+ env, calls, output = setup
+ process_file = tmp_path / "stage-processes.json"
+ env.update(ENGINE_BLOCK_STAGE=stage, ENGINE_PROCESS_FILE=str(process_file))
+ if ignore_term:
+ env["ENGINE_IGNORE_TERM"] = "1"
+ process = subprocess.Popen(["bash", str(SCRIPT)], env=env, start_new_session=True,
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ owned = []
+ try:
+ deadline = time.monotonic() + 5
+ while not process_file.exists() and time.monotonic() < deadline:
+ time.sleep(0.01)
+ assert process_file.exists(), "container-client stage did not start"
+ owned = json.loads(process_file.read_text())
+ assert os.getpgid(owned[0]) == owned[0]
+ process.send_signal(signum)
+ stdout, stderr = process.communicate(timeout=10)
+ assert process.returncode == 128 + signum, (stdout, stderr)
+ # The direct client must be reaped, not left a zombie owned by Bash.
+ with pytest.raises(ProcessLookupError):
+ os.kill(owned[0], 0)
+ # A helper can briefly await host PID 1's reap; it must not be running.
+ status = Path(f"/proc/{owned[1]}/stat")
+ if status.exists():
+ assert status.read_text().rsplit(")", 1)[1].split()[0] == "Z"
+ assert [call[0] for call in recorded(calls)] == (
+ ["info"] if stage == "info" else ["info", "build"]
+ )
+ if stage == "info":
+ assert not output.exists()
+ finally:
+ # Ensure a failed regression cannot leave its own blocking substitutes.
+ for pgid in ([owned[0]] if owned else []) + [process.pid]:
+ try:
+ os.killpg(pgid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ process.wait(timeout=5)
diff --git a/tests/test_w5_generation_reference.py b/tests/test_w5_generation_reference.py
new file mode 100644
index 00000000..97ccc84b
--- /dev/null
+++ b/tests/test_w5_generation_reference.py
@@ -0,0 +1,162 @@
+"""Small, model-free checks of the independent W5 reference oracle."""
+
+from dataclasses import replace
+
+import numpy as np
+import pytest
+
+from probes.w5_generation import run
+from scratchv.runtime.llm_inputs import prepare_inputs
+
+
+class Tokenizer:
+ model_vocab_size = 8
+ pad_token_id = 0
+ generation_eos_token_ids = (5, 6)
+
+ def encode(self, text, *, add_special_tokens):
+ assert text == "raw prompt" and add_special_tokens is False
+ return [1, 2]
+
+ def validate_token_ids(self, tokens):
+ if any(not 0 <= token < 7 for token in tokens):
+ raise ValueError("undecodable token ID")
+ return list(tokens)
+
+ def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces):
+ assert clean_up_tokenization_spaces is False
+ pieces = {0: "", 1: "raw", 2: "prompt", 3: "Paris", 4: "London",
+ 5: "", 6: ""}
+ return "".join(pieces[token] for token in tokens
+ if not skip_special_tokens or token not in (0, 5, 6))
+
+
+class Session:
+ def __init__(self):
+ self.calls = []
+
+ def run(self, names, feed):
+ assert names == ["logits"]
+ # The final query exposes all valid keys, including any valid pad ID.
+ length = int(np.count_nonzero(feed["attention_mask"][0, 0, -1] == 0))
+ self.calls.append(tuple(feed["input_ids"][0, :length]))
+ logits = np.zeros((1, 256, 8), dtype=np.float32)
+ logits[0, :, 7] = 100 # An incorrect padding-row selection cannot pass.
+ row = logits[0, length - 1]
+ row[:] = 0
+ if length == 2:
+ row[3] = row[4] = 10 # The independent argmax must prefer lower ID.
+ else:
+ row[6] = 10
+ return [logits]
+
+
+def case():
+ return {"name": "small", "prompt": "raw prompt", "max_new_tokens": 3,
+ "required_substring": "Paris"}
+
+
+def frozen():
+ return {**case(), "prompt_utf8_hex": "raw prompt".encode().hex(),
+ "prompt_token_ids": [1, 2], "generated_token_ids": [3, 6],
+ "generated_text": "Paris", "stop_reason": "eos"}
+
+
+@pytest.mark.parametrize("key,replacement", [
+ ("prompt", "different"), ("prompt_utf8_hex", "00"),
+ ("required_substring", "London"), ("max_new_tokens", 4),
+ ("prompt_token_ids", [2, 1]), ("generated_token_ids", [4, 6]),
+ ("generated_text", "London"), ("stop_reason", "max_new_tokens"),
+])
+def test_frozen_comparison_rejects_each_contract_field_change(key, replacement):
+ expected = frozen()
+ actual = {**expected, key: replacement}
+ assert run.compare_frozen(actual, expected) == {
+ "passed": False, "different_fields": [key],
+ }
+ assert expected[key] != replacement
+
+
+def test_frozen_comparison_ignores_measurement_fields():
+ expected = frozen()
+ actual = {**expected, "wall_seconds": 2.0, "cpu_seconds": 3.0,
+ "process_peak_rss_kib": 100, "steps": [{"timing": 99}]}
+ assert run.compare_frozen(actual, expected) == {
+ "passed": True, "different_fields": [],
+ }
+
+
+def test_run_case_checks_actual_greedy_rows_eos_and_host_append():
+ session = Session()
+ result = run.run_case(case(), Tokenizer(), session)
+ assert session.calls == [(1, 2), (1, 2, 3)]
+ assert result["generated_token_ids"] == [3, 6]
+ assert result["generated_text"] == "Paris"
+ assert result["stop_reason"] == "eos"
+ assert result["semantic_passed"] and result["host_oracle_passed"]
+ assert [step["valid_length"] for step in result["steps"]] == [2, 3]
+ assert [step["token_id"] for step in result["steps"]] == [3, 6]
+ assert result["steps"][0]["top1_top2_margin"] == 0
+ assert run.compare_frozen(result, frozen())["passed"]
+
+
+def test_independent_input_oracle_rejects_wrong_host_append(monkeypatch):
+ def broken_generate(_prompt, *, tokenizer, forward, max_new_tokens):
+ forward(prepare_inputs([1, 2], pad_token_id=0, vocab_size=8))
+ forward(prepare_inputs([1, 2, 4], pad_token_id=0, vocab_size=8))
+
+ monkeypatch.setattr(run, "generate", broken_generate)
+ session = Session()
+ with pytest.raises(ValueError, match="input_ids"):
+ run.run_case(case(), Tokenizer(), session)
+ # checked_inputs must fail before a second session call.
+ assert session.calls == [(1, 2)]
+
+
+def test_independent_stop_oracle_rejects_extra_host_forward(monkeypatch):
+ original_generate = run.generate
+
+ def broken_generate(prompt, *, tokenizer, forward, max_new_tokens):
+ result = original_generate(prompt, tokenizer=tokenizer, forward=forward,
+ max_new_tokens=max_new_tokens)
+ forward(prepare_inputs([1, 2, 3, 6], pad_token_id=0, vocab_size=8))
+ return result
+
+ monkeypatch.setattr(run, "generate", broken_generate)
+ session = Session()
+ with pytest.raises(AssertionError, match="after the independent stop"):
+ run.run_case(case(), Tokenizer(), session)
+ assert session.calls == [(1, 2), (1, 2, 3)]
+
+
+@pytest.mark.parametrize("field,value", [
+ ("prompt_token_ids", (2, 1)), ("generated_token_ids", (4, 6)),
+ ("generated_text", "London"), ("stop_reason", "max_new_tokens"),
+])
+def test_independent_final_oracle_rejects_corrupted_host_result(monkeypatch, field, value):
+ original_generate = run.generate
+
+ def broken_generate(*args, **kwargs):
+ result = original_generate(*args, **kwargs)
+ return replace(result, **{field: value})
+
+ monkeypatch.setattr(run, "generate", broken_generate)
+ with pytest.raises(AssertionError, match="Host"):
+ run.run_case(case(), Tokenizer(), Session())
+
+
+def test_reference_diagnostic_margin_handles_all_finite_fp32_logits():
+ import json
+
+ class ExtremeSession:
+ def run(self, names, feed):
+ logits = np.full((1, 256, 8), -np.finfo(np.float32).max, dtype=np.float32)
+ logits[0, 1, 3] = np.finfo(np.float32).max
+ return [logits]
+
+ with np.errstate(over="raise"):
+ result = run.run_case({**case(), "max_new_tokens": 1}, Tokenizer(), ExtremeSession())
+ assert result["generated_token_ids"] == [3]
+ assert result["generated_text"] == "Paris"
+ assert result["steps"][0]["top1_top2_margin"] == 2.0 * float(np.finfo(np.float32).max)
+ json.dumps(result, allow_nan=False)
diff --git a/tests/test_w5_inference_cli.py b/tests/test_w5_inference_cli.py
new file mode 100644
index 00000000..a0d238c4
--- /dev/null
+++ b/tests/test_w5_inference_cli.py
@@ -0,0 +1,849 @@
+"""W5 CLI orchestration boundaries without a model, compiler or QEMU."""
+
+from importlib.util import module_from_spec, spec_from_file_location
+import json
+from pathlib import Path
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+
+from scratchv.runtime.conversation import Conversation
+from scratchv.runtime.generation import GenerationCancelledError
+
+
+@pytest.fixture
+def cli():
+ path = Path(__file__).resolve().parents[1] / "scripts/run_inference.py"
+ spec = spec_from_file_location("w5_inference_cli_tests", path)
+ module = module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+class Tokenizer:
+ model_vocab_size = 8
+ tokenizer_vocab_size = 7
+ pad_token_id = 0
+ generation_eos_token_ids = (6,)
+
+ def __init__(self):
+ self.prompts = []
+
+ def encode(self, text, *, add_special_tokens):
+ assert add_special_tokens is False
+ self.prompts.append(text)
+ return [1, 2]
+
+ def validate_token_ids(self, tokens):
+ if any(type(token) is not int or not 0 <= token < 7 for token in tokens):
+ raise ValueError("unknown token")
+ return list(tokens)
+
+ def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces):
+ return "x" * sum(token == 3 for token in tokens)
+
+
+def logits(token=3):
+ result = np.zeros((1, 256, 8), dtype=np.float32)
+ result[0, :, token] = 1
+ return result
+
+
+@pytest.fixture
+def runtime(cli, monkeypatch):
+ tokenizer = Tokenizer()
+ executable = SimpleNamespace(
+ evidence={"matmul_policy": "sequential", "qemu_workspace_layout": "none"},
+ toolchain=SimpleNamespace(cc="unused-cc", qemu="unused-qemu"), cores=1,
+ )
+ state = SimpleNamespace(tokenizer=tokenizer, exe=executable, prepared=[], adapters=[],
+ saved=[], fma=[], failure=None, oracle=None, cores=[])
+
+ def prepare(args, out, cores):
+ state.prepared.append((args, out, cores))
+ return executable
+
+ class Adapter:
+ def __init__(self, exe, output, *, timeout):
+ self.exe, self.output, self.timeout = exe, output, timeout
+ self.vocab_size = tokenizer.model_vocab_size
+ self.runs, self.inputs = [], []
+ state.adapters.append(self)
+
+ def __call__(self, prepared):
+ self.inputs.append(prepared)
+ if state.failure is not None:
+ self.runs.append({"guest_completed": False})
+ raise state.failure
+ self.runs.append({"guest_completed": True})
+ return logits()
+
+ def fma(*args, **kwargs):
+ state.fma.append((args, kwargs))
+ return {"passed": True, "status": "PASS"}
+
+ monkeypatch.setattr(cli.Qwen3Tokenizer, "from_directory", lambda path: tokenizer)
+ monkeypatch.setattr(cli, "prepare_executable", prepare)
+ monkeypatch.setattr(cli, "save_executable", lambda exe, out: state.saved.append((exe, out)) or "fixture-metadata-identity")
+ monkeypatch.setattr(cli, "QemuForward", Adapter)
+ monkeypatch.setattr(cli, "run_fma_probe", fma)
+ monkeypatch.setattr(cli, "oracle_session", lambda path: SimpleNamespace(run=lambda names, feed: [state.oracle]))
+ monkeypatch.setattr(cli, "resolve_cores", lambda requested: state.cores.append(requested) or (4 if requested == "auto" else requested))
+ return state
+
+
+def argv(tmp_path, *extra):
+ return ["--model", "small.onnx", "--tokenizer-dir", "tokenizer", "--output-dir", str(tmp_path / "run"), *extra]
+
+
+def report(tmp_path):
+ return json.loads((tmp_path / "run/report.json").read_text(encoding="utf-8"))
+
+
+@pytest.mark.parametrize("flag", ["--cores", "--harts"])
+@pytest.mark.parametrize("requested,resolved", [(None, 1), ("1", 1), ("2", 2), ("4", 4), ("auto", 4)])
+def test_raw_multitoken_one_build_one_adapter_and_cores_resolved_once(cli, runtime, tmp_path, flag, requested, resolved):
+ options = [] if requested is None else [flag, requested]
+ assert cli.main(argv(tmp_path, "--prompt", " raw text ", "--max-new-tokens", "3", *options)) == 0
+ value = report(tmp_path)
+ assert value["status"] == "PASS" and value["passed"]
+ assert value["full_model_gate"] == "NOT_RUN"
+ assert value["qemu_workspace_layout"] == "none" and value["mean_policy"] == "sequential"
+ assert value["nonlinear_policy"] == "libm"
+ assert value["executable_sha256"] == "fixture-metadata-identity"
+ assert value["cores"] == value["harts"] == resolved
+ assert value["cores_requested"] == value["harts_requested"]
+ assert runtime.cores == ["auto" if requested == "auto" else int(requested or 1)]
+ assert len(runtime.prepared) == len(runtime.saved) == len(runtime.adapters) == 1
+ assert len(runtime.fma) == 1
+ assert runtime.adapters[0].exe is runtime.exe
+ assert [item.valid_length for item in runtime.adapters[0].inputs] == [2, 3, 4]
+ assert runtime.tokenizer.prompts == [" raw text "]
+ assert value["cases"][0]["generated_token_ids"] == [3, 3, 3]
+ assert value["qemu_forwards"] == 3
+
+
+def test_three_chat_turns_reuse_adapter_and_commit_ordered_history(cli, runtime, tmp_path, monkeypatch):
+ template = "{% for m in messages %}{{ m.role }}:{{ m.content }}|{% endfor %}assistant:"
+ chat = Conversation(runtime.tokenizer, template, system_prompt="brief")
+ monkeypatch.setattr(Conversation, "from_directory", lambda *a, **kw: chat)
+ assert cli.main(argv(tmp_path, "--mode", "chat", "--system-prompt", "brief",
+ "--chat-turn", "first", "--chat-turn", "second", "--chat-turn", "third",
+ "--max-new-tokens", "2")) == 0
+ value = report(tmp_path)
+ assert [x["content"] for x in value["history"]] == ["brief", "first", "xx", "second", "xx", "third", "xx"]
+ assert runtime.tokenizer.prompts == [
+ "system:brief|user:first|assistant:",
+ "system:brief|user:first|assistant:xx|user:second|assistant:",
+ "system:brief|user:first|assistant:xx|user:second|assistant:xx|user:third|assistant:",
+ ]
+ assert len(runtime.adapters) == len(runtime.prepared) == 1
+ assert value["qemu_forwards"] == 6 and len(value["cases"]) == 3
+
+
+def test_build_only_never_executes_fma_or_forward_and_not_pass(cli, runtime, tmp_path):
+ assert cli.main(argv(tmp_path, "--build-only")) == 0
+ value = report(tmp_path)
+ assert value["status"] == "BUILD_ONLY" and value["passed"] is False
+ assert value["full_model_gate"] == "NOT_RUN" and value["qemu_forwards"] == 0
+ assert runtime.fma == [] and runtime.adapters[0].inputs == []
+
+
+@pytest.mark.parametrize("error,expected,status", [
+ (GenerationCancelledError("cancelled"), 130, "CANCELLED"),
+ (KeyboardInterrupt(), 130, "CANCELLED"),
+ (TimeoutError("deadline expired"), 1, "FAIL"),
+ (RuntimeError("backend failed"), 1, "FAIL"),
+])
+def test_cancel_failure_report_retains_attempt_without_claiming_success(cli, runtime, tmp_path, error, expected, status):
+ runtime.failure = error
+ assert cli.main(argv(tmp_path, "--prompt", "text", "--max-new-tokens", "2")) == expected
+ value = report(tmp_path)
+ assert value["status"] == status and value["passed"] is False
+ assert value["cases"] == [] and value["full_model_gate"] == "NOT_RUN"
+ assert value["forward_attempts"] == 1 and value["qemu_forwards"] == 0
+ assert type(error).__name__ in value["error"]
+
+
+def test_numeric_oracle_failure_is_reported_before_committing_tokens(cli, runtime, tmp_path):
+ runtime.oracle = logits()
+ runtime.oracle[0, -1, 0] = 0.1
+ assert cli.main(argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1", "--verify-onnx", "small.onnx")) == 1
+ value = report(tmp_path)
+ assert value["status"] == "FAIL" and not value["passed"]
+ assert value["numeric_steps"][0]["passed"] is False
+ assert value["cases"] == []
+
+
+def test_numeric_comparison_checks_padding_rows_and_equal_argmax(cli):
+ expected = logits()
+ bad_padding = expected.copy()
+ bad_padding[0, -1, 1] = cli.ATOL * 2
+ assert cli.compare_step(bad_padding, expected, 2, 8)["passed"] is False
+ nonfinite = expected.copy()
+ nonfinite[0, -1, 0] = np.inf
+ with pytest.raises(ValueError):
+ cli.compare_step(nonfinite, expected, 2, 8)
+ expected[0, 1, 4] = 1 + cli.ATOL / 4
+ actual = expected.copy()
+ actual[0, 1, 3], actual[0, 1, 4] = expected[0, 1, 4], expected[0, 1, 3]
+ comparison = cli.compare_step(actual, expected, 2, 8)
+ assert comparison["max_abs"] < cli.ATOL
+ assert comparison["qemu_token_id"] != comparison["ort_token_id"]
+ assert not comparison["passed"]
+
+
+def matching_reuse_metadata(cli, monkeypatch, path):
+ path.parent.mkdir(parents=True)
+ path.write_text("{}", encoding="utf-8")
+ path.with_name("report.json").write_text(json.dumps({"executable_sha256": "matching"}), encoding="utf-8")
+ monkeypatch.setattr(cli, "sha256_file", lambda metadata: "matching")
+
+
+@pytest.mark.parametrize("changes,cores", [
+ ({"cores": 4}, 1),
+ ({"matmul_policy": "blocked_fma"}, 1),
+ ({"matmul_tile": "2x4"}, 1),
+ ({"matmul_rhs_layout": "column-panel"}, 1),
+ ({"qemu_workspace_layout": "two-buffer-pad-4k"}, 1),
+])
+def test_reuse_rejects_changed_configuration_or_padding(cli, monkeypatch, tmp_path, changes, cores):
+ args = cli.parse_args(["--reuse-build", str(tmp_path / "old/executable.json"),
+ "--tokenizer-dir", "tokens", "--output-dir", str(tmp_path / "new"),
+ "--prompt", "text"])
+ matching_reuse_metadata(cli, monkeypatch, args.reuse_build)
+ evidence = {"matmul_policy": "sequential", "matmul_tile": "1x4", "matmul_rhs_layout": "original", "qemu_workspace_layout": "none"}
+ evidence.update({k: v for k, v in changes.items() if k != "cores"})
+ exe = SimpleNamespace(cores=changes.get("cores", 1), evidence=evidence)
+ monkeypatch.setattr(cli, "load_executable", lambda path: exe)
+ with pytest.raises(ValueError, match="differs"):
+ cli.prepare_executable(args, tmp_path, cores)
+
+
+def test_matching_reuse_returns_same_executable_without_compile(cli, monkeypatch, tmp_path):
+ args = cli.parse_args(["--reuse-build", str(tmp_path / "old/executable.json"),
+ "--tokenizer-dir", "tokens", "--output-dir", str(tmp_path / "new"),
+ "--prompt", "text"])
+ matching_reuse_metadata(cli, monkeypatch, args.reuse_build)
+ exe = SimpleNamespace(cores=1, evidence={"matmul_policy": "sequential"})
+ monkeypatch.setattr(cli, "load_executable", lambda path: exe)
+ monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("must not rebuild"))
+ assert cli.prepare_executable(args, tmp_path, 1) is exe
+
+
+@pytest.mark.parametrize("options", [
+ ["--reference-suite"],
+ ["--reference-suite", "--verify-onnx", "model.onnx", "--max-new-tokens", "8"],
+ ["--reference-suite", "--verify-onnx", "model.onnx", "--mode", "chat"],
+ ["--chat-turn", "hello"],
+ ["--prompt", "hello", "--qemu-workspace-layout", "two-buffer-pad-4k"],
+])
+def test_cli_rejects_mutated_reference_contract_or_implicit_chat_padding(cli, tmp_path, options):
+ with pytest.raises(SystemExit) as caught:
+ cli.parse_args(argv(tmp_path, *options))
+ assert caught.value.code == 2
+
+
+def test_reference_session_refuses_small_graph_for_full_model_gate(cli, monkeypatch, tmp_path):
+ import onnx
+ from probes.w2_qwen3_parse import validation
+ fixture = json.loads((cli.ROOT / "probes/w5_generation/references.json").read_text(encoding="utf-8"))
+ monkeypatch.setattr(onnx, "load", lambda *a, **kw: object())
+ monkeypatch.setattr(validation, "audit_graph_structure", lambda graph: {"passed": True, "layer_count": 2})
+ with pytest.raises(ValueError, match="complete 28-layer"):
+ cli.reference_session(tmp_path / "onnx/model.onnx", fixture)
+
+
+@pytest.mark.parametrize("audit,shape", [
+ (None, (1, 256, 8)),
+ (None, (1, 256, 151936)),
+ ({"passed": True, "layer_count": 28}, (1, 256, 8)),
+ ({"passed": True, "layer_count": 2}, (1, 256, 151936)),
+ ({"passed": False, "layer_count": 28}, (1, 256, 151936)),
+])
+def test_full_gate_requires_executable_audit_and_complete_output(cli, audit, shape):
+ evidence = {} if audit is None else {"w5_graph_audit": audit}
+ exe = SimpleNamespace(evidence=evidence, output=SimpleNamespace(shape=shape))
+ with pytest.raises(ValueError, match="complete graph evidence"):
+ cli.require_full_executable(exe)
+
+
+def test_full_gate_accepts_native_w5_complete_graph_audit(cli):
+ exe = SimpleNamespace(evidence={"w5_graph_audit": {"passed": True, "layer_count": 28}},
+ output=SimpleNamespace(shape=(1, 256, 151936)))
+ assert cli.require_full_executable(exe) is None
+
+
+def test_legacy_w4_evidence_survives_two_reuses_without_rechecking_identity(cli, monkeypatch, tmp_path):
+ from dataclasses import replace
+ from scratchv.runtime.riscv_external import ExternalExecutable
+ from scratchv.runtime.riscv_tensor import RiscVToolchain
+ from scratchv.ir import DataType
+ from scratchv.backend.tensor_c_codegen import TensorSpec
+
+ legacy = tmp_path / "w4"
+ legacy.mkdir()
+ original = ExternalExecutable(
+ tmp_path / "model.elf", tmp_path / "weights", {}, {}, (),
+ TensorSpec("logits", DataType.FLOAT32, (1, 256, 151936)), (),
+ RiscVToolchain(("cc",), "qemu"), {"harts": 4, "matmul_tile": "2x4",
+ "matmul_rhs_layout": "column-panel", "qemu_workspace_layout": "none"}, harts=4,
+ )
+ identity = cli.save_executable(original, legacy)
+ audit = {"passed": True, "layer_count": 28, "graph": "fixed Qwen3 graph"}
+ (legacy / "report.json").write_text(json.dumps({"executable_sha256": identity,
+ "matmul_policy": "blocked_fma", "graph_audit": audit}), encoding="utf-8")
+ checked = []
+ hash_file = cli.sha256_file
+ monkeypatch.setattr(cli, "sha256_file", lambda path: checked.append(path) or hash_file(path))
+ # Exercise the real metadata writer/identity link; lower-level ELF validation
+ # is covered by W4. No compiler, model or guest process is needed here.
+ monkeypatch.setattr(cli, "load_executable", lambda path: replace(original,
+ evidence=json.loads(path.read_text(encoding="utf-8"))["evidence"]))
+ monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("must not rebuild"))
+ current = legacy / "executable.json"
+ for name in ("w5-first", "w5-second"):
+ out = tmp_path / name
+ out.mkdir()
+ args = cli.parse_args(["--reuse-build", str(current), "--tokenizer-dir", "tokens",
+ "--output-dir", str(out), "--reference-suite", "--verify-onnx", "onnx/model.onnx",
+ "--harts", "4", "--matmul-policy", "blocked_fma", "--matmul-tile", "2x4",
+ "--matmul-rhs-layout", "column-panel"])
+ exe = cli.prepare_executable(args, out, 4)
+ assert exe.evidence["matmul_policy"] == "blocked_fma"
+ assert exe.evidence["w5_graph_audit"] == audit
+ assert cli.require_full_executable(exe) is None
+ assert exe.elf_path == original.elf_path and exe.bundle_dir == original.bundle_dir
+ identity = cli.save_executable(exe, out)
+ # W5's report contains no legacy graph_audit/matmul_policy fallback.
+ (out / "report.json").write_text(json.dumps({"executable_sha256": identity}), encoding="utf-8")
+ current = out / "executable.json"
+ assert checked == [legacy / "executable.json", tmp_path / "w5-first/executable.json"]
+ assert "matmul_policy" not in original.evidence and "w5_graph_audit" not in original.evidence
+
+
+def test_verified_legacy_report_cannot_overwrite_native_failed_audit(cli, monkeypatch, tmp_path):
+ from dataclasses import dataclass
+
+ @dataclass
+ class Executable:
+ evidence: dict
+ cores: int = 1
+
+ metadata = tmp_path / "old/executable.json"
+ matching_reuse_metadata(cli, monkeypatch, metadata)
+ record = json.loads(metadata.with_name("report.json").read_text())
+ record.update(matmul_policy="sequential", graph_audit={"passed": True, "layer_count": 28})
+ metadata.with_name("report.json").write_text(json.dumps(record))
+ exe = Executable({"w5_graph_audit": {"passed": False, "layer_count": 2}})
+ monkeypatch.setattr(cli, "load_executable", lambda path: exe)
+ args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens",
+ "--output-dir", str(tmp_path / "new"), "--prompt", "text"])
+ reused = cli.prepare_executable(args, tmp_path, 1)
+ assert reused.evidence["matmul_policy"] == "sequential"
+ assert reused.evidence["w5_graph_audit"] == exe.evidence["w5_graph_audit"]
+
+
+def test_reference_suite_rejects_small_executable_before_fma_or_oracle(cli, runtime, tmp_path):
+ runtime.exe.output = SimpleNamespace(shape=(1, 256, 8))
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1
+ value = report(tmp_path)
+ assert value["status"] == "FAIL" and value["full_model_gate"] == "NOT_RUN"
+ assert "complete graph evidence" in value["error"]
+ assert runtime.adapters == [] and runtime.fma == []
+
+
+def test_signal_during_fma_cannot_be_swallowed_by_exception_handler(cli, runtime, tmp_path, monkeypatch):
+ import signal
+ previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)}
+ swallowed = []
+
+ def fma(*args, **kwargs):
+ try:
+ signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None)
+ except Exception as exc:
+ swallowed.append(exc)
+ return {"passed": False, "status": "FAIL"}
+ pytest.fail("signal did not abort FMA")
+
+ monkeypatch.setattr(cli, "run_fma_probe", fma)
+ assert cli.main(argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1")) == 130
+ value = report(tmp_path)
+ assert value["status"] == "CANCELLED" and not value["passed"]
+ assert "KeyboardInterrupt" in value["error"]
+ assert value["stage"] == "fma_preflight" and value["qemu_forwards"] == 0
+ assert not swallowed
+ assert all(signal.getsignal(sig) is handler for sig, handler in previous.items())
+
+
+@pytest.mark.parametrize("record", [None, {}, {"executable_sha256": "drifted"}])
+def test_reuse_identity_is_required_before_loading_or_compiling(cli, monkeypatch, tmp_path, record):
+ metadata = tmp_path / "old/executable.json"
+ metadata.parent.mkdir()
+ metadata.write_text("{}", encoding="utf-8")
+ if record is not None:
+ metadata.with_name("report.json").write_text(json.dumps(record), encoding="utf-8")
+ args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens",
+ "--output-dir", str(tmp_path / "new"), "--prompt", "text"])
+ monkeypatch.setattr(cli, "sha256_file", lambda path: "matching")
+ monkeypatch.setattr(cli, "load_executable", lambda *a: pytest.fail("identity must be checked before load"))
+ monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("identity failure must not compile"))
+ with pytest.raises(FileNotFoundError if record is None else ValueError):
+ cli.prepare_executable(args, tmp_path, 1)
+
+
+@pytest.fixture
+def reference_suite(cli, runtime, monkeypatch, tmp_path):
+ root = tmp_path / "fixture-root"
+ (root / "probes/w5_generation").mkdir(parents=True)
+ cases = [{"name": name, "prompt": name, "max_new_tokens": 2,
+ "prompt_token_ids": [1, 2], "generated_token_ids": [3, 3],
+ "generated_text": "xx", "stop_reason": "max_new_tokens",
+ "required_substring": "x"} for name in ("france", "germany", "japan")]
+ fixture = {"status": "frozen", "tokenizer": {"pad_token_id": 0,
+ "generation_eos_token_ids": [6], "model_vocab_size": 8, "tokenizer_vocab_size": 7},
+ "cases": cases}
+ (root / "probes/w5_generation/references.json").write_text(json.dumps(fixture), encoding="utf-8")
+ monkeypatch.setattr(cli, "ROOT", root)
+ runtime.exe.evidence["w5_graph_audit"] = {"passed": True, "layer_count": 28}
+ runtime.exe.output = SimpleNamespace(shape=(1, 256, 151936))
+ runtime.oracle = logits()
+ monkeypatch.setattr(cli, "reference_session", lambda *a: SimpleNamespace(run=lambda names, feed: [runtime.oracle]))
+ snapshots = []
+ atomic = cli.atomic_text
+
+ def save(path, text):
+ if path.name == "report.json":
+ snapshots.append(json.loads(text))
+ atomic(path, text)
+
+ monkeypatch.setattr(cli, "atomic_text", save)
+ return SimpleNamespace(fixture=fixture, snapshots=snapshots)
+
+
+def test_full_suite_gate_records_running_then_pass(cli, runtime, reference_suite, tmp_path):
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 0
+ value = report(tmp_path)
+ assert value["full_model_gate"] == "PASS" and value["status"] == "PASS"
+ assert len(value["numeric_steps"]) == 6 and len(value["cases"]) == 3
+ assert all(row["reference_passed"] for row in value["cases"])
+ assert all(row["numeric_step_count"] == row["forward_count"] == 2 for row in value["cases"])
+ assert [(row["case"], row["step"], row["valid_length"]) for row in value["numeric_steps"]] == [
+ (name, step, 2 + step) for name in ("france", "germany", "japan") for step in range(2)]
+ assert reference_suite.snapshots[0]["full_model_gate"] == "RUNNING"
+ assert reference_suite.snapshots[0]["qemu_forwards"] == 0
+
+
+@pytest.mark.parametrize("failure,code,gate", [
+ (TimeoutError("expired"), 1, "FAIL"),
+ (GenerationCancelledError("cancelled"), 130, "CANCELLED"),
+ (KeyboardInterrupt(), 130, "CANCELLED"),
+])
+def test_full_suite_running_error_has_terminal_gate(cli, runtime, reference_suite, tmp_path, failure, code, gate):
+ runtime.failure = failure
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == code
+ value = report(tmp_path)
+ assert value["full_model_gate"] == gate and value["status"] == gate
+ assert value["stage"] == "generation" and value["forward_attempts"] == 1
+ assert value["cases"] == []
+ assert reference_suite.snapshots[0]["full_model_gate"] == "RUNNING"
+
+
+def test_full_suite_numeric_failure_marks_failed_not_not_run(cli, runtime, reference_suite, tmp_path):
+ runtime.oracle[0, -1, 0] = cli.ATOL * 2
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1
+ value = report(tmp_path)
+ assert value["full_model_gate"] == "FAIL" and value["status"] == "FAIL"
+ assert value["qemu_forwards"] == 1 and value["cases"] == []
+ assert value["numeric_steps"][0]["passed"] is False
+
+
+def test_full_suite_output_reference_failure_marks_failed(cli, runtime, reference_suite, tmp_path):
+ fixture = reference_suite.fixture
+ fixture["cases"][0]["generated_text"] = "wrong frozen text"
+ (cli.ROOT / "probes/w5_generation/references.json").write_text(json.dumps(fixture), encoding="utf-8")
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1
+ value = report(tmp_path)
+ assert value["full_model_gate"] == "FAIL" and value["qemu_forwards"] == 2
+ assert value["cases"][0]["reference_passed"] is False
+ assert all(row["passed"] for row in value["numeric_steps"])
+
+
+@pytest.mark.parametrize("stage,cancel", [
+ ("prepare", False), ("fma_preflight", False),
+ ("fma_preflight", True), ("oracle_prepare", False),
+])
+def test_full_suite_pre_generation_failure_remains_not_run(cli, runtime, reference_suite, tmp_path, monkeypatch, stage, cancel):
+ def fail(*args, **kwargs):
+ raise KeyboardInterrupt("cancel") if cancel else ValueError("setup failed")
+
+ target = {"prepare": "prepare_executable", "fma_preflight": "run_fma_probe",
+ "oracle_prepare": "reference_session"}[stage]
+ monkeypatch.setattr(cli, target, fail)
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == (130 if cancel else 1)
+ value = report(tmp_path)
+ assert value["full_model_gate"] == "NOT_RUN" and value["stage"] == stage
+ assert value["status"] == ("CANCELLED" if cancel else "FAIL")
+ assert value["qemu_forwards"] == 0 and value["numeric_steps"] == []
+ assert all(row["full_model_gate"] == "NOT_RUN" for row in reference_suite.snapshots)
+
+
+@pytest.mark.parametrize("calls", [0, 1, 3])
+def test_full_suite_rejects_missing_or_extra_forward_evidence(cli, runtime, reference_suite, tmp_path, monkeypatch, calls):
+ from scratchv.runtime.generation import GenerationResult
+ from scratchv.runtime.llm_inputs import prepare_inputs
+
+ def faulty_generate(prompt, *, forward, **kwargs):
+ history = [1, 2]
+ for _ in range(calls):
+ forward(prepare_inputs(history, pad_token_id=0, vocab_size=8))
+ history.append(3)
+ return GenerationResult((1, 2), (3, 3), "xx", "max_new_tokens")
+
+ monkeypatch.setattr(cli, "generate", faulty_generate)
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1
+ value = report(tmp_path)
+ assert value["full_model_gate"] == "FAIL" and not value["passed"]
+ assert value["qemu_forwards"] == min(calls, 2)
+
+
+@pytest.mark.parametrize("fault", ["wrong_length", "stale_history", "mask", "padding"])
+def test_full_suite_rejects_wrong_input_history_even_when_logits_match(cli, runtime, reference_suite, tmp_path, monkeypatch, fault):
+ from dataclasses import replace
+ from scratchv.runtime.generation import GenerationResult
+ from scratchv.runtime.llm_inputs import prepare_inputs
+
+ def faulty_generate(prompt, *, forward, **kwargs):
+ prepared = prepare_inputs([1, 2], pad_token_id=0, vocab_size=8)
+ if fault == "wrong_length":
+ prepared = replace(prepared, valid_length=1)
+ elif fault == "stale_history":
+ prepared.input_ids[0, 1] = 4
+ elif fault == "mask":
+ prepared.attention_mask[0, 0, 0, 1] = 0
+ else:
+ prepared.input_ids[0, -1] = 5
+ forward(prepared)
+ forward(prepare_inputs([1, 2, 3], pad_token_id=0, vocab_size=8))
+ return GenerationResult((1, 2), (3, 3), "xx", "max_new_tokens")
+
+ monkeypatch.setattr(cli, "generate", faulty_generate)
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1
+ value = report(tmp_path)
+ assert value["full_model_gate"] == "FAIL" and value["qemu_forwards"] == 0
+
+
+@pytest.mark.parametrize("same_argmax", [False, True])
+def test_numeric_comparison_rejects_wrong_selected_row_independently(cli, monkeypatch, same_argmax):
+ expected = logits()
+ expected[0, 0, 3 if same_argmax else 4] = 2
+ monkeypatch.setattr(cli, "last_valid_logits", lambda array, length, **kwargs: array[0, 0].copy())
+ result = cli.compare_step(expected.copy(), expected, 2, 8)
+ assert result["max_abs"] == 0 and result["qemu_token_id"] == result["ort_token_id"]
+ assert result["selected_row_passed"] is False and result["passed"] is False
+
+
+def test_full_suite_rejects_frozen_token_drift_with_equal_qemu_ort_logits(cli, runtime, reference_suite, tmp_path, monkeypatch):
+ runtime.oracle = logits(4)
+ original = cli.QemuForward
+
+ class Adapter(original):
+ def __call__(self, prepared):
+ super().__call__(prepared)
+ return logits(4)
+
+ monkeypatch.setattr(cli, "QemuForward", Adapter)
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1
+ value = report(tmp_path)
+ row = value["numeric_steps"][0]
+ assert value["full_model_gate"] == "FAIL" and value["qemu_forwards"] == 1
+ assert row["max_abs"] == 0 and row["selected_row_passed"] is True
+ assert row["qemu_token_id"] == row["ort_token_id"] == 4
+ assert row["reference_token_id"] == 3 and row["reference_token_passed"] is False
+ assert row["case"] == "france" and row["step"] == 0 and row["valid_length"] == 2
+
+
+@pytest.mark.parametrize("error,code,status", [
+ (TimeoutError("expired after first forward"), 1, "FAIL"),
+ (GenerationCancelledError("cancelled after first forward"), 130, "CANCELLED"),
+])
+def test_full_suite_abort_retains_completed_step_evidence(cli, runtime, reference_suite, tmp_path, monkeypatch, error, code, status):
+ generate = cli.generate
+
+ def abort_after_forward(prompt, *, forward, **kwargs):
+ def guarded(prepared):
+ forward(prepared)
+ raise error
+ return generate(prompt, forward=guarded, **kwargs)
+
+ monkeypatch.setattr(cli, "generate", abort_after_forward)
+ assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == code
+ value = report(tmp_path)
+ assert value["full_model_gate"] == status and value["status"] == status
+ assert value["cases"] == [] and value["qemu_forwards"] == 1
+ assert value["numeric_steps"][0]["passed"] is True
+ assert value["numeric_steps"][0]["case"] == "france"
+
+
+@pytest.mark.parametrize("requested,cpu_count,resolved", [
+ (None, 8, 1), ("1", 8, 1), ("2", 8, 2), ("4", 1, 4),
+ ("auto", 1, 1), ("auto", 3, 2), ("auto", 12, 4),
+])
+def test_cli_uses_real_core_resolver_without_changing_default(cli, runtime, tmp_path, monkeypatch, requested, cpu_count, resolved):
+ from scratchv.runtime import riscv_harts
+ affinity_reads = []
+ monkeypatch.setattr(riscv_harts.os, "sched_getaffinity", lambda pid: affinity_reads.append(pid) or set(range(cpu_count)))
+ monkeypatch.setattr(cli, "resolve_cores", riscv_harts.resolve_cores)
+ options = [] if requested is None else ["--cores", requested]
+ assert cli.main(argv(tmp_path, "--build-only", *options)) == 0
+ value = report(tmp_path)
+ assert value["cores"] == resolved and runtime.prepared[0][2] == resolved
+ assert value["qemu_workspace_layout"] == "none" and value["qemu_forwards"] == 0
+ assert affinity_reads == ([0] if requested == "auto" else [])
+
+
+@pytest.mark.parametrize("built", [None, "sequential", "two-four-lane-last-axis"])
+@pytest.mark.parametrize("requested", ["sequential", "two-four-lane-last-axis"])
+def test_w5_reuse_mean_policy_requires_explicit_matching_build(cli, monkeypatch, tmp_path, built, requested):
+ metadata = tmp_path / "old/executable.json"
+ matching_reuse_metadata(cli, monkeypatch, metadata)
+ record = json.loads(metadata.with_name("report.json").read_text())
+ if built is not None:
+ record["mean_policy"] = built
+ metadata.with_name("report.json").write_text(json.dumps(record))
+ evidence = {"matmul_policy": "sequential"}
+ if built is not None:
+ evidence["mean_policy"] = built
+ exe = SimpleNamespace(cores=1, evidence=evidence)
+ monkeypatch.setattr(cli, "load_executable", lambda path: exe)
+ monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("reuse must not compile"))
+ args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens",
+ "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--mean-policy", requested])
+ if (built or "sequential") == requested:
+ assert cli.prepare_executable(args, tmp_path, 1) is exe
+ else:
+ with pytest.raises(ValueError, match="Mean policy"):
+ cli.prepare_executable(args, tmp_path, 1)
+
+
+def test_w5_reuse_mean_evidence_and_sibling_report_must_agree(cli, monkeypatch, tmp_path):
+ metadata = tmp_path / "old/executable.json"
+ matching_reuse_metadata(cli, monkeypatch, metadata)
+ exe = SimpleNamespace(cores=1, evidence={"matmul_policy": "sequential", "mean_policy": "two-four-lane-last-axis"})
+ monkeypatch.setattr(cli, "load_executable", lambda path: exe)
+ args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens",
+ "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--mean-policy", "two-four-lane-last-axis"])
+ with pytest.raises(ValueError, match="Mean policy"):
+ cli.prepare_executable(args, tmp_path, 1)
+
+
+@pytest.mark.parametrize("policy", ["sequential", "two-four-lane-last-axis"])
+def test_w5_mean_policy_reaches_real_codegen_without_compiler(cli, monkeypatch, tmp_path, policy):
+ from scratchv.frontend import onnx_parser
+ from scratchv.ir.builder import IRBuilder
+ from scratchv.ir.types import Value
+ from scratchv.runtime.riscv_external import ExternalExecutable
+ from scratchv.runtime.riscv_tensor import RiscVToolchain
+ builder = IRBuilder()
+ x = Value("x", shape=(2, 8))
+ builder.new_function("main", [x]); builder.new_block("entry"); builder.ret(builder.neg(x))
+ monkeypatch.setattr(onnx_parser, "ONNXParser", lambda: SimpleNamespace(initializers={}, parse=lambda *a, **kw: builder.program))
+ toolchain = RiscVToolchain(("unused-cc",), "unused-qemu")
+ monkeypatch.setattr(cli, "discover_toolchain", lambda *a: toolchain)
+ built = []
+
+ def build(artifact, bundle, out, tools, **kwargs):
+ built.append(artifact)
+ return ExternalExecutable(out / "model.elf", bundle, {}, {}, artifact.inputs,
+ artifact.output, artifact.external_weights, tools, {"mean_policy": artifact.mean_policy})
+
+ monkeypatch.setattr(cli, "build_external", build)
+ args = cli.parse_args(argv(tmp_path, "--build-only", "--mean-policy", policy))
+ exe = cli.prepare_executable(args, tmp_path, 1)
+ assert len(built) == 1 and built[0].mean_policy == policy
+ assert exe.evidence["mean_policy"] == policy and exe.cores == 1
+ assert built[0].matmul_policy == "sequential" and built[0].qemu_workspace_layout == "none"
+
+
+def test_w5_mean_policy_is_explicit_in_report(cli, runtime, tmp_path):
+ assert cli.main(argv(tmp_path, "--build-only", "--mean-policy", "two-four-lane-last-axis")) == 0
+ assert report(tmp_path)["mean_policy"] == "two-four-lane-last-axis"
+ assert runtime.prepared[0][0].mean_policy == "two-four-lane-last-axis"
+
+
+@pytest.mark.parametrize("built", [None, "libm", "fp32-reference"])
+@pytest.mark.parametrize("requested", ["libm", "fp32-reference"])
+def test_w5_reuse_nonlinear_policy_requires_explicit_matching_build(cli, monkeypatch, tmp_path, built, requested):
+ metadata = tmp_path / "old/executable.json"
+ matching_reuse_metadata(cli, monkeypatch, metadata)
+ record = json.loads(metadata.with_name("report.json").read_text())
+ if built is not None:
+ record["nonlinear_policy"] = built
+ metadata.with_name("report.json").write_text(json.dumps(record))
+ evidence = {"matmul_policy": "sequential"}
+ if built is not None:
+ evidence["nonlinear_policy"] = built
+ exe = SimpleNamespace(cores=1, evidence=evidence)
+ monkeypatch.setattr(cli, "load_executable", lambda path: exe)
+ monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("reuse must not compile"))
+ args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens",
+ "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--nonlinear-policy", requested])
+ if (built or "libm") == requested:
+ assert cli.prepare_executable(args, tmp_path, 1) is exe
+ else:
+ with pytest.raises(ValueError, match="nonlinear policy"):
+ cli.prepare_executable(args, tmp_path, 1)
+
+
+def test_w5_reuse_nonlinear_evidence_and_sibling_report_must_agree(cli, monkeypatch, tmp_path):
+ metadata = tmp_path / "old/executable.json"
+ matching_reuse_metadata(cli, monkeypatch, metadata)
+ exe = SimpleNamespace(cores=1, evidence={"matmul_policy": "sequential", "nonlinear_policy": "fp32-reference"})
+ monkeypatch.setattr(cli, "load_executable", lambda path: exe)
+ args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens",
+ "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--nonlinear-policy", "fp32-reference"])
+ with pytest.raises(ValueError, match="nonlinear policy"):
+ cli.prepare_executable(args, tmp_path, 1)
+
+
+@pytest.mark.parametrize("policy", ["libm", "fp32-reference"])
+def test_w5_nonlinear_policy_reaches_real_codegen_without_compiler(cli, monkeypatch, tmp_path, policy):
+ from scratchv.frontend import onnx_parser
+ from scratchv.ir.builder import IRBuilder
+ from scratchv.ir.types import Value
+ from scratchv.runtime.riscv_external import ExternalExecutable
+ from scratchv.runtime.riscv_tensor import RiscVToolchain
+ builder = IRBuilder()
+ x = Value("x", shape=(2, 8))
+ builder.new_function("main", [x]); builder.new_block("entry"); builder.ret(builder.neg(x))
+ monkeypatch.setattr(onnx_parser, "ONNXParser", lambda: SimpleNamespace(initializers={}, parse=lambda *a, **kw: builder.program))
+ toolchain = RiscVToolchain(("unused-cc",), "unused-qemu")
+ monkeypatch.setattr(cli, "discover_toolchain", lambda *a: toolchain)
+ built = []
+
+ def build(artifact, bundle, out, tools, **kwargs):
+ built.append(artifact)
+ return ExternalExecutable(out / "model.elf", bundle, {}, {}, artifact.inputs,
+ artifact.output, artifact.external_weights, tools, {"nonlinear_policy": artifact.nonlinear_policy})
+
+ monkeypatch.setattr(cli, "build_external", build)
+ args = cli.parse_args(argv(tmp_path, "--build-only", "--nonlinear-policy", policy))
+ exe = cli.prepare_executable(args, tmp_path, 1)
+ assert len(built) == 1 and built[0].nonlinear_policy == policy
+ assert exe.evidence["nonlinear_policy"] == policy and exe.cores == 1
+ assert built[0].matmul_policy == "sequential" and built[0].qemu_workspace_layout == "none"
+
+
+def test_w5_nonlinear_policy_is_explicit_in_report(cli, runtime, tmp_path):
+ assert cli.main(argv(tmp_path, "--build-only", "--nonlinear-policy", "fp32-reference")) == 0
+ assert report(tmp_path)["nonlinear_policy"] == "fp32-reference"
+ assert runtime.prepared[0][0].nonlinear_policy == "fp32-reference"
+
+
+@pytest.mark.parametrize("policy", ["unknown", "sequential", ""])
+def test_w5_unknown_nonlinear_policy_fails_before_output(cli, tmp_path, policy):
+ with pytest.raises(SystemExit) as exc:
+ cli.main(argv(tmp_path, "--build-only", "--nonlinear-policy", policy))
+ assert exc.value.code == 2 and not (tmp_path / "run").exists()
+
+
+def test_finite_extreme_logits_have_json_safe_diagnostic_margin(cli):
+ expected = np.full((1, 256, 8), -np.finfo(np.float32).max, dtype=np.float32)
+ expected[0, 1, 3] = np.finfo(np.float32).max
+ with np.errstate(over="raise"):
+ comparison = cli.compare_step(expected.copy(), expected, 2, 8)
+ assert comparison["passed"] and comparison["max_abs"] == 0.0
+ assert comparison["qemu_token_id"] == comparison["ort_token_id"] == 3
+ assert comparison["ort_top2_margin"] == 2.0 * float(np.finfo(np.float32).max)
+ json.dumps(comparison, allow_nan=False)
+
+
+@pytest.mark.parametrize("flags", [
+ ["--cores", "4", "--harts", "4"], ["--harts", "auto", "--cores", "auto"],
+ ["--harts", "1", "--harts", "4", "--cores", "4"],
+ ["--cores", "1", "--cores", "4", "--harts", "4"],
+])
+def test_equal_cli_aliases_resolve_once(cli, runtime, tmp_path, flags):
+ assert cli.main(argv(tmp_path, "--build-only", *flags)) == 0
+ saved = report(tmp_path)
+ assert saved["cores"] == saved["harts"] == 4
+ assert len(runtime.cores) == len(runtime.prepared) == 1
+
+
+@pytest.mark.parametrize("flags", [
+ ["--cores", "4", "--harts", "1"], ["--harts", "4", "--cores", "1"],
+ ["--cores", "auto", "--harts", "4"], ["--harts", "auto", "--cores", "1"],
+ ["--harts", "1", "--harts", "4", "--cores", "1"],
+ ["--cores", "1", "--cores", "4", "--harts", "1"],
+ ["--cores", "3"], ["--harts", "0"],
+])
+def test_core_alias_conflict_or_invalid_count_has_no_side_effects(cli, runtime, tmp_path, flags):
+ with pytest.raises(SystemExit) as exc:
+ cli.main(argv(tmp_path, "--build-only", *flags))
+ assert exc.value.code == 2
+ assert not (tmp_path / "run").exists()
+ assert runtime.prepared == runtime.cores == []
+
+
+@pytest.mark.parametrize("fields", [{"cores": 4}, {"harts": 4}, {"cores": 1, "harts": 4}])
+def test_reused_report_rejects_inconsistent_core_aliases(cli, monkeypatch, tmp_path, fields):
+ metadata = tmp_path / "old/executable.json"
+ matching_reuse_metadata(cli, monkeypatch, metadata)
+ previous = metadata.with_name("report.json")
+ record = json.loads(previous.read_text())
+ record.update(fields)
+ previous.write_text(json.dumps(record))
+ monkeypatch.setattr(cli, "load_executable", lambda path: SimpleNamespace(
+ cores=1, evidence={"matmul_policy": "sequential"}))
+ args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens",
+ "--output-dir", str(tmp_path / "new"), "--prompt", "text"])
+ with pytest.raises(ValueError, match="core|hart"):
+ cli.prepare_executable(args, tmp_path, cores=1)
+
+
+@pytest.mark.parametrize("flag", ["--cores", "--harts"])
+@pytest.mark.parametrize("first,last,expected", [("1", "4", 4), ("4", "1", 1), ("1", "auto", 4)])
+def test_repeated_core_option_preserves_argparse_last_value(cli, runtime, tmp_path, flag, first, last, expected):
+ assert cli.main(argv(tmp_path, "--build-only", flag, first, flag, last)) == 0
+ saved = report(tmp_path)
+ requested = "auto" if last == "auto" else int(last)
+ assert saved["cores"] == saved["harts"] == expected
+ assert saved["cores_requested"] == saved["harts_requested"] == requested
+ assert runtime.cores == [requested]
+ assert len(runtime.prepared) == 1
+
+
+@pytest.mark.parametrize("flag", ["--cores", "--harts"])
+@pytest.mark.parametrize("requested", [None, "1", "2", "4", "auto"])
+def test_parse_args_retains_legacy_harts_attribute_and_string_type(cli, tmp_path, flag, requested):
+ flags = [] if requested is None else [flag, requested]
+ args = cli.parse_args(argv(tmp_path, "--build-only", *flags))
+ assert args.cores == args.harts == (requested or "1")
+ assert type(args.cores) is type(args.harts) is str
+
+
+def test_parsed_core_attributes_stay_synchronized_when_callers_update_either_name(cli, tmp_path):
+ args = cli.parse_args(argv(tmp_path, "--build-only"))
+ args.harts = "4"
+ assert args.cores == args.harts == "4"
+ args.cores = "auto"
+ assert args.harts == args.cores == "auto"
+ # W4 resolves its selected count on the namespace after parsing.
+ args.cores = 4
+ assert args.harts == args.cores == 4
+
+
+@pytest.mark.parametrize("value,expected", [("4", 4), ("auto", 4)])
+def test_old_wrapper_harts_assignment_reaches_core_selection_once(
+ cli, runtime, monkeypatch, tmp_path, value, expected):
+ args = cli.parse_args(argv(tmp_path, "--build-only"))
+ args.harts = value
+ monkeypatch.setattr(cli, "parse_args", lambda _argv: args)
+ assert cli.main([]) == 0
+ saved = report(tmp_path)
+ assert saved["cores"] == saved["harts"] == expected
+ assert saved["cores_requested"] == saved["harts_requested"] == (
+ value if value == "auto" else int(value))
+ assert runtime.cores == [value if value == "auto" else int(value)]
+ assert len(runtime.prepared) == 1