From 259fc8e0da75975089f922232517dc8c65a3aa2b Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Fri, 23 Oct 2026 15:25:31 +0800 Subject: [PATCH 1/7] feat(w5): add QEMU token generation and reproducible end-to-end gates --- .github/workflows/ci.yml | 2 +- .github/workflows/llm-deploy.yml | 104 ++- .github/workflows/w4-full-numeric.yml | 273 ++++++- THIRD_PARTY_NOTICES.txt | 7 +- deployment/w5/Dockerfile | 35 + deployment/w5/Dockerfile.dockerignore | 27 + docs/INDEX.md | 1 + docs/llm-deploy-v1.0/W4/runtime-contract.md | 27 + docs/llm-deploy-v1.0/W5/README.md | 279 +++++++ docs/llm-deploy-v1.0/W5/chat.md | 27 + docs/llm-deploy-v1.0/W5/deployment.md | 201 +++++ docs/llm-deploy-v1.0/W5/qemu-generation.md | 579 ++++++++++++++ ...00\345\217\221\350\256\241\345\210\222.md" | 2 +- probes/w4_qwen3_full/run.py | 24 +- probes/w5_generation/README.md | 41 + probes/w5_generation/references.json | 422 ++++++++++ probes/w5_generation/run.py | 228 ++++++ pyproject.toml | 3 +- scratchv/backend/tensor_c_codegen.py | 83 +- scratchv/backend/tensor_c_mean.py | 77 ++ scratchv/backend/tensor_c_nonlinear.py | 171 ++++ scratchv/runtime/conversation.py | 153 ++++ scratchv/runtime/generation.py | 133 +++ scratchv/runtime/qemu_generation.py | 140 ++++ scratchv/runtime/riscv_external.py | 101 ++- scripts/check_w5_ci.py | 139 ++++ scripts/ci_scopes.json | 30 +- scripts/deploy_w5.sh | 87 ++ scripts/run_inference.py | 368 +++++++++ tests/test_ci_routing.py | 11 + tests/test_ci_workflows.py | 34 +- tests/test_conversation.py | 272 +++++++ tests/test_generation.py | 431 ++++++++++ tests/test_qemu_generation.py | 422 ++++++++++ tests/test_riscv_external_contract.py | 124 +++ tests/test_riscv_mean_policy.py | 72 ++ tests/test_riscv_nonlinear_policy.py | 67 ++ tests/test_tensor_c_mean_policy.py | 175 ++++ tests/test_tensor_c_nonlinear_policy.py | 239 ++++++ tests/test_w4_ci_summary.py | 2 +- tests/test_w4_full_probe_contract.py | 124 ++- tests/test_w4_workflows.py | 286 ++++++- tests/test_w5_ci.py | 216 +++++ tests/test_w5_deployment.py | 243 ++++++ tests/test_w5_generation_reference.py | 162 ++++ tests/test_w5_inference_cli.py | 757 ++++++++++++++++++ 46 files changed, 7343 insertions(+), 58 deletions(-) create mode 100644 deployment/w5/Dockerfile create mode 100644 deployment/w5/Dockerfile.dockerignore create mode 100644 docs/llm-deploy-v1.0/W5/README.md create mode 100644 docs/llm-deploy-v1.0/W5/chat.md create mode 100644 docs/llm-deploy-v1.0/W5/deployment.md create mode 100644 docs/llm-deploy-v1.0/W5/qemu-generation.md create mode 100644 probes/w5_generation/README.md create mode 100644 probes/w5_generation/references.json create mode 100644 probes/w5_generation/run.py create mode 100644 scratchv/backend/tensor_c_mean.py create mode 100644 scratchv/backend/tensor_c_nonlinear.py create mode 100644 scratchv/runtime/conversation.py create mode 100644 scratchv/runtime/generation.py create mode 100644 scratchv/runtime/qemu_generation.py create mode 100644 scripts/check_w5_ci.py create mode 100755 scripts/deploy_w5.sh create mode 100644 scripts/run_inference.py create mode 100644 tests/test_conversation.py create mode 100644 tests/test_generation.py create mode 100644 tests/test_qemu_generation.py create mode 100644 tests/test_riscv_mean_policy.py create mode 100644 tests/test_riscv_nonlinear_policy.py create mode 100644 tests/test_tensor_c_mean_policy.py create mode 100644 tests/test_tensor_c_nonlinear_policy.py create mode 100644 tests/test_w5_ci.py create mode 100644 tests/test_w5_deployment.py create mode 100644 tests/test_w5_generation_reference.py create mode 100644 tests/test_w5_inference_cli.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8eceda99..f57037d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: uses: ./.github/workflows/topic06-benchmark.yml w4: - name: W4 small QEMU checks + name: W4/W5 small QEMU checks needs: scope if: github.event_name == 'pull_request' && needs.scope.outputs.w4 == 'true' uses: ./.github/workflows/w4-full-numeric.yml diff --git a/.github/workflows/llm-deploy.yml b/.github/workflows/llm-deploy.yml index 3b20f7ce..eeaa3937 100644 --- a/.github/workflows/llm-deploy.yml +++ b/.github/workflows/llm-deploy.yml @@ -37,6 +37,11 @@ on: required: false default: false + run_w5_validation: + description: W5 only; mutually exclusive with W3/W4 and skips legacy W1/W2 full runs + type: boolean + required: false + default: false run_w4_validation: description: Also build and execute one complete W4 RV64 QEMU forward type: boolean @@ -69,20 +74,51 @@ env: QWEN3_PY: output/qwen3-probe-venv/bin/python jobs: + validation-selection: + name: Validate explicit full-run selection + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Reject conflicting W5 and legacy full selections + env: + W5_REQUESTED: ${{ inputs.run_w5_validation }} + W3_REQUESTED: ${{ inputs.run_w3_validation }} + W4_REQUESTED: ${{ inputs.run_w4_validation }} + run: | + python3 - <<'PY' + import os + selected = lambda name: os.environ.get(name, "").casefold() == "true" + if selected("W5_REQUESTED") and (os.environ["GITHUB_EVENT_NAME"] != "workflow_dispatch" + or selected("W3_REQUESTED") or selected("W4_REQUESTED")): + raise SystemExit("W5 requires a manual W5-only run; disable W3 and W4 selections") + PY + + w5-validation: + name: "W5 / explicit container generation only" + needs: validation-selection + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w5_validation }} + uses: ./.github/workflows/w4-full-numeric.yml + with: + run_w5: true + w3-validation: + needs: validation-selection name: "W3 / explicit full validation" - if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w3_validation }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w3_validation && !inputs.run_w5_validation }} uses: ./.github/workflows/w3-nightly.yml w4-validation: + needs: validation-selection name: "W4 / explicit full QEMU validation" - if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w4_validation }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_w4_validation && !inputs.run_w5_validation }} uses: ./.github/workflows/w4-full-numeric.yml with: run_full: true case: ${{ inputs.w4_case || 'full_seed_0' }} llm-deploy: + needs: validation-selection + if: ${{ !inputs.run_w5_validation }} name: "LLM Deploy v1.0 / W1 + W2" runs-on: ubuntu-24.04 timeout-minutes: 60 @@ -129,6 +165,31 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -v -e ".[all,llm-deploy]" + - name: W5 official three-round conversation template + run: | + set -euo pipefail + mkdir -p output/w5-multiturn + python3 -B -m pytest \ + tests/test_conversation.py::test_official_three_round_rendering_matches_transformers \ + -q -p no:cacheprovider --junitxml output/w5-multiturn/tests.xml + python3 - <<'PY' + import xml.etree.ElementTree as ET + root = ET.parse("output/w5-multiturn/tests.xml").getroot() + for suffix in ("None", "Give brief answers."): + name = f"test_official_three_round_rendering_matches_transformers[{suffix}]" + rows = [node for node in root.iter("testcase") if node.get("name") == name] + if len(rows) != 1 or any(rows[0].find(tag) is not None for tag in ("skipped", "error", "failure")): + raise SystemExit(f"Official three-round template must execute and pass: {name}") + PY + - name: Upload W5 conversation template evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: w5-multiturn-${{ github.run_id }}-${{ github.run_attempt }} + if-no-files-found: warn + retention-days: 14 + path: output/w5-multiturn/tests.xml + # ── docs:interfaces / docs:risks ────────────────────────────────── - name: "docs:interfaces + docs:risks" id: docs @@ -461,10 +522,11 @@ jobs: } full-qwen3-onnx: + needs: validation-selection name: "W1 / full Qwen3 ONNX release validation" # This fetches the pinned LFS objects and executes the complete FP32 model. It is not # counted as a passing gate on PRs where it did not execute. - if: github.event_name != 'pull_request' + if: github.event_name != 'pull_request' && !inputs.run_w5_validation runs-on: ubuntu-24.04 timeout-minutes: 60 steps: @@ -522,6 +584,8 @@ jobs: full-qwen3-frontend: + needs: validation-selection + if: ${{ !inputs.run_w5_validation }} name: "W2 / complete Qwen3 frontend" # Required on PRs as well as manual/nightly runs. No full-model ORT or IR inference. runs-on: ubuntu-24.04 @@ -580,7 +644,7 @@ jobs: w2-acceptance: name: "W2 / required acceptance" needs: [llm-deploy, full-qwen3-frontend] - if: always() + if: always() && !inputs.run_w5_validation runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 @@ -694,3 +758,35 @@ jobs: if not (passed or not_run): raise SystemExit("Explicit W4 validation must actually run and pass") PY + + w5-execution-summary: + name: "W5 / explicit execution scope" + needs: [validation-selection, w5-validation, llm-deploy, full-qwen3-onnx, full-qwen3-frontend, w3-validation, w4-validation] + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require the requested W5-only execution + env: + W5_REQUESTED: ${{ inputs.run_w5_validation }} + W5_JOB_RESULTS: ${{ toJSON(needs) }} + run: | + python3 - <<'PY' + import json, os + requested = (os.environ["GITHUB_EVENT_NAME"] == "workflow_dispatch" + and os.environ.get("W5_REQUESTED", "").casefold() == "true") + results = {name: item.get("result") for name, item in json.loads(os.environ["W5_JOB_RESULTS"]).items()} + selected = results.get("validation-selection") == "success" + passed = requested and selected and results.get("w5-validation") == "success" + if requested: + passed = passed and all(results.get(name) == "skipped" for name in ( + "llm-deploy", "full-qwen3-onnx", "full-qwen3-frontend", "w3-validation", "w4-validation")) + not_run = not requested and selected and results.get("w5-validation") == "skipped" + label = "PASS" if passed else "NOT RUN" if not_run else "FAIL" + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream: + stream.write(f"## W5 execution scope\n\n**{label}** — explicit container generation, three frozen prompts and 24 forwards.\n\n") + stream.write("Only the selected W5 path may execute; the reusable workflow requires real small QEMU, " + "official three-round template and full-generation evidence. NOT RUN is not W5 acceptance.\n") + if not (passed or not_run): + raise SystemExit("Requested W5-only validation must pass without concurrent legacy full runs") + PY diff --git a/.github/workflows/w4-full-numeric.yml b/.github/workflows/w4-full-numeric.yml index b39efc7f..5aa47017 100644 --- a/.github/workflows/w4-full-numeric.yml +++ b/.github/workflows/w4-full-numeric.yml @@ -1,8 +1,35 @@ -name: W4 full RV64 numeric +name: W4 and W5 RV64 validation on: + workflow_dispatch: + inputs: + mode: + description: Small regressions by default; full modes are mutually exclusive + type: choice + default: small + options: + - small + - w4-forward + - w5-generation + case: + description: W4-only case; ignored by the frozen W5 three-prompt suite + type: choice + default: full_seed_0 + options: + - full_seed_0 + - full_seed_42 + - one_token + - short_17 + - short_255 + - changed_future + - changed_padding workflow_call: inputs: + run_w5: + description: Explicit manual W5-only generation from an existing dispatch entry + type: boolean + required: false + default: false run_full: description: Explicitly execute one complete QEMU forward; false runs only unit and small QEMU regressions type: boolean @@ -18,7 +45,7 @@ permissions: contents: read concurrency: - group: w4-full-numeric-${{ github.ref }}-${{ inputs.run_full && 'full' || 'unit' }}-${{ inputs.case || 'full_seed_0' }} + group: w4-full-numeric-${{ github.ref }}-${{ inputs.run_full && 'full' || 'unit' }}-${{ inputs.case || 'full_seed_0' }}-${{ inputs.run_w5 && 'w5' || inputs.mode || 'call' }} cancel-in-progress: false env: @@ -33,6 +60,19 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 30 steps: + - name: Reject conflicting or nonmanual full-generation selections + env: + W5_REQUESTED: ${{ inputs.run_w5 }} + W4_REQUESTED: ${{ inputs.run_full }} + RUN_MODE: ${{ inputs.mode }} + run: | + python3 - <<'PY' + import os + w5 = os.environ.get("W5_REQUESTED", "").casefold() == "true" or os.environ.get("RUN_MODE") == "w5-generation" + w4 = os.environ.get("W4_REQUESTED", "").casefold() == "true" or os.environ.get("RUN_MODE") == "w4-forward" + if w5 and (w4 or os.environ["GITHUB_EVENT_NAME"] != "workflow_dispatch"): + raise SystemExit("W5 must be explicitly manual and cannot share a run with W4 full inference") + PY - uses: actions/checkout@v4 with: ref: ${{ github.sha }} @@ -79,6 +119,8 @@ jobs: tests/test_w4_fma_conformance.py tests/test_w4_fma_cross_backend.py \ tests/test_riscv_process_cleanup.py \ tests/test_riscv_weight_transport.py \ + tests/test_qemu_generation.py::test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel \ + tests/test_riscv_mean_policy.py tests/test_riscv_nonlinear_policy.py \ tests/test_w4_workflows.py tests/test_w4_ci_summary.py \ -q -p no:cacheprovider --basetemp output/w4-unit/pytest \ --junitxml output/w4-unit/tests.xml @@ -96,7 +138,19 @@ jobs: "test_post_execution_transport_corruption_fails_and_retains_copies", "test_success_copy_cleanup_failure_is_not_a_pass", "test_actual_shared_library_executes_kernel_calls_and_unloads[sequential]", - "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]") + "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]", + "test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel[1]", + "test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel[4]", + "test_explicit_mean_order_rv64_and_context_reuse[13-1]", + "test_explicit_mean_order_rv64_and_context_reuse[13-4]", + "test_explicit_mean_order_rv64_and_context_reuse[128-1]", + "test_explicit_mean_order_rv64_and_context_reuse[128-4]", + "test_explicit_mean_order_rv64_and_context_reuse[1024-1]", + "test_explicit_mean_order_rv64_and_context_reuse[1024-4]", + "test_explicit_mean_overflow_propagates_and_build_remains_reusable[1]", + "test_explicit_mean_overflow_propagates_and_build_remains_reusable[4]", + "test_reference_nonlinear_rv64_contexts_and_error_recovery[1]", + "test_reference_nonlinear_rv64_contexts_and_error_recovery[4]") for name in required: real = [node for node in root.iter("testcase") if node.get("name") == name] if len(real) != 1 or any(real[0].find(tag) is not None for tag in ("skipped", "error", "failure")): @@ -128,8 +182,8 @@ jobs: full-numeric: name: One complete Qwen3 RV64 forward needs: unit - # Caller event alone never enables the expensive full-model forward. - if: ${{ inputs.run_full }} + # Keep old reusable callers; a direct manual dispatch chooses only one full mode. + if: ${{ !inputs.run_w5 && inputs.mode != 'w5-generation' && (inputs.run_full || (github.event_name == 'workflow_dispatch' && inputs.mode == 'w4-forward')) }} runs-on: ubuntu-24.04 timeout-minutes: 240 env: @@ -393,16 +447,197 @@ jobs: output/w4-ci/**/input.npy output/w4-ci/evidence-retention.json + w5-generation: + name: Manual W5 three-prompt generation in a clean container + needs: unit + # Never automatic, and never paired with the W4 complete forward. + if: ${{ github.event_name == 'workflow_dispatch' && (inputs.run_w5 || inputs.mode == 'w5-generation') && !inputs.run_full && inputs.mode != 'w4-forward' }} + runs-on: ubuntu-24.04 + timeout-minutes: 350 + env: + SCRATCHV_W5_IMAGE: scratchv-w5:${{ github.run_id }}-${{ github.run_attempt }} + SCRATCHV_W5_ASSETS: ${{ github.workspace }}/output/w5-assets + SCRATCHV_W5_OUTPUT: ${{ github.workspace }}/output/w5-ci + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.sha }} + persist-credentials: false + lfs: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + - name: Record W5 runner resources before installation + run: | + set -euo pipefail + mkdir -p output/w5-ci + { + printf 'commit=%s\nrun_id=%s\nattempt=%s\n' "$GITHUB_SHA" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" + cat /etc/os-release + uname -a + lscpu + free -b + df -B1 . + docker version + } > output/w5-ci/runner-resources.txt + - name: Build the clean W5 deployment image + timeout-minutes: 25 + run: | + set -euo pipefail + docker build --file deployment/w5/Dockerfile \ + --build-arg BASE_IMAGE=ubuntu:24.04 --tag "$SCRATCHV_W5_IMAGE" . + docker image inspect "$SCRATCHV_W5_IMAGE" > output/w5-ci/container-image.json + docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" -m pip freeze \ + > output/w5-ci/dependencies.txt + docker run --rm --entrypoint qemu-system-riscv64 "$SCRATCHV_W5_IMAGE" --version \ + >> output/w5-ci/runner-resources.txt + - name: Acquire only the fixed W5 ONNX and tokenizer assets + timeout-minutes: 15 + run: | + set -euo pipefail + git lfs install --local + git lfs pull --include="assets/qwen3-0.6b/tokenizer/*" --exclude="" + python - <<'PY' + import json, shutil + from pathlib import Path + from probes.w1_qwen3_export.run import acquire_model + assets = Path("output/w5-assets") + result = acquire_model(assets / "onnx") + shutil.copytree("assets/qwen3-0.6b/tokenizer", assets / "tokenizer") + Path("output/w5-ci/acquisition.json").write_text(json.dumps(result, indent=2) + "\n") + PY + - name: Require the official three-round template in the clean W5 image + timeout-minutes: 5 + run: | + set -euo pipefail + docker run --rm --init --user "$(id -u):$(id -g)" --env HOME=/tmp \ + --env SCRATCHV_QWEN3_TOKENIZER_DIR=/assets/tokenizer \ + --mount "type=bind,src=$SCRATCHV_W5_ASSETS,dst=/assets,readonly" \ + --mount "type=bind,src=$SCRATCHV_W5_OUTPUT,dst=/output" \ + --mount "type=bind,src=$GITHUB_WORKSPACE/tests/test_conversation.py,dst=/tmp/test_conversation.py,readonly" \ + --entrypoint python "$SCRATCHV_W5_IMAGE" \ + -m pytest -q -c /dev/null -p no:cacheprovider \ + /tmp/test_conversation.py::test_official_three_round_rendering_matches_transformers \ + --junitxml /output/multiturn-tests.xml + python - <<'PY' + import xml.etree.ElementTree as ET + root = ET.parse("output/w5-ci/multiturn-tests.xml").getroot() + for suffix in ("None", "Give brief answers."): + name = f"test_official_three_round_rendering_matches_transformers[{suffix}]" + rows = [node for node in root.iter("testcase") if node.get("name") == name] + if len(rows) != 1 or any(rows[0].find(tag) is not None for tag in ("skipped", "error", "failure")): + raise SystemExit(f"Official three-round template must execute and pass: {name}") + PY + - name: Require four CPUs and capacity for 24 retained logits + run: | + set -euo pipefail + python - <<'PY' + import json, os, shutil + from pathlib import Path + mem = {line.split(":")[0]: int(line.split()[1]) * 1024 + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith(("MemTotal:", "MemAvailable:"))} + evidence = {"stage": "after_image_and_assets", + "available_cpus": len(os.sched_getaffinity(0)), "memory": mem, + "disk_free_bytes": shutil.disk_usage(".").free, + "minimum_available_cpus": 4, + "minimum_available_memory_bytes": 10 * 1024**3, + "minimum_disk_free_bytes": 10 * 1024**3, + "estimated_remaining_payload_bytes": 2 * 2384319432 + 24 * 256 * 151936 * 4, + "estimate_scope": "Bundle plus one temporary transport copy and all 24 logits, excluding build/cache/log overhead; image/assets already exist."} + evidence["passed"] = (evidence["available_cpus"] >= evidence["minimum_available_cpus"] + and mem.get("MemAvailable", 0) >= evidence["minimum_available_memory_bytes"] + and evidence["disk_free_bytes"] >= evidence["minimum_disk_free_bytes"]) + Path("output/w5-ci/resource-preflight.json").write_text(json.dumps(evidence, indent=2) + "\n") + if not evidence["passed"]: + raise SystemExit("Insufficient W5 CPU/RAM/disk; generation NOT RUN. See resource-preflight.json") + PY + - name: Run the frozen W5 suite through the deployment script + id: generation + timeout-minutes: 285 + run: | + set -euo pipefail + # Image preparation above owns dependency installation. The original + # deploy entry verifies its build using the same now-cached image. + set +e + timeout --signal=INT --kill-after=30s 280m \ + /usr/bin/time -v -o output/w5-ci/process-time.txt \ + bash scripts/deploy_w5.sh \ + --model /assets/onnx/model.onnx --tokenizer-dir /assets/tokenizer \ + --reference-suite --verify-onnx /assets/onnx/model.onnx \ + --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ + --matmul-rhs-layout column-panel \ + --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ + --timeout 1800 --total-timeout 16500 \ + 2>&1 | tee output/w5-ci/runner.log + status=${PIPESTATUS[0]} + set -e + printf '%s\n' "$status" > output/w5-ci/runner-exit-code.txt + exit "$status" + - name: Reap any container left by an interrupted Docker client + if: always() + timeout-minutes: 2 + run: | + set -euo pipefail + # This tag belongs only to this run/attempt; never stop unrelated containers. + containers=$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE") + for container in $containers; do + docker stop --time 30 "$container" + done + test -z "$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE")" + - name: Require complete W5 generation evidence + id: w5_gate + if: always() + run: | + python -B -X utf8 scripts/check_w5_ci.py \ + --report output/w5-ci/run/report.json \ + --exit-code-file output/w5-ci/runner-exit-code.txt \ + --output output/w5-ci/ci-gate.json + - name: Record final W5 resource and retention scope + if: always() + run: | + mkdir -p output/w5-ci + { free -b; df -B1 .; } > output/w5-ci/final-resources.txt + cat > output/w5-ci/evidence-retention.json <<'JSON' + { + "reports_retention_days": 30, + "raw_logits_uploaded": false, + "weights_uploaded": false, + "scope": "Three frozen raw-completion prompts, eight greedy tokens each; one ELF/bundle, 4 hart, explicit Mean and nonlinear reference policies, padding off.", + "container_scope": "The original deployment script exit, CLI gate and workflow job together establish container acceptance. CLI container_reproduction=NOT_ASSERTED is preserved.", + "timing_scope": "GNU time measures the Docker client, not container CPU/RSS; per-forward QEMU elapsed/RSS evidence is in run.json. This CI run is not a calibrated speedup benchmark.", + "failure_scope": "Normal failures retain reports. Runner loss, hard job timeout or cancellation can prevent cleanup and artifact upload.", + "defaults_changed": false + } + JSON + - name: Upload W5 reports and failure diagnostics only + if: always() + uses: actions/upload-artifact@v4 + with: + name: w5-generation-reports-${{ github.run_id }}-${{ github.run_attempt }} + if-no-files-found: warn + retention-days: 30 + path: | + output/w5-ci/**/*.xml + output/w5-ci/**/*.json + output/w5-ci/**/*.txt + output/w5-ci/**/*.log + output/w5-ci/**/*.stderr + output/w5-ci/**/*.stdout + !output/w5-ci/.zig-cache/** + ci-status: - name: Summarize W4 gates, including failed or skipped dependencies + name: Summarize W4/W5 gates, including failed or skipped dependencies if: ${{ always() }} - needs: [unit, full-numeric] + needs: [unit, full-numeric, w5-generation] runs-on: ubuntu-24.04 timeout-minutes: 5 env: UNIT_RESULT: ${{ needs.unit.result }} FULL_NUMERIC_RESULT: ${{ needs.full-numeric.result }} - FULL_REQUESTED: ${{ inputs.run_full }} + FULL_REQUESTED: ${{ !inputs.run_w5 && inputs.mode != 'w5-generation' && (inputs.run_full || (github.event_name == 'workflow_dispatch' && inputs.mode == 'w4-forward')) }} + W5_GENERATION_RESULT: ${{ needs.w5-generation.result }} + W5_REQUESTED: ${{ github.event_name == 'workflow_dispatch' && (inputs.run_w5 || inputs.mode == 'w5-generation') && !inputs.run_full && inputs.mode != 'w4-forward' }} SELECTED_CASE: ${{ inputs.case || 'full_seed_0' }} steps: - name: Record W4 CI result without promoting skipped gates @@ -413,29 +648,37 @@ jobs: import json, os from pathlib import Path requested = os.environ.get("FULL_REQUESTED", "").casefold() == "true" - unit, full = os.environ["UNIT_RESULT"], os.environ["FULL_NUMERIC_RESULT"] - passed = unit == "success" and full == ("success" if requested else "skipped") + w5_requested = os.environ.get("W5_REQUESTED", "").casefold() == "true" + unit, full, w5 = (os.environ[key] for key in ("UNIT_RESULT", "FULL_NUMERIC_RESULT", "W5_GENERATION_RESULT")) + passed = (not (requested and w5_requested) and unit == "success" + and full == ("success" if requested else "skipped") + and w5 == ("success" if w5_requested else "skipped")) row = {"schema_version": 1, "passed": passed, "status": "PASS" if passed else "FAIL", - "scope": "One complete RV64 case plus retained-array audit" if requested else "Unit and small-graph regressions only", + "scope": ("Three frozen W5 cases and 24 complete QEMU forwards in the deployment container" + if w5_requested else "One complete RV64 case plus retained-array audit" + if requested else "Unit and small-graph regressions only"), "commit": os.environ["GITHUB_SHA"], "run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": os.environ["GITHUB_RUN_ATTEMPT"], "case": os.environ["SELECTED_CASE"] if requested else None, "full_numeric_requested": requested, - "job_results": {"unit": unit, "full-numeric": full}, + "w5_generation_requested": w5_requested, + "w5_generation_accepted": w5_requested and passed, + "job_results": {"unit": unit, "full-numeric": full, "w5-generation": w5}, "full_numeric_accepted": requested and passed, "team_accepted": False, "independent_reproduction": False, "seven_case_coverage": False} Path("output/w4-status/status.json").write_text(json.dumps(row, indent=2) + "\n") labels = {"success": "PASS", "failure": "FAIL", "cancelled": "CANCELLED", "skipped": "NOT RUN (未运行)"} with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as stream: - stream.write(f"## W4 CI {row['status']}\n\nCommit: `{row['commit']}`. {row['scope']}.\n\n" + stream.write(f"## W4/W5 CI {row['status']}\n\nCommit: `{row['commit']}`. {row['scope']}.\n\n" "| 检查 | 本次结果 |\n| --- | --- |\n" f"| 单测与真实小图 QEMU | {labels.get(unit, 'UNKNOWN')} |\n" - f"| 完整 28 层 QEMU 与数组审计 | {labels.get(full, 'UNKNOWN')} |\n\n" + f"| W4 完整 28 层 QEMU 与数组审计 | {labels.get(full, 'UNKNOWN')} |\n" + f"| W5 容器三例 24 步生成 | {labels.get(w5, 'UNKNOWN')} |\n\n" "数值和耗时见已执行任务的验收摘要；未运行项目没有本次数值结果。\n\n" "
验收范围\n\n" "PR 默认只执行单测与小图，完整前向通过手动工作流运行。" "This does not approve team acceptance or independent reproduction.\n\n
\n") if not passed: - raise SystemExit("Required W4 jobs failed, were cancelled, or did not run") + raise SystemExit("Required W4/W5 jobs failed, were cancelled, or did not run") PY - name: Upload W4 CI status even on gate failure if: always() diff --git a/THIRD_PARTY_NOTICES.txt b/THIRD_PARTY_NOTICES.txt index 6eaafd6c..18a0ed60 100644 --- a/THIRD_PARTY_NOTICES.txt +++ b/THIRD_PARTY_NOTICES.txt @@ -3,12 +3,15 @@ file-specific third-party material. Package metadata lists both applicable licenses; it does not replace these file-specific notices. scratchv/verification/fp32_eigen.py +scratchv/backend/tensor_c_mean.py (including its emitted C helper) +scratchv/backend/tensor_c_nonlinear.py (including its emitted trigonometric helper) Derived from Eigen at commit 1d8b82b0740839c0de7f1242a3585e3390ff5f33. - This file is licensed under MPL-2.0. Its source contains the original + These files are licensed under MPL-2.0. Their sources contain the original copyright notices and source file references. The full license text is distributed as LICENSES/Eigen-MPL-2.0.txt. scratchv/verification/fp32_reference.py +scratchv/backend/tensor_c_nonlinear.py (Sigmoid and Softmax portions, including emitted C) Contains polynomial constants and evaluation adapted from Microsoft ONNX Runtime v1.22.1 (MLAS), under MIT. The source file includes the Microsoft - copyright notice and full license text. + copyright notice and full license text; these notices also accompany emitted C. diff --git a/deployment/w5/Dockerfile b/deployment/w5/Dockerfile new file mode 100644 index 00000000..9888d338 --- /dev/null +++ b/deployment/w5/Dockerfile @@ -0,0 +1,35 @@ +# Build from the repository root. +ARG BASE_IMAGE=ubuntu:24.04 +FROM ${BASE_IMAGE} + +ENV DEBIAN_FRONTEND=noninteractive \ + PATH=/opt/venv/bin:$PATH \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + OMP_NUM_THREADS=1 \ + OPENBLAS_NUM_THREADS=1 \ + MKL_NUM_THREADS=1 \ + NUMEXPR_NUM_THREADS=1 + +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + ca-certificates python3.12 python3.12-venv qemu-system-misc libgomp1 \ + && rm -rf /var/lib/apt/lists/* \ + && python3.12 -m venv /opt/venv + +WORKDIR /opt/scratchv +COPY pyproject.toml README.md LICENSE THIRD_PARTY_NOTICES.txt ./ +COPY LICENSES ./LICENSES +COPY scratchv ./scratchv +COPY scratchv_dag ./scratchv_dag +COPY benchmarks ./benchmarks +COPY scripts/run_inference.py ./scripts/run_inference.py +COPY probes ./probes + +# Reuse the project's frozen Python/Zig deployment environment. +# Model/tokenizer assets and compiled artifacts are mounted at runtime. +RUN python -m pip install --no-cache-dir -e ".[llm-deploy]" \ + && python -c "from scratchv.runtime.riscv_tensor import discover_toolchain; discover_toolchain()" \ + && qemu-system-riscv64 --version + +ENTRYPOINT ["python", "/opt/scratchv/scripts/run_inference.py"] diff --git a/deployment/w5/Dockerfile.dockerignore b/deployment/w5/Dockerfile.dockerignore new file mode 100644 index 00000000..4b14a867 --- /dev/null +++ b/deployment/w5/Dockerfile.dockerignore @@ -0,0 +1,27 @@ +# Dockerfile-specific allowlist: never send weights, ELF, venvs, Git or output. +** +!pyproject.toml +!README.md +!LICENSE +!LICENSES/ +!LICENSES/** +!THIRD_PARTY_NOTICES.txt +!scratchv/ +!scratchv/** +!scratchv_dag/ +!scratchv_dag/** +!benchmarks/ +!benchmarks/** +!scripts/ +!scripts/run_inference.py +!probes/ +!probes/**/ +!probes/**/*.py +!probes/**/*.json +!deployment/ +!deployment/w5/ +!deployment/w5/Dockerfile +**/__pycache__/ +**/*.pyc +probes/**/out/ +probes/**/output/ diff --git a/docs/INDEX.md b/docs/INDEX.md index 628ca94b..cda8f449 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -44,6 +44,7 @@ | [developer_guide.md](reference/developer_guide.md) | 开发者指南：代码规范、测试、CI | | [optimization_guide.md](reference/optimization_guide.md) | 优化指南：从哪里入手、怎么验证 | | [verification.md](reference/verification.md) | 验证方法：如何确认编译结果正确 | +| [W5 生成合同与当前进度](llm-deploy-v1.0/W5/README.md) | Host token 循环、固定参考、完整生成/容器证据与 CI 验收边界 | | [CODING_STANDARDS.md](reference/CODING_STANDARDS.md) | 代码风格规范 | | [CONTRIBUTING.md](../CONTRIBUTING.md) | 贡献指南 | diff --git a/docs/llm-deploy-v1.0/W4/runtime-contract.md b/docs/llm-deploy-v1.0/W4/runtime-contract.md index 4ef42c86..ced771fd 100644 --- a/docs/llm-deploy-v1.0/W4/runtime-contract.md +++ b/docs/llm-deploy-v1.0/W4/runtime-contract.md @@ -63,6 +63,33 @@ int scratchv_run_external(const void *const inputs[], blocked_fma；后者按 128 项 K 分块、块内显式融合乘加、块间 FP32 相加。保持非 fast-math、禁止隐式 FMA 合并，不能将两种累加顺序声称为逐位等价。 +`mean_policy` 默认为 `sequential`。W4/W5 CLI 可显式传 +`--mean-policy two-four-lane-last-axis`，只对 FP32 单一末轴 ReduceMean 使用已有 +W3 参考的双四路固定求和顺序；这是便携标量 C，不要求宿主或 guest 支持 SSE/RVV。 +非末轴归约、MatMul 和 Softmax 不随此选项改变。它可能改变 FP32 舍入结果， +不能当作与旧顺序逐位等价，也不能标成完整的 `numpy-fp32-reference-v4` 实现。 +构建/运行报告记录该策略，旧构建缺字段只解释为 `sequential`；不同策略的 ELF +不得混用。默认值、有限性检查、错误传播和工作区要求保持不变。 + +`nonlinear_policy` 默认为 `libm`，保留原 Sin/Cos、Sigmoid 和 Softmax 路径。 +显式 `--nonlinear-policy fp32-reference` 使用已有参考的非融合三角多项式、 +Sigmoid 有理多项式及 Softmax 的 16 路求和/倒数乘法顺序；以标量 C 和显式 FP32 FMA +实现，不要求 guest 支持 AVX512，也不按宿主 ISA 自动切换。三角函数在 +`abs(x) >= 18000` 时保留平台 libm 回退；不承诺该范围与所有平台逐位一致。 +它与 Mean 选项独立，既不改变 MatMul 策略，也不改变 Pow。两个选项都可能改变 +舍入结果；不能把组合称为完整 FP32 reference profile 的逐位实现或任意 ORT 版本的保证。 +原有限性检查、错误传播、Softmax 行并行/回调复用和工作区契约仍适用。 + +参考 Sigmoid 将输入夹到 [-18, 18]、输出夹到 [0, 1]；Softmax 将减去行最大值后的 +指数输入下限固定为约 -103.972084。原非有限输入（包括 -inf mask）以及有限值相减 +产生的非有限结果仍被拒绝。这是固定数值策略，不代表通用精度提升；数值对照以 +IEEE FP32、最近偶数舍入（RNE）、保留次正规数及显式 FMA、禁止隐式融合/fast-math +为前提。现有验证覆盖当前 Host 与 RV64GC/QEMU，不构成任意 ORT 版本、CPU dispatch、 +NPU 或 flush-to-zero 模式的等价保证。 + +构建/运行证据记录此策略，缺字段的旧 ELF 只解释为 `libm`；CLI 复用必须显式匹配。 +数值策略需要重新生成 ELF，兼容的原权重 bundle 可以复用。 + `riscv_external.py` 验证 ELF 架构、ABI、入口、装载段和布局，启动裸机 QEMU。guest 输出完成帧后，host 用 QMP 停止并转储精确大小输出，再退出；输出不是由 host C/NumPy 替算。实际 QEMU 在完整运行前须通过严格 FMA 位模式/FCSR 检查。 diff --git a/docs/llm-deploy-v1.0/W5/README.md b/docs/llm-deploy-v1.0/W5/README.md new file mode 100644 index 00000000..b9285181 --- /dev/null +++ b/docs/llm-deploy-v1.0/W5/README.md @@ -0,0 +1,279 @@ +# W5：Host 生成合同与端到端验收 + +合同版本：`w5-generation-v1`，冻结日期：2026-10-23。W5 从 PR104 已合入的 +`main`（`61bded377097900f4c61323269fd6aed41f333d5`）开始，独立分支为 +`codex/w5-generation`。阶段目标和责任分工沿用[开发计划](../开发计划.md)。 + +W5 的生成合同、Host 循环、QEMU 适配/命令行、三用例验收入口、三轮历史和容器部署文件 +现已实现。P0/P1 的冻结 ORT 参考与上轮 1/4 hart 小图记录保留。初始旧算术配置的 +4 hart 完整模型三例均在首步完整 logits 检查失败，这些历史结果仍完整保留。 +后续显式启用 Mean 与非线性参考策略后，Host 三例预检与正式 4 hart QEMU 套件均已 +**24/24 步、3/3 用例通过**。QEMU 每步完整 logits、冻结 token IDs、最终文本和停止 +原因均通过，全部步骤最大绝对误差为 **0.000303983688**，保持严格 `<1e-3` 门槛。 +通过结果限定于下述显式配置，未将默认算术路径标为通过，详见 +[完整模型实跑报告](qemu-generation.md)。本轮干净容器的默认 1 hart、手动 4 hart、 +auto 小图和真实超时/取消、Docker 构建取消检查已通过；完整 28 层 France 单 token +容器部署也已通过，实际输出 ` Paris`，完整 logits 最大误差为 **0.000172138214**。 +**W5 本地功能出口已满足**；W5 CI 已接入既有流程并通过本地相关检查，远端运行 **NOT_RUN**； +WSL 配置未改，所有 W5 改动仍未提交、未推送。 + +使用入口：[QEMU 生成与本轮报告](qemu-generation.md)、[三轮历史](chat.md)、 +[干净容器部署](deployment.md)。 + +## 固定生成合同 + +| 项目 | v1 约定 | +|---|---| +| 模型 | `Qwen/Qwen3-0.6B`，revision `c1899de289a04d12100db370d81485cdf75e47ca` | +| 资产身份 | 沿用 [W1 manifest](../../../probes/w1_qwen3_export/manifest.json) 中固定 ONNX、权重和 tokenizer；不另建校验体系 | +| 计算范围 | FP32、batch=1、固定序列容量 256、无 KV cache；每个新 token 重新执行完整前向 | +| 输入模式 | raw completion：使用 prompt 原文，不插入 BOS/EOS、系统提示或 chat 模板；tokenizer 按既有 NFC 规范化 | +| 输入张量 | INT64 `input_ids[1,256]`，右侧使用固定 `pad_token_id` 补齐；FP32 `attention_mask[1,1,256,256]`，沿用 W2 causal/key-padding 语义 | +| 有效长度 | 由实际 token 数确定；prompt 中出现 pad/EOS ID 不改变长度，不据此停止 | +| 前向输出 | 完整 FP32 `logits[1,256,151936]`；包括未使用 padding 行在内，所有值都必须有限 | +| 选择规则 | 取 `valid_length - 1` 行，直接 greedy argmax；平局选较小 ID，不额外计算 Softmax | +| 可解码范围 | 选中 ID 必须属于固定 tokenizer 的真实 ID 集合；不能仅以小于模型词表宽度为准 | +| 追加规则 | 每次前向最多追加一个合法 token；不可解码 ID 报错，不屏蔽、不替换、不重采样 | +| 停止顺序 | 新生成 EOS → 总长度达到 256 → 已生成数量达到 `max_new_tokens` | +| EOS | 使用固定 tokenizer 的 `generation_eos_token_ids`：`151645` 和 `151643`；只有新生成 token 触发 EOS | +| 输出文本 | 仅解码新生成 IDs，`skip_special_tokens=True`、`clean_up_tokenization_spaces=False`；完整新生成 IDs 是验证依据，EOS ID 保留在其中 | +| 返回数据 | prompt IDs、新生成 IDs、生成文本和停止原因组成不可变结果；无隐式会话历史 | + +生成核心复用 [W2 输入与 greedy 辅助函数](../../../scratchv/runtime/llm_inputs.py) +和[本地 tokenizer](../../../scratchv/runtime/qwen3_tokenizer.py)。模型词表为 151936， +固定 tokenizer 实际包含 151669 个 ID；这两者不能混用。主循环不调整数值误差门槛， +也不会只检查用于 argmax 的那一行而漏掉其他输出中的 NaN/Inf。 + +`max_new_tokens` 是上限，不预留或扩展容量。prompt 非空且不超过 256 个 token； +空 prompt、超长 prompt、非法 token、负数/布尔预算明确报错，不静默截断。 +0 预算不会调用前向；初始长度已为 256 时也不调用前向，按容量原因返回。 +若一次追加同时触发 EOS、容量和预算，按上表顺序报告原因。生成 EOS 仍计入预算。 + +## Host 调用及失败语义 + +[Host 核心](../../../scratchv/runtime/generation.py) 的公开入口为： + +```python +generate(prompt, *, tokenizer, forward, max_new_tokens, + cancelled=None, timeout_seconds=None) -> GenerationResult +``` + +它接受一个 `PreparedInputs -> np.ndarray` 的同步 `forward` 回调。 +回调可接 ORT 或已实现的 `QemuForward` 适配器；核心不编译模型、不管理 QEMU，也不为每个 +新 token 重建 tokenizer。每一步从本次生成的 token 历史重新准备输入，复用后端输出 +缓冲区不会改变已经选出的 token；不同调用不共享生成状态。 + +失败行为属于合同： + +- 后端原始异常向调用方传播；非法形状、dtype、非有限输出及不可解码 ID 都使本次生成失败。 +- 取消抛 `GenerationCancelledError`（`RuntimeError` 子类），超时抛 `TimeoutError`，不返回看似成功的部分结果。下一次调用从新的 prompt 开始，不能继承失败调用的 token。 +- `cancelled` 是可选取消检查回调；检查发生在同步前向前后及返回前。它不强制中断正在阻塞的前向。 +- `timeout_seconds` 是可选有限非负秒数，布尔值不合法；0 表示立即超时，即使 token 预算为 0。 +- 超时从输入与配置验证完成后开始，以单调时钟作协作式检查；取消与超时同时触发时先报告取消。正在运行的同步后端由后端自身负责限时、取消和进程清理；Host 合同不声称能终止任意阻塞回调。 + +本轮不会为每个 token 新建线程来模拟可中断执行，也不会吞掉失败后继续选 token。 +QEMU 适配已复用现有运行器，在 UART/QMP 等待中响应取消，并验证超时/取消后的进程及进程组回收。失败运行按既有策略保留 transport 诊断副本。 + +## 3 条固定参考用例 + +以下三条原文和预算在 ORT 执行前选定；Paris 原文与 W5 硬门槛保持一致。 + +| 用例 | prompt 原文 | `max_new_tokens` | 预先约定的内容检查 | +|---|---|---:|---| +| `france` | `The capital of France is` | 8 | 新生成文本包含 `Paris` | +| `germany` | `The capital of Germany is` | 8 | 新生成文本包含 `Berlin` | +| `japan` | `The capital of Japan is` | 8 | 新生成文本包含 `Tokyo` | + +参考必须记录 prompt IDs、新生成 IDs、文本、EOS/停止原因和实际 ORT 版本； +不能只填写上述预期词。未来 QEMU 对照优先比较逐 token ID；若出现分歧，保留相同 +输入的数值误差和 top-1/top-2 margin，定位原因，不用“语义合理”覆盖数值失败。 +原始全量 logits 是运行中验证对象，无需将每一步约 148 MiB 输出都加入 Git。 + +[固定参考文件](../../../probes/w5_generation/references.json)保存三条输入及冻结后的 +精确输出，检查入口为 [run.py](../../../probes/w5_generation/run.py)。在仓库根目录， +激活已按 [Linux 复现说明](../LINUX_REPRODUCTION.md)准备的环境后执行： + +```bash +python probes/w5_generation/run.py \ + --assets assets/qwen3-0.6b \ + --output-dir output/w5-generation-reference +``` + +这条命令使用固定资产和同一 ORT CPU session 验证三条参考，并在输出目录保存 +`reference-run.json`；不会启动 QEMU。默认检查已冻结参考，不自动覆盖它。 +初次采集需要显式 `--record`，并按预先固定的内容标准确认结果后才冻结；不能为消除 +失败任意重录参考。QEMU 用户入口已实现为 `scripts/run_inference.py`；与本节 ORT 参考工具用途不同，见 [QEMU 说明](qemu-generation.md)。 +2026-10-23 使用同一个 ORT CPU session 实跑，三条均完成 8 个 token，停止原因为 +`max_new_tokens`。独立逐步输入/greedy/停止 oracle 与 Host 核心一致，三个预先固定的 +内容检查均通过，精确 token IDs 已冻结到上述 JSON。 + +| 用例 | 实际生成的新文本（保留前导空格） | 耗时 | CPU 时间 | +|---|---|---:|---:| +| `france` | ` Paris. The capital of Italy is Rome` | 44.672 s | 44.649 s | +| `germany` | ` Berlin. The capital of France is Paris` | 42.910 s | 42.894 s | +| `japan` | ` Tokyo. The capital of the United States` | 42.406 s | 42.394 s | + +全轮耗时 **130.676 s**，CPU 时间 **130.611 s**；进程峰值 RSS **5.002 GiB** +（`ru_maxrss` 为整个进程累计峰值，不能当作每例独立分配或增量）。模型 session +加载耗时 0.307 s；这些是 ORT 参考实测，不是 QEMU 生成速度。 + +P0/P1 阶段只执行一次真实 ORT 采集并在每步使用独立 oracle 验证，随后冻结参考，完成离线 +字段一致性检查；未重复执行“冻结后第二次 ORT 回放”。精简公开记录、逐 token +margin 与环境位于 JSON 的 `baseline` / `cases`，详细本地产物为 +`output/w5-generation-reference/reference-run.json`。原始续写可以继续列举其他事实， +并因 8-token 预算停止；这三条只验证约定的 raw completion 内容，不宣称 chat 或多轮能力。 + +## 保留的执行配置 + +既有 RV64 路径保持默认 **1 hart**，显式接受 **1 / 2 / 4 / auto**。 +`auto` 只在构建/启动时按进程可用 CPU 解析一次，上限为 4，不在 token 之间动态改数。 +复用 ELF 的 hart 数必须与构建记录一致。Host 生成核心不覆盖这套配置，也不会把 +ORT 的 CPU 线程数误当成 guest hart 数。 + +QEMU 工作区布局保持 `qemu_workspace_layout=none`，即 padding 关闭。 +均值默认仍为 `sequential`；显式 `--mean-policy two-four-lane-last-axis` +可用于数值对照，但单独启用尚不能让完整三例通过，见续作报告。 +非线性路径默认仍为 `libm`；显式 `--nonlinear-policy fp32-reference` 选择已有参考的 +Sin/Cos、Sigmoid、Softmax 算术顺序，不改变 hart 选择，也不要求目标具有 x86 SIMD。 +本轮完整 QEMU 三例通过的配置为 `4 hart / blocked_fma / 2x4 / column-panel`，同时 +显式启用 `--mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference`， +padding 为 `none`；24 步复用同一 ELF/bundle。默认 `1 hart / sequential / 1x4 / original`、 +Mean `sequential` 和非线性 `libm` 均未改变，不能把新策略的验收结果归给旧算术配置。 +复用时必须显式匹配构建策略；实验 padding、RVV、KV cache、动态长度和量化不纳入本轮。 + +## 阶段与验收状态 + +| 阶段/门槛 | 本轮状态与后续要求 | +|---|---| +| P0：合同与 3 条参考 | **PASS（本地）**；合同冻结，3/3 真实 ORT/Host 参考与独立 oracle 通过，参考 IDs/text/停止原因冻结 | +| P1：Host token 循环 | **PASS（本地）**；核心 55 项及下述直接相关验证通过 | +| P2：QEMU 生成适配/CLI | **PASS（本地小图）**；构建一次，同一 ELF/bundle 跨 token 和跨命令复用；1/4 hart 数值与中断清理通过 | +| `e2e:qemu-paris` | **PASS（本地、显式新策略 4 hart）**；France 完整 8 token 的数值、IDs、文本和停止原因通过，输出包含 Paris。历史旧配置首步 FAIL 的 `max_abs=0.003272057` 保留于实跑报告 | +| `e2e:qemu-prompts` | **PASS（本地、显式新策略 4 hart）**；3/3 用例、24/24 完整 QEMU 前向及逐步数值通过，IDs/text/停止原因与冻结参考一致。历史旧配置初始轮仅检查 3/24 步且均 FAIL，余下 21 步当时未执行 | +| `unit:multiturn` | **PASS（本地）**；40 项会话测试含官方模板/IDs 六份对照；raw 与显式 chat 分开，失败不提交历史 | +| `deploy:script` | **PASS（本地真实容器）**；默认 1/手动 4/auto 小图、真实 QEMU 超时/取消复用与 Docker 构建取消通过；完整 28 层 France 单 token 实际生成 ` Paris`，数值与进程清理通过，不重复宿主三例套件 | +| W5 CI 接入 | **本地实现与检查 PASS；远端 NOT_RUN**；提交前 811 项相关检查通过，最终 11 项重叠边界复查通过；14 个真实必跑节点复用既有证据，本轮未重新执行模型；未提交/推送 | + +W4 完整单次前向的已有证据不替代 W5 token 循环；本页 W5 结果来自上述独立完整生成验收。 +W4 独立复现、Nightly 和团队签收仍按 [W4 验收说明](../W4/validation.md)分别确认。 + +完整 QEMU 三例验收已通过，原始结果位于 +`output/w5-followup-20261023/qemu-reference-4hart/run/report.json`。 +gate 失败状态、旧 W4 构建审计传递、逐步验收证据及取消清理已补强。 +本地功能出口已有完整证据：宿主三例 24 步、宿主独立前向，以及干净容器镜像、 +小图、中断清理和完整 Paris 单 token 均已通过，见 [部署记录](deployment.md)。 +容器此次只补一次完整前向，不将其写成容器三例套件；CI 已完成本地接入和相关检查，远端 +**NOT_RUN**，本地功能完成不等于已经取得远端 Actions 或发布验收记录。 +不把候选 token 一致或静态部署检查写成实跑 PASS。当前每 token 启动 QEMU，但复用构建； +持久 guest、权重驻留、KV cache 等不属于 W5 v1 必需实现。 + + +## CI 接入与远端执行边界 + +W5 复用 [w4-full-numeric.yml](../../../.github/workflows/w4-full-numeric.yml)，没有再建立 +一套并行的完整模型工作流。手动 `workflow_dispatch` 的 `mode` 为互斥选择： + +| mode | 执行范围 | +|---|---| +| `small`（默认） | 相关单测和真实小图，不运行完整模型 | +| `w4-forward` | 原 W4 完整单次前向 | +| `w5-generation` | 原 `deploy_w5.sh` 在真实容器中运行三例 3×8 token，再校验完整报告 | + +已有 `workflow_call` 的 `run_full/case` 接口保留，新增 `run_w5=false`，不自动开启生成。 +W5 选择与 W4 完整前向冲突，或在非手动事件请求 W5 时，先于模型执行明确失败。 + +**合并前的手动入口**使用 main 上已经存在的 +[llm-deploy.yml](../../../.github/workflows/llm-deploy.yml)，显式传 +`run_w5_validation=true`；该模式跳过旧 W1/W2 完整任务与 W3/W4 完整执行， +只调用同一提交的 W5 验证。W5 与 W3/W4 同时勾选会直接失败，不同时跑多套模型。 +GitHub 要求手动工作流存在于默认分支，见 +[官方手动运行说明](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manually-run-a-workflow)。 +审查时远端 main 的 `w4-full-numeric.yml` 尚无 `workflow_dispatch`，因此不能把新文件的 +直接手动入口当作仅推送 W5 分支后必然可用；不要为了建立入口先合并未验收的 W5。 + +下列命令仅供后续获得提交、推送和远端验证授权后使用，本轮没有执行；还需要仓库写权限： + +```bash +gh workflow run llm-deploy.yml --ref codex/w5-generation \ + -f run_w5_validation=true -f run_w3_validation=false -f run_w4_validation=false +``` + +新 `w4-full-numeric.yml` 手动入口进入默认分支后，也可使用上表的 `mode`。 +W5 完整 job/步骤/CLI 总预算分别为 +**350 / 285 / 275 分钟**；镜像和固定资产落盘后才检查 **至少 4 CPU、10 GiB 可用 RAM、 +10 GiB 剩余磁盘**，不足即明确失败，不通过缩小模型或跳过 token 凑齐门槛。 +远端 runner 是否实际满足资源和时限，仍须授权提交/推送后实跑确认。 + +W4 unit 增加 **12 个真实 QEMU 必跑节点**（生成与取消 2、Mean 8、非线性 2）； +[llm-deploy.yml](../../../.github/workflows/llm-deploy.yml) 在现有 tokenizer 环境补 **2 个 +官方三轮模板节点**。W5-only 跳过普通 llm-deploy job 时，使用已经构建的同一 +W5 镜像、只读 tokenizer 与测试文件执行这两个模板节点，并保留必跑 JUnit 检查； +不会因此再安装一套依赖或重复生成。普通 W5 单测继续由既有通用 CI 承担。 +最终汇总区分未选择的 `NOT_RUN` 与请求后失败/取消/skip，不能把缺失证据作为通过。 + +上一轮 CI 接入的 **753 项轻量检查通过（19.64s）**，覆盖工作流结构、旧 W4 接口、路径路由、 +JUnit 缺失/skip/fail、资源短缺、退出码与汇总；其中严格报告 gate 为 44 项。 +上述 **14 个必跑节点在 CI 接入阶段仅 collect-only 核对 ID**，该阶段没有再次执行这些节点或完整模型； +已有真实容器生成/取消 2 节点通过，Mean/非线性沿用此前真实小图证据，不把收集当实跑。 +[严格报告检查器](../../../scripts/check_w5_ci.py) 对已有真实三例 24 步报告 +`output/w5-followup-20261023/qemu-reference-4hart/run/report.json` 离线检查通过。 +本地 CI 检查记录为 `output/w5-ci-integration-20261023/validation.json`， +同目录保存测试日志、14 个收集节点 ID 以及 98 段内嵌 shell 的语法检查结果。 +这些都是本地实现/证据检查，**远端 W5 CI 仍为 NOT_RUN**，未提交、未推送。 + +## 提交前检查与收尾（2026-10-23） + +已核对 W5 全部待提交改动：生成合同与冻结参考、Host/会话循环、QEMU 适配与 CLI、 +显式数值策略、运行器清理、容器打包、第三方声明、CI 路由和报告。 +**本地可执行的 W5 工作已完成，未发现尚未处理的本地阻塞；远端 CI 验收仍待执行。** +本轮没有重新运行完整模型或 QEMU 小图，没有暂存、提交或推送。 + +本轮修复和澄清： + +- 两个有限 FP32 logits 的诊断差值原先可能溢出为 Infinity，导致严格 JSON 报告写入失败。 + CLI 和 Host 参考工具现先转为 Python float 再相减；只改变诊断计算，不改变模型、 + argmax、数值容差、冻结 IDs/text 或每行计算顺序。新增正负 FP32 最大值边界测试。 +- 补齐合并前可用的 W5-only 手动 CI 入口：复用已注册的 llm-deploy，拒绝与 W3/W4 + 同时选择，保留真实小图、官方三轮模板和完整生成门槛。job 预算增加到 350 分钟， + 给准备、清理和报告上传留余量，生成步骤与 CLI 的时限未放宽。 +- 修正测量口径：宿主/进程树的 21,032 次采样无读取错误，但 24 步中有 17 步的 + 独立 QEMU RSS 各缺一次读取；RSS 峰值是观测值。容器运行器记录的 430.481 秒 + 不包含前置输入打包/写入和 ELF/bundle 校验，原始测量值没有改动。 + +| 本轮必要验证 | 结果与边界 | +|---|---| +| CLI 与 Host 参考工具 | 107 passed，0 skip；包含两处溢出复现及严格 JSON 边界 | +| 数值策略、external 运行器及 W4 契约 | 280 passed，0 skip；默认值、错误传播、上下文/工作区复用和取消清理 | +| CI、部署与打包相关检查 | 811 passed，0 skip；最终 11 项边界复查与此集合重叠，不重复累计 | +| 工作流与脚本静态检查 | actionlint 1.7.12 检查三份工作流、Bash 语法及 diff 检查通过；未运行外部 ShellCheck/Pyflakes | +| 既有三例 24 步报告 | 严格 CI 检查器离线核验 PASS；未重新推理 | + +本轮证据在工作树下的 `output/w5-precommit-20261023/`： +`generation-audit.json`、`numeric-audit.json`、`ci-audit.json` 及对应 JUnit。 +前后轮测试集合存在交叉，不将数量累加为独立覆盖总数。 +默认 1 hart、手动 1/2/4 与启动时 auto 保持正常，W5 padding 强制为 none； +Docker 本轮未启动，WSL 配置未改。 + +剩余事项按性质区分： + +1. **合入/发布前仍须取得远端 CI 证据**：后续获准推送后，使用上节 W5-only 入口； + runner 资源不足、失败或取消必须明确失败，不能用本地 PASS 替代。 +2. **数值验收配置限制**：完整模型 PASS 仅属于已记录的显式参考算术组合； + 原默认 Mean sequential/nonlinear libm 的历史完整 logits 超差仍保留，不能宣称默认路径也通过。 + 本轮没有修改算术，因此未重复完整模型。 +3. **已知运行边界**：同步构建、传输和 ORT 调用按阶段检查预算，不承诺硬中断任意 ORT 调用。 + KV cache、持久 guest/权重驻留、RVV 和量化属于后续阶段，不属于 W5 v1 未完成的出口项。 + +## P0/P1 已有验证（2026-10-23，未重复执行） + +| 验证 | 结果 | 覆盖边界 | +|---|---:|---| +| [Host 生成核心](../../../tests/test_generation.py) | 55 passed | 多步输入与结果、0/负数/布尔预算、EOS/容量优先级、非法输出/ID、第二步错误、取消/超时、不同上下文及输出缓冲区复用 | +| [参考工具](../../../tests/test_w5_generation_reference.py) | 16 passed | 独立逐步输入/greedy/停止 oracle、冻结字段差异、语义检查和错误识别 | +| [输入辅助函数](../../../tests/test_llm_inputs.py) / [tokenizer](../../../tests/test_qwen3_tokenizer.py) | 148 passed | 既有固定 ABI、mask、词表、解码和严格边界保持正常 | +| [hart 选择辅助函数](../../../tests/test_riscv_external_harts.py) | 28 passed | 默认/手动/auto、CPU 可用性回退、非法计数和栈规划；11 项不相关测试未选中 | +| [W4 CLI 配置边界](../../../tests/test_w4_full_probe_contract.py) | 10 passed | mock 构建覆盖默认单核、手动 2/4、auto 一次解析及复用 hart 匹配；未启动 QEMU | + +P0/P1 共 **257 passed**；本轮新增的适配、CLI、多轮、部署及相关运行器检查见 [续作报告](qemu-generation.md)。不将前后轮重叠测试简单累加。 +这组结果验证新 Host 核心及已有配置边界，不代表真实 QEMU 的多 token 端到端门槛通过。 +文档检查确认本页本地链接和索引入口可解析，未增加 AI 审查。 diff --git a/docs/llm-deploy-v1.0/W5/chat.md b/docs/llm-deploy-v1.0/W5/chat.md new file mode 100644 index 00000000..397e0fcb --- /dev/null +++ b/docs/llm-deploy-v1.0/W5/chat.md @@ -0,0 +1,27 @@ +# W5 Host 三轮文本对话 + +状态：Host 历史管理和官方模板一致性已实现；这里的验证不加载模型权重，不代表完整 QEMU 多轮语义验收。 + +## 接口和语义 + +`Conversation.from_directory(tokenizer_directory, system_prompt=None)` 从同一固定本地 snapshot 加载 tokenizer 与 `tokenizer_config.json` 中的官方 `chat_template`。生产运行只需 `tokenizers` 和 `Jinja2`，不依赖 Transformers 或 Torch。模板通过 Jinja2 的不可变沙箱渲染，固定 `add_generation_prompt=True`、`enable_thinking=False`，不手写替代 Qwen3 的模板。 + +`turn(user_text, forward=backend, max_new_tokens=..., cancelled=..., timeout_seconds=...)` 复用 Host `generate`，返回同一 `GenerationResult`。仅支持文本 system/user/assistant：system 只能在构造时提供，后续 user/assistant 顺序由成功的 turn 维护，不支持工具、图片或其他角色。`history` 是不可变快照；`reset()` 仅清除已完成回合，保留初始 system 文本。 + +每轮将完整历史重新套用官方模板并编码，不保留 KV cache，不暗中删除早期消息，也不截断到 256 tokens。超过容量会在调用 backend 前报错。原始完成模式的三条参考用例保持 raw prompt，不自动套用聊天模板。 + +一次 turn 只有成功返回且至少生成一个 token，才同时提交 user 和 assistant 消息。异常、取消、超时、零 token 预算，以及恰好占满 256-token 上下文且没有生成空间的情况，均不修改历史。EOS-only 回复可作为合法空 assistant 回复提交。预算或容量中止后的回复保留返回的文本，`stop_reason` 明确记录其终止原因。历史存储文本；下轮重新编码，未承诺历史 token IDs 原样拼接。 + +超时与取消沿用 Host 生成合同。该层的协作检查不能中断正在阻塞的 backend；QEMU adapter 负责运行中的取消、超时以及进程清理。一个 Conversation 实例不能并发或重入使用，活动 turn 中拒绝另一个 turn 或 reset。 + +## 相关验证 + +```bash +python -m pytest tests/test_conversation.py -q +``` + +固定本地 tokenizer 存在时，测试还会使用 `AutoTokenizer.from_pretrained(..., local_files_only=True)` 独立核对官方 Transformers 的 `apply_chat_template`，无网络下载、无模型前向。可通过 `SCRATCHV_QWEN3_TOKENIZER_DIR` 指定本地快照，默认是仓库 `assets/qwen3-0.6b/tokenizer`。 + +2026-10-23：40 项通过，无跳过。覆盖三轮精确历史顺序、非 thinking 格式、两会话隔离、reset、不可变快照、角色/文本校验、零预算、EOS-only、容量边界、异常原样传播、取消/超时回滚以及重入保护。其中官方 tokenizer 对照覆盖有/无 system 的各三轮，共六份渲染文本和 token 序列；assistant 文本在此模板测试中为预设值，未作完整模型生成。 + +默认单核、手动/自动 hart 选择及 padding 策略由已有 backend 保持，本模块不改变它们。 diff --git a/docs/llm-deploy-v1.0/W5/deployment.md b/docs/llm-deploy-v1.0/W5/deployment.md new file mode 100644 index 00000000..5742154a --- /dev/null +++ b/docs/llm-deploy-v1.0/W5/deployment.md @@ -0,0 +1,201 @@ +# W5：Linux 容器部署入口 + +本页说明 [Dockerfile](../../../deployment/w5/Dockerfile) 和 +[一键脚本](../../../scripts/deploy_w5.sh) 的使用方式。 +生成合同、固定参考和阶段门槛见 [W5 总览](README.md)。 + +## 部署范围 + +镜像使用 Ubuntu 24.04 / Python 3.12，安装发行版 `qemu-system-misc`，并直接复用 +`python -m pip install -e ".[llm-deploy]"` 固定的依赖，其中包括 Zig 0.14.1。 +不再维护第二份 Python/Zig 版本列表。镜像中的 QEMU 由 Ubuntu 软件源提供； +正式验收须记录实际版本，不能将它的耗时与另一 QEMU 版本的历史记录直接比较。 + +基础镜像默认仍为 `ubuntu:24.04`。Docker Hub 不可达时，可显式设置 +`SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu:24.04`，选择 Canonical 发布的 +[Amazon ECR Public Ubuntu 镜像](https://gallery.ecr.aws/ubuntu/ubuntu)； +来源说明见 [Ubuntu 官方容器文档](https://ubuntu.com/docs/oci-registries/)。 +脚本将该值作为一个完整的 `--build-arg BASE_IMAGE=...` 参数传入 Dockerfile； +未设置或为空时保留默认值，不自动切换来源、不重标记 `ubuntu` 镜像，也不修改引擎或代理配置。 +例如，在下方部署命令前增加： + +```bash +SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu:24.04 \ + bash scripts/deploy_w5.sh --help +``` + +此例仍会通过原脚本构建镜像，再运行 CLI 帮助；实际生成时追加下方原有参数即可。 + +[构建上下文白名单](../../../deployment/w5/Dockerfile.dockerignore) 仅包含源码、安装 +元数据和命令行入口所需文件，模型权重、tokenizer、虚拟环境、ELF、Git 和历史输出 +均不随镜像构建上下文发送。模型资产在运行时只读挂载到 `/assets`，构建和报告写入 +可写挂载 `/output`。脚本按调用者的 Linux UID/GID 运行，避免产生 root 所有的报告。 + +脚本只做容器预检、构建与启动，所有生成参数原样传给已有 CLI，不再实现一套 +采样、token 循环、模型下载或校验流程。未准备资产时明确失败，不自动下载权重。 +Linux 主机需要 `setsid`（通常由 util-linux 提供）。引擎预检和镜像构建客户端在各自 +进程组内运行；脚本收到 SIGINT/SIGTERM 时会向客户端和辅助进程转发终止、等待清理， +5 秒后对仍存在的进程组强制结束，并以 130/143 返回，不继续启动模型。 +容器启动后保留 `exec ... run --init` 的信号转发路径。本轮已额外验证真实 Docker +构建取消时客户端进程组和引擎侧 RUN 进程均消失，实测范围和证据见下文。 +Docker 是本页的部署目标；替换容器 CLI 前需确认其支持 Dockerfile 专用 ignore 文件， +本轮未验证 Podman 兼容性。 + +## 小图部署验证 + +先在正常 Linux 开发环境准备一个满足生成 ABI 的小图及对应本地 tokenizer，放在同一 +资产目录下：`model.onnx`、`tokenizer/`；若 ONNX 使用外置数据，数据文件也必须位于 +该挂载目录中。小图使用与正式生成相同的固定长度输入和 logits 接口，但不能当作 +Qwen3 的语言能力或 Paris 门槛证据。 + +本轮已有真实 CLI 小图资产 `output/w5-cli-small-20261023/model.onnx` 和 +`output/w5-cli-small-20261023/tokenizer/`，可在同一工作树中复用。先复制到统一的 +只读资产挂载目录，再执行下面的容器命令；本轮真实容器结果见页末： + +```bash +mkdir output/w5-small-assets +cp -a output/w5-cli-small-20261023/model.onnx \ + output/w5-cli-small-20261023/tokenizer output/w5-small-assets/ + +SCRATCHV_W5_ASSETS="$PWD/output/w5-small-assets" \ +SCRATCHV_W5_OUTPUT="$PWD/output/w5-container-smoke" \ + bash scripts/deploy_w5.sh \ + --model /assets/model.onnx \ + --tokenizer-dir /assets/tokenizer \ + --prompt 1 --max-new-tokens 8 \ + --matmul-policy blocked_fma \ + --harts 1 --timeout 30 --total-timeout 120 \ + --verify-onnx /assets/model.onnx +``` + +2026-10-23，以上资产已通过宿主 Linux CLI 的真实 QEMU 对照：1 和 4 hart 均 +**PASS**，各运行 4 次前向，生成 IDs 同为 `[2, 3, 4, 6]`，文本为 `2 3 4`，由 EOS +停止；每步进行 ORT 数值/greedy 对照。使用显式 `blocked_fma / 1x4 / original`、 +padding `none`。证据位于 `output/w5-cli-final-20261023/hart1/report.json` 和 +`output/w5-cli-final-20261023/hart4/report.json`，这两份是本地诊断产物，不提交 Git。 +上述两份记录是**宿主 CLI 小图通过**，本轮新增的真实容器证据另列于页末。宿主完整模型的初始旧算术 +配置曾使三例均在首步完整 logits 检查失败；该历史结果保留于 [实跑报告](qemu-generation.md)。 +随后显式启用 Mean 与非线性参考策略的正式 4 hart QEMU 套件已完成 **3/3 用例、24/24 步通过**， +数值、冻结 IDs、最终文本与停止原因均满足合同。通过配置对应下方完整验收示例， +默认单核、Mean `sequential`、非线性 `libm` 与 padding `none` 不变。 +这些宿主结果不替代容器证据；本轮容器小图和清理、完整 Paris 单 token 均已有独立实跑通过记录。 + +脚本默认将 CLI 输出写入 `/output/run`，该子目录必须尚不存在；重复验证时显式 +增加 `--output-dir /output/run-h4` 等新目录，并传 `--harts 4`。不重复使用已有输出目录， +也不自动删除旧报告。宿主输出目录可以包含空格，挂载路径不可包含逗号。 + +没有指定 `--harts` 时仍为单核，手动 `1/2/4` 和 `auto` 原样交给 CLI。`auto` 在构建时 +解析一次，不在 token 之间改变 ELF 的 hart 数。padding 始终关闭。运行器每个 token +可以重新启动 QEMU，但复用同一轮构建得到的 ELF 和权重 bundle，不按 token 重新编译。 + +`--reuse-build` 仅复用相同容器路径下的已保存构建。构建元数据可能包含绝对路径， +不要直接把宿主生成的 ELF/bundle 元数据拿到 `/output` 下宣称可跨环境复用；跨环境 +从 `--model` 重新构建。在容器内先构建、后复用时，保持挂载目标和构建路径一致。 + +## 容器完整模型最短生成命令（本轮已通过） + +固定真实模型资产沿用 W1 manifest 和 W5 三条参考；目录结构是 +`onnx/model.onnx`、外置权重和 `tokenizer/`。本轮仅补 France 的一个真实生成 token， +不在容器重复已经通过的宿主三例 24 步： + +```bash +SCRATCHV_W5_ASSETS="$PWD/assets/qwen3-0.6b" \ +SCRATCHV_W5_OUTPUT="$PWD/output/w5-container-acceptance" \ +SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu:24.04 \ + bash scripts/deploy_w5.sh \ + --model /assets/onnx/model.onnx \ + --tokenizer-dir /assets/tokenizer \ + --prompt 'The capital of France is' --max-new-tokens 1 \ + --verify-onnx /assets/onnx/model.onnx --harts 4 \ + --matmul-policy blocked_fma --matmul-tile 2x4 \ + --matmul-rhs-layout column-panel \ + --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ + --timeout 1800 --total-timeout 3600 +``` + +该命令执行一遍完整 28 层模型，验证容器内实际生成的 ` Paris`，不会运行 3×8 token。 +本轮实测产物位于 `output/w5-container-20261023/paris/`，已成功生成 ` Paris`，进程退出码为 `0`。 +单 prompt 命令的 `full_model_gate=NOT_RUN` 表示没有再次运行三例 suite； +不能把它误读为没执行完整模型，也不能将单次成功计为容器三例通过。 +宿主三例和独立单次前向记录继续保留于 [实跑报告](qemu-generation.md)。 + +可设置 `SCRATCHV_CONTAINER_ENGINE` 为 Docker 可执行文件， +`SCRATCHV_W5_IMAGE` 为本地镜像标签；默认分别为 `docker` 和 `scratchv-w5:local`。 +一键脚本会先检查引擎存在且可用，再构建镜像；不会安装引擎、修改 WSL 或 Windows。 + +## 本轮真实容器验证（2026-10-23） + +早期检查时环境没有容器引擎，历史静态检查不冒充容器验收。本轮已使用 **Docker +Engine 29.9.2 / Buildx 0.38.1** 真正构建镜像并运行容器。引擎仅手动启动； +验证结束后 `docker.service / docker.socket / containerd.service` 均已 +**inactive + disabled**，停止前无运行容器。临时 socket 权限恢复为 `root:docker 0660`， +未新增永久组成员，WSL 配置原文保持不变；已安装引擎保留，不开启自启。 + +Docker Hub 的认证请求出现 TLS EOF，IPv4/TLS 1.2 复查仍失败；本轮因此显式设置 +`SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu:24.04`，使用上方 Canonical +官方来源。默认仍为 `ubuntu:24.04`，没有重标记该镜像或修改代理配置。 +基础镜像和部署镜像实际拉取/构建成功；小图镜像内 Python **3.12.3**、Zig **0.14.1**、 +QEMU **8.2.2（Ubuntu 包 1:8.2.2+ds-0ubuntu1.18）**、ORT **1.22.1**、 +NumPy **2.2.6**，Torch 为 **2.7.1+cpu**。 + +| 真实容器检查 | 结果 | +|---|---| +| 默认单 hart CLI 小图 | PASS；4 次前向，IDs `[2,3,4,6]`，文本 `2 3 4`，EOS 停止 | +| 手动 4 hart CLI 小图 | PASS；同上，完整 logits 逐位 ORT 一致 | +| `auto` 复用 4 hart 构建 | PASS；解析为 4，4 次前向，IDs/text/EOS 一致 | +| 三组小图数值与布局 | 全部 `max_abs=0`，padding `none` | +| 真实 QEMU 1/4 hart 多 token、超时、取消、复用 | 2 项测试 PASS、0 skip；每个配置 8 次 guest 启动（6 次成功、2 次预期中断），16 次进程/进程组均回收 | +| Docker 实际构建 SIGINT/SIGTERM | 2/2 PASS；退出码 130/143，客户端组与引擎侧 RUN 进程均消失，未启动模型 | +| 完整 28 层 France 单 token 生成 | **PASS**；IDs `[12095]`、文本 ` Paris`，预算停止，完整 logits `max_abs=0.000172138214 < 0.001`；28 层图审计、FMA、QEMU 回收均通过 | + +QEMU 清理测试直接复用已有 `test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel`， +不同上下文生成 `[4]`，超时/取消后再次生成 `[6]`，均使用同一配置内的 ELF/bundle。 +真实 Docker 构建取消使用原部署脚本的逐字节相同副本和独立小型诊断上下文， +在实际引擎侧 RUN 进程已运行后分别发 SIGINT/SIGTERM；两次约 **0.117s / 0.115s** +观察到清理完成，延后复查无残留。它验证实际 Docker 构建生命周期，不运行模型或下载资产。 + +之前的 Bash 语法及 **17 项**隔离/打包测试继续有效，包含默认基础镜像、空值回退、 +ECR 与字面量参数边界及客户端取消；本轮真实引擎结果另列，不把替身检查重复累计成真实容器测试。 +完整 Paris 验收由实际 Docker 调用独立执行：容器内从只读 ONNX 资产重新构建 ELF/bundle， +显式使用 `4 hart / blocked_fma / 2x4 / column-panel / mean two-four-lane-last-axis / +nonlinear fp32-reference / padding none`，仅生成 1 token。输入 prompt IDs 为 +`[785,6722,315,9625,374]`，输出 `[12095]`、` Paris`，停止为 `max_new_tokens`。 +全部 **38,895,616** 个 logits 对当轮 ORT 的最大误差为 **0.000172138214**，没有跳过补齐行。 + +| 容器 Paris 实测范围 | 结果 | +|---|---:| +| CLI 总耗时（含独立重新构建、FMA、ORT 和核验） | 547.454 s | +| UART 前向耗时（含 QEMU 启动和装载） | 415.058 s | +| QEMU 完整进程耗时（另含 stop/dump/quit） | 415.478 s | +| 运行器记录耗时（权重传输、QEMU、后检查及清理） | 430.481 s | +| QEMU RSS 观测峰值 | 5.398 GiB | +| 模型 QEMU 前向次数 / 成功回收 | 1 / 1 | + +运行器记录耗时不含计时起点之前的输入打包、输入文件写入及 ELF/bundle 校验，不能当作整个 adapter 调用耗时。 + +RSS 来自 0.2s 周期采样，共 2071 个有效样本；有 **1 次 RSS 读取失败**， +记录为 `partial`，上述值是观测峰值，不声称完整无缺失采样。本次没有独立采集 +QEMU/Host CPU 总时间，不用 Docker 客户端时间或宿主前向记录冒充容器 CPU。 +完整 **155,582,464 字节**输出已直接与既有宿主独立完整前向比较，**逐字节一致**， +没有新增哈希检查。 +正式 CLI 保留 `full_model_gate=NOT_RUN` 和 `container_reproduction=NOT_ASSERTED`： +单 prompt 没有重跑三例 gate，而容器身份不由内部 CLI 自行宣称。外层真实 Docker 构建、 +容器调用、退出码与完整结果共同构成 **deploy:script PASS（本地真实容器）** 的证据； +宿主三例 24 步继续单独记录，不扩张为容器三例通过。 + +本地证据路径相对仓库根： + +- `output/w5-container-20261023/engine.json`、`image.json`：引擎和镜像内实际工具版本。 +- 同目录 `default-h1.log`、`base-pull.log`、`default-h1-ecr.log`：Docker Hub 失败、官方 ECR 拉取与实际构建记录。 +- 同目录 `small-summary.json` 与 `small-runs/{default-h1,manual-h4,auto-reuse}/report.json`：三组真实 CLI 小图。 +- 同目录 `small-runs/real-qemu-tests.xml`、`real-qemu-tests.log`，以及 `small-runs/real-qemu-tests/` 下两份 `generation-check.json`：真实 QEMU 数值、中断及复用。 +- 同目录 `build-cancel/report.json`、`build-cancel/delayed-recheck.json`：Docker 实际构建取消与延后清理检查。 +- 同目录 `paris.log`、`paris-summary.json`、`paris/run/report.json` 和 `paris/run/steps/step-0000/run.json`：本轮唯一完整模型容器生成、图审计/FMA/数值与回收证据。 +- 同目录 `services-final.json`：容器清空、服务 inactive/disabled、临时 socket 权限恢复与 WSL 配置未变。 + +W5 CI 已复用既有流程接入；容器收尾时 753 项本地轻量检查通过，新增必跑节点仅收集核对。 +后续提交前检查补齐 W5-only 手动入口和 job 余量，811 项相关检查通过，最终 11 项重叠 +边界复查通过，未重新执行完整模型，见 [提交前收尾](README.md#提交前检查与收尾2026-10-23)。 +具体模式、资源要求与证据边界见 [CI 接入说明](README.md#ci-接入与远端执行边界)。 +远端 **NOT_RUN**，未提交、未推送，不能将本地结果写成远端 CI 已通过。 +W5 本地功能出口与本轮容器收尾已完成；默认配置及 WSL 配置不变。 diff --git a/docs/llm-deploy-v1.0/W5/qemu-generation.md b/docs/llm-deploy-v1.0/W5/qemu-generation.md new file mode 100644 index 00000000..01153225 --- /dev/null +++ b/docs/llm-deploy-v1.0/W5/qemu-generation.md @@ -0,0 +1,579 @@ +# W5 QEMU 生成入口与续作报告 + +日期：2026-10-23。分支 `codex/w5-generation`，基于已合并 PR104 的 `61bded3`。 +所有变更仍在独立工作树内，未暂存、提交、推送。WSL 配置未修改；W5 CI 已接入既有流程并通过本地相关检查，远端尚未运行。 +生成合同及参考用例见 [总览](README.md)。 + +当前进展：初始旧算术配置的三例首步 FAIL 作为历史记录保留；后续显式 Mean 与非线性 +参考策略已完成 Host 与正式 4 hart QEMU **各 24/24 步、3/3 用例通过**，未放宽数值门槛。 +对应套件 CLI 的 `status` 和 `full_model_gate` 均为 `PASS`。额外的独立完整模型单次前向 +也已通过，完整输出与套件 France 首步逐字节一致；它单独记录，不计入这 24 步验收。 +下文旧失败表不代表新策略的最终状态。 + +## 实现范围 + +- [命令行入口](../../../scripts/run_inference.py)：从 ONNX 构建一次，或通过 `--reuse-build` 载入经过现有 ELF/权重/布局/生成源码检查的 `executable.json`。 +- [QemuForward](../../../scratchv/runtime/qemu_generation.py)：同一 ELF/bundle 跨 token、不同上下文复用；每次只更新输入并运行现有 QEMU stop/dump/quit 流程，完整输出后按有效位置 greedy。不会每 token 重新编译、写权重 bundle、运行 FMA 预检或准备 ORT session。 +- [运行器](../../../scratchv/runtime/riscv_external.py)：保留 UART 事件通知、QMP 停止/转储/退出；增加等待期间取消检查，任何异常均经过原有进程清理。50ms 是有取消回调时的最长检查间隔，UART/QMP 数据到达会立即唤醒，不额外给完成计时加 50ms。 +- [三轮历史](chat.md)：官方本地模板、显式 non-thinking、成功后提交 user/assistant；错误、取消、超时不污染后续历史。 +- [部署文件](deployment.md)：Ubuntu24.04/Python3.12 镜像、固定依赖与薄启动脚本。真实容器小图、QEMU 中断复用、Docker 构建取消及完整 Paris 单 token 均已通过。 + +保持默认 `1 hart / sequential / 1x4 / original / padding none`。 +Mean 默认仍为 `sequential`，非线性策略默认仍为 `libm`。 +手动 `--harts 1/2/4` 和 `auto` 均保留，auto 在构建/启动时只解析一次。 +复用 ELF 必须显式匹配其 hart、算术策略、tile 和 RHS 布局；不能因请求 auto 或省略参数 +悄悄换成另一种编译配置。1 与 4 hart 各有自己的编译产物，每种配置内部复用同一个 ELF/bundle。 + +## 命令行用法 + +在已准备 W4 固定环境的仓库根目录使用以下参数。这里以符合 ABI 的小图为例： + +```bash +python scripts/run_inference.py \ + --model output/w5-cli-small-20261023/model.onnx \ + --tokenizer-dir output/w5-cli-small-20261023/tokenizer \ + --prompt 1 --max-new-tokens 8 --harts 4 \ + --matmul-policy blocked_fma \ + --verify-onnx output/w5-cli-small-20261023/model.onnx \ + --output-dir output/w5-small-h4 --timeout 30 --total-timeout 120 + +python scripts/run_inference.py \ + --reuse-build output/w5-small-h4/executable.json \ + --tokenizer-dir output/w5-cli-small-20261023/tokenizer \ + --prompt 2 --max-new-tokens 8 --harts 4 \ + --matmul-policy blocked_fma \ + --verify-onnx output/w5-cli-small-20261023/model.onnx \ + --output-dir output/w5-small-h4-reuse +``` + +输出目录必须为新目录。小图资产由本轮测试准备，属于本地 `output`，不是预训练模型。 +单独准备构建可用 `--build-only`，其状态为 `BUILD_ONLY`，不会伪装成生成 PASS。 + +`--mode chat --chat-turn 'first question' --chat-turn 'second question' --chat-turn 'third question'` +可在一条命令内进行三轮，另可传 `--system-prompt`；三轮共用同一个 backend/构建。 +默认 `--prompt` 是 raw completion，不隐式插入聊天模板。 + +`--verify-onnx` 在同一个 ORT CPU session 上比较每步相同输入的完整 logits，并要求 +最大绝对误差严格小于已有 W4 的 `1e-3`，greedy ID 也必须一致；保留 top-2 margin。 +上轮小图全部逐位一致，实际 `max_abs=0`，并非仅靠误差阈值通过。 +ORT 是可选诊断 oracle，不参与生成输出的替代计算。QEMU 仍执行全部客体算术。 + +真实三用例入口为 `--reference-suite --verify-onnx <固定onnx/model.onnx>`，使用冻结的 +三个 prompt、预算和参考；同时要求可执行文件的完整图证据、ORT 的完整 28 层结构 +以及固定资产接口。缺失图证据或只有小图时拒绝给出完整模型 PASS。W4 旧产物沿用 +已有 metadata 身份链接，未新增每 token 的指纹或溯源体系。 +完整模型验收示例显式传入 `--mean-policy two-four-lane-last-axis` 和 +`--nonlinear-policy fp32-reference`，配合 `blocked_fma / 2x4 / column-panel / padding none`。 +`--reference-suite` 不会自动启用这两项；复用该配置的 ELF 时也必须显式匹配。 +上轮未执行完整模型；本轮实跑进展见下节。完整命令见 [部署说明](deployment.md)。 + +## 初始旧算术配置的完整模型 4 hart 验收（2026-10-23，历史结果） + +**初始旧配置结果为 FAIL：三条固定用例均在首步完整 logits 数值检查失败，未完成 3×8 token 生成。** +France 首步使正式 `--reference-suite` 按合同退出。随后用同一入口分别运行 Germany、 +Japan，并在 output 计量包装中执行相同冻结参考逐步检查；两例也在首步失败。 +共完成 **3 次完整前向**，其余 **21 步没有执行**，没有可报告的完整生成文本或停止原因。 +没有跳过 padding 行、放宽阈值或将候选 token 一致解释为生成验收通过。 + +三次均复用同一份既有的完整 Qwen3-0.6B **28 层** ELF 和权重 bundle，显式配置为 +`4 hart / blocked_fma / 2x4 / column-panel / padding none`。未重新编译模型，默认单核、 +手动/auto 选择、CI、WSL 配置不变。该初始验收阶段只新增 output 诊断产物和更新文档，未改正式实现；后续显式策略实现与验证见下文。 +每个命令沿用一次小型 FMA 预检，不计入模型前向；由于失败后独立运行剩余用例，ORT +session 在各命令分别初始化，未声称三个命令共用同一个 session 或驻留 QEMU 进程。 + +### 三条用例实测结果 + +检查每步完整 `[1,256,151936]` 的 **38,895,616** 个 FP32 logits： +与同输入的本轮单线程 ORT 比较，要求 `max_abs < 1e-3`、`rtol=0`，greedy ID 必须一致。 +冻结 JSON 没有保存全部 logits，因此另核对冻结 ID、有效长度和选中 logit； +最终文本、全部新 IDs 和停止原因属于用例末尾检查，本次因首步失败未到达。 + +下表均为**首步完整前向**，不是每例完整 8-token 生成的成本： + +| 固定用例 | UART 前向耗时 s | QEMU CPU s | QEMU RSS 观测峰值 GiB | 完整 logits max_abs | 数值结果 | +|---|---:|---:|---:|---:|---| +| France | 432.607 | 1492.427 | 5.393 | 0.003272057 | FAIL | +| Germany | 400.287 | 1435.645 | 5.401 | 0.001770496 | FAIL | +| Japan | 399.097 | 1428.588 | 5.399 | 0.001197100 | FAIL | + +三例首步候选分别为 **Paris / Berlin / Tokyo**，QEMU、当轮 ORT 和冻结 ID 一致。 +QEMU 选中 logit 与冻结值的误差分别为 `1.335144e-5 / 1.716614e-5 / 1.144409e-5`； +当轮 ORT 选中 logit 与冻结值均精确一致。这不改变完整 logits 已失败的结论。 + +| 用例 | Host 生成调用耗时 s | Host CPU s | QEMU 完整进程耗时 s | QEMU 平均 CPU | Host RSS 峰值 GiB | 同时树 RSS 峰值 GiB | +|---|---:|---:|---:|---:|---:|---:| +| France | 453.789 | 20.722 | 432.969 | 344.7% | 4.068 | 6.699 | +| Germany | 421.424 | 20.245 | 400.650 | 358.3% | 4.100 | 8.372 | +| Japan | 422.524 | 22.998 | 399.460 | 357.6% | 4.099 | 6.399 | + +- UART 前向时间从 QEMU 启动到完成帧验证，包含启动和权重装载；完整进程时间另含停止、转储和退出。 +- QEMU CPU 为每步 `RUSAGE_CHILDREN` user+system 差值，该窗口唯一子进程为 QEMU；平均 CPU 以完整进程墙钟为分母，100% 表示一个逻辑 CPU。 +- Host 生成调用时间另含 ORT、输入/权重传输和数值检查。Host CPU 与 QEMU CPU 分列；采样线程开销计入 Host。 +- QEMU RSS 沿用运行器 0.2s 采样，Host/同时进程树 RSS 为 0.5s 采样，均为观测峰值。树 RSS 求和可能重复共享页，不能将独立的 Host/QEMU 峰值相加当作同时峰值。 +- 三次命令含准备/FMA 预检分别耗时 458.113s、428.556s、430.072s。全部子进程 CPU 还包含 FMA 编译/运行，不能全称模型 CPU。 + +三次 QEMU 均 `qemu_reaped=true`，最终检查 PID 和 POSIX 进程组均不存在，成功 transport +副本已清理。数值失败在正常回收之后发生。采样中 perf 与 MONOTONIC_RAW、UTC 与 perf +的最大差均小于 **0.000362s**，未观察到历史跳时。本轮不计算加速比，也不将这些失败首步 +外推为完整生成速度。 + +Germany 的命令准备/Host 阶段与一次 France 离线 ORT 诊断重叠约 10 秒；该诊断在首次 +采样到 Germany 模型 QEMU 前约 7.4 秒已结束。没有证据显示模型 QEMU 窗口重叠， +但 Germany 全程数据仍不作为无干扰性能基线。诊断只执行一次，未重跑任何 QEMU 前向。 + +### France 错误定位：有效行通过，全输出失败 + +使用保留的 QEMU 输出和一次同输入 ORT 离线重放进行定位。实际 token IDs 和 attention +mask 与冻结合同的 `prepare_inputs` 逐位相同；ORT 仍为 intra/inter 1 线程、 +`ORT_DISABLE_ALL`、关闭 CPU memory arena。 + +| 输出范围 | 最大绝对误差 | 大于等于 1e-3 的元素数 | +|---|---:|---:| +| 有效 query 行 0–4 | 0.000104904 | 0 | +| 用于选 token 的 row 4 | 0.0000572205 | 0 | +| padding query 行 5–255 | 0.003272057 | 20,641 | + +超阈值只出现在 padding query 行 **23、101、250**，分别有 555、6、20,080 个元素。 +最大坐标 `[0,250,45743]`：QEMU `-5.272073746`，ORT `-5.275345802`， +相对误差约 `0.0620%`。全部 256 行 argmax 一致，选取行 top-2 margin 为 `3.161660194`。 +Germany/Japan 本轮未追加逐行定位，不能直接将 France 的行分布推广到另两例。 + +这里的 padding 指**序列补齐位置**，与本轮保持关闭的 **4KiB 工作区 padding 选项**不是同一件事。 +既有 mask 允许这些 query 关注前面的有效 key；不能直接改成全遮挡以规避误差。 +目前只确定误差位置，尚未确定首次产生/放大误差的算子，也未证明是否与 4 hart 并行有关。 +后续补做的同配置 1/4 hart、同源码 Host 和逐层诊断见下节。 + +### 产物及发现的记录问题 + +本地路径相对当前 W5 工作树： + +- `output/w5-full-reference-4hart-20261023/summary.json`：三例联合汇总，明确 FAIL、3/24 前向和 21 步未执行。 +- `output/w5-full-reference-4hart-20261023/`：France；`output/w5-full-reference-4hart-20261023-germany/`、`output/w5-full-reference-4hart-20261023-japan/`：其余两例。 +- 每个目录的 `run/report.json`、`steps.json`、`cases.json`、`numeric-steps.json`：原始 CLI 状态、逐步资源与数值结果；`resources.json`、`resources.jsonl` 为资源汇总/采样。 +- 每个目录的 `run/steps/step-0000/output.bin`：完整 QEMU 输出；France 另有 `step-0000-error-localization.json`、`step-0000-ort-logits.npy`。 +- `output/run_w5_full_reference.py`、`output/diagnose_w5_step0_error.py`：计量/离线定位工具。没有修改原始失败报告或冻结参考。 + +首次完整验收确认了两个 CLI 证据记录问题；原始报告保持原样，下节续作已修复这两项： + +1. 正式 suite 失败后 `status=FAIL`，但 `full_model_gate` 仍保留初始化的 `NOT_RUN`。联合汇总根据实际前向和数值失败记为 FAIL，不能将该字段误读为本轮没运行完整模型。 +2. 从旧 W4 构建载入的完整图审计只用于当次检查，没有带入新保存的 W5 metadata/report。对新副本离线调用 `require_full_executable` 会报缺少完整图证据；原始 W4 metadata/report 仍可复用，本轮三例均使用原始路径，因此该问题没有改变此次计算结果。 + +## 持续完善与数值定位（2026-10-23） + +本轮从北京时间 **2026-10-23 09:35:57**（UTC 01:35:57）开始，持续处理 W5 的剩余问题。 +截至最终输出重复性和进程清理核查的 **13:41:27**（UTC 05:41:27），已持续 **4 小时 5 分 30 秒**； +按分钟约为 09:35–13:41、4 小时 6 分钟。下文保留诊断过程和最终验收结果。 +冻结合同和 `max_abs < 1e-3` 不变，没有忽略 padding query 行或使用参考输出替代 QEMU。 +下列修改均未提交、未推送，CI、WSL、默认单核及手动/auto 选择不变。 + +### 已修复的边界与验收证据 + +- 完整生成开始时将 `full_model_gate` 从 `NOT_RUN` 改为 `RUNNING`，失败/取消分别记为 + `FAIL`/`CANCELLED`。准备失败和 build-only 保留 `NOT_RUN`，不会声称运行了模型。 +- 通过已有身份检查载入旧 W4 构建后，把其完整图审计保存到 W5 元数据；真实 28 层构建 + 连续转换、复用两次成功（均为 build-only，没有新前向）。 +- 参考 suite 将冻结历史、实际前向次数、每步完整 logits 对照、独立选择行和冻结 token + 逐步绑定；缺步、额外步、错误历史/长度/mask/padding 或选错行不得只凭最终文本一致通过。 +- QEMU 失败时，损坏、非对象、不可读或含非有限数的诊断 JSON 不再覆盖原始超时/取消异常。 + 后续 backend 复用和会话历史回滚保持正常。 +- 部署脚本对 info/build 客户端按进程组转发取消并回收，避免脚本退出后编译客户端残留。 + 守护进程侧的构建取消仍需真实容器引擎验证。 + +本轮 CLI **66 项**相关单测通过；适配器边界及跨层历史/缓冲区复用 **27 项**检查通过（2 项真实 QEMU 测试未重跑）； +部署脚本 **14 项**检查通过。以上是各自相关验证，不与历史 216 项重复累加。 +真实 CLI 另复用既有小图构建，分别完成默认 1 hart、手动 4 hart、auto→4 的两 token +调用：三组均产生 `[2,3]`、按预算停止，6 次前向完整输出与 ORT 逐位一致，全部 QEMU 已回收。 +小图仅证明接口/边界，不用于性能加速结论。 + +产物：`output/w5-followup-20261023/cli-evidence-roundtrip/summary.json`、 +`output/w5-followup-20261023/cli-regression/summary.json`。 + +### 三例错误分布补齐 + +复用三例原始 QEMU 输出；Germany/Japan 各进行一次相同输入 ORT 离线对照，没有重跑 QEMU。 + +| 用例 | 有效行最大误差 | 序列补齐行最大误差 | 超阈值 query 行 | 全 256 行 argmax | +|---|---:|---:|---|---| +| France | 0.000104904 | 0.003272057 | 23、101、250 | 全部一致 | +| Germany | 0.000104904 | 0.001770496 | 59、69、121、130 | 全部一致 | +| Japan | 0.000104904 | 0.001197100 | 23、55、58、122、167、177 | 全部一致 | + +各例输入 IDs/mask 均与固定合同逐位相同。对应各例原目录下新增 +`step-0000-error-localization.json` 和 `step-0000-ort-logits.npy`；首步失败记录没有被覆盖。 +有效行通过仍不代表完整 logits 门槛通过。 + +### 同源码与逐层证据 + +相同 `2x4 / column-panel` 生成 C、权重和输入在 Host 顺序执行，完整 logits 与 ORT +最大误差为 **0.002274513**，与先前 4 hart QEMU 的最大差为 **0.001323700**。 +因此存在独立于多 hart 调度的数值差异；Host 与 RV64 的数学库、目标架构不同，不能 +直接把这两个差值归给某一种原因。Host 该次前向 20.819s、CPU 20.814s、RSS 峰值约 3.168GiB， +只作为诊断记录，不替代 QEMU 验收。 + +在同源码 C 增加 embedding、28 层输出、最终归一化共 30 个观测点，额外缓存 30MiB； +原工作区和 bundle 不变。Host 观测版与其原始 logits 逐位一致；ORT 观测版与已保存的 +原始 ORT logits 也逐位一致，排除了这次增加观测点改变最终结果的影响。 + +embedding 完全一致，第一层补齐行最大误差约 `6.68e-6`。row 250 在第 10–13 层输出的 +误差依次为 `0.000189 → 0.000610 → 0.001075 → 0.001394`，提示这一段对上游误差敏感。 +第 2 层有较早超过中间诊断阈值的有效行，但对应约 6543 的激活，仅相差少量 FP32 ULP， +不能把“首次过阈值”直接当根因。随后用同一中间输入拆分第 12 层，区分局部计算与累积，结果见下文。 + +产物：`output/w5-followup-20261023/host-same-source/summary.json`、 +`output/w5-followup-20261023/checkpoints/comparison.json`。 +同配置 1 hart QEMU 完整前向已结束，与 4 hart 的 **38,895,616 个 FP32 logits 逐位一致**， +相对 ORT 的最大误差同为 `0.003272057`；这个输入上的失败不由并行引入。 +单 hart QEMU 进程耗时 **1320.992s**、CPU **1318.380s**、RSS 观测峰值 **5.320GiB**， +平均约 99.8% CPU；进程及进程组已回收。该运行与 Host 诊断编译/执行有重叠， +仅记录实测成本，不用它和历史 4 hart 时间计算严格加速比。 +产物：`output/w5-followup-20261023/qemu-1hart/summary.json`。 + +进一步把同源 C 编译为静态 x86_64-musl，确认无动态装载器，CPU feature 集合与原 Host +编译一致；所得全部 logits 与 QEMU 也逐位一致。其前向 21.622s、CPU 21.497s、RSS 约 +2.951GiB，是本机 Host 诊断而非 QEMU 加速后的时间。它与 glibc Host 存在差别， +说明数学库路径值得继续拆分，但不能据此认定某一个 libm 函数已是唯一根因。 +产物：`output/w5-followup-20261023/host-musl-same-source/summary.json`。 + +同一 ORT 第 11 层输出送入第 12 层切片时，切片 ORT 与原整图该层逐位一致， +Host 开/关观测点也逐位一致；该层独立注入的 padding 最大误差仅 `1.72e-5`、row250 +仅 `2.15e-6`，远低于整图该行累计 `0.001075`。这表明主要存在上游差异和敏感放大， +不能把第 12 层指认为单一错误算子。后续平方、均值及非线性路径的完整 Host 单变量对照见下文。 +产物：`output/w5-followup-20261023/layer12-same-input-v5/report.json`。 + +### 可选 Mean 策略与多 token 收益边界 + +已实现 `--mean-policy sequential|two-four-lane-last-axis`，默认仍为 `sequential`。 +新策略只改变 FP32 末轴 ReduceMean 的求和顺序，对齐既有 W3 的 `mean_last` 参考； +它不启用 SIMD 指令，也不改变 MatMul、Softmax 或 Pow。构建 evidence、运行报告和 +W4/W5 复用检查都携带该字段。旧 ELF 缺字段只能按旧顺序解释，新策略必须重新构建， +不得给旧产物改标签。helper 保留原有 MPL 许可证与第三方声明。 + +| 受控 France 首步 Host 候选 | 对 ORT 全 logits max_abs | 判断 | +|---|---:|---| +| 原顺序、静态 musl | 0.003272057 | FAIL，且与旧 QEMU 逐位一致 | +| 仅 Pow 精确平方 | 0.003275394 | FAIL；局部平方更准确不保证整图误差减小，未纳入正式修复 | +| 仅末轴 Mean 固定求和顺序 | 0.000959873 | 首步低于阈值，余量较窄 | + +Mean 的 4 组真实算子输入和合成边界共 158 个诊断样本与既有 FP32 参考逐位一致。 +正式实现新增 20 项 Host 测试并通过原有 5 项相关默认归约回归；真实 1/4 hart 小图 +8 项通过，覆盖宽 13/128/1024、不同上下文、符号零、溢出报错后同构建复用。 +追加的错误码/进程组断言两项通过，属于这 8 项的复测，不重复累加。 +CLI/运行器合同相关三文件合计 271 项通过，替代本轮早先对应子集的计数。 + +真实新 CLI 对带 ReduceMean 的小图完成默认单核、手动 4、auto→4 的两 token 对照， +完整 logits 逐位一致；从新策略构建复用时若省略策略、退回默认，则在启动任何前向前拒绝。 +这些结果只说明新选项与复用边界正确，不等于完整模型已验收。 + +随后复用同一静态 musl 候选、bundle 和一个 ORT session，按冻结历史进行三例多 token +Host 预检。**3/3 用例仍失败**，共检查 7 步、4 步通过，每例遇到严格数值失败立即停止： + +| 用例 | 已通过 step（零基） | 停止 step | 停止步 full max_abs | +|---|---|---:|---:| +| France | 0、1 | 2 | 0.002007961 | +| Germany | 0 | 1 | 0.001288414 | +| Japan | 0 | 1 | 0.001218975 | + +7 步中 France 首步复用先前已核对输入的成功输出，新增 6 次 Host 前向；没有重跑 QEMU。 +所有已检查步的有效选择行和冻结 ID 一致，但不能绕过完整 logits 门槛继续生成或宣布通过。 +因此此 Mean 选项只能作为显式的算术对照策略，单独启用尚未解除 W5 阻塞。 +下一步以 France step 2 的同一输入分别对照三角函数、Sigmoid、Softmax，避免将多个改动 +同时加入后无法归因。正式默认与 Softmax 行顺序当前都未改变。 + +资源口径：该多 token 诊断从持有 ORT 的父进程启动 native driver，Linux `ru_maxrss` +可能带入父进程高水位，不能当作 driver 独立峰值；原始 stdout 保留并加注，后续采用 +独立轻量父进程或 `/proc` 观测。其 CPU/墙钟和数值判断不受这个 RSS 字段问题影响。 + +产物均相对当前 W5 工作树： + +- `output/w5-followup-20261023/host-mean-candidate/summary.json` +- `output/w5-followup-20261023/host-pow2-candidate/summary.json` +- `output/w5-followup-20261023/mean-host-reference-suite/report.json` +- `output/w5-followup-20261023/mean-rv64-tests/`、`mean-rv64-overflow-final/` +- `output/w5-followup-20261023/mean-cli/summary.json` + +### 非线性算子单变量对照（Host 诊断） + +Mean-only 在 France step02（7 个有效 token）失败后，使用同一保存输入、原 bundle 和 +静态 musl driver，分别只替换 Sin/Cos、Sigmoid 或 Softmax 的参考算术。每个候选的 +原语先对照现有 FP32 reference；完整前向没有改变冻结参考或 `max_abs < 1e-3` 门槛。 + +| 显式诊断候选 | France step02 完整 logits max_abs | 耗时 / CPU 时间 | driver 峰值 RSS | +|---|---:|---:|---:| +| Mean-only（前述基线） | 0.002007961 | 见原逐步记录 | 不使用继承的高水位作独立峰值 | +| Mean + Sin/Cos 参考路径 | 0.000732422 | 21.330 / 21.327 s | 3,258,296 KiB | +| Mean + Sigmoid 参考路径 | 0.000707626 | 19.876 / 19.874 s | 3,258,368 KiB | +| Mean + Softmax 参考路径 | 0.000941038 | 20.056 / 20.048 s | 3,258,232 KiB | + +三种单变量候选都让这一 Host 步骤通过，选中 ID 均为冻结参考的 `576`；这不代表 +整套生成或 QEMU 已通过。继续运行 Mean + Sin/Cos 三例预检时，France 首步又出现 +`max_abs=0.0010201931`，已按原门槛停止并保留失败结果，证明不能从单步外推收益。 +上述单变量候选保留在 `output`。后续已将组合路径接为显式选项，见下节；默认实现未改变。参考路径使用标量 C；Softmax +的 16-lane/ldexp 描述固定的 FP32 运算顺序，不要求 RISC-V 目标具有 AVX512 指令。 +Sin/Cos 仅在已验证区间使用参考多项式，区间外保留原 libm 回退。 + +原语和完整单步证据： + +- `output/w5-followup-20261023/trig-candidate/primitive-validation.json` +- `output/w5-followup-20261023/host-mean-trig-france-step02/summary.json` +- `output/w5-followup-20261023/sigmoid-diagnostic/micro-summary.json` +- `output/w5-followup-20261023/host-mean-sigmoid-step02/summary.json` +- `output/w5-followup-20261023/softmax-candidate/validation.json` +- `output/w5-followup-20261023/host-mean-softmax-candidate/step02-summary.json` +- `output/w5-followup-20261023/mean-trig-host-reference-suite/report.json`（多 token 收益边界） + +### 显式非线性策略与正式入口验证 + +正式后端新增 `nonlinear_policy="libm"|"fp32-reference"`，W4/W5 CLI 对应 +`--nonlinear-policy`。默认仍为 `libm`；参考路径需显式选择，Mean 另由 +`--mean-policy two-four-lane-last-axis` 控制。原始 MatMul/Pow/独立 Exp 和默认 hart +均不随选项改变。旧 ELF 缺字段按 `libm` 解释，策略不匹配在运行前拒绝；构建、运行 +和复用报告记录实际策略，不增加新的哈希链。许可证与范围见 +[运行合同](../W4/runtime-contract.md)。 + +- 新后端直接检查 **21 passed**，相关既有 Softmax/Mean 检查 **48 passed**； + 六份修改前默认 C 源码快照逐字节相同，包含 inline/external、回调与 Mean 配置组合。 +- runtime/W4/W5 三份相关 contract 文件 **328 passed**，覆盖新策略、旧元数据、 + 双向复用不匹配、默认/手动/auto 及 Mean 组合；本组包含前述重叠检查，不重复累计。 +- 正式生成代码的真实 RV64 混合图 **2 passed**（1/4 hart）：含 Sin/Cos/Sigmoid、 + 末轴与非末轴 Softmax、Mean，两个上下文全部与固定参考逐位一致；中途发生有限输入 + 差值溢出时，仍传播 guest `status 2` 并回收进程，原 ELF/bundle 随后可正常复用。 +- 真实 CLI 的默认 1 hart、手动 4 hart、auto→4 hart 各完成两步 `[2,3]`， + 共 6 次前向全部与 ORT 逐位一致，QEMU 进程及进程组均已消失。 +- 此前 output-only Softmax 诊断另验证窄/宽行、非末轴、回调上下文与工作区恢复； + 注入零和/NaN 归一化错误后调用方输出不污染。它不计为完整模型验收。 + +正式验证产物为 `output/w5-followup-20261023/backend-nonlinear-validation.json`、 +`nonlinear-rv64-tests/` 和 `nonlinear-cli/summary.json`。 + +组合 Host 三例预检已完成 **24/24 步、3/3 用例通过**；其中 France 首步仅按同一 +驱动及逐字节相同输入复用一次，其余 23 次新执行。每步检查完整 logits、所选行及 +冻结 ID，三条最终文本和停止原因完全一致，未改变误差门槛或过滤 padding 行。 + +| Host 诊断用例 | 全 logits 最大误差 | 有效行最大误差 | 8 次前向耗时 / CPU 时间 | +|---|---:|---:|---:| +| France | 0.000303984 | 0.000008583 | 158.853 / 158.753 s | +| Germany | 0.000262737 | 0.000020981 | 159.154 / 159.040 s | +| Japan | 0.000197887 | 0.000012875 | 158.336 / 158.239 s | + +上表 France 含已完成首步的原始测量。预检这次实际总耗时 577.597 s（包含 ORT 与 +对照，且不重复执行已复用首步），不能与上表求和混为同一计时口径。独立驱动通过 +`/proc` 观测的峰值 RSS 最大 3,258,564 KiB，ORT 父进程累计峰值 5,290,672 KiB。 +原 `getrusage` 子进程高水位继承父进程值的问题已明确标注，不用它冒充独立 driver 峰值。 + +证据为 `output/w5-followup-20261023/mean-trig-sigmoid-softmax-host-reference-suite/report.json`。 +上述数据属于完整 Host 数值预检，**不能替代 QEMU 验收**。后续使用正式 codegen 的新 +4 hart ELF、复用原 bundle 完成了三例 24 步 QEMU 验证；其独立实测结果见下节。 + +### 正式组合策略 QEMU 完整生成（3/3 用例、24/24 步通过） + +正式 codegen 已构建新的完整 Qwen3-0.6B **28 层** ELF；三条冻结用例复用同一 +ELF 和原权重 bundle，逐 token 只更新输入并启动 QEMU。显式配置为 +`4 hart / blocked_fma / 2x4 / column-panel / mean two-four-lane-last-axis / nonlinear fp32-reference / padding none`。 +没有将 Host 输出替代 QEMU 计算，也没有修改默认单核、手动/auto hart 选择或冻结参考。 +24 份运行记录中的既有 ELF/bundle 身份字段完全一致，没有新增哈希检查。 + +三例各完整生成 8 token，**3/3 用例、24/24 步数值及冻结参考检查通过**，命令退出码为 +`0`，正式 CLI 的 `status=PASS / full_model_gate=PASS / qemu_forwards=24`。 +每步都比较完整 `[1,256,151936]` 的 **38,895,616** 个 FP32 logits，严格要求 +`max_abs < 1e-3`、`rtol=0`，没有跳过序列补齐行。全套最大误差为 +`0.000303983688`；这表示容差一致，不表示全部 logits 逐位相等。 + +| 固定用例 | 已完成步数 | 生成调用墙钟 s | QEMU CPU s | Host CPU s | 全 logits 最大误差 | 当前状态 | +|---|---:|---:|---:|---:|---:|---| +| France | 8/8 | 3375.024 | 11517.789 | 167.657 | 0.000303983688 | 整例通过 | +| Germany | 8/8 | 3668.852 | 12574.174 | 190.673 | 0.000262737274 | 整例通过 | +| Japan | 8/8 | 3489.655 | 11958.881 | 178.759 | 0.000197887421 | 整例通过 | + +三例新 token IDs、完整文本和停止原因均与冻结参考一致，停止原因均为 `max_new_tokens`： + +| 用例 | 新 token IDs | 生成文本 | +|---|---|---| +| France | `[12095, 13, 576, 6722, 315, 15344, 374, 21718]` | ` Paris. The capital of Italy is Rome` | +| Germany | `[19846, 13, 576, 6722, 315, 9625, 374, 12095]` | ` Berlin. The capital of France is Paris` | +| Japan | `[26194, 13, 576, 6722, 315, 279, 3639, 4180]` | ` Tokyo. The capital of the United States` | + +各例最大完整 logits 误差分别位于本例 step 3、3、1；冻结所选 token 的 logit 最大误差 +分别为 `1.907349e-6 / 5.722046e-6 / 5.722046e-6`。全部 24 个不同 QEMU 进程 +均记录 `qemu_reaped=true`；套件结束后只读核查这 24 个 PID 和 POSIX 进程组均已消失。 + +| 每例 8 步资源口径 | France | Germany | Japan | +|---|---:|---:|---:| +| UART 前向耗时合计 / 每步均值 | 3197.261 s / 399.658 s | 3480.225 s / 435.028 s | 3312.868 s / 414.109 s | +| 单步 UART 前向耗时范围 | 395.824–407.598 s | 423.936–453.608 s | 406.769–428.952 s | +| QEMU 完整进程耗时合计 | 3199.695 s | 3482.999 s | 3315.397 s | +| QEMU 平均 CPU 占用（100% 为一个逻辑 CPU） | 359.97% | 361.02% | 360.71% | +| QEMU RSS 观测峰值 | 5.406 GiB | 5.406 GiB | 5.406 GiB | +| Host RSS 观测峰值 | 4.254 GiB | 4.102 GiB | 4.098 GiB | +| 同时进程树 RSS 观测峰值 | 7.565 GiB | 7.644 GiB | 7.411 GiB | + +整次测量命令墙钟 **10540.190s（约 2 小时 55 分 40 秒）**，包含准备、ORT 与核验； +24 步 UART 前向合计 **9990.355s**、平均 **416.265s/token**，完整 QEMU 进程合计 +**9998.091s**。模型 QEMU CPU 合计 **36050.844s**，平均 CPU 占用 **360.58%**； +Host CPU 全程 **540.717s**，其中三例生成阶段合计 **537.089s**。 +所有子进程 CPU 全程为 **36050.974s**，另含准备/FMA 检查，不能全部算作模型 CPU。 + +计量口径沿用前文：UART 时间包括 QEMU 启动和权重装载，完整进程时间另含 +stop/dump/quit；平均 CPU 用 QEMU CPU 总时间除以完整进程耗时之和。 +生成调用墙钟和此表 Host CPU 包含本例 ORT 与数值核验，逐步 adapter 的 Host CPU +仅覆盖 adapter 调用，两者不能混用。QEMU RSS 为 0.2s 采样，Host/同时进程树为 +0.5s 采样；全程 Host 与同时树峰值分别为 **4.254 / 7.644 GiB**。 +不能将逐步峰值相加，也不能将独立 Host/QEMU 峰值相加当作同时峰值；树 RSS 可能重复共享页。 +Host/同时进程树采样器全程 21,032 次采样、0 次错误。独立的 QEMU RSS 采样中， +24 步有 **17 步为 `partial`，各有 1 次读取错误**；5.406 GiB 仍只是观测峰值， +不能据此声称 QEMU 内存采样完全无缺失。`perf−MONOTONIC_RAW` 与 `UTC−perf` 的 +最大绝对漂移分别为 **0.000773s / 0.000504s**，未观察到历史跳时。 +本次不使用旧计时异常或不同算术策略的记录计算加速比。 + +本地产物路径相对当前 W5 工作树： + +- `output/w5-followup-20261023/formal-reference-4hart/executable.json`：本次正式 ELF 的复用入口。 +- `output/w5-followup-20261023/formal-reference-4hart/report.json`：构建与原 bundle/布局匹配记录；该构建报告本身不代表生成验收通过。 +- `../w4-harts-short-20261021/optimization-completion-20261022/integration-full-4hart/run/weights/`：继续复用的原权重 bundle。 +- `output/w5-followup-20261023/qemu-reference-4hart/metadata.json`：实际命令、环境与测量配置。 +- 同目录 `cases.json`、`steps.json`、`numeric-steps.json`：三例结果、24 步资源及完整 logits/冻结参考对照。 +- 同目录 `resources.jsonl` 和 `resources.json`：全程资源采样与最终汇总。该次命令使用原计量包装，不要求后来单次前向包装新增的 `measurement_passed` 字段；终态以 CLI、3 例输出和 24 步数值证据共同确认。 +- 同目录 `run/report.json` 及 `run/steps/step-0000/run.json` 至 `step-0023/run.json`：CLI 最终状态与逐步运行证据。 + +### 独立完整模型单次前向（已通过） + +三例通过后，另用相同正式 4 hart ELF/bundle、France 固定 prompt 完成独立的 **1 token / +1 次完整前向**计量，沿用相同算术策略和 `padding none`。命令退出码 `0`，生成 +`[12095]`、文本 ` Paris`，按 `max_new_tokens` 停止，与冻结首 token、文本前缀及预算停止规则一致。 +选中 logit 为 `17.498851776123047`，与当轮 ORT 和冻结值完全一致；完整 +**38,895,616** 个 logits 对 ORT 的 `max_abs=0.000172138214 < 0.001`。 + +| 单次完整前向指标 | 实测 | +|---|---:| +| UART 完成耗时 | 418.954 s，约 6 分 59 秒 | +| QEMU 完整进程耗时 | 419.383 s | +| QEMU CPU 时间 / 平均 CPU 占用 | 1498.851 s / 357.39% | +| 生成调用墙钟 / Host CPU | 441.033 s / 21.604 s | +| 整次测量墙钟 / Host CPU（含准备） | 448.263 s / 25.332 s | +| QEMU RSS 观测峰值 | 5.398 GiB | +| Host RSS 观测峰值 | 4.119 GiB | +| 同时进程树 RSS 观测峰值 | 7.210 GiB | + +本次启动新 QEMU 进程，但**没有清空系统文件缓存，因此不称为冷启动测量**。 +完整 **155,582,464 字节**输出与先前三例套件 France 首步逐字节一致。 +已有 ELF/bundle 身份也相同，没有新增哈希；最终确认套件与本次共 **25 个 QEMU PID +及其进程组全部消失**。单次采样 0 次错误，`perf−MONOTONIC_RAW` 与 `UTC−perf` +最大绝对漂移为 **0.000160s / 0.000144s**。 + +此命令范围为 `full-model single forward`，没有再次执行三例参考 suite，故正式 CLI +`full_model_gate=NOT_RUN` 是正常范围标记，**不是没有运行完整模型**。 +计量结果 `measurement_passed=true`，`full_reference_suite_evaluated=false`； +三例 gate 的 `PASS` 来自前述独立的 24 步验收，不能把这一次重复前向计为第 25 个 suite 步骤。 + +本地产物目录：`output/w5-followup-20261023/qemu-final-single-forward-4hart/`。 +其中 `steps.json`、`cases.json`、`numeric-steps.json`、`resources.json` 分别记录单步、 +生成调用、数值和资源结果；`run/report.json` 保留正式 CLI 范围。 +`repeat-and-cleanup-check.json` 记录同 ELF/bundle、完整输出逐字节一致和 25 个进程/进程组清理核查。 + +### 真实容器部署续验（本地通过） + +本轮已用 Docker Engine 29.9.2 / Buildx 0.38.1 真正构建并启动干净镜像。 +Docker Hub 认证 TLS EOF 后显式选择 Canonical 官方 ECR Ubuntu 24.04，默认基础镜像 +仍为 `ubuntu:24.04`。引擎仅手动启动，验证后 Docker 服务、socket 和 containerd +均为 **inactive + disabled**；停止前无运行容器，临时权限已恢复，WSL 配置未变。 +工具版本、完整命令与证据见 [部署记录](deployment.md)。 + +默认 1 hart、手动 4 hart、`auto→4` 的容器 CLI 小图各 4 次前向均通过， +IDs `[2,3,4,6]`、EOS 停止，完整 logits 与 ORT 逐位一致，padding `none`。 +已有真实 QEMU 超时/取消及失败后复用测试在镜像内 **2/2 通过、0 skip**， +1/4 hart 共 16 次 guest 启动均完成进程/进程组回收。实际 Docker 构建 +SIGINT/SIGTERM 另 **2/2 通过**，客户端组及引擎侧 RUN 进程均消失，延后复查无残留。 + +本轮只追加一次容器内完整 28 层 France 单 token 前向，现已 **PASS、退出码 0**： +输出 `[12095]`、文本 ` Paris`、预算停止，完整 **38,895,616** 个 logits 的 +`max_abs=0.000172138214 < 0.001`；图审计确认 28 层，FMA 与 QEMU 回收通过。 +容器从只读模型资产独立重新构建，CLI 总耗时 **547.454s**（含构建/FMA/ORT/核验）， +UART 前向 **415.058s**、QEMU 完整进程 **415.478s**、运行器记录耗时 **430.481s**， +后者含权重传输、QEMU、后检查与清理，不含计时起点之前的输入打包/写入及 ELF/bundle 校验。 +QEMU RSS 观测峰值 **5.398GiB**。RSS 有 2071 个有效样本，**1 次读取失败**， +可用性为 `partial`；本次未独立测量 CPU 总时间，不借用宿主或 Docker 客户端数据。 +容器输出 **155,582,464 字节**与既有宿主独立完整前向直接比较逐字节一致，未新增哈希。 + +内部 CLI 的 `full_model_gate=NOT_RUN / container_reproduction=NOT_ASSERTED` 保留： +单 prompt 不重跑三例 gate，实际容器身份由外层 Docker 构建/运行和退出证据确认。 +它与既有宿主三例 24 步及宿主独立前向分别记录，构成 **deploy:script 本地真实容器 PASS**， +不声称容器三例通过。**W5 本地功能出口已满足**；容器收尾时 CI 接入的 753 项轻量 +检查通过，14 个新增必跑节点仅收集核对、未重跑；已有真实 24 步报告的严格检查通过。 +后续提交前检查与修复见 [收尾记录](README.md#提交前检查与收尾2026-10-23)。 +模式和资源门槛见 [CI 接入说明](README.md#ci-接入与远端执行边界)； +未提交/推送，远端 **NOT_RUN**。 + +证据相对仓库根：`output/w5-container-20261023/small-summary.json`、 +`small-runs/real-qemu-tests.xml`（同目录）、`build-cancel/report.json`、 +`build-cancel/delayed-recheck.json` 及 `paris.log`、`paris-summary.json`、`paris/run/report.json`、`services-final.json`。 + +## 上轮小图数值与清理验证 + +适配器使用真实 Gather + MatMul 小图：输入 `[1,256]` token IDs，输出完整 `[1,256,8]` +FP32 logits，随历史输入改变。合成图声明完整 mask ABI 但不消费 mask；mask 的精确 +causal/key-padding 语义由已有 P1 测试验证，不能将该小图称为 Transformer 语言验收。 + +| 对照 | 1 hart | 4 hart | +|---|---|---| +| 同一 ELF/bundle 的多 token | `[2,3,4,6]`，EOS 停止 | 相同 | +| 独立上下文与预算停止 | `[4]`，`max_new_tokens` | 相同 | +| 每次成功前向的完整输出 | 2048 元素与 ORT 逐位相等 | 相同 | +| 正在运行的 WFI 测试 guest 超时 | 抛 TimeoutError；进程和进程组已回收 | 相同 | +| 正在运行时取消 | 抛 GenerationCancelledError；进程和进程组已回收 | 相同 | +| 中断后复用 backend/ELF | 再生成 `[6]`，EOS | 相同 | +| 独立真实启动数 | 8 次：6 成功、1 超时、1 取消 | 相同 | + +这组 16 次真实启动全部核对 PID 与 POSIX 进程组消失，无清理错误。 +失败运行按原有策略各保留 8192 字节 transport 诊断副本；这是故意保留的文件，不是 +仍在运行的编译/QEMU 进程。成功运行的 transport 副本已清理。 + +另外从真实 CLI（本地小 tokenizer，并非 Python mock）完成四次命令： + +| CLI 调用 | 新 token IDs | 前向次数 | 完整输出 max_abs | 结果 | +|---|---|---:|---:|---| +| 1 hart 首次构建，prompt `1` | `[2,3,4,6]` | 4 | 0 | PASS | +| 4 hart 首次构建，prompt `1` | `[2,3,4,6]` | 4 | 0 | PASS | +| 1 hart 复用构建，prompt `2` | `[3,4,6]` | 3 | 0 | PASS | +| 4 hart 复用构建，prompt `2` | `[3,4,6]` | 3 | 0 | PASS | + +表中引用最终入口的复测产物；早期小图记录保留，不与最终矩阵重复累计。 +跨命令的 ELF 和 bundle 路径确实相同，所有 QEMU 均回收。复用前沿用已有 +report 到 executable metadata 的身份校验，缺失或漂移先于加载/编译失败。每条生成命令仅在准备阶段 +做一次独立的小型 FMA 预检；它不算模型 token 前向。默认配置另做真实小图 build-only， +未启动 QEMU，记录默认 1 hart、sequential 和 padding none。 + +这些约 20ms 的 guest 是正确性冒烟。未采集 CPU 总时间，0.2s 的 RSS 周期只捕捉到 +启动样本，不能据此宣称加速或峰值内存改善；未使用历史异常计时或完整模型速度作推算。 + +## 上轮相关检查与产物 + +| 检查 | 通过数量 | +|---|---:| +| 新 QEMU 适配/取消/QMP 边界与真实 1/4 hart 小图 | 18 | +| 既有 external runtime 与 W4 契约相关回归 | 112 | +| 三轮历史及官方模板文本/token IDs 对照 | 40 | +| CLI 参数、复用、数值、错误/信号与完整图门槛 | 37 | +| 部署脚本/打包/失败路径隔离检查 | 9 | + +上轮共 **216 项相关检查通过**；与 P0/P1 的旧记录分开列出，不将重叠项重复累计。 +另完成 12 个 Python 文件语法检查、部署脚本 Bash 语法检查、6 份相关文档的 +53 个本地链接检查（含 33 个 Markdown/HTML 链接），未引入重复文档名称； +最终四份 CLI 报告与汇总、构建复用路径及进程回收记录离线核对一致。 +上轮未运行完整模型；初始完整验收阶段仅进行上述完整验收和必要离线定位，未重跑这组 216 项单测或全量回归。后续显式策略的相关检查另列于持续完善部分；未追加 AI 审查。CI/WSL 未改，代码未提交推送。 + +本地诊断路径（相对当前 W5 工作树）： + +- `output/w5-qemu-generation-summary.json`：适配器 1/4 hart 汇总。 +- `output/w5-qemu-generation-real/`：真实逐步输出、超时/取消和回收证据。 +- `output/w5-cli-final-20261023/summary.json`：四次 CLI 构建/复用对照。 +- `output/w5-cli-final-20261023/{hart1,hart4,hart1-reuse,hart4-reuse}/report.json`：逐 token ORT/QEMU 记录。 +- `output/w5-cli-small-20261023/default-build-only/report.json`：默认配置构建检查。 + +## 剩余问题和明确未执行项 + +1. **默认算术配置的数值边界仍需保留**：显式组合策略已经完成 Host 和 QEMU 各三例 24/24 步验收；初始 `mean sequential / nonlinear libm` 配置的三例首步仍有完整 logits 超过 `1e-3` 的历史失败，不能把显式策略 PASS 推广为默认路径 PASS。默认值未改变，完整验收需显式选择本文配置。独立单次前向也已通过，并已单独记录，不能与 24 步套件混算。 +2. **本地功能与远端门槛分开**：真实容器镜像、小图、中断清理及唯一追加的完整 France 单 token 已通过，W5 本地功能出口已满足；没有重复容器三例套件。引擎及 socket 已停止并禁用自启，临时权限已恢复，WSL 配置不变。 +3. **CI 本地实现与检查通过，远端 NOT_RUN**：提交前补齐现有 llm-deploy 的 W5-only 手动入口，811 项相关检查及最终 11 项重叠边界复查通过；保留 12 个真实 QEMU 小图、2 个官方模板及完整生成门槛。14 个真实节点本轮没有重新运行；远端资源和时限仍待授权提交/推送后验证，不把本地通过写成远端 Actions 已通过。详见 [提交前收尾](README.md#提交前检查与收尾2026-10-23)。 +4. **同步边界**：QEMU UART/QMP 等待可响应取消并回收进程；构建/传输/可选 ORT 校验等同步工作仍在各自阶段边界检查总预算，Host deadline 不承诺硬中断任意 ORT 调用。SIGINT/SIGTERM 会经已有编译/QEMU清理路径退出，不返回成功的部分生成。 + +固定 256 长度、无 KV cache 意味着每个新 token 重算完整前向；当前实现复用构建， +不宣称 QEMU 进程或权重驻留。持久 guest、KV/RVV/量化等属于后续优化，不是本轮漏做的生成链路。 diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index be84d5e5..fd2c1914 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -157,7 +157,7 @@ Host 端循环： | Nightly 完整门禁 | 每日夜间 | 完整 ONNX、完整 IR 数值、QEMU 单次前向、端到端 | 2–4 h | 阻塞里程碑 | | Weekly/Release 门禁 | 每周五 / tag | 多轮、文档复现、性能回归、交付物 | 4–8 h | 阻塞发布 | -上表是分层目标，不代表所有未来 job 已交付。当前 `.github/workflows/llm-deploy.yml` 使用 GitHub 托管 Ubuntu runner：PR 小模型任务和手动/定时完整 ONNX 任务各有 60 分钟 job 上限，QEMU 小模型步骤另有超时。W3 已提供 `w3-preparation.yml`、`w3-full-numeric.yml` 和显式启用的 `w3-nightly.yml`，并可通过既有 `llm-deploy.yml` 手动设置 `run_w3_validation=true`，从同一提交调用 W3 编排。作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 通过同提交两套 W3 gate 及汇总，完整 raw 已下载验真、audit 通过，定时 schedule 未启用。通用 PR CI 全绿不能替代这些专项记录。W4 的 PR 小图回归由统一 CI 调用 `w4-full-numeric.yml`；`llm-deploy.yml` 手动设置 `run_w4_validation=true` 才调用完整前向，另有默认关闭的 `w4-nightly.yml`；所有 Nightly 入口需仓库变量 `W4_NIGHTLY_ENABLED=true`。配置和本地检查不代表已取得远端成功记录，完整门槛、独立复现与团队验收分别记录。生成门禁仍属 W5 待实现。实际工具为 Zig 0.14.1、`qemu-system-riscv64`（裸机）和固定 Python 依赖；旧独立测试仍使用 `qemu-riscv32`，不能互换。更大内存的自托管 runner、容器封装与缓存策略是完整模型阶段的资源计划，不宣称已经部署。 +上表是分层目标，不代表所有未来 job 已交付。当前 `.github/workflows/llm-deploy.yml` 使用 GitHub 托管 Ubuntu runner：PR 小模型任务和手动/定时完整 ONNX 任务各有 60 分钟 job 上限，QEMU 小模型步骤另有超时。W3 已提供 `w3-preparation.yml`、`w3-full-numeric.yml` 和显式启用的 `w3-nightly.yml`，并可通过既有 `llm-deploy.yml` 手动设置 `run_w3_validation=true`，从同一提交调用 W3 编排。作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 通过同提交两套 W3 gate 及汇总，完整 raw 已下载验真、audit 通过，定时 schedule 未启用。通用 PR CI 全绿不能替代这些专项记录。W4 的 PR 小图回归由统一 CI 调用 `w4-full-numeric.yml`；`llm-deploy.yml` 手动设置 `run_w4_validation=true` 才调用完整前向，另有默认关闭的 `w4-nightly.yml`；所有 Nightly 入口需仓库变量 `W4_NIGHTLY_ENABLED=true`。配置和本地检查不代表已取得远端成功记录，完整门槛、独立复现与团队验收分别记录。W5 生成门禁已经实现，并以显式参考算术策略完成本地完整 QEMU 三例 24/24 步验收，详见 [W5 状态与证据](W5/README.md)；W5 CI 已接入既有工作流，并补齐合并前 W5-only 手动入口；全部待提交改动的本地检查与修复已完成，详见 [W5 提交前收尾](W5/README.md#提交前检查与收尾2026-10-23)。远端尚未提交运行，不能与本地 PASS 混同。实际工具为 Zig 0.14.1、`qemu-system-riscv64`（裸机）和固定 Python 依赖；旧独立测试仍使用 `qemu-riscv32`，不能互换。更大内存的自托管 runner 与缓存策略仍按资源计划推进；W5 容器封装已经实现，真实容器小图、中断清理和完整 Paris 单 token 均已通过，本地功能出口已满足，见 [部署记录](W5/deployment.md)。这些记录不替代远端 CI 或正式发布验收。 ### 4.2 硬门槛 CI Job diff --git a/probes/w4_qwen3_full/run.py b/probes/w4_qwen3_full/run.py index e9eee48b..ad5fa9c0 100644 --- a/probes/w4_qwen3_full/run.py +++ b/probes/w4_qwen3_full/run.py @@ -143,6 +143,12 @@ def spec(value): or type(exe.evidence.get("harts", 1)) is not int or exe.evidence.get("harts", 1) != exe.harts): raise ValueError("Reused hart count does not match build evidence") + mean_policy = exe.evidence.get("mean_policy", "sequential") + if not isinstance(mean_policy, str) or mean_policy not in ("sequential", "two-four-lane-last-axis"): + raise ValueError("Unsupported reused Mean policy") + nonlinear_policy = exe.evidence.get("nonlinear_policy", "libm") + if not isinstance(nonlinear_policy, str) or nonlinear_policy not in ("libm", "fp32-reference"): + raise ValueError("Unsupported reused nonlinear policy") manifest = validate_weight_bundle(exe.bundle_dir, exe.weights) if manifest != exe.manifest: raise ValueError("Reused weights do not match build manifest") @@ -197,6 +203,10 @@ def main(argv=None): cli.add_argument("--build-timeout", type=float, default=900) cli.add_argument("--build-only", action="store_true") cli.add_argument("--matmul-policy", choices=("sequential", "blocked_fma"), default="sequential") + cli.add_argument("--mean-policy", choices=("sequential", "two-four-lane-last-axis"), default="sequential", + help="FP32 last-axis Mean order; explicit W3 reference component, not its full profile") + cli.add_argument("--nonlinear-policy", choices=("libm", "fp32-reference"), default="libm", + help="Explicit FP32 Trig/Sigmoid/Softmax order; default preserves platform libm") cli.add_argument("--matmul-tile", choices=("1x4", "2x4", "1x8"), default="1x4", help="MatMul output tile (default: 1x4); 2x4/1x8 require blocked_fma") cli.add_argument("--reuse-build", type=Path, help="Local executable.json; verified before every run") @@ -218,7 +228,8 @@ def main(argv=None): report = {"schema_version": 1, "gate": "numeric:qemu-full-qwen3", "passed": False, "status": "FAIL", "stage": "initialization", "stages_seconds": {}, "cases": [], "harts_requested": requested_harts, "harts": args.harts, - "matmul_tile": args.matmul_tile, + "matmul_tile": args.matmul_tile, "mean_policy": args.mean_policy, + "nonlinear_policy": args.nonlinear_policy, "matmul_rhs_layout": args.matmul_rhs_layout, "qemu_workspace_layout_requested": args.qemu_workspace_layout, "gates": {"build:riscv-full": False, "smoke:qemu-full": False, @@ -261,6 +272,14 @@ def stage(name, operation): exe = stage("load_existing_build", lambda: load_executable(args.reuse_build)) if exe.harts != args.harts: raise ValueError("Reused build hart count differs from requested --harts") + if (exe.evidence.get("mean_policy", "sequential") != args.mean_policy + or previous.get("mean_policy", "sequential") != args.mean_policy): + raise ValueError("Reused build Mean policy differs from requested --mean-policy; " + "rebuild the executable") + if (exe.evidence.get("nonlinear_policy", "libm") != args.nonlinear_policy + or previous.get("nonlinear_policy", "libm") != args.nonlinear_policy): + raise ValueError("Reused build nonlinear policy differs from requested --nonlinear-policy; " + "rebuild the executable") if (exe.evidence.get("matmul_tile", "1x4") != args.matmul_tile or previous.get("matmul_tile", "1x4") != args.matmul_tile): raise ValueError("Reused build MatMul tile differs from requested --matmul-tile; " @@ -304,6 +323,7 @@ def stage(name, operation): initializers=parser.initializers, constant_storage="external", max_constant_bytes=3 * 1024**3, max_workspace_bytes=1024**3, kernel_calls=True, matmul_policy=args.matmul_policy, matmul_tile=args.matmul_tile, + mean_policy=args.mean_policy, nonlinear_policy=args.nonlinear_policy, matmul_rhs_layout=args.matmul_rhs_layout, qemu_workspace_layout=args.qemu_workspace_layout).generate()) report["codegen"] = {"source_bytes": len(artifact.source.encode()), @@ -313,6 +333,8 @@ def stage(name, operation): "qemu_workspace_buffers": artifact.qemu_workspace_buffers, "weights": len(artifact.external_weights), "kernel_calls": artifact.kernel_calls, + "mean_policy": artifact.mean_policy, + "nonlinear_policy": artifact.nonlinear_policy, "matmul_tile": artifact.matmul_tile, "matmul_rhs_layout": artifact.matmul_rhs_layout, "matmul_rhs_panels": list(artifact.matmul_rhs_panels), diff --git a/probes/w5_generation/README.md b/probes/w5_generation/README.md new file mode 100644 index 00000000..9b6b0d1c --- /dev/null +++ b/probes/w5_generation/README.md @@ -0,0 +1,41 @@ +# W5 Host 生成参考 + +本探针验证 Host token 循环；实际推理后端是固定完整 Qwen3-0.6B ONNX 的 ONNX Runtime CPU。它不启动 QEMU，不能代替 W5 的 QEMU Paris / 三 prompt 验收。生成合同见 [W5 合同](../../docs/llm-deploy-v1.0/W5/README.md)。 + +## 固定参考 + +[references.json](references.json) 固定以下三个原始 completion prompt；它们在第一次运行前选定，每条预算 8 个新 token，分别检查输出包含 Paris、Berlin、Tokyo。 + +- `The capital of France is` +- `The capital of Germany is` +- `The capital of Japan is` + +prompt 按 UTF-8 原文编码，`add_special_tokens=False`，不套 chat template。模型身份引用既有 [W1 manifest](../w1_qwen3_export/manifest.json)，不增加另一套哈希清单。运行前检查 ONNX 文件长度，避免把 LFS pointer 当模型；资产完整性沿用已有下载/验收流程。本探针不宣称长度检查等价于 SHA256 校验。 + +参考保存 prompt 字节及 token IDs、所有生成 IDs、文本、EOS 集、停止原因和逐步 top-1/top-2 间距。使用 full FP32 logits 的最后有效行；检查完整输出（包括 padding 行）有限性。argmax 覆盖整个模型词表；采样到 tokenizer 不支持的 padded ID 会失败，不做过滤。 + +## 运行 + +先按 [Linux 复现约定](../../docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md) 安装选定环境并获取固定模型、tokenizer 资产。在仓库根目录运行： + +```bash +python probes/w5_generation/run.py \ + --assets assets/qwen3-0.6b \ + --output-dir output/w5-generation-reference +``` + +正常运行只读冻结参考，写 `output/w5-generation-reference/reference-run.json`。`--assets` 可指向已有完整资产目录，避免工作树重复下载。默认模式要求生成 token IDs、文本和停止原因与冻结参考完全一致；浮点 top-2 间距用于诊断，不把它当跨后端数值误差标准。公开 CLI 的 `--record` 只向 output 写候选参考，**不会覆盖 references.json**；更换冻结参考必须单独审查 prompt、模型身份及验收条件的变化。 + +一个 ORT session 顺序处理三个用例，沿用 W4 参考设置：CPU provider、intra/inter-op 均为 1、禁用 graph optimization 和 CPU memory arena。每次 forward 都由独立 oracle 检查： + +1. 当前 token 历史、最后有效位置、右 padding 和 causal/key padding mask。 +2. 完整 logits 的 FP32 ABI 与有限性，直接索引最后有效行并用 NumPy argmax 计算 token。 +3. 独立的 EOS → 容量 → token 预算停止规则、生成 IDs、解码文本和调用次数。 + +这样一次真实 ORT 运行同时验证 Host 循环和参考输出，不再重复跑同一套完整参考。结果中 `qemu_e2e: NOT_RUN` 明确区分本轮覆盖范围。 + +## 计时与内存 + +每步同时记录 ORT forward wall/CPU 时间；每条用例另记包括 Host 输入准备、检查、采样及解码的 wall/CPU 时间。session 加载单独计时。CPU 时间含进程内所有线程；Linux `ru_maxrss` 是进程截至当时的累计峰值，不能作为每个用例的独立内存增量。这些是 ORT/Host 数据，不能与既有 QEMU 计时混用或推算 QEMU 加速比。 + +本探针不读写 hart、padding、CI 或 WSL 设置。QEMU 的默认 1 hart、手动 1/2/4、启动时 auto 和默认关闭的 padding 仍由既有后端负责。 diff --git a/probes/w5_generation/references.json b/probes/w5_generation/references.json new file mode 100644 index 00000000..35aaea10 --- /dev/null +++ b/probes/w5_generation/references.json @@ -0,0 +1,422 @@ +{ + "schema_version": 1, + "status": "frozen", + "model_manifest": "probes/w1_qwen3_export/manifest.json", + "model_id": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "contract": { + "batch_size": 1, + "sequence_length": 256, + "dtype": "float32", + "use_cache": false, + "prompt_format": "raw_completion", + "add_special_tokens": false, + "sampling": "greedy_argmax_lowest_id", + "skip_special_tokens": true, + "clean_up_tokenization_spaces": false, + "stop_priority": [ + "eos", + "capacity", + "max_new_tokens" + ] + }, + "cases": [ + { + "name": "france", + "prompt": "The capital of France is", + "prompt_utf8_hex": "546865206361706974616c206f66204672616e6365206973", + "required_substring": "Paris", + "max_new_tokens": 8, + "prompt_token_ids": [ + 785, + 6722, + 315, + 9625, + 374 + ], + "generated_token_ids": [ + 12095, + 13, + 576, + 6722, + 315, + 15344, + 374, + 21718 + ], + "generated_text": " Paris. The capital of Italy is Rome", + "stop_reason": "max_new_tokens", + "semantic_passed": true, + "host_oracle_passed": true, + "steps": [ + { + "step": 0, + "valid_length": 5, + "token_id": 12095, + "decoded_token": " Paris", + "top1_logit": 17.498851776123047, + "top2_logit": 14.337191581726074, + "top1_top2_margin": 3.1616601943969727, + "forward_wall_seconds": 5.923962489000019, + "forward_cpu_seconds": 5.923590624 + }, + { + "step": 1, + "valid_length": 6, + "token_id": 13, + "decoded_token": ".", + "top1_logit": 19.274662017822266, + "top2_logit": 19.083740234375, + "top1_top2_margin": 0.19092178344726562, + "forward_wall_seconds": 5.199123996000026, + "forward_cpu_seconds": 5.199001791 + }, + { + "step": 2, + "valid_length": 7, + "token_id": 576, + "decoded_token": " The", + "top1_logit": 16.967761993408203, + "top2_logit": 15.490779876708984, + "top1_top2_margin": 1.4769821166992188, + "forward_wall_seconds": 5.950034952000067, + "forward_cpu_seconds": 5.946767153999998 + }, + { + "step": 3, + "valid_length": 8, + "token_id": 6722, + "decoded_token": " capital", + "top1_logit": 18.299823760986328, + "top2_logit": 14.856491088867188, + "top1_top2_margin": 3.4433326721191406, + "forward_wall_seconds": 5.595218937000027, + "forward_cpu_seconds": 5.592634477000001 + }, + { + "step": 4, + "valid_length": 9, + "token_id": 315, + "decoded_token": " of", + "top1_logit": 20.05550193786621, + "top2_logit": 17.614667892456055, + "top1_top2_margin": 2.4408340454101562, + "forward_wall_seconds": 5.342139397999972, + "forward_cpu_seconds": 5.338031255000001 + }, + { + "step": 5, + "valid_length": 10, + "token_id": 15344, + "decoded_token": " Italy", + "top1_logit": 16.641294479370117, + "top2_logit": 16.617063522338867, + "top1_top2_margin": 0.02423095703125, + "forward_wall_seconds": 5.288755011000035, + "forward_cpu_seconds": 5.285648705 + }, + { + "step": 6, + "valid_length": 11, + "token_id": 374, + "decoded_token": " is", + "top1_logit": 22.20791244506836, + "top2_logit": 16.946107864379883, + "top1_top2_margin": 5.261804580688477, + "forward_wall_seconds": 5.632552263999969, + "forward_cpu_seconds": 5.624142145 + }, + { + "step": 7, + "valid_length": 12, + "token_id": 21718, + "decoded_token": " Rome", + "top1_logit": 20.907331466674805, + "top2_logit": 17.204931259155273, + "top1_top2_margin": 3.7024002075195312, + "forward_wall_seconds": 5.547119829000053, + "forward_cpu_seconds": 5.546546575000001 + } + ], + "wall_seconds": 44.67187527400006, + "cpu_seconds": 44.649298945, + "process_peak_rss_kib": 5244604 + }, + { + "name": "germany", + "prompt": "The capital of Germany is", + "prompt_utf8_hex": "546865206361706974616c206f66204765726d616e79206973", + "required_substring": "Berlin", + "max_new_tokens": 8, + "prompt_token_ids": [ + 785, + 6722, + 315, + 9856, + 374 + ], + "generated_token_ids": [ + 19846, + 13, + 576, + 6722, + 315, + 9625, + 374, + 12095 + ], + "generated_text": " Berlin. The capital of France is Paris", + "stop_reason": "max_new_tokens", + "semantic_passed": true, + "host_oracle_passed": true, + "steps": [ + { + "step": 0, + "valid_length": 5, + "token_id": 19846, + "decoded_token": " Berlin", + "top1_logit": 16.521493911743164, + "top2_logit": 14.624658584594727, + "top1_top2_margin": 1.8968353271484375, + "forward_wall_seconds": 5.496049604999939, + "forward_cpu_seconds": 5.495774664999999 + }, + { + "step": 1, + "valid_length": 6, + "token_id": 13, + "decoded_token": ".", + "top1_logit": 19.563419342041016, + "top2_logit": 19.295778274536133, + "top1_top2_margin": 0.2676410675048828, + "forward_wall_seconds": 5.400778742000057, + "forward_cpu_seconds": 5.400431184999995 + }, + { + "step": 2, + "valid_length": 7, + "token_id": 576, + "decoded_token": " The", + "top1_logit": 16.98853874206543, + "top2_logit": 15.723114013671875, + "top1_top2_margin": 1.2654247283935547, + "forward_wall_seconds": 5.322351349999963, + "forward_cpu_seconds": 5.322156993 + }, + { + "step": 3, + "valid_length": 8, + "token_id": 6722, + "decoded_token": " capital", + "top1_logit": 18.370086669921875, + "top2_logit": 14.746591567993164, + "top1_top2_margin": 3.623495101928711, + "forward_wall_seconds": 4.931711173000053, + "forward_cpu_seconds": 4.928969291000001 + }, + { + "step": 4, + "valid_length": 9, + "token_id": 315, + "decoded_token": " of", + "top1_logit": 20.154003143310547, + "top2_logit": 18.079017639160156, + "top1_top2_margin": 2.0749855041503906, + "forward_wall_seconds": 5.378948186000002, + "forward_cpu_seconds": 5.3775705759999965 + }, + { + "step": 5, + "valid_length": 10, + "token_id": 9625, + "decoded_token": " France", + "top1_logit": 18.578645706176758, + "top2_logit": 17.20067024230957, + "top1_top2_margin": 1.3779754638671875, + "forward_wall_seconds": 5.244032164000032, + "forward_cpu_seconds": 5.239271584999997 + }, + { + "step": 6, + "valid_length": 11, + "token_id": 374, + "decoded_token": " is", + "top1_logit": 21.884960174560547, + "top2_logit": 16.38650894165039, + "top1_top2_margin": 5.498451232910156, + "forward_wall_seconds": 5.602864303999922, + "forward_cpu_seconds": 5.602449051000008 + }, + { + "step": 7, + "valid_length": 12, + "token_id": 12095, + "decoded_token": " Paris", + "top1_logit": 21.248014450073242, + "top2_logit": 16.446701049804688, + "top1_top2_margin": 4.801313400268555, + "forward_wall_seconds": 5.380494970000086, + "forward_cpu_seconds": 5.373863341999993 + } + ], + "wall_seconds": 42.910345698000015, + "cpu_seconds": 42.893603793, + "process_peak_rss_kib": 5244636 + }, + { + "name": "japan", + "prompt": "The capital of Japan is", + "prompt_utf8_hex": "546865206361706974616c206f66204a6170616e206973", + "required_substring": "Tokyo", + "max_new_tokens": 8, + "prompt_token_ids": [ + 785, + 6722, + 315, + 6323, + 374 + ], + "generated_token_ids": [ + 26194, + 13, + 576, + 6722, + 315, + 279, + 3639, + 4180 + ], + "generated_text": " Tokyo. The capital of the United States", + "stop_reason": "max_new_tokens", + "semantic_passed": true, + "host_oracle_passed": true, + "steps": [ + { + "step": 0, + "valid_length": 5, + "token_id": 26194, + "decoded_token": " Tokyo", + "top1_logit": 16.132610321044922, + "top2_logit": 15.243704795837402, + "top1_top2_margin": 0.8889055252075195, + "forward_wall_seconds": 5.337803195999982, + "forward_cpu_seconds": 5.3336678580000125 + }, + { + "step": 1, + "valid_length": 6, + "token_id": 13, + "decoded_token": ".", + "top1_logit": 18.725910186767578, + "top2_logit": 18.47475242614746, + "top1_top2_margin": 0.2511577606201172, + "forward_wall_seconds": 5.40962941600003, + "forward_cpu_seconds": 5.409200784999996 + }, + { + "step": 2, + "valid_length": 7, + "token_id": 576, + "decoded_token": " The", + "top1_logit": 16.345678329467773, + "top2_logit": 15.184026718139648, + "top1_top2_margin": 1.161651611328125, + "forward_wall_seconds": 5.2799776159999965, + "forward_cpu_seconds": 5.279397790000004 + }, + { + "step": 3, + "valid_length": 8, + "token_id": 6722, + "decoded_token": " capital", + "top1_logit": 17.322513580322266, + "top2_logit": 14.836996078491211, + "top1_top2_margin": 2.4855175018310547, + "forward_wall_seconds": 5.683535881000012, + "forward_cpu_seconds": 5.678628095000008 + }, + { + "step": 4, + "valid_length": 9, + "token_id": 315, + "decoded_token": " of", + "top1_logit": 19.100204467773438, + "top2_logit": 17.763046264648438, + "top1_top2_margin": 1.337158203125, + "forward_wall_seconds": 5.363878753999984, + "forward_cpu_seconds": 5.363313796 + }, + { + "step": 5, + "valid_length": 10, + "token_id": 279, + "decoded_token": " the", + "top1_logit": 16.005170822143555, + "top2_logit": 15.628267288208008, + "top1_top2_margin": 0.3769035339355469, + "forward_wall_seconds": 5.153919003999931, + "forward_cpu_seconds": 5.153602902000003 + }, + { + "step": 6, + "valid_length": 11, + "token_id": 3639, + "decoded_token": " United", + "top1_logit": 16.810474395751953, + "top2_logit": 15.270747184753418, + "top1_top2_margin": 1.5397272109985352, + "forward_wall_seconds": 5.270241868000085, + "forward_cpu_seconds": 5.270055357000004 + }, + { + "step": 7, + "valid_length": 12, + "token_id": 4180, + "decoded_token": " States", + "top1_logit": 20.563379287719727, + "top2_logit": 17.63241195678711, + "top1_top2_margin": 2.930967330932617, + "forward_wall_seconds": 4.756456833000016, + "forward_cpu_seconds": 4.755680379999987 + } + ], + "wall_seconds": 42.40610255299998, + "cpu_seconds": 42.394194961, + "process_peak_rss_kib": 5244696 + } + ], + "tokenizer": { + "pad_token_id": 151643, + "generation_eos_token_ids": [ + 151645, + 151643 + ], + "model_vocab_size": 151936, + "tokenizer_vocab_size": 151669 + }, + "baseline": { + "recorded_at_utc": "2026-10-23T00:42:50.852643+00:00", + "acquisition": "One ORT run with independent per-step Host/input/argmax/stop oracle; not a second frozen replay.", + "environment": { + "python": "3.12.3", + "platform": "Linux-6.18.40.1-microsoft-standard-WSL2-x86_64-with-glibc2.39", + "packages": { + "numpy": "2.2.6", + "onnxruntime": "1.22.1", + "tokenizers": "0.21.4" + }, + "provider": "CPUExecutionProvider", + "intra_op_threads": 1, + "inter_op_threads": 1, + "graph_optimization": "ORT_DISABLE_ALL", + "cpu_memory_arena": false + }, + "session_load_wall_seconds": 0.3070653280000215, + "session_load_cpu_seconds": 0.29269802199999995, + "wall_seconds": 130.67631491099996, + "cpu_seconds": 130.61070972899998, + "process_peak_rss_kib": 5244696, + "memory_note": "Linux ru_maxrss is cumulative process peak, not per-case allocation.", + "qemu_e2e": "NOT_RUN" + } +} diff --git a/probes/w5_generation/run.py b/probes/w5_generation/run.py new file mode 100644 index 00000000..8281ba52 --- /dev/null +++ b/probes/w5_generation/run.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Run three fixed W5 Host generation references with the pinned full ORT model. + +Default: compare token IDs/text/stop reasons with references.json. +--record: collect a candidate reference in output only; never rewrite the fixture. +This is an ORT/Host test, not the QEMU W5 end-to-end gate. +""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import importlib.metadata +import json +from pathlib import Path +import platform +import resource +import sys +import time + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from probes.w2_runtime_model.run import checked_inputs +from scratchv.runtime.generation import generate +from scratchv.runtime.llm_inputs import SEQ_LENGTH +from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer + +FIXTURE = Path(__file__).with_name("references.json") + + +def reference_stop(length, generated, budget, eos_ids): + """Independent stop oracle; the prompt EOS is deliberately ignored.""" + if generated and generated[-1] in eos_ids: + return "eos" + if length == SEQ_LENGTH: + return "capacity" + if len(generated) >= budget: + return "max_new_tokens" + return None + + +def session_for(assets, manifest): + import onnxruntime as ort + + # Reuse the existing artifact identity/size manifest, without rehashing GBs. + # Size is an availability/LFS-pointer check, not a replacement integrity hash. + for item in manifest["files"]: + path = assets / "onnx" / item["name"] + if not path.is_file() or path.stat().st_size != item["bytes"]: + raise ValueError(f"Missing/incomplete fixed ONNX asset: {item['name']}") + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL + options.enable_cpu_mem_arena = False + session = ort.InferenceSession( + str(assets / "onnx/model.onnx"), options, providers=["CPUExecutionProvider"]) + inputs = [(item.name, item.type, item.shape) for item in session.get_inputs()] + outputs = [(item.name, item.type, item.shape) for item in session.get_outputs()] + if inputs != [("input_ids", "tensor(int64)", [1, 256]), + ("attention_mask", "tensor(float)", [1, 1, 256, 256])]: + raise ValueError(f"Unexpected model input ABI: {inputs}") + if outputs != [("logits", "tensor(float)", [1, 256, 151936])]: + raise ValueError(f"Unexpected model output ABI: {outputs}") + return session + + +def run_case(case, tokenizer, session): + """Exercise Host generate, checked by independent per-call and final oracles.""" + prompt = case["prompt"] + budget = case["max_new_tokens"] + prompt_ids = tokenizer.encode(prompt, add_special_tokens=False) + history = list(prompt_ids) + independently_generated = [] + steps = [] + eos_ids = tokenizer.generation_eos_token_ids + start_wall, start_cpu = time.perf_counter(), time.process_time() + + def forward(prepared): + # A wrong append, stale input, padding value, mask, or extra call fails. + if reference_stop(len(history), independently_generated, budget, eos_ids): + raise AssertionError("Host called forward after the independent stop condition") + feed = checked_inputs(history, prepared, tokenizer.pad_token_id) + begin_wall, begin_cpu = time.perf_counter(), time.process_time() + logits = session.run(["logits"], feed)[0] + elapsed_wall = time.perf_counter() - begin_wall + elapsed_cpu = time.process_time() - begin_cpu + if (logits.shape != (1, SEQ_LENGTH, tokenizer.model_vocab_size) + or logits.dtype != np.float32 or not np.isfinite(logits).all()): + raise ValueError("ORT returned invalid complete FP32 logits") + # Deliberately independent of last_valid_logits/greedy_token. + vector = logits[0, len(history) - 1, :] + token = int(np.argmax(vector)) + tokenizer.validate_token_ids([token]) + top_two = np.partition(vector, -2)[-2:] + steps.append({ + "step": len(steps), "valid_length": len(history), + "token_id": token, + "decoded_token": tokenizer.decode([token], skip_special_tokens=False, + clean_up_tokenization_spaces=False), + "top1_logit": float(vector[token]), + "top2_logit": float(top_two[0]), + "top1_top2_margin": float(vector[token]) - float(top_two[0]), + "forward_wall_seconds": elapsed_wall, "forward_cpu_seconds": elapsed_cpu, + }) + independently_generated.append(token) + history.append(token) + return logits + + actual = generate(prompt, tokenizer=tokenizer, forward=forward, max_new_tokens=budget) + wall_seconds = time.perf_counter() - start_wall + cpu_seconds = time.process_time() - start_cpu + expected_stop = reference_stop(len(history), independently_generated, budget, eos_ids) + expected_text = tokenizer.decode(independently_generated, skip_special_tokens=True, + clean_up_tokenization_spaces=False) + if tuple(actual.prompt_token_ids) != tuple(prompt_ids): + raise AssertionError("Host prompt IDs differ from the independent tokenizer result") + if tuple(actual.generated_token_ids) != tuple(independently_generated): + raise AssertionError("Host sampled IDs differ from independent ORT argmax") + if actual.stop_reason != expected_stop or actual.generated_text != expected_text: + raise AssertionError("Host stop reason/text differs from the independent oracle") + if len(steps) != len(independently_generated): + raise AssertionError("Forward count differs from generated token count") + return { + "name": case["name"], "prompt": prompt, + "prompt_utf8_hex": prompt.encode("utf-8").hex(), + "required_substring": case["required_substring"], "max_new_tokens": budget, + "prompt_token_ids": prompt_ids, "generated_token_ids": independently_generated, + "generated_text": expected_text, "stop_reason": expected_stop, + "semantic_passed": case["required_substring"] in expected_text, + "host_oracle_passed": True, "steps": steps, + "wall_seconds": wall_seconds, "cpu_seconds": cpu_seconds, + "process_peak_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, + } + + +def compare_frozen(actual, expected): + keys = ("prompt", "prompt_utf8_hex", "required_substring", "max_new_tokens", + "prompt_token_ids", "generated_token_ids", "generated_text", "stop_reason") + differences = [key for key in keys if actual[key] != expected[key]] + return {"passed": not differences, "different_fields": differences} + + +def main(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument("--assets", type=Path, default=ROOT / "assets/qwen3-0.6b") + cli.add_argument("--output-dir", type=Path, required=True) + cli.add_argument("--record", action="store_true", + help="Record a candidate reference in output; do not edit frozen references") + args = cli.parse_args(argv) + fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + manifest = json.loads((ROOT / fixture["model_manifest"]).read_text(encoding="utf-8")) + if (manifest["revision"] != fixture["revision"] + or manifest["model_id"] != fixture["model_id"]): + raise ValueError("Reference fixture does not match the fixed model manifest") + if not args.record and fixture["status"] != "frozen": + raise ValueError("Freeze and review the initial reference before verification mode") + args.output_dir.mkdir(parents=True, exist_ok=True) + output = args.output_dir / "reference-run.json" + report = { + "schema_version": 1, "status": "running", + "started_at_utc": datetime.now(timezone.utc).isoformat(), + "mode": "record_candidate" if args.record else "verify_frozen", + "model_manifest": fixture["model_manifest"], + "model_id": manifest["model_id"], "revision": manifest["revision"], + "asset_identity": "existing manifest/revision; files checked for expected size, not rehashed", + "contract": fixture["contract"], + "environment": { + "python": platform.python_version(), "platform": platform.platform(), + "packages": {name: importlib.metadata.version(name) + for name in ("numpy", "onnxruntime", "tokenizers")}, + "provider": "CPUExecutionProvider", "intra_op_threads": 1, + "inter_op_threads": 1, "graph_optimization": "ORT_DISABLE_ALL", + "cpu_memory_arena": False, + }, + "memory_note": "Linux ru_maxrss is cumulative process peak, not per-case allocation.", + "qemu_e2e": "NOT_RUN", "cases": [], + } + started_wall, started_cpu = time.perf_counter(), time.process_time() + + def save(): + output.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + save() + try: + tokenizer = Qwen3Tokenizer.from_directory(args.assets / "tokenizer") + report["tokenizer"] = { + "pad_token_id": tokenizer.pad_token_id, + "generation_eos_token_ids": list(tokenizer.generation_eos_token_ids), + "model_vocab_size": tokenizer.model_vocab_size, + "tokenizer_vocab_size": tokenizer.tokenizer_vocab_size, + } + if not args.record and report["tokenizer"] != fixture["tokenizer"]: + raise ValueError("Tokenizer IDs/vocabulary changed from the frozen reference") + begin_wall, begin_cpu = time.perf_counter(), time.process_time() + session = session_for(args.assets, manifest) + report["session_load_wall_seconds"] = time.perf_counter() - begin_wall + report["session_load_cpu_seconds"] = time.process_time() - begin_cpu + for case in fixture["cases"]: + print(f"Running {case['name']}: {case['max_new_tokens']} token budget", flush=True) + result = run_case(case, tokenizer, session) + result["frozen_comparison"] = None if args.record else compare_frozen(result, case) + report["cases"].append(result) + save() + print(json.dumps({key: result[key] for key in + ("name", "generated_text", "stop_reason", "wall_seconds", + "cpu_seconds", "semantic_passed")}, ensure_ascii=False), flush=True) + report["status"] = "passed" if all( + case["semantic_passed"] and case["host_oracle_passed"] + and (args.record or case["frozen_comparison"]["passed"]) + for case in report["cases"]) else "failed" + except BaseException as exc: + report["status"] = "failed" + report["error"] = f"{type(exc).__name__}: {exc}" + raise + finally: + report["wall_seconds"] = time.perf_counter() - started_wall + report["cpu_seconds"] = time.process_time() - started_cpu + report["process_peak_rss_kib"] = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + save() + print(f"{report['status'].upper()}: {output}", flush=True) + return 0 if report["status"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 076e3464..f35b4816 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ classifiers = [ riscv = ["tinyfive"] llvm = ["llvmlite"] # optional: LLVM IR JIT execution verify = ["onnxruntime"] # optional: ONNX Runtime comparison -llm = ["tokenizers>=0.21,<0.22"] # optional: local Qwen3 text tokenizer; no Torch +llm = ["tokenizers>=0.21,<0.22", "jinja2>=3.1.6,<4"] # optional: local Qwen3 text tokenizer; no Torch # Pinned deployment environment: Ubuntu 24.04 x86_64 / CPython 3.12. # Use the official CPU wheel so one install does not pull CUDA dependencies. llm-deploy = [ @@ -50,6 +50,7 @@ llm-deploy = [ "safetensors==0.5.3", "protobuf==5.29.5", "tokenizers==0.21.4", + "jinja2==3.1.6", "pytest==9.1.1", "ziglang==0.14.1", "PyYAML==6.0.3", diff --git a/scratchv/backend/tensor_c_codegen.py b/scratchv/backend/tensor_c_codegen.py index bd9f784b..a4edec3d 100644 --- a/scratchv/backend/tensor_c_codegen.py +++ b/scratchv/backend/tensor_c_codegen.py @@ -19,6 +19,16 @@ Supported element types are FP32, INT32 and INT64. Control flow, dynamic shapes, empty reductions and nonfinite arithmetic are rejected explicitly. This backend does not silently lower tensor operations to scalar integer instructions. + +The default Mean policy preserves sequential FP32 accumulation. The explicit +``mean_policy="two-four-lane-last-axis"`` option changes only single-last-axis +FP32 ReduceMean to the existing reference's canonical two-four-lane order. +It uses portable scalar C, preserves finite checks and requires no SIMD ISA. +The separate nonlinear_policy="fp32-reference" option selects fixed FP32 +Sin/Cos, Sigmoid and Softmax arithmetic; its default "libm" is unchanged. +The reference uses no host ISA dispatch, preserves all numeric checks, and +retains libm trigonometry outside abs(x) < 18000. Pow and standalone Exp are +unaffected. Neither explicit policy guarantees every ORT build's exact bits. """ from __future__ import annotations @@ -30,6 +40,10 @@ import numpy as np +from scratchv.backend.tensor_c_mean import TWO_FOUR_LANE_LAST_AXIS_C +from scratchv.backend.tensor_c_nonlinear import ( + TRIG_REFERENCE_C, SIGMOID_REFERENCE_C, SOFTMAX_REFERENCE_C, +) from scratchv.ir.types import DataType, OpCode, Program, Value from scratchv.verification.ir_numpy_ops import OpError, check_instruction @@ -82,6 +96,8 @@ class TensorCArtifact: matmul_rhs_panels: tuple[dict[str, object], ...] = () qemu_workspace_layout: str = "none" qemu_workspace_buffers: tuple[tuple[int, int], ...] = () + mean_policy: str = "sequential" + nonlinear_policy: str = "libm" def _shape(shape): @@ -202,7 +218,8 @@ def __init__(self, program: Program, initializers: Mapping[str, np.ndarray] | No *, max_workspace_bytes=256 * 1024 * 1024, max_constant_bytes=256 * 1024 * 1024, constant_storage="inline", kernel_calls=False, matmul_policy="sequential", matmul_tile="1x4", - qemu_workspace_layout="none", matmul_rhs_layout="original"): + qemu_workspace_layout="none", matmul_rhs_layout="original", + mean_policy="sequential", nonlinear_policy="libm"): self.program = program self.initializers = {} if initializers is None else dict(initializers) self.max_workspace_bytes = max_workspace_bytes @@ -210,6 +227,8 @@ def __init__(self, program: Program, initializers: Mapping[str, np.ndarray] | No self.constant_storage = constant_storage self.kernel_calls = kernel_calls self.matmul_policy = matmul_policy + self.mean_policy = mean_policy + self.nonlinear_policy = nonlinear_policy self.matmul_tile = matmul_tile self.matmul_rhs_layout = matmul_rhs_layout self.qemu_workspace_layout = qemu_workspace_layout @@ -222,6 +241,14 @@ def generate(self) -> TensorCArtifact: raise TensorCCodegenError("kernel_calls must be boolean") if self.kernel_calls and self.constant_storage != "external": raise TensorCCodegenError("kernel_calls requires external constant storage") + if (not isinstance(self.mean_policy, str) + or self.mean_policy not in ("sequential", "two-four-lane-last-axis")): + raise TensorCCodegenError( + "mean_policy must be 'sequential' or 'two-four-lane-last-axis'") + if (not isinstance(self.nonlinear_policy, str) + or self.nonlinear_policy not in ("libm", "fp32-reference")): + raise TensorCCodegenError( + "nonlinear_policy must be 'libm' or 'fp32-reference'") if self.matmul_policy not in ("sequential", "blocked_fma"): raise TensorCCodegenError("matmul_policy must be 'sequential' or 'blocked_fma'") if self.matmul_policy != "sequential" and self.constant_storage != "external": @@ -269,6 +296,8 @@ def generate(self) -> TensorCArtifact: self.external_scalar_checks = [] self.kernels = [] self._softmax_callbacks = {} + self._mean_last_used = False + self._nonlinear_used = set() self.constant_bytes = 0 self._counter = 0 defined = set() @@ -413,8 +442,12 @@ def generate(self) -> TensorCArtifact: self.body += ["if (!output) return 1;", f"for (size_t i=0; i<{output.size}ULL; ++i) (({ctype} *)output)[i] = {self.names[output.name]}[i];", "return 0;"] + mean_prelude = TWO_FOUR_LANE_LAST_AXIS_C if self._mean_last_used else "" + nonlinear_prelude = "".join(source for name, source in ( + ("trig", TRIG_REFERENCE_C), ("sigmoid", SIGMOID_REFERENCE_C), + ("softmax", SOFTMAX_REFERENCE_C)) if name in self._nonlinear_used) if self.constant_storage == "external": - source = (_PRELUDE + _EXTERNAL_PRELUDE + "".join(self.kernels) + source = (_PRELUDE + mean_prelude + nonlinear_prelude + _EXTERNAL_PRELUDE + "".join(self.kernels) + "#if defined(_WIN32)\n__declspec(dllexport)\n#endif\n" + "int scratchv_run_external(const void *const inputs[], " "const void *const weights[], void *workspace, size_t workspace_bytes, " @@ -428,18 +461,21 @@ def generate(self) -> TensorCArtifact: external_initializers=tuple(self.external_initializers), kernel_calls=self.kernel_calls, matmul_policy=self.matmul_policy, + mean_policy=self.mean_policy, + nonlinear_policy=self.nonlinear_policy, matmul_tile=self.matmul_tile, matmul_rhs_layout=self.matmul_rhs_layout, matmul_rhs_panels=tuple(self.matmul_rhs_panels), qemu_workspace_layout=self.qemu_workspace_layout, qemu_workspace_buffers=qemu_workspace_buffers, ) - source = (_PRELUDE + "\n" + "\n".join(self.constants) + source = (_PRELUDE + mean_prelude + nonlinear_prelude + "\n" + "\n".join(self.constants) + f"\nstatic union {{ uint64_t align; unsigned char bytes[{max(high, 8)}]; }} sv_arena;\n" + "#if defined(_WIN32)\n__declspec(dllexport)\n#endif\n" + "int scratchv_run(const void *const inputs[], void *output) {\n " + "\n ".join(self.body) + "\n}\n") - return TensorCArtifact(source, inputs, output, high, self.constant_bytes) + return TensorCArtifact(source, inputs, output, high, self.constant_bytes, + mean_policy=self.mean_policy, nonlinear_policy=self.nonlinear_policy) def _panel_inputs(self, program, initializers): """Select immutable 2D RHS storage; copy only IR containers we change. @@ -888,6 +924,13 @@ def _emit(self, instruction, output, details): if op == OpCode.SQRT: checks = f"if ({a}<0.0f) return 2; " expression = f"{function}({a})" + if self.nonlinear_policy == "fp32-reference": + if op in (OpCode.SIN, OpCode.COS): + self._nonlinear_used.add("trig") + expression = f"sv_fp32_reference_trig({a}, {int(op == OpCode.SIN)})" + elif op == OpCode.SIGMOID: + self._nonlinear_used.add("sigmoid") + expression = f"sv_fp32_reference_sigmoid({a})" self.body.append(f"for (size_t i=0; i<{size}ULL; ++i) {{ {checks}{dst}[i]={expression}; }}") return x, src = xs[0], names[0] @@ -1036,6 +1079,18 @@ def _emit(self, instruction, output, details): f"{dst}[batch*{m*n}ULL+m*{n}ULL+n]=acc; }}") elif op == OpCode.REDUCE_MEAN: axes = details["axes"] + if (self.mean_policy == "two-four-lane-last-axis" + and x.dtype == DataType.FLOAT32 and x.shape + and axes == (len(x.shape) - 1,)): + # Explicitly changes the reduction order; no SIMD ISA required. + # Shape, not the caller's address, determines the row prefix. + self._mean_last_used = True + width = x.shape[-1] + self.body.append( + f"for (size_t i=0; i<{size}ULL; ++i) " + f"{dst}[i]=sv_two_four_lane_mean_last(" + f"{src}+i*{width}ULL, {width}ULL, i);") + return # Keep the existing finite check and error propagation. remaining = tuple(a for a in range(len(x.shape)) if a not in axes) outer_shape = tuple(x.shape[a] for a in remaining) reduced_shape = tuple(x.shape[a] for a in axes) @@ -1049,14 +1104,24 @@ def _emit(self, instruction, output, details): axis = details["axis"] outer, dimension, inner = math.prod(x.shape[:axis]), x.shape[axis], math.prod(x.shape[axis+1:]) off = f"o*{dimension*inner}ULL+j*{inner}ULL+t" - # A row retains its original max, exp/sum and normalization order. # Only independent (outer, inner) rows may run on different harts. - row = ("float maximum=-0x1.fffffep+127f, sum=0.0f; " + # The explicitly selected arithmetic is identical in both paths. + initialization, exponential = "sum=0.0f", "expf" + accumulate, finish, normalize = "sum+=v;", "", "/=sum" + if self.nonlinear_policy == "fp32-reference": + self._nonlinear_used.add("softmax") + initialization = "sv_reference_lanes[16]={0.0f}" + exponential = "sv_fp32_reference_exp_nonpositive" + accumulate = "sv_reference_lanes[j%16ULL]+=v;" + finish = ("float sum=sv_fp32_reference_sum_sixteen(sv_reference_lanes); " + "float sv_reference_reciprocal=1.0f/sum; ") + normalize = "*=sv_reference_reciprocal" + row = (f"float maximum=-0x1.fffffep+127f, {initialization}; " f"for (size_t j=0; j<{dimension}ULL; ++j) if ({src}[{off}]>maximum) maximum={src}[{off}]; " f"for (size_t j=0; j<{dimension}ULL; ++j) {{ float shifted={src}[{off}]-maximum; " - "if (!sv_finite(shifted)) return 2; float v=expf(shifted); " - f"{dst}[{off}]=v; sum+=v; }} " - f"for (size_t j=0; j<{dimension}ULL; ++j) {{ {dst}[{off}]/=sum; " + f"if (!sv_finite(shifted)) return 2; float v={exponential}(shifted); " + f"{dst}[{off}]=v; {accumulate} }} " + f"{finish}for (size_t j=0; j<{dimension}ULL; ++j) {{ {dst}[{off}]{normalize}; " f"if (!sv_finite({dst}[{off}])) return 2; }} ") serial = (f"for (size_t o=0; o<{outer}ULL; ++o) " f"for (size_t t=0; t<{inner}ULL; ++t) {{ " + row + "}") diff --git a/scratchv/backend/tensor_c_mean.py b/scratchv/backend/tensor_c_mean.py new file mode 100644 index 00000000..720932fd --- /dev/null +++ b/scratchv/backend/tensor_c_mean.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: MPL-2.0 +# Adapted from Eigen (https://gitlab.com/libeigen/eigen), commit +# 1d8b82b0740839c0de7f1242a3585e3390ff5f33: +# Eigen/src/Core/Redux.h +# Eigen/src/Core/arch/SSE/PacketMath.h +# Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h +# Copyright (C) 2006-2008 Benoit Jacob +# Copyright (C) 2007 Julien Pommier +# Copyright (C) 2008-2009 Gael Guennebaud +# Copyright (C) 2009-2019 Gael Guennebaud +# Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) +# +# This Source Code Form is subject to the terms of the Mozilla +# Public License, v. 2.0. If a copy of the MPL was not distributed +# with this file, You can obtain one at https://mozilla.org/MPL/2.0/. +# The full license is also distributed in LICENSES/Eigen-MPL-2.0.txt. +"""C source for the explicitly selected FP32 last-axis Mean order. + +This portable scalar C follows the existing FP32 Eigen reference's two four-lane +accumulators, canonical row prefix, pairwise lane fold and tail. It changes +floating-point summation order; it is not a host SIMD or hardware requirement. +Finite input/output checks remain the tensor backend's responsibility. +""" + +TWO_FOUR_LANE_LAST_AXIS_C = r''' +/* + * SPDX-License-Identifier: MPL-2.0 + * Adapted from Eigen (https://gitlab.com/libeigen/eigen), commit + * 1d8b82b0740839c0de7f1242a3585e3390ff5f33: + * Eigen/src/Core/Redux.h + * Eigen/src/Core/arch/SSE/PacketMath.h + * Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h + * Copyright (C) 2006-2008 Benoit Jacob + * Copyright (C) 2007 Julien Pommier + * Copyright (C) 2008-2009 Gael Guennebaud + * Copyright (C) 2009-2019 Gael Guennebaud + * Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) + * + * This Source Code Form is subject to the terms of the Mozilla + * Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at https://mozilla.org/MPL/2.0/. + * The full license is also distributed in LICENSES/Eigen-MPL-2.0.txt. + */ + +/* Nonempty contiguous FP32 row. Canonical row alignment is independent of the + * caller's actual address. Compile without fast-math or contraction. */ +static float sv_two_four_lane_mean_last(const float *x, size_t width, size_t row_index) { + const size_t start = (width < 4 || width % 4 == 0) + ? 0 : (4 - ((row_index % 4) * (width % 4)) % 4) % 4; + float sum; + if (width - start < 4) { + sum = x[0]; + for (size_t k=1; k 4) { + float b0=x[start+4], b1=x[start+5], b2=x[start+6], b3=x[start+7]; + for (size_t k=start+8; k pair_end) { + a0=a0+x[pair_end]; a1=a1+x[pair_end+1]; + a2=a2+x[pair_end+2]; a3=a3+x[pair_end+3]; + } + } + sum = (a0+a2)+(a1+a3); + for (size_t k=0; k= 18000), fused bounded-rational +Sigmoid and polynomial Softmax with sixteen accumulators, pairwise lane fold, +ldexpf scaling and reciprocal multiplication. It requires no SIMD ISA. +Input/output finiteness and error propagation remain the backend's contract. +""" + +TRIG_REFERENCE_C = r''' +/* + * SPDX-License-Identifier: MPL-2.0 + * Adapted from Eigen (https://gitlab.com/libeigen/eigen), commit + * 1d8b82b0740839c0de7f1242a3585e3390ff5f33: + * Eigen/src/Core/Redux.h + * Eigen/src/Core/arch/SSE/PacketMath.h + * Eigen/src/Core/arch/Default/GenericPacketMathFunctions.h + * Copyright (C) 2006-2008 Benoit Jacob + * Copyright (C) 2007 Julien Pommier + * Copyright (C) 2008-2009 Gael Guennebaud + * Copyright (C) 2009-2019 Gael Guennebaud + * Copyright (C) 2014 Pedro Gonnet (pedro.gonnet@gmail.com) + * + * This Source Code Form is subject to the terms of the Mozilla + * Public License, v. 2.0. If a copy of the MPL was not distributed + * with this file, You can obtain one at https://mozilla.org/MPL/2.0/. + * The full license is also distributed in LICENSES/Eigen-MPL-2.0.txt. + */ + +/* Explicit FP32 non-FMA trig order from fp32_eigen.trig. + * Outside the conservative interval, retain the original platform libm. + * Portable scalar C; no host SIMD or RVV requirement. */ +static float sv_fp32_reference_trig(float value, int sine) { + union {float f; uint32_t u;} input={.f=value}, absolute, rounded_bits, output; + absolute.u=input.u & 0x7fffffffU; + if (!(absolute.f < 18000.0f)) return sine ? sinf(value) : cosf(value); + float reduced=absolute.f; + float rounded=reduced * 0x1.45f3060000000p-1f; + rounded=rounded + 12582912.0f; + rounded_bits.f=rounded; + uint32_t quadrant=rounded_bits.u; + float multiple=rounded-12582912.0f; + reduced=multiple*-0x1.9200000000000p+0f+reduced; + reduced=multiple*-0x1.fb80000000000p-12f+reduced; + reduced=multiple*0x1.5dc0000000000p-23f+reduced; + reduced=multiple*0x1.e973dc0000000p-35f+reduced; + float squared=reduced*reduced; + float cosine=0x1.98e6160000000p-16f; + cosine=cosine*squared+-0x1.6c06dc0000000p-10f; + cosine=cosine*squared+0x1.55553c0000000p-5f; + cosine=cosine*squared-0.5f; + cosine=cosine*squared+1.0f; + float sinpoly=-0x1.9ae1940000000p-13f; + sinpoly=sinpoly*squared+0x1.110ba60000000p-7f; + sinpoly=sinpoly*squared+-0x1.5555500000000p-3f; + sinpoly=sinpoly*squared; + sinpoly=sinpoly*reduced+reduced; + int even=(quadrant & 1U)==0; + output.f=sine ? (even ? sinpoly : cosine) : (even ? cosine : sinpoly); + uint32_t sign=sine ? (input.u ^ (quadrant<<30)) & 0x80000000U + : ((quadrant+1U)<<30) & 0x80000000U; + output.u ^= sign; + return output.f; +} +''' + +SIGMOID_REFERENCE_C = r''' +/* +MLAS polynomial constants/evaluation adapted from ONNX Runtime v1.22.1 +core/mlas/lib/logistic.cpp, via scratchv.verification.fp32_reference.sigmoid. +Copyright (c) Microsoft Corporation. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +static float sv_fp32_reference_sigmoid(float x) { + if (x < -18.0f) x=-18.0f; if (x > 18.0f) x=18.0f; + float squared=x*x; + float numerator=0x1.806aa20000000p-35f; + float denominator=0x1.5789ea0000000p-41f; + numerator=__builtin_fmaf(numerator,squared,0x1.f09d960000000p-24f); + numerator=__builtin_fmaf(numerator,squared,0x1.fe82760000000p-15f); + numerator=__builtin_fmaf(numerator,squared,0x1.16fab00000000p-7f); + numerator=__builtin_fmaf(numerator,squared,0x1.fc7e640000000p-3f); + denominator=__builtin_fmaf(denominator,squared,0x1.8be4f60000000p-28f); + denominator=__builtin_fmaf(denominator,squared,0x1.a62fba0000000p-18f); + denominator=__builtin_fmaf(denominator,squared,0x1.be2a7e0000000p-10f); + denominator=__builtin_fmaf(denominator,squared,0x1.de7c300000000p-4f); + denominator=__builtin_fmaf(denominator,squared,0x1.fc7e680000000p-1f); + float value=(numerator*x)/denominator+0.5f; + if (value < 0.0f) value=0.0f; if (value > 1.0f) value=1.0f; + return value; +} +''' + +SOFTMAX_REFERENCE_C = r''' +/* + * MLAS polynomial constants/evaluation are adapted from ONNX Runtime v1.22.1: + * core/mlas/lib/logistic.cpp, compute.cpp, x86_64/TransKernelFma3.S and + * x86_64/TransKernelAvx512F.S (including their amd64 Windows equivalents). + * Copyright (c) Microsoft Corporation. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Fixed sixteen-lane reference arithmetic, scalar FP32 implementation. + * This chooses numerical order, not the runtime host ISA. */ +extern float fmaf(float,float,float); +extern float ldexpf(float,int); +static float sv_fp32_reference_exp_nonpositive(float x) { + const float lower=-103.9720840454f, bias=12582912.0f; + if (x < lower) x=lower; + float rounded=fmaf(x,1.44269504088896341f,bias); + float exponent=rounded-bias; + x=fmaf(exponent,-6.93145752e-1f,x); + x=fmaf(exponent,-1.42860677e-6f,x); + float p=0x1.694000p-10f; + p=fmaf(p,x,0x1.125edcp-7f); + p=fmaf(p,x,0x1.555b5ap-5f); + p=fmaf(p,x,0x1.555450p-3f); + p=fmaf(p,x,0x1.fffff6p-2f); + p=fmaf(p,x,0x1.000000p+0f); + p=fmaf(p,x,0x1.000000p+0f); + return ldexpf(p,(int)exponent); +} +static float sv_fp32_reference_sum_sixteen(const float *s) { + float a0=s[0]+s[8],a1=s[1]+s[9],a2=s[2]+s[10],a3=s[3]+s[11]; + float a4=s[4]+s[12],a5=s[5]+s[13],a6=s[6]+s[14],a7=s[7]+s[15]; + float first=(a0+a1)+(a2+a3); + float last=(a4+a5)+(a6+a7); + return first+last; +} +''' diff --git a/scratchv/runtime/conversation.py b/scratchv/runtime/conversation.py new file mode 100644 index 00000000..a84adf18 --- /dev/null +++ b/scratchv/runtime/conversation.py @@ -0,0 +1,153 @@ +"""Host text conversations using the supplied Qwen3 chat template. + +The template is loaded locally and rendered without Transformers or Torch. This +layer owns text history only: the existing generation loop and forward backend +continue to own token generation and execution resources. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +import json +from pathlib import Path +from typing import TYPE_CHECKING + +import numpy as np + +from .generation import GenerationResult, generate +from .llm_inputs import PreparedInputs + +if TYPE_CHECKING: + from .qwen3_tokenizer import Qwen3Tokenizer + + +@dataclass(frozen=True) +class ChatMessage: + """One text-only message; tools and multimodal content are unsupported.""" + + role: str + content: str + + def __post_init__(self) -> None: + if self.role not in ("system", "user", "assistant"): + raise ValueError("role must be system, user or assistant") + if not isinstance(self.content, str): + raise TypeError("message content must be str") + self.content.encode("utf-8", errors="strict") + + +class Conversation: + """Transactional text history with explicit non-thinking generation. + + A successful turn with at least one generated token commits its user and + assistant messages together. Errors, cancellation, timeout, zero budget and + a full prompt with no room for output leave history unchanged. A response + stopped by token budget or capacity is retained as the returned partial + text; its ``GenerationResult.stop_reason`` remains explicit. + + History is re-rendered and encoded for every turn, without truncation or + implicit removal of old messages. There is no guest session or KV cache. + The same instance must not be used concurrently or reentrantly. + """ + + def __init__( + self, + tokenizer: Qwen3Tokenizer, + chat_template: str, + *, + system_prompt: str | None = None, + ) -> None: + if not isinstance(chat_template, str): + raise TypeError("chat_template must be str") + if not chat_template.strip(): + raise ValueError("chat_template must not be empty") + # Match the official template renderer's whitespace and sandbox rules; + # undefined optional message fields intentionally remain false/empty. + from jinja2 import TemplateError + from jinja2.sandbox import ImmutableSandboxedEnvironment + + def raise_exception(message: str) -> None: + raise TemplateError(message) + + environment = ImmutableSandboxedEnvironment( + trim_blocks=True, lstrip_blocks=True, + extensions=["jinja2.ext.loopcontrols"], + ) + environment.globals["raise_exception"] = raise_exception + self._template = environment.from_string(chat_template) + self.tokenizer = tokenizer + self._initial = (() if system_prompt is None else + (ChatMessage("system", system_prompt),)) + self._history = self._initial + self._busy = False + + @classmethod + def from_directory( + cls, directory: str | Path, *, system_prompt: str | None = None, + ) -> Conversation: + """Load the official template and tokenizer from the same snapshot.""" + from .qwen3_tokenizer import Qwen3Tokenizer + + directory = Path(directory) + config = json.loads((directory / "tokenizer_config.json").read_text(encoding="utf-8")) + if not isinstance(config, dict): + raise ValueError("tokenizer_config.json must contain a JSON object") + template = config.get("chat_template") + if not isinstance(template, str) or not template.strip(): + raise ValueError("tokenizer_config.json must contain one nonempty chat_template") + return cls(Qwen3Tokenizer.from_directory(directory), template, + system_prompt=system_prompt) + + @property + def history(self) -> tuple[ChatMessage, ...]: + """An immutable snapshot of committed text messages.""" + return self._history + + def reset(self) -> None: + """Discard completed turns while retaining the initial system prompt.""" + if self._busy: + raise RuntimeError("conversation has an active turn") + self._history = self._initial + + def render_prompt(self, user_text: str) -> str: + """Render committed history plus a pending user turn without mutation.""" + user = ChatMessage("user", user_text) + messages = [dict(role=item.role, content=item.content) + for item in (*self._history, user)] + return self._template.render( + messages=messages, tools=None, + add_generation_prompt=True, enable_thinking=False, + ) + + def turn( + self, + user_text: str, + *, + forward: Callable[[PreparedInputs], np.ndarray], + max_new_tokens: int, + cancelled: Callable[[], bool] | None = None, + timeout_seconds: float | None = None, + ) -> GenerationResult: + """Generate one reply and atomically commit it on success. + + Timeout/cancellation semantics are those of :func:`generate`; a backend + remains responsible for interrupting and cleaning up blocking work. + The full rendered prompt must fit the fixed 256-token ABI. + """ + if self._busy: + raise RuntimeError("conversation has an active turn") + self._busy = True + try: + prompt = self.render_prompt(user_text) + result = generate( + prompt, tokenizer=self.tokenizer, forward=forward, + max_new_tokens=max_new_tokens, cancelled=cancelled, + timeout_seconds=timeout_seconds, + ) + if result.generated_token_ids: + self._history += (ChatMessage("user", user_text), + ChatMessage("assistant", result.generated_text)) + return result + finally: + self._busy = False diff --git a/scratchv/runtime/generation.py b/scratchv/runtime/generation.py new file mode 100644 index 00000000..59cc63b5 --- /dev/null +++ b/scratchv/runtime/generation.py @@ -0,0 +1,133 @@ +"""Synchronous Host greedy generation for the fixed-length Qwen3 ABI. + +The caller owns the forward backend and its resources. This module neither +builds a model nor starts a process, and never changes hart or workspace policy. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +import math +from numbers import Real +import time +from typing import TYPE_CHECKING + +import numpy as np + +from .llm_inputs import ( + PreparedInputs, + StopReason, + generation_stop_reason, + greedy_token, + last_valid_logits, + prepare_inputs, +) + +if TYPE_CHECKING: + from .qwen3_tokenizer import Qwen3Tokenizer + + +class GenerationCancelledError(RuntimeError): + """The caller cancelled generation; no successful result is returned.""" + + +@dataclass(frozen=True) +class GenerationResult: + """Owned token IDs and the generated continuation, excluding the prompt. + + IDs retain generated EOS/special tokens. Display text skips special tokens + and disables whitespace cleanup; IDs are the authoritative comparison. + """ + + prompt_token_ids: tuple[int, ...] + generated_token_ids: tuple[int, ...] + generated_text: str + stop_reason: StopReason + + +def generate( + prompt: str, + *, + tokenizer: Qwen3Tokenizer, + forward: Callable[[PreparedInputs], np.ndarray], + max_new_tokens: int, + cancelled: Callable[[], bool] | None = None, + timeout_seconds: float | None = None, +) -> GenerationResult: + """Generate a raw completion without KV cache, templates or truncation. + + ``forward`` receives fresh right-padded inputs at each step and returns the + full finite FP32 logits. It may reuse its output buffer between calls; this + loop retains no output arrays. Each call to ``generate`` has isolated state. + + Cancellation and timeout are cooperative checks before/after forward and + before returning success. They cannot interrupt a blocking backend. The + backend must enforce its own execution timeout and clean up its processes; + backend exceptions (including KeyboardInterrupt) propagate unchanged. + A finite nonnegative timeout starts after input/configuration validation. + Zero expires immediately, even when the token budget is zero. + """ + if not isinstance(prompt, str): + raise TypeError("prompt must be str") + if not callable(forward): + raise TypeError("forward must be callable") + if cancelled is not None and not callable(cancelled): + raise TypeError("cancelled must be callable or None") + if timeout_seconds is not None: + if isinstance(timeout_seconds, (bool, np.bool_)) or not isinstance(timeout_seconds, Real): + raise TypeError("timeout_seconds must be a real number or None") + timeout_seconds = float(timeout_seconds) + if not math.isfinite(timeout_seconds) or timeout_seconds < 0: + raise ValueError("timeout_seconds must be finite and nonnegative") + + prompt_ids = tuple(tokenizer.validate_token_ids( + tokenizer.encode(prompt, add_special_tokens=False) + )) + vocab_size = tokenizer.model_vocab_size + pad_token_id = tokenizer.pad_token_id + eos_ids = tuple(tokenizer.validate_token_ids(tokenizer.generation_eos_token_ids)) + history = list(prompt_ids) + prepared = prepare_inputs(history, pad_token_id=pad_token_id, vocab_size=vocab_size) + # Validate the budget even if capacity or EOS would otherwise end the loop. + stop = generation_stop_reason( + valid_length=len(history), generated_tokens=0, + max_new_tokens=max_new_tokens, eos_token_ids=eos_ids, vocab_size=vocab_size, + ) + deadline = None if timeout_seconds is None else time.monotonic() + timeout_seconds + + def check_abort() -> None: + if cancelled is not None and cancelled(): + raise GenerationCancelledError("generation cancelled") + if deadline is not None and time.monotonic() >= deadline: + raise TimeoutError("generation timeout expired") + + generated: list[int] = [] + while True: + check_abort() + if stop is not None: + text = tokenizer.decode( + generated, skip_special_tokens=True, clean_up_tokenization_spaces=False + ) + check_abort() + return GenerationResult(prompt_ids, tuple(generated), text, stop) + + logits = forward(prepared) + check_abort() + row = last_valid_logits(logits, len(history), vocab_size=vocab_size) + # Release the full output before the next call allocates/reuses a buffer. + del logits + next_id = greedy_token(row) + del row + # The model has unused vocabulary rows: never mask or resample them. + tokenizer.validate_token_ids([next_id]) + check_abort() + generated.append(next_id) + history.append(next_id) + stop = generation_stop_reason( + valid_length=len(history), generated_tokens=len(generated), + max_new_tokens=max_new_tokens, last_token_id=next_id, + eos_token_ids=eos_ids, vocab_size=vocab_size, + ) + if stop is None: + prepared = prepare_inputs(history, pad_token_id=pad_token_id, vocab_size=vocab_size) diff --git a/scratchv/runtime/qemu_generation.py b/scratchv/runtime/qemu_generation.py new file mode 100644 index 00000000..49c152fb --- /dev/null +++ b/scratchv/runtime/qemu_generation.py @@ -0,0 +1,140 @@ +"""Fixed-ABI Host generation over one reusable RV64 executable and bundle. + +Each token launches a fresh QEMU guest with updated inputs. The ELF and bundle +are built once by the caller; this adapter neither recompiles nor changes hart +selection, code generation policy, weights, or workspace layout. +""" +from __future__ import annotations + +from collections.abc import Callable +import json +import math +from numbers import Real +from pathlib import Path +import threading +import time + +import numpy as np + +from .generation import GenerationCancelledError +from .llm_inputs import PreparedInputs, SEQ_LENGTH +from .riscv_external import ExternalExecutable, run_external + + +class QemuForward: + """Callable backend for ``generation.generate`` using the fixed W5 ABI. + + ``timeout`` bounds each guest invocation; ``deadline`` optionally bounds the + entire generation using an absolute ``time.monotonic()`` deadline. A shared + cancellation callback is checked during guest UART/QMP waits. Instances are + intentionally serial, but may be reused by independent generation contexts. + Step directories are unique and failed invocations retain diagnostic data. + """ + + def __init__(self, executable: ExternalExecutable, output_dir, *, + timeout: float = 3600.0, cancelled: Callable[[], bool] | None = None, + deadline: float | None = None): + for name, value in (("timeout", timeout), ("deadline", deadline)): + if value is None and name == "deadline": + continue + if isinstance(value, (bool, np.bool_)) or not isinstance(value, Real): + raise TypeError(f"{name} must be a real number") + if not math.isfinite(value) or (name == "timeout" and value <= 0): + raise ValueError(f"{name} must be finite" + (" and positive" if name == "timeout" else "")) + if cancelled is not None and not callable(cancelled): + raise TypeError("cancelled must be callable or None") + if executable.evidence.get("qemu_workspace_layout", "none") != "none": + raise ValueError("W5 generation requires qemu_workspace_layout=none") + specs = {spec.name: spec for spec in executable.inputs} + expected = {"input_ids": ((1, SEQ_LENGTH), np.dtype("int64")), + "attention_mask": ((1, 1, SEQ_LENGTH, SEQ_LENGTH), np.dtype("float32"))} + if len(specs) != len(executable.inputs) or set(specs) != set(expected): + raise ValueError("W5 requires exactly input_ids and attention_mask inputs") + for name, (shape, dtype) in expected.items(): + if tuple(specs[name].shape) != shape or specs[name].numpy_dtype != dtype: + raise ValueError(f"W5 {name} shape/dtype differs from the fixed ABI") + output = executable.output + if (len(output.shape) != 3 or tuple(output.shape[:2]) != (1, SEQ_LENGTH) + or output.shape[2] < 1 or output.numpy_dtype != np.dtype("float32")): + raise ValueError("W5 output must be full FP32 [1,256,vocab] logits") + self.executable = executable + self.vocab_size = output.shape[2] + self.output_dir = Path(output_dir).resolve() + # Refuse stale step/report reuse instead of overwriting previous runs. + self.output_dir.mkdir(parents=True, exist_ok=False) + self.timeout = float(timeout) + self.cancelled = cancelled + self.deadline = None if deadline is None else float(deadline) + self.runs: list[dict] = [] + self._lock = threading.Lock() + self._next_step = 0 + + def _cancelled(self): + if self.cancelled is not None and self.cancelled(): + return True + if self.deadline is not None and time.monotonic() >= self.deadline: + raise TimeoutError("generation timeout expired") + return False + + def __call__(self, prepared: PreparedInputs) -> np.ndarray: + if not self._lock.acquire(blocking=False): + raise RuntimeError("QemuForward does not support overlapping invocations") + try: + if not isinstance(prepared, PreparedInputs): + raise TypeError("QemuForward requires PreparedInputs") + if (isinstance(prepared.valid_length, (bool, np.bool_)) + or not isinstance(prepared.valid_length, (int, np.integer)) + or not 1 <= prepared.valid_length <= SEQ_LENGTH): + raise ValueError("valid_length must be in [1,256]") + if self._cancelled(): + raise GenerationCancelledError("generation cancelled") + timeout = self.timeout + if self.deadline is not None: + timeout = min(timeout, self.deadline - time.monotonic()) + if timeout <= 0: + raise TimeoutError("generation timeout expired") + step_dir = self.output_dir / f"step-{self._next_step:04d}" + self._next_step += 1 + checkpoint = self._cancelled if self.cancelled is not None or self.deadline is not None else None + try: + output, report = run_external( + self.executable, prepared.as_feed(), step_dir, + timeout=timeout, cancelled=checkpoint, + ) + # run_external already verifies every output element is finite; + # generate retains its full-output check before selecting a row. + if output.shape != (1, SEQ_LENGTH, self.vocab_size) or output.dtype != np.float32: + raise ValueError("QEMU returned logits outside the fixed ABI") + if self._cancelled(): + raise GenerationCancelledError("generation cancelled") + self.runs.append(dict(report, step=len(self.runs), run_dir=str(step_dir), + valid_length=int(prepared.valid_length))) + return output + except BaseException as exc: + report_path = step_dir / "run.json" + report = {} + try: + def finite_number(value): + number = float(value) + if not math.isfinite(number): + raise ValueError("run.json must contain finite JSON numbers") + return number + + candidate = json.loads(report_path.read_text(encoding="utf-8"), + parse_constant=finite_number, parse_float=finite_number) + if not isinstance(candidate, dict): + raise ValueError("run.json must contain a JSON object") + report = candidate + except FileNotFoundError: + # Failures before guest launch need not produce a report. + pass + except Exception as report_error: + # Diagnostic failures must not replace the backend error, + # including cancellation/interrupts, or lose this attempt. + report["report_read_error"] = f"{type(report_error).__name__}: {report_error}" + self.runs.append(dict(report, passed=False, status="FAIL", step=len(self.runs), + run_dir=str(step_dir), valid_length=int(prepared.valid_length), + error=f"{type(exc).__name__}: {exc}")) + raise + finally: + self._lock.release() diff --git a/scratchv/runtime/riscv_external.py b/scratchv/runtime/riscv_external.py index 3a9c6985..594d82f3 100644 --- a/scratchv/runtime/riscv_external.py +++ b/scratchv/runtime/riscv_external.py @@ -14,6 +14,7 @@ import os from pathlib import Path import select +import selectors import signal import socket import stat @@ -176,6 +177,12 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, harts = resolve_harts(harts) if artifact.constant_storage != "external": raise ValueError("External runtime requires external code generation") + mean_policy = getattr(artifact, "mean_policy", "sequential") + if not isinstance(mean_policy, str) or mean_policy not in ("sequential", "two-four-lane-last-axis"): + raise ValueError("Unsupported external Mean policy") + nonlinear_policy = getattr(artifact, "nonlinear_policy", "libm") + if not isinstance(nonlinear_policy, str) or nonlinear_policy not in ("libm", "fp32-reference"): + raise ValueError("Unsupported external nonlinear policy") manifest = validate_weight_bundle(bundle_dir, artifact.external_weights) _, input_bytes = input_layout(artifact.inputs) layout = plan_guest_memory(workspace_bytes=artifact.workspace_bytes, @@ -233,6 +240,7 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, raise RuntimeError(result.stderr.decode(errors="replace")[-12000:]) evidence = validate_external_elf(build_dir / "model.elf", layout) evidence.update(compile_seconds=time.perf_counter() - start, command=command, harts=harts, + mean_policy=mean_policy, nonlinear_policy=nonlinear_policy, matmul_tile=artifact.matmul_tile, matmul_rhs_layout=artifact.matmul_rhs_layout, matmul_rhs_panels=list(artifact.matmul_rhs_panels), @@ -248,9 +256,17 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, toolchain, evidence, harts) +def _check_cancelled(cancelled): + if cancelled is not None and cancelled(): + from .generation import GenerationCancelledError + raise GenerationCancelledError("generation cancelled") + + class QMP: """A bounded localhost QMP client that ignores asynchronous events.""" - def __init__(self, port, deadline): + def __init__(self, port, deadline, *, cancelled=None): + self.cancelled = cancelled + self.receive_buffer = bytearray() self.deadline = deadline self.serial = 0 self.socket = socket.create_connection(("127.0.0.1", port), timeout=self.remaining()) @@ -268,12 +284,15 @@ def __init__(self, port, deadline): raise def remaining(self): + _check_cancelled(getattr(self, "cancelled", None)) value = self.deadline - time.monotonic() if value <= 0: raise TimeoutError("QMP deadline exceeded") return min(value, 30.0) def read(self): + if getattr(self, "cancelled", None) is not None: + return self._read_cancellable() self.socket.settimeout(self.remaining()) line = self.stream.readline(1024 * 1024) if not line: @@ -282,6 +301,34 @@ def read(self): raise RuntimeError("Truncated/oversized QMP response") return json.loads(line) + def _read_cancellable(self): + # A timed-out makefile.readline cannot safely be resumed. Keep partial + # lines ourselves and use readiness notifications with bounded abort + # checkpoints; no successful frame is delayed by the checkpoint period. + limit = 1024 * 1024 + with selectors.DefaultSelector() as selector: + selector.register(self.socket, selectors.EVENT_READ) + while True: + remaining = self.remaining() + newline = self.receive_buffer.find(b"\n") + if newline >= 0: + if newline + 1 >= limit: + raise RuntimeError("Truncated/oversized QMP response") + line = bytes(self.receive_buffer[:newline + 1]) + del self.receive_buffer[:newline + 1] + return json.loads(line) + if len(self.receive_buffer) >= limit: + raise RuntimeError("Truncated/oversized QMP response") + if not selector.select(min(remaining, 0.05)): + continue + self.socket.settimeout(remaining) + chunk = self.socket.recv(min(65536, limit - len(self.receive_buffer))) + if not chunk: + if self.receive_buffer: + raise RuntimeError("Truncated/oversized QMP response") + raise EOFError("QMP connection closed before response") + self.receive_buffer.extend(chunk) + def command(self, name, arguments=None): self.serial += 1 request = {"execute": name, "id": self.serial} @@ -446,8 +493,9 @@ def _drain(self): if mask & (0x400 | 0x800 | 0x8000): raise RuntimeError("UART completion watch lost its directory") - def wait(self, process, deadline, timeout): + def wait(self, process, deadline, timeout, *, cancelled=None): while True: + _check_cancelled(cancelled) if time.monotonic() >= deadline: raise TimeoutError(f"RV64 forward exceeded {timeout}s") # Inspect before checking exit: a valid final write may accompany exit. @@ -483,6 +531,8 @@ def wait(self, process, deadline, timeout): poller.register(self.fd, select.POLLIN) poller.register(self.pidfd, select.POLLIN) timeout_ms = min(2_147_483_647, max(0, deadline - time.monotonic()) * 1000) + if cancelled is not None: + timeout_ms = min(timeout_ms, 50) for fd, events in poller.poll(timeout_ms): if events & select.POLLNVAL: raise OSError(errno.EBADF, "UART completion wait descriptor is invalid") @@ -550,12 +600,26 @@ def report(self): } -def run_external(executable, inputs, run_dir, *, timeout=3600.0): - """Run a fresh guest, require completion, stop, dump exact output, then quit.""" +def run_external(executable, inputs, run_dir, *, timeout=3600.0, cancelled=None): + """Run a fresh guest, require completion, stop, dump exact output, then quit. + + An optional cancellation callback is observed during UART/QMP waits and + between transport stages. Raising or returning true unwinds process cleanup. + Existing event timing remains unchanged without a cancellation callback. + """ + if cancelled is not None and not callable(cancelled): + raise TypeError("cancelled must be callable or None") + _check_cancelled(cancelled) if not np.isfinite(timeout) or timeout <= 0: raise ValueError("timeout must be positive and finite") harts = validate_harts(getattr(executable, "harts", 1)) built_harts = validate_harts(executable.evidence.get("harts", 1)) + mean_policy = executable.evidence.get("mean_policy", "sequential") + if not isinstance(mean_policy, str) or mean_policy not in ("sequential", "two-four-lane-last-axis"): + raise ValueError("Unsupported executable Mean policy") + nonlinear_policy = executable.evidence.get("nonlinear_policy", "libm") + if not isinstance(nonlinear_policy, str) or nonlinear_policy not in ("libm", "fp32-reference"): + raise ValueError("Unsupported executable nonlinear policy") if harts != built_harts or executable.layout["regions"]["stack"]["nbytes"] != harts * MIB: raise ValueError("Executable hart count disagrees with its build or stack layout") if sha256_file(executable.elf_path) != executable.evidence["sha256"]: @@ -582,7 +646,7 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0): options["start_new_session"] = True started = time.perf_counter() report = {"passed": False, "status": "FAIL", "command": command, "timeout_s": timeout, - "harts": harts, + "harts": harts, "mean_policy": mean_policy, "nonlinear_policy": nonlinear_policy, "qemu_workspace_layout": executable.evidence.get("qemu_workspace_layout", "none"), "transport": "QMP physical memory dump after guest completion", "guest_completed": False} transport_dir = run_dir / "weight-transport" @@ -604,6 +668,7 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0): path = str(transport_dir / row["file"]).replace(",", ",,") command += ["-device", f"loader,file={path},addr=0x{layout['weights_base'] + row['offset']:x},force-raw=on"] # Empty bundles intentionally emit no raw loader: QEMU rejects 0-byte files. + _check_cancelled(cancelled) report["stage"] = "qemu" deadline = time.monotonic() + timeout completion = _UartCompletionWaiter(run_dir, executable.output.nbytes) @@ -611,12 +676,17 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0): qemu_started = time.perf_counter() process = subprocess.Popen(command, cwd=run_dir, stdout=stdout, stderr=stderr, **options) job = _windows_job(process) + report["qemu_pid"] = process.pid monitor = _QemuMemoryMonitor(process) monitor.start() - completion.wait(process, deadline, timeout) + if cancelled is None: + completion.wait(process, deadline, timeout) + else: + completion.wait(process, deadline, timeout, cancelled=cancelled) report.update(guest_completed=True, forward_wall_s=time.perf_counter() - qemu_started, forward_wall_scope="QEMU launch through host validation of the UART completion frame") - qmp = QMP(port, deadline) + qmp = (QMP(port, deadline) if cancelled is None else + QMP(port, deadline, cancelled=cancelled)) qmp.command("stop") # A fast event must not hide bytes appended after the first valid frame. decode_completion((run_dir / "uart.bin").read_bytes(), executable.output.nbytes) @@ -633,7 +703,19 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0): raise ValueError("Output dump size mismatch") report["output_dump_s"] = time.perf_counter() - dump_started report["qmp_quit_reply_received"] = qmp.command("quit") is not None - process.wait(timeout=min(10, max(0.1, deadline - time.monotonic()))) + if cancelled is None: + process.wait(timeout=min(10, max(0.1, deadline - time.monotonic()))) + else: + while True: + _check_cancelled(cancelled) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("QEMU shutdown deadline exceeded") + try: + process.wait(timeout=min(0.05, remaining)) + break + except subprocess.TimeoutExpired: + continue qemu_exited = time.perf_counter() if process.returncode: raise RuntimeError(f"QEMU shutdown failed ({process.returncode})") @@ -645,9 +727,11 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0): raise ValueError("Nonfinite RV64 output") if validate_weight_bundle(executable.bundle_dir, executable.weights) != manifest: raise ValueError("Weight bundle changed during execution") + _check_cancelled(cancelled) report["stage"] = "weight_transport_postcheck" report["weight_transport"]["post_validation_elapsed_s"] = validate_transport( transport_dir, report["weight_transport"]) + _check_cancelled(cancelled) report.update(passed=True, status="PASS", exit_code=process.returncode, output_sha256=sha256_file(output_path), elf_sha256=executable.evidence["sha256"], weight_sha256=manifest["sha256"], input_sha256=sha256_file(run_dir / "inputs.bin")) @@ -697,6 +781,7 @@ def cleanup(label, action): cleanup("process.wait fallback", lambda: process.wait(timeout=5)) if process.returncode is not None and qemu_exited is None: qemu_exited = time.perf_counter() + report["qemu_reaped"] = process is not None and process.returncode is not None report["qemu_wall_s"] = (qemu_exited - qemu_started if qemu_exited is not None and qemu_started is not None else None) if job is not None: diff --git a/scripts/check_w5_ci.py b/scripts/check_w5_ci.py new file mode 100644 index 00000000..ba14f742 --- /dev/null +++ b/scripts/check_w5_ci.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Check the existing W5 CLI report; never execute or rehash model artifacts.""" +from __future__ import annotations + +import argparse +import json +import math +import os +from pathlib import Path +import re + +ROOT = Path(__file__).resolve().parents[1] +POLICIES = {"harts": 4, "matmul_policy": "blocked_fma", "matmul_tile": "2x4", + "matmul_rhs_layout": "column-panel", "mean_policy": "two-four-lane-last-axis", + "nonlinear_policy": "fp32-reference", "qemu_workspace_layout": "none"} + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def validate_report(report, reference): + """Require all three frozen eight-token cases and their actual QEMU evidence.""" + require(isinstance(report, dict) and isinstance(reference, dict), "Expected report objects") + require(reference.get("status") == "frozen", "Reference is not frozen") + cases = reference["cases"] + require([case["name"] for case in cases] == ["france", "germany", "japan"] + and all(case["max_new_tokens"] == len(case["generated_token_ids"]) == 8 + for case in cases), "Expected the frozen three-by-eight contract") + require(report.get("passed") is True and report.get("status") == "PASS" + and report.get("stage") == "complete" and report.get("full_model_gate") == "PASS" + and report.get("mode") == "raw" and not report.get("error"), + "Generation did not complete the full reference gate") + build = report.get("build", {}) + for key, value in POLICIES.items(): + require(type(report.get(key)) is type(value) and report[key] == value + and type(build.get(key)) is type(value) and build[key] == value, + f"Unexpected {key} in report/build") + audit = build.get("w5_graph_audit", {}) + require(audit.get("passed") is True and type(audit.get("layer_count")) is int + and audit["layer_count"] == 28, "Missing complete 28-layer graph audit") + require(all(type(report.get(key)) is int and report[key] == 24 + for key in ("forward_attempts", "qemu_forwards")), "Expected 24 executed forwards") + for key, size in (("cases", 3), ("numeric_steps", 24), ("runs", 24)): + require(isinstance(report.get(key), list) and len(report[key]) == size, + f"Incomplete or extra {key}") + fma = report.get("fma_conformance", {}) + comparison = fma.get("comparison", {}) + rows = comparison.get("rows", []) + require(fma.get("passed") is True and fma.get("status") == "PASS" + and comparison.get("passed") is True + and comparison.get("criterion") == "exact binary32 bits and FCSR" + and type(comparison.get("failed_rows")) is int and comparison["failed_rows"] == 0 + and isinstance(rows, list) and len(rows) == 12 + and all(all(row.get(key) is True for key in ("passed", "bits_match", "csr_match")) + for row in rows), "FMA exact bit/FCSR conformance did not pass") + # Compare existing identities only: no new hashes or extra artifact reads. + elf, weights = build.get("sha256"), report["runs"][0].get("weight_sha256") + require(all(isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) + for value in (elf, weights)), "Missing existing ELF/bundle identity") + index = 0 + for actual, frozen in zip(report["cases"], cases): + require(actual.get("reference_passed") is True + and all(actual.get(key) == frozen[key] for key in ( + "name", "prompt", "max_new_tokens", "prompt_token_ids", + "generated_token_ids", "generated_text", "stop_reason")) + and frozen["required_substring"] in actual["generated_text"] + and all(type(actual.get(key)) is int and actual[key] == 8 + for key in ("numeric_step_count", "forward_count")), + f"Frozen output mismatch: {frozen['name']}") + for local_step, token in enumerate(frozen["generated_token_ids"]): + numeric, run = report["numeric_steps"][index], report["runs"][index] + valid_length = len(frozen["prompt_token_ids"]) + local_step + require(numeric.get("case") == frozen["name"] + and type(numeric.get("step")) is int and numeric["step"] == local_step + and type(numeric.get("valid_length")) is int + and numeric["valid_length"] == valid_length + and all(numeric.get(key) is True for key in ( + "passed", "selected_row_passed", "reference_token_passed")) + and all(type(numeric.get(key)) is int and numeric[key] == token + for key in ("qemu_token_id", "ort_token_id", "reference_token_id")), + f"Wrong or missing token evidence at {frozen['name']}:{local_step}") + error = numeric.get("max_abs") + require(type(error) in (int, float) and math.isfinite(error) and 0 <= error < 1e-3 + and type(numeric.get("atol")) in (int, float) and numeric["atol"] == 1e-3 + and type(numeric.get("rtol")) in (int, float) and numeric["rtol"] == 0 + and type(numeric.get("elements_compared")) is int + and numeric["elements_compared"] == 256 * 151936, + f"Full-logits strict tolerance failed at {frozen['name']}:{local_step}") + require(run.get("passed") is True and run.get("status") == "PASS" + and run.get("guest_completed") is True and run.get("qemu_reaped") is True + and type(run.get("exit_code")) is int and run["exit_code"] == 0 + and not run.get("cleanup_errors") and not run.get("error") + and type(run.get("step")) is int and run["step"] == index + and type(run.get("valid_length")) is int and run["valid_length"] == valid_length + and run.get("elf_sha256") == elf and run.get("weight_sha256") == weights + and all(type(run.get(key)) is type(POLICIES[key]) and run[key] == POLICIES[key] + for key in ("harts", "mean_policy", "nonlinear_policy", "qemu_workspace_layout")), + f"QEMU completion, cleanup or reuse failed at forward {index}") + index += 1 + + +def main(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + cli.add_argument("--report", type=Path, required=True) + cli.add_argument("--reference", type=Path, default=ROOT / "probes/w5_generation/references.json") + cli.add_argument("--exit-code-file", type=Path, required=True) + cli.add_argument("--output", type=Path, required=True) + args = cli.parse_args(argv) + result = {"status": "FAIL", "passed": False, "scope": "Three frozen cases, 24 full QEMU forwards", + "commit": os.environ.get("GITHUB_SHA"), "report": str(args.report), + "defaults_changed": False, "raw_logits_uploaded": False} + try: + require(args.exit_code_file.read_text().strip() == "0", "Generation process did not exit successfully") + report = json.loads(args.report.read_text(encoding="utf-8")) + validate_report(report, json.loads(args.reference.read_text(encoding="utf-8"))) + result.update(status="PASS", passed=True, numeric_steps=24, cases=3, + max_abs=max(row["max_abs"] for row in report["numeric_steps"]), + elapsed_seconds=report.get("elapsed_seconds")) + except (OSError, ValueError, TypeError, KeyError, AttributeError) as exc: + result["error"] = f"{type(exc).__name__}: {exc}" + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, allow_nan=False) + "\n", encoding="utf-8") + text = (f"## W5 generation {result['status']}\n\n{result['scope']}. " + "4 hart; explicit Mean/nonlinear reference policies; padding off. " + "This is not acceptance of the unchanged default arithmetic configuration.\n\n" + + (f"24/24 numeric steps, 3/3 frozen outputs; max_abs={result['max_abs']:.9g}; " + f"elapsed={result['elapsed_seconds']} s.\n" if result["passed"] + else f"Not accepted: {result['error']}\n")) + print(text) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream: + stream.write(text) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci_scopes.json b/scripts/ci_scopes.json index 87b87c36..d0bf470e 100644 --- a/scripts/ci_scopes.json +++ b/scripts/ci_scopes.json @@ -30,7 +30,18 @@ "tests/test_w2_*.py", "tests/test_numeric_metrics.py", "tests/test_numeric_summary.py", - ".github/workflows/llm-deploy.yml" + ".github/workflows/llm-deploy.yml", + "scripts/run_inference.py", + "scripts/deploy_w5.sh", + "scripts/check_w5_ci.py", + "deployment/w5/**", + "probes/w5_generation/**", + "tests/test_generation.py", + "tests/test_conversation.py", + "tests/test_qemu_generation.py", + "tests/test_w5_*.py", + "tests/test_riscv_mean_policy.py", + "tests/test_riscv_nonlinear_policy.py" ], "w4": [ ".github/workflows/w4-full-numeric.yml", @@ -59,6 +70,21 @@ "tests/test_riscv_external*.py", "tests/test_riscv_process_cleanup.py", "tests/test_riscv_weight_transport.py", - "requirements/qwen3-small-probe.txt" + "requirements/qwen3-small-probe.txt", + "scripts/run_inference.py", + "scripts/deploy_w5.sh", + "scripts/check_w5_ci.py", + "deployment/w5/**", + "probes/w5_generation/**", + "tests/test_generation.py", + "tests/test_conversation.py", + "tests/test_qemu_generation.py", + "tests/test_w5_*.py", + "tests/test_riscv_mean_policy.py", + "tests/test_riscv_nonlinear_policy.py", + "pyproject.toml", + "assets/qwen3-0.6b/**", + "tests/test_tensor_c_mean_policy.py", + "tests/test_tensor_c_nonlinear_policy.py" ] } diff --git a/scripts/deploy_w5.sh b/scripts/deploy_w5.sh new file mode 100755 index 00000000..1c25c955 --- /dev/null +++ b/scripts/deploy_w5.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Thin clean-container entry point; generation arguments belong to run_inference.py. +set -euo pipefail + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +engine="${SCRATCHV_CONTAINER_ENGINE:-docker}" +image="${SCRATCHV_W5_IMAGE:-scratchv-w5:local}" +base_image="${SCRATCHV_W5_BASE_IMAGE:-ubuntu:24.04}" +assets="${SCRATCHV_W5_ASSETS:-$repo_root/assets/qwen3-0.6b}" +output="${SCRATCHV_W5_OUTPUT:-$repo_root/output/w5-deployment}" + +if ! command -v -- "$engine" >/dev/null 2>&1; then + printf 'Container engine is unavailable: %s. Install/start it separately; no system configuration was changed.\n' "$engine" >&2 + exit 127 +fi +if ! command -v setsid >/dev/null 2>&1; then + printf 'Linux setsid is required to clean up cancelled container-client processes.\n' >&2 + exit 127 +fi + +# Own each preflight/build client process group while Bash waits. Killing only +# this script must not leave the client (or its helpers) building in background. +# The container engine owns daemon-side cancellation; a real-engine check is +# still required to verify that side of the protocol. +run_engine_stage() { + local client_pid="" status attempt + cancel_engine_stage() { + local exit_status="$1" target="${client_pid:-${!:-}}" + trap '' INT TERM + if [[ -n "$target" ]]; then + # The direct-PID fallback covers a signal before setsid takes effect. + kill -TERM -- "-$target" 2>/dev/null || kill -TERM "$target" 2>/dev/null || true + for ((attempt = 0; attempt < 50; attempt++)); do + if ! kill -0 -- "-$target" 2>/dev/null; then + break + fi + sleep 0.1 + done + kill -KILL -- "-$target" 2>/dev/null || true + wait "$target" 2>/dev/null || true + fi + exit "$exit_status" + } + trap 'cancel_engine_stage 130' INT + trap 'cancel_engine_stage 143' TERM + setsid -- "$engine" "$@" & + client_pid=$! + if wait "$client_pid"; then + status=0 + else + status=$? + fi + trap - INT TERM + return "$status" +} + +if ! run_engine_stage info >/dev/null 2>&1; then + printf 'Container engine is not ready: %s. No model execution was started.\n' "$engine" >&2 + exit 1 +fi +if [[ ! -d "$assets" ]]; then + printf 'Assets directory does not exist: %s\n' "$assets" >&2 + exit 2 +fi +assets="$(cd -- "$assets" && pwd)" +mkdir -p -- "$output" +output="$(cd -- "$output" && pwd)" +# Docker/Podman --mount use comma-delimited fields. +if [[ "$assets" == *","* || "$output" == *","* ]]; then + printf 'Container mount paths must not contain commas.\n' >&2 + exit 2 +fi + +run_engine_stage build \ + --file "$repo_root/deployment/w5/Dockerfile" \ + --build-arg "BASE_IMAGE=$base_image" \ + --tag "$image" "$repo_root" + +# Numeric IDs keep report/build files owned by the invoking Linux developer. +# --init forwards signals/reaps children; the existing runner owns QEMU cleanup. +exec "$engine" run --rm --init \ + --user "$(id -u):$(id -g)" \ + --env HOME=/tmp \ + --env SCRATCHV_ZIG_CACHE=/output/.zig-cache \ + --mount "type=bind,src=$assets,dst=/assets,readonly" \ + --mount "type=bind,src=$output,dst=/output" \ + "$image" --output-dir /output/run "$@" diff --git a/scripts/run_inference.py b/scripts/run_inference.py new file mode 100644 index 00000000..2dab02e7 --- /dev/null +++ b/scripts/run_inference.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +"""Build/reuse one RV64 ELF and weight bundle for Host-driven QEMU generation. + +Raw completion is the default; chat and the frozen reference suite are explicit. +Each forward uses the existing bounded QEMU runner; weights/ELF are not rebuilt. +""" +from __future__ import annotations + +import argparse +from dataclasses import asdict, replace +import gc +import json +from pathlib import Path +import signal +import sys +import threading +import time + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from probes.w3_common import atomic_text +from probes.w2_runtime_model.run import checked_inputs +from probes.w4_qwen3_full.run import load_executable, save_executable, ATOL +from probes.w4_qwen3_full.fma_conformance import run_probe as run_fma_probe +from scratchv.backend.tensor_c_codegen import TensorCCodegen +from scratchv.runtime.generation import GenerationCancelledError, generate +from scratchv.runtime.llm_inputs import last_valid_logits, greedy_token +from scratchv.runtime.qemu_generation import QemuForward +from scratchv.runtime.qwen3_tokenizer import Qwen3Tokenizer +from scratchv.runtime.riscv_external import build_external, sha256_file +from scratchv.runtime.riscv_harts import resolve_harts +from scratchv.runtime.riscv_tensor import discover_toolchain +from scratchv.runtime.weight_bundle import write_weight_bundle + + +def parse_args(argv=None): + cli = argparse.ArgumentParser(description=__doc__) + source = cli.add_mutually_exclusive_group(required=True) + source.add_argument("--model", type=Path, help="Static ONNX graph; compile once") + source.add_argument("--reuse-build", type=Path, help="Existing W4/W5 executable.json") + cli.add_argument("--tokenizer-dir", type=Path, required=True) + cli.add_argument("--output-dir", type=Path, required=True, help="New output directory") + cli.add_argument("--mode", choices=("raw", "chat"), default="raw") + prompt = cli.add_mutually_exclusive_group() + prompt.add_argument("--prompt") + prompt.add_argument("--chat-turn", action="append", help="Repeat for ordered user turns") + prompt.add_argument("--reference-suite", action="store_true", help="Run all 3 frozen W5 prompts") + cli.add_argument("--system-prompt") + cli.add_argument("--max-new-tokens", type=int) + cli.add_argument("--harts", choices=("1", "2", "4", "auto"), default="1") + cli.add_argument("--timeout", type=float, default=3600, help="Per-forward QEMU timeout, seconds") + cli.add_argument("--total-timeout", type=float, default=10800, help="Generation/suite timeout, seconds") + cli.add_argument("--build-timeout", type=float, default=900) + cli.add_argument("--build-only", action="store_true") + cli.add_argument("--cc") + cli.add_argument("--qemu") + cli.add_argument("--matmul-policy", choices=("sequential", "blocked_fma"), default="sequential") + cli.add_argument("--mean-policy", choices=("sequential", "two-four-lane-last-axis"), default="sequential", + help="FP32 last-axis Mean order; default preserves sequential reduction") + cli.add_argument("--nonlinear-policy", choices=("libm", "fp32-reference"), default="libm", + help="Explicit FP32 Trig/Sigmoid/Softmax order; default preserves platform libm") + cli.add_argument("--matmul-tile", choices=("1x4", "2x4", "1x8"), default="1x4") + cli.add_argument("--matmul-rhs-layout", choices=("original", "column-panel"), default="original") + cli.add_argument("--verify-onnx", type=Path, help="Optional ORT numeric/greedy oracle for every step") + args = cli.parse_args(argv) + if any(not np.isfinite(t) or t <= 0 for t in (args.timeout, args.total_timeout, args.build_timeout)): + cli.error("timeouts must be positive and finite") + if args.max_new_tokens is not None and args.max_new_tokens < 0: + cli.error("--max-new-tokens must be nonnegative") + if not args.build_only and args.prompt is None and not args.chat_turn and not args.reference_suite: + cli.error("generation requires --prompt, --chat-turn, or --reference-suite") + if (args.chat_turn or args.system_prompt is not None) and args.mode != "chat": + cli.error("--chat-turn/--system-prompt require --mode chat") + if args.reference_suite and (args.mode != "raw" or args.max_new_tokens is not None or not args.verify_onnx): + cli.error("--reference-suite requires raw mode, --verify-onnx and its frozen token budgets") + if (args.matmul_tile != "1x4" or args.matmul_rhs_layout != "original") and args.matmul_policy != "blocked_fma": + cli.error("nondefault tile/RHS layout requires --matmul-policy blocked_fma") + if args.reuse_build and args.cc: + cli.error("--cc cannot change a reused build") + return args + + +def prepare_executable(args, out, harts): + """Reuse existing metadata validation, or compile/write weights exactly once.""" + if args.reuse_build: + previous = args.reuse_build.with_name("report.json") + record = json.loads(previous.read_text(encoding="utf-8")) + if record.get("executable_sha256") != sha256_file(args.reuse_build): + raise ValueError("Reused executable metadata identity is missing or changed") + exe = load_executable(args.reuse_build) + # W4 stored arithmetic policy in the sibling report; W5 also keeps it + # in the validated executable metadata. + policy = exe.evidence.get("matmul_policy", record.get("matmul_policy")) + expected = {"matmul_tile": args.matmul_tile, "matmul_rhs_layout": args.matmul_rhs_layout, + "qemu_workspace_layout": "none"} + defaults = {"matmul_tile": "1x4", "matmul_rhs_layout": "original", "qemu_workspace_layout": "none"} + if exe.harts != harts or policy != args.matmul_policy: + raise ValueError("Reused build hart count/arithmetic policy differs from requested configuration") + if (exe.evidence.get("mean_policy", "sequential") != args.mean_policy + or record.get("mean_policy", "sequential") != args.mean_policy): + raise ValueError("Reused build Mean policy differs from requested --mean-policy") + if (exe.evidence.get("nonlinear_policy", "libm") != args.nonlinear_policy + or record.get("nonlinear_policy", "libm") != args.nonlinear_policy): + raise ValueError("Reused build nonlinear policy differs from requested --nonlinear-policy") + if any(exe.evidence.get(k, defaults[k]) != value for k, value in expected.items()): + raise ValueError("Reused build layout/tile differs from requested configuration; padding must be none") + # Preserve validated W4 evidence when this executable is saved as W5 + # metadata. A later reuse must not depend on the original W4 report. + evidence = dict(exe.evidence) + evidence.setdefault("matmul_policy", policy) + if "w5_graph_audit" not in evidence and isinstance(record.get("graph_audit"), dict): + evidence["w5_graph_audit"] = dict(record["graph_audit"]) + if evidence != exe.evidence: + exe = replace(exe, evidence=evidence) + if args.qemu: + exe = replace(exe, toolchain=discover_toolchain(exe.toolchain.cc, args.qemu)) + return exe + from scratchv.frontend.onnx_parser import ONNXParser + parser = ONNXParser() + program = parser.parse(str(args.model.resolve()), mmap_external_data=True) + artifact = TensorCCodegen( + program, initializers=parser.initializers, constant_storage="external", + max_constant_bytes=3 * 1024**3, max_workspace_bytes=1024**3, kernel_calls=True, + matmul_policy=args.matmul_policy, matmul_tile=args.matmul_tile, mean_policy=args.mean_policy, + nonlinear_policy=args.nonlinear_policy, + matmul_rhs_layout=args.matmul_rhs_layout, qemu_workspace_layout="none", + ).generate() + write_weight_bundle(artifact.external_weights, artifact.external_initializers, out / "weights") + exe = build_external(artifact, out / "weights", out / "build", + discover_toolchain(args.cc, args.qemu), timeout=args.build_timeout, harts=harts) + audit = {"passed": False, "layer_count": None} + if tuple(exe.output.shape) == (1, 256, 151936): + import onnx + from probes.w2_qwen3_parse.validation import audit_graph_structure, Qwen3StructureError + try: + found = audit_graph_structure(onnx.load(str(args.model), load_external_data=False)) + audit = {key: found[key] for key in ("passed", "layer_count", "graph", "tied_embeddings")} + except Qwen3StructureError as exc: + audit["error"] = str(exc) + return replace(exe, evidence={**exe.evidence, "matmul_policy": args.matmul_policy, + "w5_graph_audit": audit}) + + +def require_full_executable(exe): + """Check executable evidence already validated/promoted by preparation.""" + audit = exe.evidence.get("w5_graph_audit") + if (not isinstance(audit, dict) or audit.get("passed") is not True + or audit.get("layer_count") != 28 or tuple(exe.output.shape) != (1, 256, 151936)): + raise ValueError("Full reference suite requires complete graph evidence for this executable") + + +def oracle_session(model_path): + import onnxruntime as ort + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL + options.enable_cpu_mem_arena = False + return ort.InferenceSession(str(model_path), options, providers=["CPUExecutionProvider"]) + + +def reference_session(model_path, fixture): + """Require the fixed complete Qwen3 graph before claiming a W5 full gate.""" + import onnx + from probes.w2_qwen3_parse.validation import audit_graph_structure + from probes.w5_generation.run import session_for + manifest = json.loads((ROOT / fixture["model_manifest"]).read_text(encoding="utf-8")) + if (manifest["model_id"] != fixture["model_id"] or manifest["revision"] != fixture["revision"] + or model_path.name != "model.onnx" or model_path.parent.name != "onnx"): + raise ValueError("Reference suite requires the fixed W1 ONNX asset layout/revision") + audit = audit_graph_structure(onnx.load(str(model_path), load_external_data=False)) + if not audit["passed"] or audit["layer_count"] != 28: + raise ValueError("Reference suite requires the complete 28-layer Qwen3 graph") + return session_for(model_path.parent.parent, manifest) + + +def compare_step(actual, expected, valid_length, vocab_size): + """Full finite-output check with the existing W4 absolute tolerance.""" + left = last_valid_logits(actual, valid_length, vocab_size=vocab_size) + right = last_valid_logits(expected, valid_length, vocab_size=vocab_size) + maximum = 0.0 + for row in range(actual.shape[1]): + delta = np.abs(actual[0, row].astype(np.float64) - expected[0, row].astype(np.float64)) + maximum = max(maximum, float(delta.max())) + ids = greedy_token(left), greedy_token(right) + selected_row_passed = (np.array_equal(left, actual[0, valid_length - 1]) + and np.array_equal(right, expected[0, valid_length - 1])) + top_two = np.partition(right, -2)[-2:] if right.size > 1 else None + # Finite FP32 endpoints can have a difference outside FP32 range. + return {"passed": maximum < ATOL and ids[0] == ids[1] and selected_row_passed, + "selected_row_passed": selected_row_passed, "max_abs": maximum, + "atol": ATOL, "rtol": 0.0, "qemu_token_id": ids[0], "ort_token_id": ids[1], + "ort_top2_margin": (float(top_two[1]) - float(top_two[0])) if top_two is not None else None, + "elements_compared": int(actual.size)} + + +def main(argv=None): + args = parse_args(argv) + out = args.output_dir.resolve() + out.mkdir(parents=True, exist_ok=False) + requested = args.harts if args.harts == "auto" else int(args.harts) + harts = resolve_harts(requested) + report = {"schema_version": 1, "status": "FAIL", "passed": False, "stage": "prepare", + "mode": args.mode, "harts_requested": requested, "harts": harts, + "qemu_workspace_layout": "none", "matmul_policy": args.matmul_policy, + "mean_policy": args.mean_policy, "nonlinear_policy": args.nonlinear_policy, + "matmul_tile": args.matmul_tile, "matmul_rhs_layout": args.matmul_rhs_layout, + "cases": [], "runs": [], "numeric_steps": [], "qemu_forwards": 0, + "full_model_gate": "NOT_RUN", "container_reproduction": "NOT_ASSERTED", + "execution": "one ELF/bundle; a fresh QEMU process per forward; no KV cache"} + started = time.perf_counter() + adapter = None + previous = {} + + def interrupted(signum, frame): + raise KeyboardInterrupt(f"generation interrupted by signal {signum}") + + def save(): + if adapter is not None: + report["runs"] = adapter.runs + report["forward_attempts"] = len(adapter.runs) + report["qemu_forwards"] = sum(bool(run.get("guest_completed")) for run in adapter.runs) + report["elapsed_seconds"] = time.perf_counter() - started + atomic_text(out / "report.json", json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + "\n") + + try: + if threading.current_thread() is threading.main_thread(): + for sig in (signal.SIGINT, signal.SIGTERM): + previous[sig] = signal.getsignal(sig) + signal.signal(sig, interrupted) + tokenizer = Qwen3Tokenizer.from_directory(args.tokenizer_dir) + exe = prepare_executable(args, out, harts) + if args.reference_suite: + require_full_executable(exe) + report["executable_sha256"] = save_executable(exe, out) + adapter = QemuForward(exe, out / "steps", timeout=args.timeout) + if adapter.vocab_size != tokenizer.model_vocab_size: + raise ValueError("Tokenizer model vocabulary differs from the executable output") + report["build"] = exe.evidence + if args.build_only: + report.update(status="BUILD_ONLY", stage="build_complete") + return 0 + report["stage"] = "fma_preflight" + fma = run_fma_probe(out / "fma-conformance", cc=exe.toolchain.cc, + qemu=exe.toolchain.qemu, timeout=min(args.build_timeout, 120)) + report["fma_conformance"] = fma + if not fma.get("passed") or fma.get("status") != "PASS": + raise ValueError("Actual RV64 execution toolchain failed FMA conformance") + fixture = None + if args.reference_suite: + report["stage"] = "reference_contract" + fixture = json.loads((ROOT / "probes/w5_generation/references.json").read_text(encoding="utf-8")) + meta = {"pad_token_id": tokenizer.pad_token_id, + "generation_eos_token_ids": list(tokenizer.generation_eos_token_ids), + "model_vocab_size": tokenizer.model_vocab_size, + "tokenizer_vocab_size": tokenizer.tokenizer_vocab_size} + if fixture["status"] != "frozen" or fixture["tokenizer"] != meta or len(fixture["cases"]) != 3: + raise ValueError("Reference suite/tokenizer differs from the frozen contract") + cases = fixture["cases"] + else: + texts = args.chat_turn or [args.prompt] + cases = [{"name": f"turn-{i}", "prompt": text, + "max_new_tokens": 8 if args.max_new_tokens is None else args.max_new_tokens} + for i, text in enumerate(texts)] + report["stage"] = "oracle_prepare" + session = (reference_session(args.verify_onnx, fixture) if fixture is not None else + oracle_session(args.verify_onnx) if args.verify_onnx else None) + conversation = None + if args.mode == "chat": + report["stage"] = "conversation_prepare" + from scratchv.runtime.conversation import Conversation + conversation = Conversation.from_directory(args.tokenizer_dir, system_prompt=args.system_prompt) + deadline = time.monotonic() + args.total_timeout + adapter.deadline = deadline + report["stage"] = "generation" + if fixture is not None: + report["full_model_gate"] = "RUNNING" + save() + + def forward(prepared): + step = len(report["numeric_steps"]) - numeric_start + if fixture is not None: + if step >= len(case["generated_token_ids"]): + raise ValueError("Host called forward after the frozen reference ended") + feed = checked_inputs(reference_history, prepared, tokenizer.pad_token_id) + else: + feed = prepared.as_feed() + expected = session.run(["logits"], feed)[0] if session is not None else None + actual = adapter(prepared) + if expected is not None: + comparison = compare_step(actual, expected, prepared.valid_length, adapter.vocab_size) + comparison.update(case=case["name"], step=step, valid_length=prepared.valid_length) + if fixture is not None: + # Select independently from frozen history, never from a + # caller-declared row or the generation sampling helper. + position = len(reference_history) - 1 + qemu_id = int(np.argmax(actual[0, position])) + ort_id = int(np.argmax(expected[0, position])) + frozen_id = case["generated_token_ids"][step] + comparison["reference_token_id"] = frozen_id + comparison["selected_row_passed"] = ( + comparison["selected_row_passed"] and comparison["qemu_token_id"] == qemu_id + and comparison["ort_token_id"] == ort_id) + comparison["reference_token_passed"] = qemu_id == ort_id == frozen_id + comparison["passed"] = (comparison["passed"] + and comparison["selected_row_passed"] and comparison["reference_token_passed"]) + report["numeric_steps"].append(comparison) + if not comparison["passed"]: + save() + raise ValueError("QEMU/ORT full logits, selected row or frozen token differ; see numeric_steps") + if fixture is not None: + reference_history.append(frozen_id) + return actual + + for case in cases: + numeric_start, run_start = len(report["numeric_steps"]), len(adapter.runs) + reference_history = list(case["prompt_token_ids"]) if fixture is not None else None + remaining = max(0.0, deadline - time.monotonic()) + kwargs = {"forward": forward, "max_new_tokens": case["max_new_tokens"], + "timeout_seconds": remaining} + result = (conversation.turn(case["prompt"], **kwargs) if conversation else + generate(case["prompt"], tokenizer=tokenizer, **kwargs)) + row = {"name": case["name"], "prompt": case["prompt"], + "max_new_tokens": case["max_new_tokens"], **asdict(result)} + if fixture is not None: + row["numeric_step_count"] = len(report["numeric_steps"]) - numeric_start + row["forward_count"] = len(adapter.runs) - run_start + row["reference_passed"] = ( + row["numeric_step_count"] == row["forward_count"] == len(case["generated_token_ids"]) + and list(result.prompt_token_ids) == case["prompt_token_ids"] + and list(result.generated_token_ids) == case["generated_token_ids"] + and result.generated_text == case["generated_text"] + and result.stop_reason == case["stop_reason"] + and case["required_substring"] in result.generated_text) + report["cases"].append(row) + save() + if fixture is not None and not row["reference_passed"]: + raise ValueError(f"Frozen reference mismatch: {case['name']}") + print(result.generated_text, flush=True) + if conversation: + report["history"] = [asdict(message) for message in conversation.history] + if fixture is not None: + report["full_model_gate"] = "PASS" + report.update(passed=True, status="PASS", stage="complete") + return 0 + except (GenerationCancelledError, KeyboardInterrupt) as exc: + report.update(status="CANCELLED", error=f"{type(exc).__name__}: {exc}") + if report["full_model_gate"] == "RUNNING": + report["full_model_gate"] = "CANCELLED" + return 130 + except Exception as exc: + report.update(error=f"{type(exc).__name__}: {exc}") + if report["full_model_gate"] == "RUNNING": + report["full_model_gate"] = "FAIL" + return 1 + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + save() + gc.collect() + print(f"{report['status']}: {out / 'report.json'}", flush=True) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_ci_routing.py b/tests/test_ci_routing.py index 835ca0b8..b3ce28db 100644 --- a/tests/test_ci_routing.py +++ b/tests/test_ci_routing.py @@ -327,3 +327,14 @@ def test_summary_cli_returns_success_only_for_complete_result(): result = subprocess.run([sys.executable, "-B", "-X", "utf8", str(ROOT / "scripts/summarize_ci.py")], env=env, capture_output=True, text=True, encoding="utf-8", timeout=10) assert result.returncode == 0 and "**PASS**" in result.stdout + + +@pytest.mark.parametrize("path", [ + "scripts/run_inference.py", "scripts/deploy_w5.sh", "scripts/check_w5_ci.py", + "deployment/w5/Dockerfile", "probes/w5_generation/references.json", + "tests/test_generation.py", "tests/test_conversation.py", "tests/test_qemu_generation.py", + "tests/test_w5_ci.py", "tests/test_riscv_mean_policy.py", "tests/test_riscv_nonlinear_policy.py", +]) +def test_w5_paths_route_to_existing_runtime_and_tokenizer_gates(path): + config = scope.read_config(ROOT / "scripts/ci_scopes.json") + assert scope.select_scopes(config, [path]) == {"llm": True, "w4": True} diff --git a/tests/test_ci_workflows.py b/tests/test_ci_workflows.py index 256a0097..cec20fe3 100644 --- a/tests/test_ci_workflows.py +++ b/tests/test_ci_workflows.py @@ -34,7 +34,7 @@ def test_reusable_jobs_share_the_callers_run_and_same_commit(): assert not ({'pull_request', 'push'} & set(workflow(filename)['on'])) assert jobs['llm']['needs'] == 'scope' assert jobs['llm']['if'] == "github.event_name == 'pull_request' && needs.scope.outputs.llm == 'true'" - assert workflow('llm-deploy.yml')['jobs']['full-qwen3-onnx']['if'] == "github.event_name != 'pull_request'" + assert workflow('llm-deploy.yml')['jobs']['full-qwen3-onnx']['if'] == "github.event_name != 'pull_request' && !inputs.run_w5_validation" assert 'workflow_dispatch' in workflow('llm-deploy.yml')['on'] @@ -96,7 +96,7 @@ def test_the_w1_w2_job_writes_no_step_summary(): data = workflow('llm-deploy.yml') writers = {name for name, job in data['jobs'].items() for step in job.get('steps', []) if 'GITHUB_STEP_SUMMARY' in step.get('run', '')} - assert writers == {'full-qwen3-onnx', 'w3-execution-summary', 'w4-execution-summary'} + assert writers == {'full-qwen3-onnx', 'w3-execution-summary', 'w4-execution-summary', 'w5-execution-summary'} def test_linux_reproduction_check_runs_without_a_path_filter(): @@ -224,3 +224,33 @@ def test_delegated_rv64_junit_check_requires_one_real_success(tmp_path, outcome) result = subprocess.run([sys.executable, '-c', checker], cwd=tmp_path, capture_output=True, text=True) assert (result.returncode == 0) is (outcome == 'success') + + +@pytest.mark.parametrize("suffix", ["None", "Give brief answers."]) +@pytest.mark.parametrize("outcome", ["pass", "skipped", "failure", "error", "missing", "duplicate"]) +def test_w5_official_three_round_template_must_execute(tmp_path, suffix, outcome): + import subprocess + import sys + import xml.etree.ElementTree as ET + + steps = workflow("llm-deploy.yml")["jobs"]["llm-deploy"]["steps"] + actual = next(step for step in steps if step.get("name") == "W5 official three-round conversation template") + assert "tests/test_conversation.py::test_official_three_round_rendering_matches_transformers" in actual["run"] + assert "if" not in actual and "continue-on-error" not in actual + checker = actual["run"].split("python3 - <<'PY'\n", 1)[1].rsplit("\nPY", 1)[0] + suite = ET.Element("testsuite") + for current in ("None", "Give brief answers."): + if current == suffix and outcome == "missing": + continue + name = f"test_official_three_round_rendering_matches_transformers[{current}]" + row = ET.SubElement(suite, "testcase", name=name) + if current == suffix: + if outcome in ("skipped", "failure", "error"): + ET.SubElement(row, outcome) + elif outcome == "duplicate": + ET.SubElement(suite, "testcase", name=name) + directory = tmp_path / "output/w5-multiturn" + directory.mkdir(parents=True) + ET.ElementTree(suite).write(directory / "tests.xml") + result = subprocess.run([sys.executable, "-c", checker], cwd=tmp_path, capture_output=True, text=True) + assert (result.returncode == 0) is (outcome == "pass"), result.stderr diff --git a/tests/test_conversation.py b/tests/test_conversation.py new file mode 100644 index 00000000..786dc13c --- /dev/null +++ b/tests/test_conversation.py @@ -0,0 +1,272 @@ +"""Host text-history tests; optional official parity loads tokenizers only.""" + +from dataclasses import FrozenInstanceError +import json +import os +from pathlib import Path + +import numpy as np +import pytest + +pytest.importorskip("jinja2") +from scratchv.runtime import conversation +from scratchv.runtime.conversation import ChatMessage, Conversation +from scratchv.runtime.generation import GenerationCancelledError, GenerationResult + + +TEMPLATE = "{%- for m in messages %}{{ m.role }}:{{ m.content }}|{% endfor -%}{{ 'assistant:' if add_generation_prompt else '' }}{{ 'nonthinking:' if enable_thinking is false else 'thinking:' }}" + + +class Tokenizer: + model_vocab_size = 258 + pad_token_id = 256 + generation_eos_token_ids = (257,) + + def encode(self, text, *, add_special_tokens): + assert add_special_tokens is False + return list(text.encode("utf-8")) + + def validate_token_ids(self, tokens): + if any(type(x) is not int or not 0 <= x < 258 for x in tokens): + raise ValueError("invalid token") + return list(tokens) + + def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces): + assert skip_special_tokens is True + assert clean_up_tokenization_spaces is False + return bytes(x for x in tokens if x < 256).decode("utf-8") + + +def logits(prepared, token=97): + value = np.zeros((1, 256, 258), dtype=np.float32) + value[0, prepared.valid_length - 1, token] = 1 + return value + + +def no_forward(_prepared): + raise AssertionError("backend must not run") + + +def test_three_rounds_preserve_exact_order_and_nonthinking_prompt(): + chat = Conversation(Tokenizer(), TEMPLATE, system_prompt="brief") + inputs = [] + for index, question in enumerate(("first", "second", "third")): + expected = "system:brief|" + "".join( + "user:" + q + "|assistant:a|" for q in ("first", "second", "third")[:index] + ) + "user:" + question + "|assistant:nonthinking:" + assert chat.render_prompt(question) == expected + + def forward(prepared): + inputs.append(bytes(prepared.input_ids[0, :prepared.valid_length].tolist()).decode("utf-8")) + return logits(prepared) + + result = chat.turn(question, forward=forward, max_new_tokens=1) + assert result.generated_text == "a" + assert result.stop_reason == "max_new_tokens" + assert inputs[-1] == expected + assert chat.history == ( + ChatMessage("system", "brief"), ChatMessage("user", "first"), ChatMessage("assistant", "a"), + ChatMessage("user", "second"), ChatMessage("assistant", "a"), + ChatMessage("user", "third"), ChatMessage("assistant", "a"), + ) + + +def test_history_snapshot_is_immutable_and_sessions_reset_independently(): + first = Conversation(Tokenizer(), TEMPLATE, system_prompt="one") + second = Conversation(Tokenizer(), TEMPLATE) + before = first.history + first.turn("first", forward=logits, max_new_tokens=1) + assert before == (ChatMessage("system", "one"),) + assert second.history == () + with pytest.raises(FrozenInstanceError): + first.history[0].content = "changed" + first.reset() + assert first.history == before + second.turn("second", forward=logits, max_new_tokens=1) + assert first.history == before + assert len(second.history) == 2 + + +@pytest.mark.parametrize("role", ["tool", "developer", "function", "", None, 1]) +def test_rejects_unsupported_roles(role): + with pytest.raises(ValueError, match="role"): + ChatMessage(role, "text") + + +@pytest.mark.parametrize("content", [None, b"text", [], {"type": "image"}, 1, True]) +def test_rejects_nontext_content_before_backend(content): + chat = Conversation(Tokenizer(), TEMPLATE) + with pytest.raises(TypeError, match="content"): + chat.turn(content, forward=no_forward, max_new_tokens=1) + assert chat.history == () + + +def test_rejects_invalid_unicode(): + with pytest.raises(UnicodeEncodeError): + ChatMessage("user", "\ud800") + + +@pytest.mark.parametrize("template", [None, [], "", " "]) +def test_requires_explicit_nonempty_template(template): + with pytest.raises((TypeError, ValueError), match="chat_template"): + Conversation(Tokenizer(), template) + + +def test_invalid_system_prompt(): + with pytest.raises(TypeError, match="content"): + Conversation(Tokenizer(), TEMPLATE, system_prompt=[]) + + +@pytest.mark.parametrize("failure", [RuntimeError("backend failed"), KeyboardInterrupt(), SystemExit(2)]) +def test_error_after_partial_generation_rolls_back_and_preserves_original_exception(failure): + chat = Conversation(Tokenizer(), TEMPLATE) + chat.turn("committed", forward=logits, max_new_tokens=1) + before = chat.history + calls = 0 + + def forward(prepared): + nonlocal calls + calls += 1 + if calls == 2: + raise failure + return logits(prepared) + + with pytest.raises(type(failure)) as caught: + chat.turn("failed", forward=forward, max_new_tokens=3) + assert caught.value is failure + assert chat.history == before + chat.turn("retry", forward=logits, max_new_tokens=1) + assert len(chat.history) == len(before) + 2 + + +def test_cancel_during_forward_rolls_back(): + chat = Conversation(Tokenizer(), TEMPLATE) + aborted = False + + def forward(prepared): + nonlocal aborted + aborted = True + return logits(prepared) + + with pytest.raises(GenerationCancelledError): + chat.turn("cancel", forward=forward, max_new_tokens=1, cancelled=lambda: aborted) + assert chat.history == () + + +def test_timeout_after_forward_rolls_back(monkeypatch): + from scratchv.runtime import generation + moments = iter((0.0, 0.0, 2.0)) + monkeypatch.setattr(generation.time, "monotonic", lambda: next(moments)) + chat = Conversation(Tokenizer(), TEMPLATE) + with pytest.raises(TimeoutError): + chat.turn("timeout", forward=logits, max_new_tokens=1, timeout_seconds=1) + assert chat.history == () + + +def test_nonfinite_output_rolls_back(): + chat = Conversation(Tokenizer(), TEMPLATE) + + def forward(prepared): + value = logits(prepared) + value[0, -1, 0] = np.nan + return value + + with pytest.raises(ValueError): + chat.turn("bad", forward=forward, max_new_tokens=1) + assert chat.history == () + + +@pytest.mark.parametrize("budget", [0, False, -1]) +def test_zero_budget_no_fake_turn_and_bad_budget_no_mutation(budget): + chat = Conversation(Tokenizer(), TEMPLATE) + if type(budget) is int and budget == 0: + result = chat.turn("valid", forward=no_forward, max_new_tokens=budget) + assert result.generated_token_ids == () + else: + with pytest.raises((TypeError, ValueError)): + chat.turn("valid", forward=no_forward, max_new_tokens=budget) + assert chat.history == () + + +def test_eos_only_reply_commits_legitimate_empty_assistant(): + chat = Conversation(Tokenizer(), TEMPLATE) + result = chat.turn("hello", forward=lambda p: logits(p, 257), max_new_tokens=2) + assert result.stop_reason == "eos" + assert chat.history == (ChatMessage("user", "hello"), ChatMessage("assistant", "")) + + +def test_capacity_checks_full_rendered_history_without_truncation(): + chat = Conversation(Tokenizer(), TEMPLATE) + chat.turn("prior", forward=logits, max_new_tokens=1) + before = chat.history + with pytest.raises(ValueError): + chat.turn("x" * 256, forward=no_forward, max_new_tokens=1) + assert chat.history == before + assert "prior" in chat.render_prompt("next") + + +def test_exact_capacity_returns_no_fake_turn_and_last_slot_commits(): + chat = Conversation(Tokenizer(), "{{ messages[-1].content }}") + result = chat.turn("x" * 256, forward=no_forward, max_new_tokens=1) + assert result.stop_reason == "capacity" + assert chat.history == () + result = chat.turn("x" * 255, forward=logits, max_new_tokens=2) + assert result.stop_reason == "capacity" + assert chat.history[-1] == ChatMessage("assistant", "a") + + +@pytest.mark.parametrize("action", ["turn", "reset"]) +def test_reentrant_state_mutation_rejected(action): + chat = Conversation(Tokenizer(), TEMPLATE) + + def forward(prepared): + if action == "turn": + chat.turn("nested", forward=no_forward, max_new_tokens=1) + else: + chat.reset() + return logits(prepared) + + with pytest.raises(RuntimeError, match="active turn"): + chat.turn("outer", forward=forward, max_new_tokens=1) + assert chat.history == () + chat.reset() + + +@pytest.mark.parametrize("config", [{}, {"chat_template": []}, [], {"chat_template": ""}]) +def test_from_directory_rejects_missing_or_ambiguous_template(tmp_path, config): + (tmp_path / "tokenizer_config.json").write_text(json.dumps(config), encoding="utf-8") + with pytest.raises(ValueError): + Conversation.from_directory(tmp_path) + + +@pytest.mark.parametrize("system_prompt", [None, "Give brief answers."]) +def test_official_three_round_rendering_matches_transformers(monkeypatch, system_prompt): + """Runs when local assets exist; does not load weights or execute a model.""" + pytest.importorskip("tokenizers") + transformers = pytest.importorskip("transformers") + root = Path(os.environ.get("SCRATCHV_QWEN3_TOKENIZER_DIR", Path(__file__).resolve().parents[1] / "assets/qwen3-0.6b/tokenizer")) + if not (root / "tokenizer.json").is_file(): + pytest.skip("official local Qwen3 tokenizer assets are not installed") + official = transformers.AutoTokenizer.from_pretrained(str(root), local_files_only=True) + chat = Conversation.from_directory(root, system_prompt=system_prompt) + replies = iter(("Paris", "Berlin", "Tokyo")) + + def fake_generate(prompt, *, tokenizer, **kwargs): + reply = next(replies) + return GenerationResult(tuple(tokenizer.encode(prompt, add_special_tokens=False)), + tuple(tokenizer.encode(reply, add_special_tokens=False)), reply, "max_new_tokens") + + monkeypatch.setattr(conversation, "generate", fake_generate) + for text in ("What is France's capital?", "And Germany's?", "What about Japan?"): + messages = [dict(role=m.role, content=m.content) for m in chat.history] + messages.append(dict(role="user", content=text)) + expected = official.apply_chat_template(messages, tokenize=False, + add_generation_prompt=True, enable_thinking=False) + rendered = chat.render_prompt(text) + assert rendered == expected + assert rendered.endswith("<|im_start|>assistant\n\n\n\n\n") + assert chat.tokenizer.encode(rendered, add_special_tokens=False) == official.apply_chat_template( + messages, tokenize=True, add_generation_prompt=True, enable_thinking=False) + assert len(chat.tokenizer.encode(rendered, add_special_tokens=False)) < 256 + chat.turn(text, forward=no_forward, max_new_tokens=1) + assert len(chat.history) == (7 if system_prompt is not None else 6) diff --git a/tests/test_generation.py b/tests/test_generation.py new file mode 100644 index 00000000..a55c6631 --- /dev/null +++ b/tests/test_generation.py @@ -0,0 +1,431 @@ +"""Host generation contract boundaries without a model, compiler or QEMU.""" + +from dataclasses import FrozenInstanceError +from numbers import Integral + +import numpy as np +import pytest + +from scratchv.runtime import generation +from scratchv.runtime.generation import GenerationCancelledError, generate +from scratchv.runtime.llm_inputs import SEQ_LENGTH + + +class Tokenizer: + """Small tokenizer with one undecodable model-vocabulary tail entry.""" + + model_vocab_size = 8 + pad_token_id = 0 + generation_eos_token_ids = (5, 6) + + def __init__(self, tokens=(1, 2)): + self.tokens = list(tokens) + self.encode_calls = [] + self.decode_calls = [] + + def encode(self, text, *, add_special_tokens): + self.encode_calls.append((text, add_special_tokens)) + return self.tokens.copy() + + def validate_token_ids(self, tokens): + checked = [] + for token in tokens: + if isinstance(token, (bool, np.bool_)) or not isinstance(token, Integral): + raise TypeError("token ID must be an integer") + if not 0 <= token < 7: + raise ValueError("token ID is absent from tokenizer vocabulary") + checked.append(int(token)) + return checked + + def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces): + tokens = self.validate_token_ids(tokens) + self.decode_calls.append((tuple(tokens), skip_special_tokens, + clean_up_tokenization_spaces)) + return " ".join(str(token) for token in tokens if token not in (0, 5, 6)) + + +def logits_for(prepared, token=3): + logits = np.zeros((1, SEQ_LENGTH, 8), dtype=np.float32) + # Padding rows deliberately prefer an undecodable model-vocabulary tail ID. + logits[0, :, 7] = 10 + logits[0, prepared.valid_length - 1, :] = 0 + logits[0, prepared.valid_length - 1, token] = 20 + return logits + + +def no_forward(_prepared): + raise AssertionError("forward must not be called") + + +def test_raw_prompt_last_valid_row_and_owned_result(): + tokenizer = Tokenizer() + inputs = [] + + def forward(prepared): + inputs.append(prepared) + return logits_for(prepared, 3 if len(inputs) == 1 else 4) + + result = generate(" raw\nCafe\u0301 ", tokenizer=tokenizer, forward=forward, + max_new_tokens=2) + assert tokenizer.encode_calls == [(" raw\nCafe\u0301 ", False)] + assert result.prompt_token_ids == (1, 2) + assert result.generated_token_ids == (3, 4) + assert result.generated_text == "3 4" + assert result.stop_reason == "max_new_tokens" + assert tokenizer.decode_calls == [((3, 4), True, False)] + assert [item.valid_length for item in inputs] == [2, 3] + np.testing.assert_array_equal(inputs[1].input_ids[0, :3], [1, 2, 3]) + assert inputs[1].attention_mask[0, 0, 2, 2] == 0 + assert inputs[1].attention_mask[0, 0, 2, 3] < 0 + tokenizer.tokens[:] = [4] + assert result.prompt_token_ids == (1, 2) + with pytest.raises((FrozenInstanceError, AttributeError)): + result.stop_reason = "eos" + + +def test_zero_budget_has_no_forward_or_generated_tokens(): + result = generate("prompt", tokenizer=Tokenizer(), forward=no_forward, + max_new_tokens=0) + assert result.generated_token_ids == () + assert result.generated_text == "" + assert result.stop_reason == "max_new_tokens" + + +@pytest.mark.parametrize("budget", [0, 1, 1000]) +def test_full_prompt_stops_at_capacity_without_forward(budget): + result = generate("prompt", tokenizer=Tokenizer([1] * SEQ_LENGTH), + forward=no_forward, max_new_tokens=budget) + assert result.prompt_token_ids == (1,) * SEQ_LENGTH + assert result.generated_token_ids == () + assert result.stop_reason == "capacity" + + +@pytest.mark.parametrize("tokens,exception", [ + ([], ValueError), ([1] * 257, ValueError), ([-1], ValueError), + ([7], ValueError), ([True], TypeError), ([1.5], TypeError), +]) +def test_invalid_prompt_rejected_even_with_zero_budget(tokens, exception): + with pytest.raises(exception): + generate("prompt", tokenizer=Tokenizer(tokens), forward=no_forward, + max_new_tokens=0) + + +@pytest.mark.parametrize("prompt", [None, b"prompt", 123, ["prompt"]]) +def test_prompt_must_be_text(prompt): + with pytest.raises(TypeError): + generate(prompt, tokenizer=Tokenizer(), forward=no_forward, + max_new_tokens=0) + + +@pytest.mark.parametrize("budget,exception", [ + (-1, ValueError), (True, TypeError), (np.bool_(False), TypeError), + (1.5, TypeError), (None, TypeError), +]) +def test_invalid_budget_rejected_before_forward(budget, exception): + with pytest.raises(exception): + generate("prompt", tokenizer=Tokenizer(), forward=no_forward, + max_new_tokens=budget) + + +@pytest.mark.parametrize("eos", [5, 6]) +def test_each_generation_eos_stops_and_is_retained_in_ids(eos): + calls = [] + + def forward(prepared): + calls.append(prepared.valid_length) + return logits_for(prepared, eos) + + result = generate("prompt", tokenizer=Tokenizer(), forward=forward, + max_new_tokens=5) + assert calls == [2] + assert result.generated_token_ids == (eos,) + assert result.generated_text == "" + assert result.stop_reason == "eos" + + +def test_eos_in_prompt_does_not_stop_generation(): + result = generate("prompt", tokenizer=Tokenizer([1, 5]), forward=logits_for, + max_new_tokens=1) + assert result.prompt_token_ids == (1, 5) + assert result.generated_token_ids == (3,) + assert result.stop_reason == "max_new_tokens" + + +@pytest.mark.parametrize("token,budget,reason", [ + (5, 1, "eos"), (5, 3, "eos"), (3, 1, "capacity"), (3, 3, "capacity"), +]) +def test_last_slot_stop_priority_and_no_extra_forward(token, budget, reason): + calls = [] + + def forward(prepared): + calls.append(prepared.valid_length) + return logits_for(prepared, token) + + result = generate("prompt", tokenizer=Tokenizer([1] * 255), forward=forward, + max_new_tokens=budget) + assert calls == [255] + assert result.generated_token_ids == (token,) + assert result.stop_reason == reason + + +def test_greedy_tie_uses_lowest_id_without_sampling(): + def forward(prepared): + logits = logits_for(prepared, 4) + logits[0, prepared.valid_length - 1, 3] = 20 + return logits + + result = generate("prompt", tokenizer=Tokenizer(), forward=forward, + max_new_tokens=1) + assert result.generated_token_ids == (3,) + + +@pytest.mark.parametrize("kind,exception", [ + ("shape", ValueError), ("dtype", TypeError), + ("valid_nan", ValueError), ("padding_inf", ValueError), +]) +def test_malformed_or_nonfinite_output_is_rejected(kind, exception): + tokenizer = Tokenizer() + + def forward(prepared): + logits = logits_for(prepared) + if kind == "shape": + return logits[:, :-1, :] + if kind == "dtype": + return logits.astype(np.float64) + if kind == "valid_nan": + logits[0, prepared.valid_length - 1, 3] = np.nan + else: + logits[0, -1, 0] = np.inf + return logits + + with pytest.raises(exception): + generate("prompt", tokenizer=tokenizer, forward=forward, max_new_tokens=1) + assert tokenizer.decode_calls == [] + + +def test_undecodable_model_tail_argmax_is_error_not_filtered_or_resampled(): + tokenizer = Tokenizer() + with pytest.raises(ValueError, match="absent"): + generate("prompt", tokenizer=tokenizer, + forward=lambda prepared: logits_for(prepared, 7), max_new_tokens=1) + assert tokenizer.decode_calls == [] + + +@pytest.mark.parametrize("error", [RuntimeError("backend failed"), TimeoutError("backend timeout")]) +def test_backend_exception_identity_is_preserved(error): + tokenizer = Tokenizer() + + def forward(_prepared): + raise error + + with pytest.raises(type(error)) as caught: + generate("prompt", tokenizer=tokenizer, forward=forward, max_new_tokens=2) + assert caught.value is error + assert tokenizer.decode_calls == [] + + +def test_cancellation_before_first_forward_and_after_forward(): + with pytest.raises(GenerationCancelledError): + generate("prompt", tokenizer=Tokenizer(), forward=no_forward, + max_new_tokens=1, cancelled=lambda: True) + + cancelled = False + tokenizer = Tokenizer() + calls = [] + + def forward(prepared): + nonlocal cancelled + calls.append(prepared.valid_length) + cancelled = True + return logits_for(prepared) + + with pytest.raises(GenerationCancelledError): + generate("prompt", tokenizer=tokenizer, forward=forward, + max_new_tokens=2, cancelled=lambda: cancelled) + assert calls == [2] + assert tokenizer.decode_calls == [] + + +@pytest.mark.parametrize("tokens,budget", [((1, 2), 0), ((1,) * 256, 2)]) +def test_cancellation_applies_to_no_forward_completion(tokens, budget): + with pytest.raises(GenerationCancelledError): + generate("prompt", tokenizer=Tokenizer(tokens), forward=no_forward, + max_new_tokens=budget, cancelled=lambda: True) + + +def test_cancellation_after_decode_does_not_return_success(): + tokenizer = Tokenizer() + cancelled = False + original_decode = tokenizer.decode + + def decode(*args, **kwargs): + nonlocal cancelled + text = original_decode(*args, **kwargs) + cancelled = True + return text + + tokenizer.decode = decode + with pytest.raises(GenerationCancelledError): + generate("prompt", tokenizer=tokenizer, forward=logits_for, + max_new_tokens=1, cancelled=lambda: cancelled) + + +@pytest.mark.parametrize("timeout,exception", [ + (-1, ValueError), (float("nan"), ValueError), (float("inf"), ValueError), + (True, TypeError), ("1", TypeError), +]) +def test_invalid_timeout_rejected_before_forward(timeout, exception): + with pytest.raises(exception): + generate("prompt", tokenizer=Tokenizer(), forward=no_forward, + max_new_tokens=0, timeout_seconds=timeout) + + +@pytest.mark.parametrize("budget", [0, 1]) +def test_zero_timeout_fails_before_forward_even_at_zero_budget(monkeypatch, budget): + monkeypatch.setattr(generation.time, "monotonic", lambda: 10.0) + with pytest.raises(TimeoutError): + generate("prompt", tokenizer=Tokenizer(), forward=no_forward, + max_new_tokens=budget, timeout_seconds=0) + + +def test_elapsed_forward_is_rejected_without_exposing_partial_success(monkeypatch): + now = [10.0] + monkeypatch.setattr(generation.time, "monotonic", lambda: now[0]) + tokenizer = Tokenizer() + + def forward(prepared): + now[0] = 12.0 + return logits_for(prepared, 5) + + with pytest.raises(TimeoutError): + generate("prompt", tokenizer=tokenizer, forward=forward, + max_new_tokens=2, timeout_seconds=1) + assert tokenizer.decode_calls == [] + + +def test_timeout_after_decode_does_not_return_success(monkeypatch): + now = [10.0] + monkeypatch.setattr(generation.time, "monotonic", lambda: now[0]) + tokenizer = Tokenizer() + original_decode = tokenizer.decode + + def decode(*args, **kwargs): + text = original_decode(*args, **kwargs) + now[0] = 11.0 + return text + + tokenizer.decode = decode + with pytest.raises(TimeoutError): + generate("prompt", tokenizer=tokenizer, forward=logits_for, + max_new_tokens=1, timeout_seconds=1) + + +def test_workspace_reuse_and_backend_input_mutation_do_not_change_history(): + tokenizer = Tokenizer() + buffer = np.empty((1, SEQ_LENGTH, 8), dtype=np.float32) + seen = [] + inputs = [] + + def forward(prepared): + seen.append(tuple(prepared.input_ids[0, :prepared.valid_length])) + inputs.append(prepared) + buffer[:] = logits_for(prepared, 3 if len(seen) == 1 else 4) + prepared.input_ids[:] = 0 + prepared.attention_mask[:] = 0 + return buffer + + result = generate("prompt", tokenizer=tokenizer, forward=forward, + max_new_tokens=2) + assert seen == [(1, 2), (1, 2, 3)] + assert inputs[0] is not inputs[1] + assert not np.shares_memory(inputs[0].input_ids, inputs[1].input_ids) + assert not np.shares_memory(inputs[0].attention_mask, inputs[1].attention_mask) + buffer[:] = np.nan + assert result.generated_token_ids == (3, 4) + assert result.prompt_token_ids == (1, 2) + + +def test_independent_calls_do_not_share_context_after_success_or_error(): + tokenizer = Tokenizer([1]) + seen = [] + + def forward(prepared): + seen.append(tuple(prepared.input_ids[0, :prepared.valid_length])) + return logits_for(prepared) + + first = generate("first", tokenizer=tokenizer, forward=forward, max_new_tokens=1) + tokenizer.tokens[:] = [2, 4] + with pytest.raises(RuntimeError, match="failed"): + generate("failed", tokenizer=tokenizer, + forward=lambda _prepared: (_ for _ in ()).throw(RuntimeError("failed")), + max_new_tokens=1) + second = generate("second", tokenizer=tokenizer, forward=forward, max_new_tokens=1) + assert seen == [(1,), (2, 4)] + assert first.prompt_token_ids == (1,) + assert second.prompt_token_ids == (2, 4) + assert first.generated_token_ids == second.generated_token_ids == (3,) + + +def test_decode_error_propagates_without_conversion_to_stop_reason(): + tokenizer = Tokenizer() + error = ValueError("decode failed") + + def decode(*_args, **_kwargs): + raise error + + tokenizer.decode = decode + with pytest.raises(ValueError) as caught: + generate("prompt", tokenizer=tokenizer, forward=logits_for, max_new_tokens=1) + assert caught.value is error + + +@pytest.mark.parametrize("error", [ + RuntimeError("second step failed"), TimeoutError("second step timeout"), + KeyboardInterrupt("interrupted second step"), +]) +def test_second_step_error_keeps_identity_and_reusable_backend_has_no_history_leak(error): + tokenizer = Tokenizer() + seen = [] + + def forward(prepared): + seen.append(tuple(prepared.input_ids[0, :prepared.valid_length])) + if len(seen) == 2: + raise error + return logits_for(prepared, 3 if len(seen) == 1 else 4) + + with pytest.raises(type(error)) as caught: + generate("first", tokenizer=tokenizer, forward=forward, max_new_tokens=3) + assert caught.value is error + assert seen == [(1, 2), (1, 2, 3)] + assert tokenizer.decode_calls == [] + + retry = generate("retry", tokenizer=tokenizer, forward=forward, max_new_tokens=1) + assert seen[-1] == (1, 2) + assert retry.generated_token_ids == (4,) + assert tokenizer.decode_calls == [((4,), True, False)] + + +def test_second_step_cancellation_discards_partial_result_and_allows_backend_reuse(): + tokenizer = Tokenizer() + cancelled = False + seen = [] + + def forward(prepared): + nonlocal cancelled + seen.append(tuple(prepared.input_ids[0, :prepared.valid_length])) + if len(seen) == 2: + cancelled = True + return logits_for(prepared, 3 if len(seen) == 1 else 4) + + with pytest.raises(GenerationCancelledError): + generate("first", tokenizer=tokenizer, forward=forward, + max_new_tokens=3, cancelled=lambda: cancelled) + assert seen == [(1, 2), (1, 2, 3)] + assert tokenizer.decode_calls == [] + + cancelled = False + retry = generate("retry", tokenizer=tokenizer, forward=forward, + max_new_tokens=1, cancelled=lambda: cancelled) + assert seen[-1] == (1, 2) + assert retry.generated_token_ids == (4,) + assert tokenizer.decode_calls == [((4,), True, False)] diff --git a/tests/test_qemu_generation.py b/tests/test_qemu_generation.py new file mode 100644 index 00000000..cd2b214b --- /dev/null +++ b/tests/test_qemu_generation.py @@ -0,0 +1,422 @@ +"""Generation ABI, live guest cancellation, and reusable 1/4-hart small graphs.""" +import json +import os +from pathlib import Path +import socket +import threading +import time +from types import SimpleNamespace + +import numpy as np +import pytest + +from scratchv.backend.tensor_c_codegen import TensorCCodegen, TensorSpec +from scratchv.frontend.onnx_parser import ONNXParser +from scratchv.ir.types import DataType +from scratchv.runtime import qemu_generation as adapter +from scratchv.runtime import riscv_external as runtime +from scratchv.runtime.generation import GenerationCancelledError, generate +from scratchv.runtime.llm_inputs import prepare_inputs +from scratchv.runtime.qemu_generation import QemuForward +from scratchv.runtime.riscv_tensor import discover_toolchain +from scratchv.runtime.weight_bundle import write_weight_bundle + + +class SmallTokenizer: + model_vocab_size = 8 + pad_token_id = 0 + generation_eos_token_ids = (6,) + + def encode(self, text, *, add_special_tokens=False): + assert not add_special_tokens + return [int(part) for part in text.split()] + + def validate_token_ids(self, tokens): + result = list(tokens) + if any(isinstance(x, (bool, np.bool_)) or not isinstance(x, (int, np.integer)) + or not 0 <= x < 8 for x in result): + raise ValueError("invalid small token") + return result + + def decode(self, tokens, *, skip_special_tokens=True, clean_up_tokenization_spaces=False): + return " ".join(str(x) for x in tokens if not skip_special_tokens or x not in (0, 6)) + + +def make_small_onnx(path): + """A real input-dependent 256-row graph, with a small vocabulary/weights.""" + import onnx + from onnx import helper, numpy_helper, TensorProto + embedding = np.zeros((8, 128), np.float32) + embedding[np.arange(8), np.arange(8)] = 1 + projection = np.zeros((128, 8), np.float32) + projection[np.arange(8), [1, 2, 3, 4, 6, 6, 6, 6]] = 8 + graph = helper.make_graph([ + helper.make_node("Gather", ["embedding", "input_ids"], ["hidden"], axis=0), + helper.make_node("MatMul", ["hidden", "projection"], ["logits"]), + ], "w5-small-generation", [ + helper.make_tensor_value_info("input_ids", TensorProto.INT64, [1, 256]), + helper.make_tensor_value_info("attention_mask", TensorProto.FLOAT, [1, 1, 256, 256]), + ], [helper.make_tensor_value_info("logits", TensorProto.FLOAT, [1, 256, 8])], [ + numpy_helper.from_array(embedding, "embedding"), + numpy_helper.from_array(projection, "projection"), + ]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 17)], ir_version=10) + onnx.checker.check_model(model) + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + onnx.save(model, path) + return path + + +def abi_executable(**changes): + base = dict(inputs=(TensorSpec("input_ids", DataType.INT64, (1, 256)), + TensorSpec("attention_mask", DataType.FLOAT32, (1, 1, 256, 256))), + output=TensorSpec("logits", DataType.FLOAT32, (1, 256, 8)), + evidence={"qemu_workspace_layout": "none"}, harts=1) + base.update(changes) + return SimpleNamespace(**base) + + +@pytest.mark.parametrize("changes,error", [ + ({"evidence": {"qemu_workspace_layout": "two-buffer-pad-4k"}}, "layout=none"), + ({"inputs": ()}, "exactly"), + ({"inputs": (TensorSpec("input_ids", DataType.INT32, (1,256)), + TensorSpec("attention_mask", DataType.FLOAT32, (1,1,256,256)))}, "shape/dtype"), + ({"output": TensorSpec("logits", DataType.FLOAT32, (1,8))}, "full FP32"), + ({"output": TensorSpec("logits", DataType.INT64, (1,256,8))}, "full FP32"), +]) +def test_rejects_incompatible_abi_before_creating_directory(tmp_path, changes, error): + with pytest.raises(ValueError, match=error): + QemuForward(abi_executable(**changes), tmp_path / "runs") + assert not (tmp_path / "runs").exists() + + +@pytest.mark.parametrize("kwargs,exception", [ + ({"timeout": 0}, ValueError), ({"timeout": float("nan")}, ValueError), + ({"timeout": True}, TypeError), ({"deadline": float("inf")}, ValueError), + ({"cancelled": 1}, TypeError), +]) +def test_invalid_options(tmp_path, kwargs, exception): + with pytest.raises(exception): + QemuForward(abi_executable(), tmp_path / "runs", **kwargs) + + +def test_serial_adapter_reuses_executable_and_keeps_full_output_contract(tmp_path, monkeypatch): + exe = abi_executable() + backend = QemuForward(exe, tmp_path / "runs") + calls = [] + def execute(executable, inputs, run_dir, **kwargs): + assert executable is exe + assert not backend._lock.acquire(blocking=False) + calls.append((inputs["input_ids"].copy(), run_dir, kwargs)) + result = np.zeros((1, 256, 8), np.float32) + if len(calls) == 2: + result = result[:, :1, :] + return result, {"passed": True} + monkeypatch.setattr(adapter, "run_external", execute) + prepared = prepare_inputs([1], pad_token_id=0, vocab_size=8) + backend(prepared) + with pytest.raises(ValueError, match="fixed ABI"): + backend(prepared) + backend(prepared) + assert [x[1].name for x in calls] == ["step-0000", "step-0001", "step-0002"] + assert [r["passed"] for r in backend.runs] == [True, False, True] + assert all(x[2]["cancelled"] is None for x in calls) + with pytest.raises(FileExistsError): + QemuForward(exe, tmp_path / "runs") + + +def test_adapter_nonoverlap_cancel_deadline_and_original_exception(tmp_path, monkeypatch): + backend = QemuForward(abi_executable(), tmp_path / "runs") + prepared = prepare_inputs([1], pad_token_id=0, vocab_size=8) + backend._lock.acquire() + try: + with pytest.raises(RuntimeError, match="overlapping"): + backend(prepared) + finally: + backend._lock.release() + backend.cancelled = lambda: True + with pytest.raises(GenerationCancelledError): + backend(prepared) + backend.cancelled = None + backend.deadline = time.monotonic() - 1 + with pytest.raises(TimeoutError, match="generation"): + backend(prepared) + assert backend.runs == [] + backend.deadline = None + error = RuntimeError("backend error") + def fail(*args, **kwargs): + raise error + monkeypatch.setattr(adapter, "run_external", fail) + with pytest.raises(RuntimeError) as actual: + backend(prepared) + assert actual.value is error + assert backend.runs[0]["status"] == "FAIL" + + +@pytest.mark.parametrize("phase", ["greeting", "partial", "reply"]) +def test_qmp_cancellation_interrupts_missing_or_partial_response(phase): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(3) + ready, stop = threading.Event(), threading.Event() + errors = [] + def serve(): + try: + with listener.accept()[0] as connection: + connection.settimeout(3) + with connection.makefile("rwb", buffering=0) as stream: + if phase != "greeting": + stream.write(b'{"QMP":{}}\n') + request = json.loads(stream.readline()) + stream.write(json.dumps({"return":{}, "id":request["id"]}).encode()+b'\n') + request = json.loads(stream.readline()) + assert request["execute"] == "stop" + if phase == "partial": + stream.write(b'{"return":') + ready.set() + assert stop.wait(3) + except BaseException as exc: + errors.append(exc) + worker = threading.Thread(target=serve) + worker.start() + client = None + began = time.monotonic() + try: + with pytest.raises(GenerationCancelledError): + client = runtime.QMP(listener.getsockname()[1], began + 5, cancelled=ready.is_set) + client.command("stop") + assert time.monotonic() - began < 1 + finally: + if client is not None: + client.close() + stop.set() + worker.join(4) + listener.close() + assert not worker.is_alive() + assert not errors + + +def _assert_reaped(report): + assert report["qemu_reaped"] + with pytest.raises(ProcessLookupError): + os.kill(report["qemu_pid"], 0) + with pytest.raises(ProcessLookupError): + os.killpg(report["qemu_pid"], 0) + assert not report.get("cleanup_errors") + + +@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="requires explicit SCRATCHV_QEMU/CC") +@pytest.mark.parametrize("harts", [1, 4]) +def test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel(tmp_path, monkeypatch, harts): + import onnxruntime as ort + path = make_small_onnx(tmp_path / "model.onnx") + parser = ONNXParser() + program = parser.parse(str(path)) + model = TensorCCodegen(program, initializers=parser.initializers, + constant_storage="external").generate() + bundle = tmp_path / "weights" + write_weight_bundle(model.external_weights, model.external_initializers, bundle) + original_harness = runtime.guest_harness + def waiting_fixture(*args): + source = original_harness(*args) + # Token 7 deliberately parks this test guest, so timeout/cancel happen + # while a real QEMU process is alive instead of before it is launched. + return source.replace(" int status=", ' if (((const long *)inputs[0])[0] == 7) for (;;) __asm__ volatile("wfi");\n int status=') + monkeypatch.setattr(runtime, "guest_harness", waiting_fixture) + exe = runtime.build_external(model, bundle, tmp_path / "build", discover_toolchain(), harts=harts) + backend = QemuForward(exe, tmp_path / "runs", timeout=10) + options = ort.SessionOptions() + options.intra_op_num_threads = options.inter_op_num_threads = 1 + session = ort.InferenceSession(str(path), sess_options=options, providers=["CPUExecutionProvider"]) + comparisons = [] + def checked(prepared): + actual = backend(prepared) + expected = session.run(None, prepared.as_feed())[0] + np.testing.assert_array_equal(actual, expected) + comparisons.append({"valid_length": prepared.valid_length, "elements": actual.size, + "max_abs": 0.0}) + return actual + first = generate("1", tokenizer=SmallTokenizer(), forward=checked, max_new_tokens=8) + assert first.generated_token_ids == (2, 3, 4, 6) and first.stop_reason == "eos" + second = generate("3", tokenizer=SmallTokenizer(), forward=checked, max_new_tokens=1) + assert second.generated_token_ids == (4,) and second.stop_reason == "max_new_tokens" + assert len(backend.runs) == 5 + for report in backend.runs: + _assert_reaped(report) + assert report["harts"] == harts and report["qemu_workspace_layout"] == "none" + assert report["weight_transport"]["retained_bytes"] == 0 + waiting = prepare_inputs([7], pad_token_id=0, vocab_size=8) + backend.timeout = 0.2 + with pytest.raises(TimeoutError, match="forward exceeded"): + backend(waiting) + _assert_reaped(backend.runs[-1]) + backend.timeout = 10 + original_wait = runtime._UartCompletionWaiter.wait + started = [] + cancel_at = [None] + def record_wait(self, process, *args, **kwargs): + assert process.poll() is None + started.append(process.pid) + cancel_at[0] = time.monotonic() + 0.05 + return original_wait(self, process, *args, **kwargs) + monkeypatch.setattr(runtime._UartCompletionWaiter, "wait", record_wait) + backend.cancelled = lambda: cancel_at[0] is not None and time.monotonic() >= cancel_at[0] + before = time.monotonic() + with pytest.raises(GenerationCancelledError): + backend(waiting) + assert time.monotonic() - before < 2 + _assert_reaped(backend.runs[-1]) + assert started == [backend.runs[-1]["qemu_pid"]] + backend.cancelled = None + third = generate("4", tokenizer=SmallTokenizer(), forward=checked, max_new_tokens=2) + assert third.generated_token_ids == (6,) and third.stop_reason == "eos" + _assert_reaped(backend.runs[-1]) + assert all(run["command"][run["command"].index("-kernel")+1] == str(exe.elf_path) + for run in backend.runs) + (tmp_path / "generation-check.json").write_text(json.dumps({ + "harts": harts, "full_model": False, "qemu_workspace_layout": "none", + "first_tokens": list(first.generated_token_ids), "different_context": list(second.generated_token_ids), + "after_abort_tokens": list(third.generated_token_ids), "numeric_comparisons": comparisons, + "one_elf_reused": True, "one_bundle_reused": True, "runs": backend.runs, + }, indent=2)+"\n") + + +def test_cancellable_qmp_preserves_fragmented_events_and_matching_id(): + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + listener.settimeout(3) + errors = [] + def serve(): + try: + with listener.accept()[0] as connection: + connection.settimeout(3) + with connection.makefile("rwb", buffering=0) as stream: + stream.write(b'{"QMP":{}}\n') + capability = json.loads(stream.readline()) + stream.write(json.dumps({"return":{}, "id":capability["id"]}).encode()+b'\n') + request = json.loads(stream.readline()) + payload = (json.dumps({"event":"STOP"})+'\n'+ + json.dumps({"return":{}, "id":request["id"]-1})+'\n'+ + json.dumps({"return":{"status":"paused"}, "id":request["id"]})+'\n').encode() + stream.write(payload[:7]) + time.sleep(0.01) + stream.write(payload[7:]) + except BaseException as exc: + errors.append(exc) + worker = threading.Thread(target=serve) + worker.start() + client = None + try: + client = runtime.QMP(listener.getsockname()[1], time.monotonic()+3, cancelled=lambda: False) + assert client.command("stop") == {"status":"paused"} + finally: + if client is not None: + client.close() + worker.join(4) + listener.close() + assert not worker.is_alive() + assert not errors + + +@pytest.mark.parametrize("contents,error", [ + ("null", TimeoutError("original timeout")), + ("[1]", GenerationCancelledError("original cancellation")), + ('"not an object"', KeyboardInterrupt("original interruption")), + ("{", RuntimeError("original backend error")), + ("[" * 2000 + "0" + "]" * 2000, TimeoutError("timeout with deeply nested report")), + ('{"elapsed_s": NaN}', TimeoutError("timeout with nonfinite report")), + ('{"samples": [Infinity, -Infinity]}', GenerationCancelledError("cancel with nonfinite report")), + (None, TimeoutError("original timeout with unreadable report")), +]) +def test_report_read_failure_preserves_backend_error_and_adapter_reuse( + tmp_path, monkeypatch, contents, error, +): + backend = QemuForward(abi_executable(), tmp_path / "runs") + prepared = prepare_inputs([1], pad_token_id=0, vocab_size=8) + calls = [] + original_read = Path.read_text + + def read(path, *args, **kwargs): + if contents is None and path.name == "run.json": + raise PermissionError("diagnostic report unreadable") + return original_read(path, *args, **kwargs) + + def execute(executable, inputs, run_dir, **kwargs): + calls.append((run_dir.name, inputs["input_ids"].copy())) + if len(calls) == 1: + run_dir.mkdir() + (run_dir / "run.json").write_text("{}" if contents is None else contents) + raise error + result = np.zeros((1, 256, 8), dtype=np.float32) + return result, {"passed": True} + + monkeypatch.setattr(Path, "read_text", read) + monkeypatch.setattr(adapter, "run_external", execute) + with pytest.raises(type(error)) as caught: + backend(prepared) + assert caught.value is error + assert len(backend.runs) == 1 + assert backend.runs[0]["passed"] is False + assert backend.runs[0]["error"] == f"{type(error).__name__}: {error}" + assert backend.runs[0]["report_read_error"] + json.dumps(backend.runs, allow_nan=False) # The CLI's finally report must still serialize. + np.testing.assert_array_equal(backend(prepared), np.zeros((1, 256, 8), np.float32)) + assert [run["passed"] for run in backend.runs] == [False, True] + assert [call[0] for call in calls] == ["step-0000", "step-0001"] + np.testing.assert_array_equal(calls[0][1], calls[1][1]) + + +@pytest.mark.parametrize("error", [ + GenerationCancelledError("cancelled during second token"), + TimeoutError("timeout during second token"), + RuntimeError("backend failed during second token"), +]) +def test_conversation_and_reused_backend_rollback_together_after_partial_turn( + tmp_path, monkeypatch, error, +): + pytest.importorskip("jinja2") + from scratchv.runtime.conversation import ChatMessage, Conversation + + backend = QemuForward(abi_executable(), tmp_path / "runs") + chat = Conversation(SmallTokenizer(), + "{% for m in messages %}{{ m.content }} {% endfor %}") + buffer = np.empty((1, 256, 8), dtype=np.float32) + seen = [] + + def execute(executable, inputs, run_dir, **kwargs): + run_dir.mkdir() + # The test's nonzero prefix denotes valid token IDs; padding is zero. + prefix = tuple(int(x) for x in inputs["input_ids"][0] if x != 0) + seen.append(prefix) + if len(seen) == 3: + (run_dir / "run.json").write_text("null") + raise error + buffer.fill(0) + token = {1: 2, 2: 4, 4: 6}[len(seen)] + buffer[0, len(prefix) - 1, token] = 10 + return buffer, {"passed": True} + + monkeypatch.setattr(adapter, "run_external", execute) + first = chat.turn("1", forward=backend, max_new_tokens=1) + committed = chat.history + assert committed == (ChatMessage("user", "1"), ChatMessage("assistant", "2")) + with pytest.raises(type(error)) as caught: + chat.turn("3", forward=backend, max_new_tokens=3) + assert caught.value is error + assert chat.history == committed + assert seen == [(1,), (1, 2, 3), (1, 2, 3, 4)] + + retry = chat.turn("5", forward=backend, max_new_tokens=2) + assert seen[-1] == (1, 2, 5) # No failed user 3 or partial assistant 4. + assert retry.generated_token_ids == (6,) and retry.stop_reason == "eos" + assert chat.history == committed + (ChatMessage("user", "5"), ChatMessage("assistant", "")) + assert [run["passed"] for run in backend.runs] == [True, True, False, True] + assert [Path(run["run_dir"]).name for run in backend.runs] == [ + "step-0000", "step-0001", "step-0002", "step-0003", + ] + buffer.fill(np.nan) + assert first.generated_token_ids == (2,) and first.generated_text == "2" + assert retry.generated_token_ids == (6,) and retry.generated_text == "" diff --git a/tests/test_riscv_external_contract.py b/tests/test_riscv_external_contract.py index 81312b9e..74244d4f 100644 --- a/tests/test_riscv_external_contract.py +++ b/tests/test_riscv_external_contract.py @@ -578,3 +578,127 @@ def inaccessible_report(path, *args, **kwargs): # Transport observation/reporting is attempted even after monitor failure. assert report["weight_transport"]["retained_observation"] == "complete" assert report["weight_transport"]["retained_bytes"] == 0 + + +@pytest.mark.parametrize("policy", ["sequential", "two-four-lane-last-axis"]) +def test_external_build_records_explicit_mean_policy_without_compiler(tmp_path, monkeypatch, policy): + import subprocess + builder = IRBuilder() + x = Value("x", shape=(2, 4)) + builder.new_function("main", [x]); builder.new_block("entry"); builder.ret(builder.neg(x)) + artifact = TensorCCodegen(builder.program, constant_storage="external", mean_policy=policy).generate() + bundle = tmp_path / "weights" + manifest = write_weight_bundle(artifact.external_weights, artifact.external_initializers, bundle) + _, input_bytes = runtime.input_layout(artifact.inputs) + layout = plan_guest_memory(workspace_bytes=artifact.workspace_bytes, + input_bytes=input_bytes, output_bytes=artifact.output.nbytes, weight_bytes=manifest["total_bytes"]) + + def compile_fixture(command, *, cwd, **kwargs): + _elf(Path(cwd) / "model.elf", layout) + return subprocess.CompletedProcess(command, 0, b"", b"") + + monkeypatch.setattr(runtime, "_run_process", compile_fixture) + monkeypatch.setattr(runtime, "toolchain_versions", lambda *_: {}) + executable = runtime.build_external(artifact, bundle, tmp_path / "build", RiscVToolchain(("mock-cc",), "mock-qemu")) + assert executable.evidence["mean_policy"] == policy + assert json.loads((tmp_path / "build/build.json").read_text())["mean_policy"] == policy + assert executable.harts == 1 and executable.evidence["qemu_workspace_layout"] == "none" + + +@pytest.mark.parametrize("policy", [None, True, "unsupported"]) +def test_external_build_rejects_unknown_mean_policy_before_creating_artifacts(tmp_path, policy): + artifact = SimpleNamespace(constant_storage="external", mean_policy=policy) + with pytest.raises(ValueError, match="Mean policy"): + runtime.build_external(artifact, tmp_path / "weights", tmp_path / "build", None) + assert not (tmp_path / "build").exists() + + +@pytest.mark.parametrize("policy", [None, "sequential", "two-four-lane-last-axis"]) +def test_external_run_reports_mean_policy_including_legacy_default(tmp_path, monkeypatch, policy): + import numpy as np + from dataclasses import replace + executable = _unlaunchable_executable(tmp_path) + if policy is not None: + executable = replace(executable, evidence={**executable.evidence, "mean_policy": policy}) + run_dir = tmp_path / "run" + # A missing QEMU executable stops before any process/model execution, but + # still exercises the runner's real failure-report publication. + with pytest.raises(OSError): + runtime.run_external(executable, {"x": np.ones(1, np.float32)}, run_dir) + report = json.loads((run_dir / "run.json").read_text()) + assert report["mean_policy"] == (policy or "sequential") + assert report["status"] == "FAIL" and report["guest_completed"] is False + + +@pytest.mark.parametrize("policy", [None, True, "unsupported"]) +def test_external_run_rejects_invalid_mean_policy_before_launch(tmp_path, policy): + import numpy as np + from dataclasses import replace + executable = _unlaunchable_executable(tmp_path) + executable = replace(executable, evidence={**executable.evidence, "mean_policy": policy}) + run_dir = tmp_path / "run" + with pytest.raises(ValueError, match="Mean policy"): + runtime.run_external(executable, {"x": np.ones(1, np.float32)}, run_dir) + assert not run_dir.exists() + + +@pytest.mark.parametrize("policy", ["libm", "fp32-reference"]) +def test_external_build_records_explicit_nonlinear_policy_without_compiler(tmp_path, monkeypatch, policy): + import subprocess + builder = IRBuilder() + x = Value("x", shape=(2, 4)) + builder.new_function("main", [x]); builder.new_block("entry"); builder.ret(builder.neg(x)) + artifact = TensorCCodegen(builder.program, constant_storage="external", nonlinear_policy=policy).generate() + bundle = tmp_path / "weights" + manifest = write_weight_bundle(artifact.external_weights, artifact.external_initializers, bundle) + _, input_bytes = runtime.input_layout(artifact.inputs) + layout = plan_guest_memory(workspace_bytes=artifact.workspace_bytes, + input_bytes=input_bytes, output_bytes=artifact.output.nbytes, weight_bytes=manifest["total_bytes"]) + + def compile_fixture(command, *, cwd, **kwargs): + _elf(Path(cwd) / "model.elf", layout) + return subprocess.CompletedProcess(command, 0, b"", b"") + + monkeypatch.setattr(runtime, "_run_process", compile_fixture) + monkeypatch.setattr(runtime, "toolchain_versions", lambda *_: {}) + executable = runtime.build_external(artifact, bundle, tmp_path / "build", RiscVToolchain(("mock-cc",), "mock-qemu")) + assert executable.evidence["nonlinear_policy"] == policy + assert json.loads((tmp_path / "build/build.json").read_text())["nonlinear_policy"] == policy + assert executable.harts == 1 and executable.evidence["qemu_workspace_layout"] == "none" + + +@pytest.mark.parametrize("policy", [None, True, "unsupported"]) +def test_external_build_rejects_unknown_nonlinear_policy_before_creating_artifacts(tmp_path, policy): + artifact = SimpleNamespace(constant_storage="external", nonlinear_policy=policy) + with pytest.raises(ValueError, match="nonlinear policy"): + runtime.build_external(artifact, tmp_path / "weights", tmp_path / "build", None) + assert not (tmp_path / "build").exists() + + +@pytest.mark.parametrize("policy", [None, "libm", "fp32-reference"]) +def test_external_run_reports_nonlinear_policy_including_legacy_default(tmp_path, monkeypatch, policy): + import numpy as np + from dataclasses import replace + executable = _unlaunchable_executable(tmp_path) + if policy is not None: + executable = replace(executable, evidence={**executable.evidence, "nonlinear_policy": policy}) + run_dir = tmp_path / "run" + # A missing QEMU executable stops before any process/model execution, but + # still exercises the runner's real failure-report publication. + with pytest.raises(OSError): + runtime.run_external(executable, {"x": np.ones(1, np.float32)}, run_dir) + report = json.loads((run_dir / "run.json").read_text()) + assert report["nonlinear_policy"] == (policy or "libm") + assert report["status"] == "FAIL" and report["guest_completed"] is False + + +@pytest.mark.parametrize("policy", [None, True, "unsupported"]) +def test_external_run_rejects_invalid_nonlinear_policy_before_launch(tmp_path, policy): + import numpy as np + from dataclasses import replace + executable = _unlaunchable_executable(tmp_path) + executable = replace(executable, evidence={**executable.evidence, "nonlinear_policy": policy}) + run_dir = tmp_path / "run" + with pytest.raises(ValueError, match="nonlinear policy"): + runtime.run_external(executable, {"x": np.ones(1, np.float32)}, run_dir) + assert not run_dir.exists() diff --git a/tests/test_riscv_mean_policy.py b/tests/test_riscv_mean_policy.py new file mode 100644 index 00000000..5947e0ce --- /dev/null +++ b/tests/test_riscv_mean_policy.py @@ -0,0 +1,72 @@ +"""Small real RV64 checks for the explicitly selected last-axis mean order.""" +import os +import json + +import numpy as np +import pytest + +from scratchv.backend.tensor_c_codegen import TensorCCodegen +from scratchv.ir.builder import IRBuilder +from scratchv.ir.types import Value +from scratchv.runtime.riscv_external import build_external, run_external +from scratchv.runtime.riscv_tensor import discover_toolchain +from scratchv.runtime.weight_bundle import write_weight_bundle +from scratchv.verification.fp32_eigen import mean_last + + +@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="explicit RV64 tools required") +@pytest.mark.parametrize("harts", [1, 4]) +@pytest.mark.parametrize("width", [13, 128, 1024]) +def test_explicit_mean_order_rv64_and_context_reuse(tmp_path, harts, width): + builder = IRBuilder() + x = Value("x", shape=(7, width)) + builder.new_function("main", [x]) + builder.new_block("entry") + builder.ret(builder.reduce_mean(x, axes=(-1,), keepdims=True)) + artifact = TensorCCodegen(builder.program, constant_storage="external", kernel_calls=True, + mean_policy="two-four-lane-last-axis").generate() + write_weight_bundle(artifact.external_weights, artifact.external_initializers, tmp_path / "weights") + executable = build_external(artifact, tmp_path / "weights", tmp_path / "build", + discover_toolchain(), harts=harts) + assert executable.evidence["mean_policy"] == "two-four-lane-last-axis" + assert executable.evidence["qemu_workspace_layout"] == "none" + rng = np.random.default_rng(1200 + width) + for index in range(2): + data = (rng.normal(size=(7, width)) * (1 if index == 0 else 256)).astype(np.float32) + # Exact signs and cancellation exercise more than a positive-only mean. + data[0] = np.float32(-0.0) + data[1, ::2] *= np.float32(-1) + expected = mean_last(data) + before = data.copy() + actual, report = run_external(executable, {"x": data}, tmp_path / f"run{index}", timeout=30) + np.testing.assert_array_equal(actual.view(np.uint32), expected.view(np.uint32)) + np.testing.assert_array_equal(data.view(np.uint32), before.view(np.uint32)) + assert report["passed"] and report["qemu_reaped"] and report["harts"] == harts + assert report["mean_policy"] == "two-four-lane-last-axis" + + +@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="explicit RV64 tools required") +@pytest.mark.parametrize("harts", [1, 4]) +def test_explicit_mean_overflow_propagates_and_build_remains_reusable(tmp_path, harts): + builder = IRBuilder() + x = Value("x", shape=(2, 16)) + builder.new_function("main", [x]) + builder.new_block("entry") + builder.ret(builder.reduce_mean(x, axes=(-1,), keepdims=False)) + artifact = TensorCCodegen(builder.program, constant_storage="external", kernel_calls=True, + mean_policy="two-four-lane-last-axis").generate() + write_weight_bundle(artifact.external_weights, artifact.external_initializers, tmp_path / "weights") + executable = build_external(artifact, tmp_path / "weights", tmp_path / "build", + discover_toolchain(), harts=harts) + with pytest.raises(RuntimeError, match="status 0x00000002"): + run_external(executable, {"x": np.full((2, 16), np.finfo(np.float32).max, np.float32)}, + tmp_path / "overflow", timeout=30) + failure = json.loads((tmp_path / "overflow/run.json").read_text()) + assert not failure["passed"] and failure["qemu_reaped"] + with pytest.raises(ProcessLookupError): + os.killpg(failure["qemu_pid"], 0) + # A valid new context uses the exact same executable after a guest error. + data = np.arange(32, dtype=np.float32).reshape(2, 16) / np.float32(8) + actual, report = run_external(executable, {"x": data}, tmp_path / "reused", timeout=30) + np.testing.assert_array_equal(actual.view(np.uint32), mean_last(data, False).view(np.uint32)) + assert report["passed"] and report["qemu_reaped"] diff --git a/tests/test_riscv_nonlinear_policy.py b/tests/test_riscv_nonlinear_policy.py new file mode 100644 index 00000000..6e3ecf50 --- /dev/null +++ b/tests/test_riscv_nonlinear_policy.py @@ -0,0 +1,67 @@ +"""Real RV64 integration of the explicit portable nonlinear/Mean policies.""" +import json +import os + +import numpy as np +import pytest + +from scratchv.backend.tensor_c_codegen import TensorCCodegen +from scratchv.ir.builder import IRBuilder +from scratchv.ir.types import Value +from scratchv.runtime.riscv_external import build_external, run_external +from scratchv.runtime.riscv_tensor import discover_toolchain +from scratchv.runtime.weight_bundle import write_weight_bundle +from scratchv.verification.fp32_eigen import mean_last, trig +from scratchv.verification.fp32_reference import sigmoid, softmax + + +@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="explicit RV64 tools required") +@pytest.mark.parametrize("harts", [1, 4]) +def test_reference_nonlinear_rv64_contexts_and_error_recovery(tmp_path, monkeypatch, harts): + # Select the saved arithmetic order, independent of the host's SIMD support. + monkeypatch.setenv("SCRATCHV_FP32_REFERENCE_CPU", "avx512") + graph = IRBuilder() + x = Value("x", shape=(128, 256)) + graph.new_function("main", [x]) + graph.new_block("entry") + graph.ret(graph.concat([ + graph.sin(x), graph.cos(x), graph.sigmoid(x), + graph.softmax(x, axis=-1), graph.softmax(x, axis=0), + graph.reduce_mean(x, axes=(-1,), keepdims=True), + ], axis=1)) + artifact = TensorCCodegen( + graph.program, constant_storage="external", kernel_calls=True, + mean_policy="two-four-lane-last-axis", nonlinear_policy="fp32-reference", + ).generate() + write_weight_bundle(artifact.external_weights, artifact.external_initializers, tmp_path / "weights") + executable = build_external(artifact, tmp_path / "weights", tmp_path / "build", + discover_toolchain(), harts=harts) + assert executable.evidence["nonlinear_policy"] == "fp32-reference" + assert executable.evidence["mean_policy"] == "two-four-lane-last-axis" + assert executable.evidence["qemu_workspace_layout"] == "none" + rng = np.random.default_rng(2718) + for context in range(2): + data = rng.uniform(-255, 255, size=x.shape).astype(np.float32) + data[0, :8] = np.array([0., -0., 1e-44, -1e-44, -18., 18., -104., 104.], np.float32) + wanted = np.concatenate([ + trig(data, sine=True), trig(data, sine=False), sigmoid([data], {}, None), + softmax([data], {"axis": -1}, None), softmax([data], {"axis": 0}, None), + mean_last(data), + ], axis=1) + saved = data.copy() + actual, record = run_external(executable, {"x": data}, tmp_path / f"context-{context}", timeout=60) + np.testing.assert_array_equal(actual.view(np.uint32), wanted.view(np.uint32)) + np.testing.assert_array_equal(data.view(np.uint32), saved.view(np.uint32)) + assert record["passed"] and record["qemu_reaped"] and record["harts"] == harts + assert record["nonlinear_policy"] == "fp32-reference" + if context == 0: + # Finite inputs whose stable Softmax subtraction overflows must + # still fail, then the same ELF/bundle must accept a fresh context. + invalid = np.zeros(x.shape, np.float32) + invalid[-1, :2] = [np.finfo(np.float32).max, -np.finfo(np.float32).max] + with pytest.raises(RuntimeError, match="status 0x00000002"): + run_external(executable, {"x": invalid}, tmp_path / "overflow", timeout=60) + failure = json.loads((tmp_path / "overflow/run.json").read_text()) + assert not failure["passed"] and failure["qemu_reaped"] + with pytest.raises(ProcessLookupError): + os.killpg(failure["qemu_pid"], 0) diff --git a/tests/test_tensor_c_mean_policy.py b/tests/test_tensor_c_mean_policy.py new file mode 100644 index 00000000..f88f6628 --- /dev/null +++ b/tests/test_tensor_c_mean_policy.py @@ -0,0 +1,175 @@ +"""Explicit FP32 Mean order; use the existing oracle and Host ABI helpers.""" + +import numpy as np +import pytest + +from scratchv.backend.tensor_c_codegen import TensorCCodegen, TensorCCodegenError +from scratchv.ir.types import DataType, Value +from scratchv.verification.fp32_eigen import mean_last +from tests.test_tensor_c_codegen import builder +from tests.test_tensor_c_external import arguments, host_cc, library, pointers + + +POLICY = "two-four-lane-last-axis" + + +def make_mean(shape, *, axes=(-1,), keepdims=True, **options): + x = Value("x", shape=shape) + graph = builder(x) + graph.ret(graph.reduce_mean(x, axes, keepdims)) + return TensorCCodegen(graph.program, **options).generate() + + +@pytest.mark.parametrize("invalid", [None, "", "eigen", True, 1, []]) +def test_mean_policy_rejects_unknown_values(invalid): + with pytest.raises(TensorCCodegenError, match="mean_policy"): + make_mean((2, 3), mean_policy=invalid) + + +@pytest.mark.parametrize("storage,kernels", [ + ("inline", False), ("external", False), ("external", True), +]) +def test_mean_policy_default_and_non_last_axes_are_unchanged(storage, kernels): + options = dict(constant_storage=storage, kernel_calls=kernels) + implicit = make_mean((2, 3, 5), **options) + explicit = make_mean((2, 3, 5), mean_policy="sequential", **options) + assert implicit == explicit + assert implicit.mean_policy == "sequential" + assert "sv_two_four_lane_mean_last" not in implicit.source + for axes in ((0,), (1,), (0, 2), None): + old = make_mean((2, 3, 5), axes=axes, **options) + selected = make_mean((2, 3, 5), axes=axes, mean_policy=POLICY, **options) + assert selected.mean_policy == POLICY + assert selected.source == old.source + assert selected.workspace_bytes == old.workspace_bytes + selected = make_mean((2, 3, 5), mean_policy=POLICY, **options) + assert "sv_two_four_lane_mean_last" in selected.source + assert "SPDX-License-Identifier: MPL-2.0" in selected.source + assert selected.workspace_bytes == implicit.workspace_bytes + # Canonical positive/negative axis spellings choose identical code. + assert selected.source == make_mean((2, 3, 5), axes=(2,), + mean_policy=POLICY, **options).source + + +@pytest.mark.parametrize("storage,kernels", [ + ("inline", False), ("external", False), ("external", True), +]) +def test_mean_policy_matches_existing_fp32_oracle_at_prefix_tail_boundaries( + tmp_path, host_cc, storage, kernels): + widths = (1, 2, 3, 4, 5, 7, 8, 9, 11, 12, 13, 15, 16, 17, 31, 127, 128, 1024) + params = [Value(f"x{width}", shape=(5, width)) for width in widths] + graph = builder(*params) + outputs = [] + for x in params: + for keepdims in (False, True): + y = graph.reduce_mean(x, (-1,), keepdims) + outputs.append(graph.reshape(y, (5,))) + graph.ret(graph.concat(outputs, axis=0)) + codegen = TensorCCodegen(graph.program, constant_storage=storage, + kernel_calls=kernels, mean_policy=POLICY) + artifact = codegen.generate() + assert artifact == codegen.generate() # Helper state resets on regeneration. + assert artifact.source.count("static float sv_two_four_lane_mean_last(") == 1 + rng = np.random.default_rng(53) + with library(tmp_path, artifact, host_cc) as run: + for kind in ("random", "cancellation", "negative-zero", "subnormal"): + feed = {} + for x in params: + # Legal 4-byte ABI alignment, deliberately shifted from 16-byte + # alignment; canonical row order must not depend on its address. + backing = np.empty(5 * x.shape[-1] + 1, np.float32) + values = backing[1:].reshape(x.shape) + if kind == "random": + values[:] = rng.normal(size=x.shape).astype(np.float32) + elif kind == "cancellation": + values[:] = np.resize(np.array([2**24, 1, -2**24, 1, -3, 2, 5], + np.float32), x.shape) + elif kind == "negative-zero": + values[:] = np.float32(-0.0) + else: + values[:] = np.resize(np.array([1, -2, 3, -4], np.float32) + * np.nextafter(np.float32(0), np.float32(1)), + x.shape) + values.flags.writeable = False + feed[x.name] = values + expected = np.concatenate([mean_last(feed[x.name], keepdims=False) + for x in params for _ in range(2)]) + saved = [values.copy() for values in feed.values()] + if storage == "external": + weights, workspace, actual, args = arguments(artifact, feed) + else: + actual = np.empty(artifact.output.shape, np.float32) + args = [pointers(list(feed.values())), actual.ctypes.data] + assert run(*args) == 0 + np.testing.assert_array_equal(actual.view(np.uint32), expected.view(np.uint32)) + for original, values in zip(saved, feed.values()): + np.testing.assert_array_equal(values.view(np.uint32), original.view(np.uint32)) + + +@pytest.mark.parametrize("shape,axes", [((2, 0), (-1,)), ((0,), None)]) +def test_mean_policy_empty_reduction_keeps_existing_rejection(shape, axes): + with pytest.raises(TensorCCodegenError, match="empty axis"): + make_mean(shape, axes=axes, mean_policy=POLICY) + + +@pytest.mark.parametrize("storage,kernels", [ + ("inline", False), ("external", False), ("external", True), +]) +def test_mean_policy_empty_leading_dimensions(tmp_path, host_cc, storage, kernels): + model = make_mean((2, 0, 7), keepdims=False, constant_storage=storage, + kernel_calls=kernels, mean_policy=POLICY) + x = np.empty((2, 0, 7), np.float32) + with library(tmp_path, model, host_cc) as run: + if storage == "external": + weights, workspace, output, args = arguments(model, x) + else: + output = np.empty(model.output.shape, np.float32) + args = [pointers([x]), output.ctypes.data] + assert run(*args) == 0 + assert output.shape == (2, 0) + np.testing.assert_array_equal(output, mean_last(x, keepdims=False)) + + +@pytest.mark.parametrize("kernels", [False, True]) +def test_mean_policy_numeric_error_propagation_and_workspace_recovery(tmp_path, host_cc, kernels): + x = Value("x", shape=(5, 13)) + graph = builder(x) + first = graph.reduce_mean(x, (-1,), True) + shifted = graph.add(x, first) + second = graph.reduce_mean(shifted, (-1,), True) + graph.ret(graph.add(second, second)) + model = TensorCCodegen(graph.program, constant_storage="external", + kernel_calls=kernels, mean_policy=POLICY).generate() + assert model.workspace_bytes < (5 * 13 + 5 * 3) * 4 + x0 = np.arange(65, dtype=np.float32).reshape(5, 13) / np.float32(10) + weights, workspace, output, args = arguments(model, x0) + with library(tmp_path, model, host_cc) as run: + for bad in (np.nan, np.inf, np.finfo(np.float32).max): + x0[:] = bad + output[:] = -123 + assert run(*args) == 2 + assert np.all(output == -123) # No final result copied on failure. + # New input pointer/context, exact same caller workspace. + x1 = np.linspace(-2, 3, 65, dtype=np.float32).reshape(5, 13) + args[0] = pointers([x1]) + assert run(*args) == 0 + expected = mean_last(x1 + mean_last(x1)) + expected = expected + expected + np.testing.assert_array_equal(output.view(np.uint32), expected.view(np.uint32)) + args[0] = pointers([x0]) + # Capacity and overlap validation must still precede touching workspace. + workspace[:] = np.uint64(0x1122334455667788) + output[:] = -123 + args[0] = pointers([x1]) + args[3] = model.workspace_bytes - 1 + assert run(*args) == 1 + assert np.all(workspace == np.uint64(0x1122334455667788)) + assert np.all(output == -123) + + +def test_mean_policy_does_not_bypass_fp32_type_contract(): + x = Value("x", DataType.INT32, shape=(2, 4)) + graph = builder(x) + graph.ret(graph.reduce_mean(x, (-1,), True)) + with pytest.raises(TensorCCodegenError): + TensorCCodegen(graph.program, mean_policy=POLICY).generate() diff --git a/tests/test_tensor_c_nonlinear_policy.py b/tests/test_tensor_c_nonlinear_policy.py new file mode 100644 index 00000000..f422d00e --- /dev/null +++ b/tests/test_tensor_c_nonlinear_policy.py @@ -0,0 +1,239 @@ +"""Explicit portable nonlinear arithmetic, using the existing FP32 oracle/ABI.""" +from dataclasses import replace +import ctypes +import re + +import numpy as np +import pytest + +from scratchv.backend.tensor_c_codegen import TensorCCodegen, TensorCCodegenError +from scratchv.ir.types import Value +from scratchv.verification import fp32_reference +from scratchv.verification.fp32_eigen import trig +from tests.test_tensor_c_codegen import builder +from tests.test_tensor_c_external import ( + arguments, host_cc, library, pointers, _PARALLEL_HOST_RUNTIME, + _softmax_pair_experiment, _softmax_experiment, +) + +POLICY = "fp32-reference" +MODES = [("inline", False), ("external", False), ("external", True)] + + +def oracle_softmax(x, axis=-1): + # Explicit reference arithmetic, independent of local/hidden CPU features. + moved = np.moveaxis(x, axis, -1) + weights = fp32_reference._exp_nonpositive( + moved - moved.max(axis=-1, keepdims=True), cpu_strategy="avx512") + result = weights * (np.float32(1) / fp32_reference._sum_lanes(weights, 16)) + return np.moveaxis(result, -1, axis) + + +def abi(model, feed): + if model.constant_storage == "external": + weights, workspace, output, args = arguments(model, feed) + return output, args, (weights, workspace) + output = np.full(model.output.shape, -999, np.float32) + return output, [pointers([feed[x.name] for x in model.inputs]), output.ctypes.data], () + + +@pytest.mark.parametrize("invalid", [None, "", "auto", "avx512", True, 1, []]) +def test_nonlinear_policy_rejects_invalid_values(invalid): + x = Value("x", shape=(4, 8)) + graph = builder(x) + graph.ret(graph.softmax(x)) + with pytest.raises(TensorCCodegenError, match="nonlinear_policy"): + TensorCCodegen(graph.program, nonlinear_policy=invalid).generate() + + +@pytest.mark.parametrize("storage,kernels", MODES) +def test_default_source_and_unused_operations_stay_unchanged(storage, kernels): + x = Value("x", shape=(4, 17)) + graph = builder(x) + graph.ret(graph.softmax(graph.sigmoid(graph.add(graph.sin(x), graph.cos(x))))) + options = dict(constant_storage=storage, kernel_calls=kernels) + old = TensorCCodegen(graph.program, **options).generate() + assert old == TensorCCodegen(graph.program, nonlinear_policy="libm", **options).generate() + assert old.nonlinear_policy == "libm" and old.mean_policy == "sequential" + assert "sv_fp32_reference_" not in old.source + generator = TensorCCodegen(graph.program, nonlinear_policy=POLICY, **options) + selected = generator.generate() + assert selected == generator.generate() + assert selected.workspace_bytes == old.workspace_bytes + for name in ("trig", "sigmoid", "exp_nonpositive", "sum_sixteen"): + assert selected.source.count(f"static float sv_fp32_reference_{name}(") == 1 + assert "SPDX-License-Identifier: MPL-2.0" in selected.source + assert "Copyright (c) Microsoft Corporation" in selected.source + generator.nonlinear_policy = "libm" + assert generator.generate() == old # A prior reference build leaves no state. + plain = builder(x) + plain.ret(plain.add(plain.exp(x), plain.pow(x, plain.make_const(2.0)))) + implicit = TensorCCodegen(plain.program, **options).generate() + explicit = TensorCCodegen(plain.program, nonlinear_policy=POLICY, **options).generate() + assert explicit.source == implicit.source # Exp/Pow are outside this policy. + assert explicit.mean_policy == "sequential" + + +@pytest.mark.parametrize("storage,kernels", MODES) +def test_trig_and_sigmoid_match_fixed_reference_and_preserve_libm_fallback( + tmp_path, host_cc, storage, kernels): + threshold = np.float32(18000) + small = np.nextafter(np.float32(0), np.float32(1)) + values = np.concatenate([ + np.linspace(-17999, 17999, 513, dtype=np.float32), + np.array([0, -0.0, small, -small, 18, -18, + np.nextafter(threshold, np.float32(0)), threshold, + np.nextafter(threshold, np.float32(np.inf)), 1e20, + np.finfo(np.float32).max], np.float32), + np.array([-threshold, -np.nextafter(threshold, np.float32(0)), + -np.nextafter(threshold, np.float32(np.inf)), -1e20, + -np.finfo(np.float32).max], np.float32), + ]) + x = Value("x", shape=values.shape) + graph = builder(x) + graph.ret(graph.concat([graph.sin(x), graph.cos(x), graph.sigmoid(x)], axis=0)) + opts = dict(constant_storage=storage, kernel_calls=kernels) + baseline = TensorCCodegen(graph.program, **opts).generate() + selected = TensorCCodegen(graph.program, nonlinear_policy=POLICY, **opts).generate() + feed = {"x": values} + with library(tmp_path / "libm", baseline, host_cc) as run: + original, args, keep = abi(baseline, feed) + assert run(*args) == 0 + with library(tmp_path / "reference", selected, host_cc) as run: + actual, args, keep = abi(selected, feed) + assert run(*args) == 0 + expected = np.concatenate([trig(values, sine=True), trig(values, sine=False), + fp32_reference.sigmoid([values], {}, None)]) + fallback = np.abs(values) >= threshold + for block in (0, 1): + # The explicit fallback is the original platform libm, not NumPy. + expected[block*values.size:(block+1)*values.size][fallback] = ( + original[block*values.size:(block+1)*values.size][fallback]) + np.testing.assert_array_equal(actual.view("u4"), expected.view("u4")) + assert np.signbit(actual[np.flatnonzero(np.signbit(values) & (values == 0))]).all() + for invalid in (np.nan, np.inf, -np.inf): + bad = values.copy() + bad[0] = invalid + args[0] = pointers([bad]) + actual.fill(-999) + assert run(*args) == 2 and np.all(actual == -999) + args[0] = pointers([values]) + assert run(*args) == 0 + np.testing.assert_array_equal(actual.view("u4"), expected.view("u4")) + + +@pytest.mark.parametrize("storage,kernels", MODES) +def test_softmax_axes_tails_empty_rows_and_fixed_cpu_independent_order( + tmp_path, host_cc, monkeypatch, storage, kernels): + specifications = [((3, n), -1) for n in (1, 7, 16, 17, 257)] + specifications += [((2, 17, 3), 1), ((2, 17, 3), 0), ((0, 17), -1)] + params = [Value(f"x{i}", shape=shape) for i, (shape, _) in enumerate(specifications)] + graph = builder(*params) + graph.ret(graph.concat([graph.reshape(graph.softmax(x, axis=axis), (int(np.prod(shape)),)) + for x, (shape, axis) in zip(params, specifications)], axis=0)) + feed = {} + rng = np.random.default_rng(217) + for x in params: + feed[x.name] = rng.normal(size=x.shape).astype("f4") * np.float32(20) + expected = np.concatenate([oracle_softmax(feed[x.name], axis).ravel() + for x, (_, axis) in zip(params, specifications)]) + # Backend selection must not consult host CPU dispatch or reference env. + monkeypatch.setenv("SCRATCHV_FP32_REFERENCE_CPU", "avx2-fma3") + model = TensorCCodegen(graph.program, constant_storage=storage, kernel_calls=kernels, + nonlinear_policy=POLICY).generate() + assert "ldexpf" in model.source and "sv_reference_lanes[16]" in model.source + actual, args, keep = abi(model, feed) + with library(tmp_path, model, host_cc) as run: + assert run(*args) == 0 + np.testing.assert_array_equal(actual.view("u4"), expected.view("u4")) + + +@pytest.mark.parametrize("kernels", [False, True]) +def test_reference_softmax_reuses_callback_with_separate_contexts_and_recovers( + tmp_path, host_cc, kernels): + program, feed = _softmax_pair_experiment() + model = TensorCCodegen(program, constant_storage="external", kernel_calls=kernels, + nonlinear_policy=POLICY).generate() + assert len(re.findall(r"static void sv_softmax_\w+_range\(", model.source)) == 1 + expected = np.concatenate([oracle_softmax(feed["x"]), oracle_softmax(feed["y"])], axis=0) + parallel = replace(model, source=model.source + _PARALLEL_HOST_RUNTIME) + for harts in (1, 4): + actual, args, keep = abi(model, feed) + with library(tmp_path / str(harts), parallel, + [*host_cc, f"-DSCRATCHV_HARTS={harts}", "-pthread"]) as run: + for inputs, wanted in [(feed, expected), + ({"x": feed["y"], "y": feed["x"]}, + np.concatenate(np.split(expected, 2)[::-1], axis=0)), + (feed, expected)]: + args[0] = pointers([inputs["x"], inputs["y"]]) + assert run(*args) == 0 + np.testing.assert_array_equal(actual.view("u4"), wanted.view("u4")) + bad = feed["y"].copy() + bad[-1, :2] = [np.finfo("f4").max, -np.finfo("f4").max] + args[0] = pointers([feed["x"], bad]) + actual.fill(-999) + assert run(*args) == 2 and np.all(actual == -999) + args[0] = pointers([feed["x"], feed["y"]]) + assert run(*args) == 0 + np.testing.assert_array_equal(actual.view("u4"), expected.view("u4")) + + +def test_reference_softmax_chain_reuses_workspace(tmp_path, host_cc): + program, x = _softmax_experiment((32, 1024), passes=4) + model = TensorCCodegen(program, constant_storage="external", kernel_calls=True, + nonlinear_policy=POLICY).generate() + assert model.workspace_bytes < 4*x.nbytes + expected = x + for _ in range(4): + expected = oracle_softmax(expected) + parallel = replace(model, source=model.source + _PARALLEL_HOST_RUNTIME) + actual, args, keep = abi(model, {"x": x}) + with library(tmp_path, parallel, [*host_cc, "-DSCRATCHV_HARTS=4", "-pthread"]) as run: + assert run(*args) == 0 + np.testing.assert_array_equal(actual.view("u4"), expected.view("u4")) + + +@pytest.mark.parametrize("kernels", [False, True]) +def test_reference_softmax_output_finite_checks_before_workspace_recovery( + tmp_path, host_cc, kernels): + shape = (17, 2049) + x, y = Value("x", shape=shape), Value("y", shape=shape) + graph = builder(x, y) + graph.softmax(x) + graph.ret(graph.softmax(y)) # No downstream finite check can hide failure. + model = TensorCCodegen(graph.program, constant_storage="external", kernel_calls=kernels, + nonlinear_policy=POLICY).generate() + feed = {"x": np.zeros(shape, "f4"), "y": np.zeros(shape, "f4")} + feed["y"][-1, 0] = -13 + expected = oracle_softmax(feed["y"]) + source = model.source.replace("float v=sv_fp32_reference_exp_nonpositive(shifted);", + "float v=sv_test_reference_exp(shifted);") + fault = r""" +int sv_test_fault, sv_test_hits; +float sv_test_reference_exp(float x) { + if (sv_test_fault && x == -13.0f) { + __atomic_fetch_add(&sv_test_hits,1,__ATOMIC_RELAXED); + if (sv_test_fault == 1) return -2048.0f; + union {uint32_t u; float f;} bad={0x7fc00000U}; return bad.f; + } + return sv_fp32_reference_exp_nonpositive(x); +} +""" + injected = replace(model, source="extern float sv_test_reference_exp(float);\n" + + source + _PARALLEL_HOST_RUNTIME + fault) + for harts in (1, 4): + actual, args, keep = abi(model, feed) + directory = tmp_path / str(harts) + with library(directory, injected, + [*host_cc, f"-DSCRATCHV_HARTS={harts}", "-pthread"]) as run: + loaded = ctypes.CDLL(str(directory / "model.so")) + mode = ctypes.c_int.in_dll(loaded, "sv_test_fault") + hits = ctypes.c_int.in_dll(loaded, "sv_test_hits") + for number in (1, 2): + mode.value, hits.value = number, 0 + actual.fill(-999) + assert run(*args) == 2 and hits.value == 1 + assert np.all(actual == -999) + mode.value, hits.value = 0, 0 + assert run(*args) == 0 and hits.value == 0 + np.testing.assert_array_equal(actual.view("u4"), expected.view("u4")) diff --git a/tests/test_w4_ci_summary.py b/tests/test_w4_ci_summary.py index 7a15cd29..4da16198 100644 --- a/tests/test_w4_ci_summary.py +++ b/tests/test_w4_ci_summary.py @@ -168,7 +168,7 @@ def test_workflow_publishes_summaries_even_on_failure_and_runs_tests(): assert step["if"] == "always()" assert "scripts/summarize_w4_ci.py" in step["run"] assert all(".outcome" in value for value in step["env"].values()) - assert set(workflow["on"]) == {"workflow_call"} + assert set(workflow["on"]) == {"workflow_call", "workflow_dispatch"} ci = yaml.load((ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8"), Loader=yaml.BaseLoader) assert "paths" not in ci["on"]["pull_request"] assert any(job.get("uses") == "./.github/workflows/w4-full-numeric.yml" diff --git a/tests/test_w4_full_probe_contract.py b/tests/test_w4_full_probe_contract.py index 9e6d1929..a1852076 100644 --- a/tests/test_w4_full_probe_contract.py +++ b/tests/test_w4_full_probe_contract.py @@ -384,7 +384,11 @@ def test_reused_executable_rejects_hart_count_changed_without_rebuild(tmp_path, @pytest.mark.parametrize("requested", [None, 2, 4, "auto"]) @pytest.mark.parametrize("layout", ["none", "two-buffer-pad-4k"]) -def test_full_cli_resolves_harts_once_before_tiny_graph_build(tmp_path, monkeypatch, requested, layout): +@pytest.mark.parametrize("mean_policy,nonlinear_policy", [ + ("sequential", "libm"), ("two-four-lane-last-axis", "libm"), + ("sequential", "fp32-reference"), ("two-four-lane-last-axis", "fp32-reference"), +]) +def test_full_cli_resolves_harts_once_before_tiny_graph_build(tmp_path, monkeypatch, requested, layout, mean_policy, nonlinear_policy): from scratchv.frontend import onnx_parser from scratchv.ir.builder import IRBuilder from scratchv.ir.types import Value @@ -415,12 +419,15 @@ def stopped_build(artifact, *a, **kw): built.append(kw["harts"]) assert type(kw["harts"]) is int and kw["harts"] == expected assert artifact.qemu_workspace_layout == layout + assert artifact.mean_policy == mean_policy + assert artifact.nonlinear_policy == nonlinear_policy raise RuntimeError("intentional tiny-graph build stop") monkeypatch.setattr(run, "build_external", stopped_build) if requested is not None: arguments += ["--harts", str(requested)] if layout != "none": arguments += ["--qemu-workspace-layout", layout] + arguments += ["--mean-policy", mean_policy, "--nonlinear-policy", nonlinear_policy] assert run.main(arguments + ["--build-only"]) == 1 saved = json.loads((output / "report.json").read_text()) assert built == [expected] @@ -428,6 +435,8 @@ def stopped_build(artifact, *a, **kw): assert saved["harts_requested"] == (requested if requested is not None else 1) assert saved["harts"] == expected assert saved["qemu_workspace_layout_requested"] == saved["codegen"]["qemu_workspace_layout"] == layout + assert saved["mean_policy"] == saved["codegen"]["mean_policy"] == mean_policy + assert saved["nonlinear_policy"] == saved["codegen"]["nonlinear_policy"] == nonlinear_policy assert "intentional tiny-graph build stop" in saved["error"] assert not saved["full_qemu_forward_executed"] @@ -705,3 +714,116 @@ def test_full_cli_rejects_rhs_layout_before_output_creation(tmp_path, layout): "--matmul-rhs-layout", layout]) assert exc.value.code == 2 assert not target.exists() + + +@pytest.mark.parametrize("policy", [None, "sequential", "two-four-lane-last-axis"]) +def test_load_retains_mean_policy_and_legacy_default(tmp_path, monkeypatch, policy): + path, record = _saved_executable(tmp_path, monkeypatch) + if policy is not None: + record["evidence"]["mean_policy"] = policy + path.write_text(json.dumps(record)) + exe = run.load_executable(path) + assert exe.evidence.get("mean_policy", "sequential") == (policy or "sequential") + if policy is None: + assert "mean_policy" not in exe.evidence # Do not relabel the legacy artifact. + + +@pytest.mark.parametrize("policy", [None, True, "reference-v4", 1]) +def test_load_rejects_invalid_explicit_mean_policy(tmp_path, monkeypatch, policy): + path, record = _saved_executable(tmp_path, monkeypatch) + record["evidence"]["mean_policy"] = policy + path.write_text(json.dumps(record)) + with pytest.raises(ValueError, match="Mean policy"): + run.load_executable(path) + + +@pytest.mark.parametrize("built", [None, "sequential", "two-four-lane-last-axis"]) +@pytest.mark.parametrize("requested", ["sequential", "two-four-lane-last-axis"]) +def test_w4_mean_policy_reuse_requires_explicit_matching_build(tmp_path, monkeypatch, built, requested): + arguments, output = _mock_preflight_and_reuse(tmp_path, monkeypatch) + existing = run.load_executable(None) + evidence = {} if built is None else {"mean_policy": built} + monkeypatch.setattr(run, "load_executable", lambda *_: replace(existing, evidence=evidence)) + previous = tmp_path / "previous/report.json" + record = json.loads(previous.read_text()) + if built is not None: + record["mean_policy"] = built + previous.write_text(json.dumps(record)) + result = run.main(arguments + ["--mean-policy", requested, "--build-only"]) + saved = json.loads((output / "report.json").read_text()) + assert saved["mean_policy"] == requested and not saved["full_qemu_forward_executed"] + if (built or "sequential") == requested: + assert result == 0 and saved["status"] == "BUILD_ONLY" + else: + assert result == 1 and "Mean policy" in saved["error"] + + +def test_w4_mean_policy_report_disagreement_rejects_reuse(tmp_path, monkeypatch): + arguments, output = _mock_preflight_and_reuse(tmp_path, monkeypatch) + existing = run.load_executable(None) + monkeypatch.setattr(run, "load_executable", lambda *_: replace(existing, + evidence={"mean_policy": "two-four-lane-last-axis"})) + assert run.main(arguments + ["--mean-policy", "two-four-lane-last-axis", "--build-only"]) == 1 + saved = json.loads((output / "report.json").read_text()) + assert "Mean policy" in saved["error"] and not saved["full_qemu_forward_executed"] + + +@pytest.mark.parametrize("policy", [None, "libm", "fp32-reference"]) +def test_load_retains_nonlinear_policy_and_legacy_default(tmp_path, monkeypatch, policy): + path, record = _saved_executable(tmp_path, monkeypatch) + if policy is not None: + record["evidence"]["nonlinear_policy"] = policy + path.write_text(json.dumps(record)) + exe = run.load_executable(path) + assert exe.evidence.get("nonlinear_policy", "libm") == (policy or "libm") + if policy is None: + assert "nonlinear_policy" not in exe.evidence # Do not relabel the legacy artifact. + + +@pytest.mark.parametrize("policy", [None, True, "reference-v4", 1]) +def test_load_rejects_invalid_explicit_nonlinear_policy(tmp_path, monkeypatch, policy): + path, record = _saved_executable(tmp_path, monkeypatch) + record["evidence"]["nonlinear_policy"] = policy + path.write_text(json.dumps(record)) + with pytest.raises(ValueError, match="nonlinear policy"): + run.load_executable(path) + + +@pytest.mark.parametrize("built", [None, "libm", "fp32-reference"]) +@pytest.mark.parametrize("requested", ["libm", "fp32-reference"]) +def test_w4_nonlinear_policy_reuse_requires_explicit_matching_build(tmp_path, monkeypatch, built, requested): + arguments, output = _mock_preflight_and_reuse(tmp_path, monkeypatch) + existing = run.load_executable(None) + evidence = {} if built is None else {"nonlinear_policy": built} + monkeypatch.setattr(run, "load_executable", lambda *_: replace(existing, evidence=evidence)) + previous = tmp_path / "previous/report.json" + record = json.loads(previous.read_text()) + if built is not None: + record["nonlinear_policy"] = built + previous.write_text(json.dumps(record)) + result = run.main(arguments + ["--nonlinear-policy", requested, "--build-only"]) + saved = json.loads((output / "report.json").read_text()) + assert saved["nonlinear_policy"] == requested and not saved["full_qemu_forward_executed"] + if (built or "libm") == requested: + assert result == 0 and saved["status"] == "BUILD_ONLY" + else: + assert result == 1 and "nonlinear policy" in saved["error"] + + +def test_w4_nonlinear_policy_report_disagreement_rejects_reuse(tmp_path, monkeypatch): + arguments, output = _mock_preflight_and_reuse(tmp_path, monkeypatch) + existing = run.load_executable(None) + monkeypatch.setattr(run, "load_executable", lambda *_: replace(existing, + evidence={"nonlinear_policy": "fp32-reference"})) + assert run.main(arguments + ["--nonlinear-policy", "fp32-reference", "--build-only"]) == 1 + saved = json.loads((output / "report.json").read_text()) + assert "nonlinear policy" in saved["error"] and not saved["full_qemu_forward_executed"] + + +@pytest.mark.parametrize("policy", ["unknown", "sequential", ""]) +def test_w4_unknown_nonlinear_policy_fails_before_output(tmp_path, policy): + target = tmp_path / "report" + with pytest.raises(SystemExit) as exc: + run.main(["--model-dir", str(tmp_path), "--output-dir", str(target), + "--nonlinear-policy", policy]) + assert exc.value.code == 2 and not target.exists() diff --git a/tests/test_w4_workflows.py b/tests/test_w4_workflows.py index 127d9b4f..505af10f 100644 --- a/tests/test_w4_workflows.py +++ b/tests/test_w4_workflows.py @@ -17,6 +17,14 @@ ROOT = Path(__file__).resolve().parents[1] +W5_REQUIRED = ( + [f"test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel[{h}]" for h in (1, 4)] + + [f"test_explicit_mean_order_rv64_and_context_reuse[{w}-{h}]" + for w in (13, 128, 1024) for h in (1, 4)] + + [f"test_explicit_mean_overflow_propagates_and_build_remains_reusable[{h}]" for h in (1, 4)] + + [f"test_reference_nonlinear_rv64_contexts_and_error_recovery[{h}]" for h in (1, 4)] +) + PATH = ROOT / ".github/workflows/w4-full-numeric.yml" NIGHTLY_PATH = ROOT / ".github/workflows/w4-nightly.yml" LLM_PATH = ROOT / ".github/workflows/llm-deploy.yml" @@ -51,14 +59,16 @@ def _guard(expression, event): def test_expensive_forward_is_not_automatic_on_pr_or_enabled_as_a_schedule(): data = workflow() - assert set(data["on"]) == {"workflow_call"} + assert set(data["on"]) == {"workflow_call", "workflow_dispatch"} run_full = data["on"]["workflow_call"]["inputs"]["run_full"] assert run_full["type"] == "boolean" and run_full["default"] == "false" assert data["permissions"] == {"contents": "read"} assert data["concurrency"]["cancel-in-progress"] == "false" full = data["jobs"]["full-numeric"] assert full["needs"] == "unit" - assert full["if"] == "${{ inputs.run_full }}" + assert full["if"] == ("${{ !inputs.run_w5 && inputs.mode != 'w5-generation' && " + "(inputs.run_full || (github.event_name == 'workflow_dispatch' " + "&& inputs.mode == 'w4-forward')) }}") assert "if" not in data["jobs"]["unit"] @@ -86,7 +96,8 @@ def test_same_ref_unit_and_full_calls_do_not_share_a_concurrency_group(): assert "${{ inputs.run_full && 'full' || 'unit' }}" in group assert concurrency["cancel-in-progress"] == "false" fixed = group.replace("${{ github.ref }}", "refs/heads/candidate").replace( - "${{ inputs.case || 'full_seed_0' }}", "full_seed_0") + "${{ inputs.case || 'full_seed_0' }}", "full_seed_0").replace( + "${{ inputs.run_w5 && 'w5' || inputs.mode || 'call' }}", "call") unit = fixed.replace("${{ inputs.run_full && 'full' || 'unit' }}", "unit") full = fixed.replace("${{ inputs.run_full && 'full' || 'unit' }}", "full") assert unit != full and "${{" not in unit + full @@ -104,20 +115,21 @@ def test_registered_llm_entry_calls_same_commit_w4_without_changing_w3_or_pr_def assert inputs["w4_case"]["options"] == ["full_seed_0", "full_seed_42", "one_token", "short_17", "short_255", "changed_future", "changed_padding"] w4 = data["jobs"]["w4-validation"] - assert w4["if"] == "${{ github.event_name == 'workflow_dispatch' && inputs.run_w4_validation }}" + assert w4["if"] == "${{ github.event_name == 'workflow_dispatch' && inputs.run_w4_validation && !inputs.run_w5_validation }}" assert w4["uses"] == "./.github/workflows/w4-full-numeric.yml" assert w4["with"] == {"run_full": "true", "case": "${{ inputs.w4_case || 'full_seed_0' }}"} assert "continue-on-error" not in w4 for event in ("pull_request", "schedule", "workflow_dispatch"): for enabled in (False, True): guard = w4["if"].removeprefix("${{").removesuffix("}}") - event_guard, flag = guard.split("&&") + event_guard, flag, w5_guard = guard.split("&&") allowed = _guard(event_guard, event) and enabled assert flag.strip() == "inputs.run_w4_validation" + assert w5_guard.strip() == "!inputs.run_w5_validation" assert allowed is (event == "workflow_dispatch" and enabled) assert inputs["run_w3_validation"]["default"] == "false" w3 = data["jobs"]["w3-validation"] - assert w3["if"] == "${{ github.event_name == 'workflow_dispatch' && inputs.run_w3_validation }}" + assert w3["if"] == "${{ github.event_name == 'workflow_dispatch' && inputs.run_w3_validation && !inputs.run_w5_validation }}" assert w3["uses"] == "./.github/workflows/w3-nightly.yml" assert data["jobs"]["w2-acceptance"]["needs"] == ["llm-deploy", "full-qwen3-frontend"] assert data["permissions"] == {"contents": "read"} @@ -309,7 +321,7 @@ def test_actual_ci_report_gate_rejects_false_or_incomplete_acceptance(tmp_path, "test_post_execution_transport_corruption_fails_and_retains_copies", "test_success_copy_cleanup_failure_is_not_a_pass", "test_actual_shared_library_executes_kernel_calls_and_unloads[sequential]", - "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]"]) + "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]"] + W5_REQUIRED) def test_actual_junit_gate_requires_real_qemu_execution(tmp_path, result, required): folder = tmp_path / "output/w4-unit" folder.mkdir(parents=True) @@ -330,7 +342,7 @@ def test_actual_junit_gate_requires_real_qemu_execution(tmp_path, result, requir "test_post_execution_transport_corruption_fails_and_retains_copies", "test_success_copy_cleanup_failure_is_not_a_pass", "test_actual_shared_library_executes_kernel_calls_and_unloads[sequential]", - "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]"): + "test_actual_shared_library_executes_kernel_calls_and_unloads[blocked_fma]") + tuple(W5_REQUIRED): if name != required: case += f'' (folder / "tests.xml").write_text(f"{case}", encoding="utf-8") @@ -556,7 +568,7 @@ def test_ci_summary_actual_bash_failure_matrix(bash, tmp_path, event, requested, enabled = requested == "true" expected = unit == "success" and full == ("success" if enabled else "skipped") env = dict(os.environ, GITHUB_EVENT_NAME=event, UNIT_RESULT=unit, FULL_NUMERIC_RESULT=full, - FULL_REQUESTED=requested, + FULL_REQUESTED=requested, W5_REQUESTED="false", W5_GENERATION_RESULT="skipped", SELECTED_CASE="full_seed_0", GITHUB_SHA="a" * 40, GITHUB_RUN_ID="123", GITHUB_RUN_ATTEMPT="1", GITHUB_STEP_SUMMARY=(tmp_path / "summary.md").as_posix()) script = step("Record W4 CI result without promoting skipped gates")["run"].replace("python3 -", '"' + sys.executable.replace("\\", "/") + '" -') @@ -593,7 +605,7 @@ def test_nightly_summary_preserves_disabled_and_rejects_every_missing_gate(bash, def test_failure_summaries_keep_json_artifacts_and_all_prerequisites(): - for path, jobname, needs in ((PATH, "ci-status", ["unit", "full-numeric"]), + for path, jobname, needs in ((PATH, "ci-status", ["unit", "full-numeric", "w5-generation"]), (NIGHTLY_PATH, "nightly-status", "full-numeric")): job = workflow(path)["jobs"][jobname] assert job["if"] == "${{ always() }}" and job["needs"] == needs @@ -602,3 +614,257 @@ def test_failure_summaries_keep_json_artifacts_and_all_prerequisites(): assert uploads[0]["with"]["if-no-files-found"] == "error" assert uploads[0]["with"]["path"].endswith("/status.json") assert all("continue-on-error" not in s for s in job["steps"]) + + +def _full_enabled(expression, event, mode, legacy, run_w5=False): + # Evaluate only the fixed, repository-owned workflow boolean expressions. + expression = expression.removeprefix("${{").removesuffix("}}").strip() + for key, value in (("github.event_name", event), ("inputs.mode", mode), ("inputs.run_full", legacy), ("inputs.run_w5", run_w5)): + expression = expression.replace(key, repr(value)) + return bool(eval(re.sub(r"!(?!=)", " not ", expression.replace("&&", " and ").replace("||", " or ")), + {"__builtins__": {}})) + + +@pytest.mark.parametrize("event,mode,legacy,w4,w5", [ + ("pull_request", "small", False, False, False), + ("pull_request", "w5-generation", False, False, False), + ("schedule", "w5-generation", False, False, False), + ("workflow_dispatch", "small", False, False, False), + ("workflow_dispatch", "w4-forward", False, True, False), + ("workflow_dispatch", "w5-generation", False, False, True), + ("workflow_call", "", True, True, False), + ("workflow_dispatch", "", True, True, False), # Existing LLM Deploy caller. + ("schedule", "", True, True, False), # Existing opt-in W4 nightly caller. + ("workflow_dispatch", "w5-generation", True, False, False), +]) +def test_w5_manual_selection_is_exclusive_and_old_callers_remain(event, mode, legacy, w4, w5): + data = workflow() + choice = data["on"]["workflow_dispatch"]["inputs"]["mode"] + assert choice == {"description": "Small regressions by default; full modes are mutually exclusive", + "type": "choice", "default": "small", + "options": ["small", "w4-forward", "w5-generation"]} + assert _full_enabled(data["jobs"]["full-numeric"]["if"], event, mode, legacy) is w4 + assert _full_enabled(data["jobs"]["w5-generation"]["if"], event, mode, legacy) is w5 + assert not (w4 and w5) + assert set(data["on"]["workflow_call"]["inputs"]) == {"run_full", "run_w5", "case"} + assert "schedule" not in data["on"] and "pull_request" not in data["on"] + + +def test_w5_manual_job_reuses_deployment_and_preserves_bounded_full_scope(): + job = workflow()["jobs"]["w5-generation"] + assert job["needs"] == "unit" + run = step("Run the frozen W5 suite through the deployment script") + assert "bash scripts/deploy_w5.sh" in run["run"] and "--reference-suite" in run["run"] + assert "--model /assets/onnx/model.onnx" in run["run"] + assert "--verify-onnx /assets/onnx/model.onnx" in run["run"] + for flag in ("--harts 4", "--matmul-policy blocked_fma", "--matmul-tile 2x4", + "--matmul-rhs-layout column-panel", "--mean-policy two-four-lane-last-axis", + "--nonlinear-policy fp32-reference"): + assert flag in run["run"] + assert "probes.w4_qwen3_full.run" not in run["run"] and "--max-new-tokens" not in run["run"] + guest = int(re.search(r"--total-timeout (\d+)", run["run"]).group(1)) + outer = int(re.search(r"--kill-after=30s (\d+)m", run["run"]).group(1)) + assert 10540 < guest < outer * 60 < int(run["timeout-minutes"]) * 60 + assert int(run["timeout-minutes"]) + 40 <= int(job["timeout-minutes"]) <= 360 + bounded_steps = sum(int(item.get("timeout-minutes", 0)) for item in job["steps"]) + assert bounded_steps + 10 <= int(job["timeout-minutes"]) # setup, diagnostics and artifact upload + assert job["steps"].index(step("Build the clean W5 deployment image")) < job["steps"].index(run) + assert job["steps"].index(step("Acquire only the fixed W5 ONNX and tokenizer assets")) < job["steps"].index(run) + upload = step("Upload W5 reports and failure diagnostics only") + assert upload["if"] == "always()" and upload["with"]["retention-days"] == "30" + assert not any(suffix in upload["with"]["path"] for suffix in (".bin", ".npy", ".onnx", ".elf", ".zip")) + gate = step("Require complete W5 generation evidence") + assert gate["if"] == "always()" and "scripts/check_w5_ci.py" in gate["run"] + assert "--exit-code-file output/w5-ci/runner-exit-code.txt" in gate["run"] + assert all("continue-on-error" not in item for item in job["steps"]) + + +@pytest.mark.parametrize("exit_code", [0, 1, 124, 130, 143]) +def test_w5_deployment_pipeline_preserves_actual_exit(bash, tmp_path, exit_code): + (tmp_path / "output/w5-ci").mkdir(parents=True) + script = step("Run the frozen W5 suite through the deployment script")["run"] + wrapper = 'timeout() { printf "%s\\n" "$@" > "$ARG_LOG"; return "$MOCK_EXIT"; }\n' + capture = tmp_path / "arguments.txt" + result = subprocess.run([bash, "--noprofile", "--norc", "-c", wrapper + script], cwd=tmp_path, + env=dict(os.environ, ARG_LOG=str(capture), MOCK_EXIT=str(exit_code)), capture_output=True, text=True) + assert result.returncode == exit_code, result.stderr + assert int((tmp_path / "output/w5-ci/runner-exit-code.txt").read_text()) == exit_code + args = capture.read_text().splitlines() + assert "scripts/deploy_w5.sh" in args and "--reference-suite" in args + assert args[args.index("--total-timeout") + 1] == "16500" + + +@pytest.mark.parametrize("cpus,ram,disk,passed", [ + (4, 10, 10, True), (8, 16, 30, True), + (3, 16, 30, False), (4, 9, 30, False), (4, 16, 9, False)]) +def test_w5_resource_shortfall_fails_before_generation(tmp_path, monkeypatch, cpus, ram, disk, passed): + (tmp_path / "output/w5-ci").mkdir(parents=True) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(os, "sched_getaffinity", lambda _: set(range(cpus))) + monkeypatch.setattr(shutil, "disk_usage", lambda _: SimpleNamespace(free=disk * 1024**3)) + real_read = Path.read_text + def memory(path, *args, **kwargs): + if path.as_posix() == "/proc/meminfo": + return f"MemTotal: 16777216 kB\\nMemAvailable: {ram * 1024**2} kB\\n".replace("\\n", "\n") + return real_read(path, *args, **kwargs) + monkeypatch.setattr(Path, "read_text", memory) + code = python_body("Require four CPUs and capacity for 24 retained logits") + if passed: + exec(compile(code, "w5-resource-preflight", "exec"), {}) + else: + with pytest.raises(SystemExit, match="generation NOT RUN"): + exec(compile(code, "w5-resource-preflight", "exec"), {}) + row = json.loads((tmp_path / "output/w5-ci/resource-preflight.json").read_text()) + assert row["passed"] is passed and row["available_cpus"] == cpus + + +@pytest.mark.parametrize("unit,full,w5,requested,expected", [ + ("success", "skipped", "success", "true", True), + ("success", "skipped", "skipped", "false", True), + ("success", "skipped", "skipped", "true", False), + ("success", "skipped", "failure", "true", False), + ("success", "skipped", "cancelled", "true", False), + ("success", "skipped", "", "true", False), + ("failure", "skipped", "success", "true", False), + ("cancelled", "skipped", "success", "true", False), + ("skipped", "skipped", "success", "true", False), + ("success", "success", "success", "true", False), + ("success", "skipped", "success", "false", False), +]) +def test_w5_summary_cannot_promote_skip_or_duplicate_full_execution( + tmp_path, unit, full, w5, requested, expected): + (tmp_path / "output/w4-status").mkdir(parents=True) + env = dict(os.environ, UNIT_RESULT=unit, FULL_NUMERIC_RESULT=full, W5_GENERATION_RESULT=w5, + FULL_REQUESTED="false", W5_REQUESTED=requested, SELECTED_CASE="full_seed_0", + GITHUB_SHA="a" * 40, GITHUB_RUN_ID="123", GITHUB_RUN_ATTEMPT="1", + GITHUB_STEP_SUMMARY=str(tmp_path / "summary.md")) + result = subprocess.run([sys.executable, "-c", python_body("Record W4 CI result without promoting skipped gates")], + cwd=tmp_path, env=env, capture_output=True, text=True) + assert (result.returncode == 0) is expected, result.stderr + row = json.loads((tmp_path / "output/w4-status/status.json").read_text()) + assert row["w5_generation_accepted"] is (expected and requested == "true") + assert row["full_numeric_accepted"] is False + + +@pytest.mark.parametrize("event,w5,w3,w4,passed", [ + ("workflow_dispatch", True, False, False, True), + ("workflow_dispatch", True, True, False, False), + ("workflow_dispatch", True, False, True, False), + ("workflow_dispatch", True, True, True, False), + ("workflow_dispatch", False, True, True, True), + ("pull_request", False, False, False, True), + ("schedule", False, False, False, True), + ("schedule", True, False, False, False), +]) +def test_registered_entry_rejects_conflicts_before_any_expensive_job(tmp_path, event, w5, w3, w4, passed): + data = workflow(LLM_PATH) + choice = data["on"]["workflow_dispatch"]["inputs"]["run_w5_validation"] + assert choice["type"] == "boolean" and choice["default"] == "false" + caller = data["jobs"]["w5-validation"] + assert caller["uses"] == "./.github/workflows/w4-full-numeric.yml" + assert caller["with"] == {"run_w5": "true"} + for name in ("w5-validation", "w3-validation", "w4-validation", "llm-deploy", + "full-qwen3-onnx", "full-qwen3-frontend"): + assert data["jobs"][name]["needs"] == "validation-selection" + for name in ("llm-deploy", "full-qwen3-onnx", "full-qwen3-frontend", "w2-acceptance"): + assert "!inputs.run_w5_validation" in data["jobs"][name]["if"] + result = subprocess.run([sys.executable, "-c", python_body("Reject conflicting W5 and legacy full selections", LLM_PATH)], + cwd=tmp_path, env=dict(os.environ, GITHUB_EVENT_NAME=event, W5_REQUESTED=str(w5).lower(), + W3_REQUESTED=str(w3).lower(), W4_REQUESTED=str(w4).lower()), + capture_output=True, text=True) + assert (result.returncode == 0) is passed, result.stderr + + +@pytest.mark.parametrize("event,mode,legacy,w5,allowed", [ + ("workflow_dispatch", "", False, True, True), + ("workflow_dispatch", "small", False, False, True), + ("workflow_dispatch", "w5-generation", False, False, True), + ("workflow_dispatch", "", True, False, True), + ("schedule", "", True, False, True), + ("workflow_dispatch", "", True, True, False), + ("workflow_dispatch", "w4-forward", False, True, False), + ("workflow_dispatch", "w5-generation", True, False, False), + ("pull_request", "", False, True, False), + ("schedule", "", False, True, False), +]) +def test_reusable_w5_input_is_manual_and_mutually_exclusive(tmp_path, event, mode, legacy, w5, allowed): + data = workflow() + option = data["on"]["workflow_call"]["inputs"]["run_w5"] + assert option["type"] == "boolean" and option["default"] == "false" + unit = data["jobs"]["unit"] + first = unit["steps"][0] + assert first["name"] == "Reject conflicting or nonmanual full-generation selections" + result = subprocess.run([sys.executable, "-c", python_body(first["name"])], cwd=tmp_path, + env=dict(os.environ, GITHUB_EVENT_NAME=event, RUN_MODE=mode, + W5_REQUESTED=str(w5).lower(), W4_REQUESTED=str(legacy).lower()), + capture_output=True, text=True) + assert (result.returncode == 0) is allowed, result.stderr + w4_enabled = _full_enabled(data["jobs"]["full-numeric"]["if"], event, mode, legacy, w5) + w5_enabled = _full_enabled(data["jobs"]["w5-generation"]["if"], event, mode, legacy, w5) + assert not (w4_enabled and w5_enabled) + if w5 and allowed: + assert w5_enabled and not w4_enabled + if not allowed: + assert not w5_enabled and not w4_enabled + + +@pytest.mark.parametrize("selection,result,legacy,requested,expected", [ + ("success", "success", "skipped", "true", "PASS"), + ("success", "skipped", "skipped", "true", "FAIL"), + ("success", "cancelled", "skipped", "true", "FAIL"), + ("success", "failure", "skipped", "true", "FAIL"), + ("failure", "skipped", "skipped", "true", "FAIL"), + ("cancelled", "skipped", "skipped", "true", "FAIL"), + ("success", "success", "success", "true", "FAIL"), + ("success", "success", "failure", "true", "FAIL"), + ("success", "success", "cancelled", "true", "FAIL"), + ("success", "skipped", "success", "false", "NOT RUN"), + ("success", "success", "skipped", "false", "FAIL"), +]) +def test_w5_entry_summary_requires_success_and_no_legacy_execution( + tmp_path, selection, result, legacy, requested, expected): + job = workflow(LLM_PATH)["jobs"]["w5-execution-summary"] + assert job["if"] == "always()" + results = {name: {"result": legacy} for name in job["needs"]} + results["validation-selection"] = {"result": selection} + results["w5-validation"] = {"result": result} + summary = tmp_path / "summary.md" + completed = subprocess.run([sys.executable, "-c", python_body("Require the requested W5-only execution", LLM_PATH)], + cwd=tmp_path, env=dict(os.environ, GITHUB_EVENT_NAME="workflow_dispatch", W5_REQUESTED=requested, + W5_JOB_RESULTS=json.dumps(results), GITHUB_STEP_SUMMARY=str(summary)), + capture_output=True, text=True) + assert (completed.returncode == 0) is (expected != "FAIL"), completed.stderr + assert f"**{expected}**" in summary.read_text() + + +@pytest.mark.parametrize("suffix", ["None", "Give brief answers."]) +@pytest.mark.parametrize("outcome", ["pass", "skipped", "failure", "error", "missing", "duplicate"]) +def test_w5_only_template_gate_uses_existing_image_and_rejects_missing_execution(tmp_path, suffix, outcome): + import xml.etree.ElementTree as ET + item = step("Require the official three-round template in the clean W5 image") + command = item["run"] + assert "docker run --rm --init" in command and '"$SCRATCHV_W5_IMAGE"' in command + assert "SCRATCHV_QWEN3_TOKENIZER_DIR=/assets/tokenizer" in command + assert "dst=/tmp/test_conversation.py,readonly" in command + assert "--reference-suite" not in command and "scripts/run_inference.py" not in command + assert "pip install" not in command and "docker build" not in command + job = workflow()["jobs"]["w5-generation"] + assert job["steps"].index(step("Acquire only the fixed W5 ONNX and tokenizer assets")) < job["steps"].index(item) + assert job["steps"].index(item) < job["steps"].index(step("Run the frozen W5 suite through the deployment script")) + suite = ET.Element("testsuite") + for current in ("None", "Give brief answers."): + if current == suffix and outcome == "missing": + continue + name = f"test_official_three_round_rendering_matches_transformers[{current}]" + row = ET.SubElement(suite, "testcase", name=name) + if current == suffix: + if outcome in ("skipped", "failure", "error"): + ET.SubElement(row, outcome) + elif outcome == "duplicate": + ET.SubElement(suite, "testcase", name=name) + target = tmp_path / "output/w5-ci/multiturn-tests.xml" + target.parent.mkdir(parents=True) + ET.ElementTree(suite).write(target) + result = subprocess.run([sys.executable, "-c", python_body(item["name"])], cwd=tmp_path, + capture_output=True, text=True) + assert (result.returncode == 0) is (outcome == "pass"), result.stderr diff --git a/tests/test_w5_ci.py b/tests/test_w5_ci.py new file mode 100644 index 00000000..115f8c28 --- /dev/null +++ b/tests/test_w5_ci.py @@ -0,0 +1,216 @@ +"""Model-free W5 CI evidence checks using the frozen generation contract.""" + +from copy import deepcopy +from importlib.util import module_from_spec, spec_from_file_location +import json +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +POLICIES = { + "harts": 4, + "qemu_workspace_layout": "none", + "matmul_policy": "blocked_fma", + "mean_policy": "two-four-lane-last-axis", + "nonlinear_policy": "fp32-reference", + "matmul_tile": "2x4", + "matmul_rhs_layout": "column-panel", +} + + +@pytest.fixture +def gate(): + spec = spec_from_file_location("w5_ci_gate_tests", ROOT / "scripts/check_w5_ci.py") + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def reference(): + return json.loads((ROOT / "probes/w5_generation/references.json").read_text()) + + +@pytest.fixture +def report(reference): + cases, numeric_steps, runs = [], [], [] + for expected in reference["cases"]: + cases.append({ + key: deepcopy(expected[key]) + for key in ("name", "prompt", "max_new_tokens", "prompt_token_ids", + "generated_token_ids", "generated_text", "stop_reason") + }) + cases[-1].update(numeric_step_count=8, forward_count=8, reference_passed=True) + for step in expected["steps"]: + numeric_steps.append({ + "passed": True, "selected_row_passed": True, "max_abs": 0.0002, + "atol": 0.001, "rtol": 0.0, "elements_compared": 38895616, + "case": expected["name"], "step": step["step"], + "valid_length": step["valid_length"], + "qemu_token_id": step["token_id"], "ort_token_id": step["token_id"], + "reference_token_id": step["token_id"], "reference_token_passed": True, + }) + runs.append({ + **POLICIES, "step": len(runs), "valid_length": step["valid_length"], + "passed": True, "status": "PASS", "guest_completed": True, + "qemu_reaped": True, "exit_code": 0, + "elf_sha256": "a" * 64, "weight_sha256": "b" * 64, + }) + rows = [] + for case, bits, inexact in ( + ("rounding_sensitive", "0xbf1825f2", 1), + ("fused_cancellation", "0xa8800000", 0), + ): + for repeat in range(6): + requested = int(repeat in (1, 4)) + rows.append({ + "case": case, "repeat": repeat, "instruction_rm": 0 if repeat < 3 else 7, + "requested_fflags": requested, "fcsr_before": requested, + "fcsr_after": requested | inexact, "expected_fcsr_after": requested | inexact, + "actual_bits": bits, "expected_bits": bits, + "bits_match": True, "csr_match": True, "passed": True, + }) + return { + **POLICIES, "schema_version": 1, "status": "PASS", "passed": True, + "stage": "complete", "mode": "raw", "harts_requested": 4, + "cases": cases, "numeric_steps": numeric_steps, "runs": runs, + "qemu_forwards": 24, "forward_attempts": 24, "full_model_gate": "PASS", + "build": { + **POLICIES, "sha256": "a" * 64, + "w5_graph_audit": {"passed": True, "layer_count": 28, "tied_embeddings": True}, + }, + "fma_conformance": { + "passed": True, "status": "PASS", + "comparison": {"passed": True, "criterion": "exact binary32 bits and FCSR", + "rows": rows, "failed_rows": 0}, + }, + } + + +def test_complete_three_by_eight_evidence_passes_without_diagnostic_extras(gate, report, reference): + assert len(report["cases"]) == 3 + assert len(report["runs"]) == len(report["numeric_steps"]) == 24 + # Production reports need no optional frozen_* diagnostics or output arrays. + assert not any(key.startswith("frozen_") for key in report["numeric_steps"][0]) + assert gate.validate_report(report, reference) is None + + +@pytest.mark.parametrize("collection", ["cases", "numeric_steps", "runs"]) +@pytest.mark.parametrize("damage", ["missing", "duplicate"]) +def test_incomplete_or_duplicate_evidence_cannot_be_hidden_by_pass(gate, report, reference, collection, damage): + if damage == "missing": + report[collection].pop() + else: + report[collection][-1] = deepcopy(report[collection][0]) + with pytest.raises(ValueError): + gate.validate_report(report, reference) + + +@pytest.mark.parametrize("field,value", [ + ("generated_token_ids", [12095] * 8), + ("generated_text", "Paris"), + ("stop_reason", "eos"), + ("prompt_token_ids", [1, 2, 3, 4, 5]), + ("numeric_step_count", 7), +]) +def test_frozen_output_and_history_must_match(gate, report, reference, field, value): + report["cases"][0][field] = value + with pytest.raises(ValueError): + gate.validate_report(report, reference) + + +@pytest.mark.parametrize("field,value", [ + ("max_abs", 0.001), + ("max_abs", float("nan")), + ("max_abs", float("inf")), + ("atol", 0.01), + ("rtol", 0.001), + ("elements_compared", 151936), + ("passed", False), + ("qemu_token_id", 0), + ("ort_token_id", 0), + ("valid_length", 6), +]) +def test_each_step_needs_strict_full_logits_and_frozen_token_evidence(gate, report, reference, field, value): + report["numeric_steps"][0][field] = value + with pytest.raises(ValueError): + gate.validate_report(report, reference) + + +@pytest.mark.parametrize("field,value", [ + ("qemu_reaped", False), + ("guest_completed", False), + ("exit_code", 130), + ("status", "CANCELLED"), + ("error", "deadline expired"), + ("cleanup_errors", ["unable to reap QEMU"]), + ("elf_sha256", "c" * 64), + ("weight_sha256", "c" * 64), +]) +def test_cancel_failure_or_different_executable_cannot_pass(gate, report, reference, field, value): + report["runs"][-1][field] = value + with pytest.raises(ValueError): + gate.validate_report(report, reference) + + +@pytest.mark.parametrize("location,field,value", [ + ("top", "harts", 1), + ("top", "qemu_workspace_layout", "two-buffer-pad-4k"), + ("top", "matmul_tile", "1x4"), + ("build", "mean_policy", "sequential"), + ("build", "nonlinear_policy", "libm"), + ("run", "mean_policy", "sequential"), + ("run", "nonlinear_policy", "libm"), +]) +def test_claimed_strategy_must_match_build_and_every_run(gate, report, reference, location, field, value): + target = report if location == "top" else report["build"] if location == "build" else report["runs"][7] + target[field] = value + with pytest.raises(ValueError): + gate.validate_report(report, reference) + + +def test_small_graph_cannot_claim_full_model_gate(gate, report, reference): + report["build"]["w5_graph_audit"]["layer_count"] = 2 + with pytest.raises(ValueError): + gate.validate_report(report, reference) + + +@pytest.mark.parametrize("damage", ["missing", "failed"]) +def test_fma_summary_cannot_mask_incomplete_or_failing_rows(gate, report, reference, damage): + rows = report["fma_conformance"]["comparison"]["rows"] + if damage == "missing": + rows.pop() + else: + rows[7]["csr_match"] = False + with pytest.raises(ValueError): + gate.validate_report(report, reference) + + +@pytest.mark.parametrize("exit_text,expected", [("0\n", 0), ("130\n", 1), ("", 1)]) +def test_cli_checks_real_exit_code_and_writes_status(gate, report, tmp_path, monkeypatch, exit_text, expected): + report_path = tmp_path / "report.json" + report_path.write_text(json.dumps(report)) + exit_path = tmp_path / "exit-code.txt" + exit_path.write_text(exit_text) + output = tmp_path / "gate.json" + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + code = gate.main(["--report", str(report_path), "--exit-code-file", str(exit_path), + "--output", str(output)]) + assert code == expected + result = json.loads(output.read_text()) + assert result["status"] == ("PASS" if expected == 0 else "FAIL") + assert result["passed"] is (expected == 0) + assert result["status"] in summary.read_text() + + +def test_missing_run_report_produces_failure_artifact(gate, tmp_path): + exit_path = tmp_path / "exit-code.txt" + exit_path.write_text("0\n") + output = tmp_path / "gate.json" + assert gate.main(["--report", str(tmp_path / "missing.json"), + "--exit-code-file", str(exit_path), "--output", str(output)]) == 1 + result = json.loads(output.read_text()) + assert result["status"] == "FAIL" and result["passed"] is False diff --git a/tests/test_w5_deployment.py b/tests/test_w5_deployment.py new file mode 100644 index 00000000..b9f038b9 --- /dev/null +++ b/tests/test_w5_deployment.py @@ -0,0 +1,243 @@ +"""Exercise deployment orchestration without requiring a container engine.""" +import json +import os +from pathlib import Path +import shlex +import shutil +import signal +import subprocess +import sys +import time + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/deploy_w5.sh" + + +@pytest.fixture +def setup(tmp_path): + assets = tmp_path / "model assets" + assets.mkdir() + output = tmp_path / "new output" + calls = tmp_path / "calls.jsonl" + engine = tmp_path / "container engine" + engine.write_text( + "#!" + sys.executable + "\n" + "import json, os, signal, subprocess, sys, time\n" + "from pathlib import Path\n" + "with open(os.environ['ENGINE_CALLS'], 'a') as f:\n" + " f.write(json.dumps(sys.argv[1:]) + '\\n')\n" + "if sys.argv[1] == os.environ.get('ENGINE_BLOCK_STAGE'):\n" + " worker_code = 'import time; time.sleep(60)'\n" + " if os.environ.get('ENGINE_IGNORE_TERM'):\n" + " signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + " worker_code = 'import signal,time; signal.signal(signal.SIGTERM,signal.SIG_IGN); time.sleep(60)'\n" + " worker = subprocess.Popen([sys.executable, '-c', worker_code])\n" + " Path(os.environ['ENGINE_PROCESS_FILE']).write_text(json.dumps([os.getpid(), worker.pid]))\n" + " time.sleep(60)\n" + "sys.exit(int(os.environ.get('ENGINE_' + sys.argv[1].upper() + '_EXIT', '0')))\n" + ) + engine.chmod(0o755) + env = dict(os.environ, SCRATCHV_CONTAINER_ENGINE=str(engine), + SCRATCHV_W5_ASSETS=str(assets), SCRATCHV_W5_OUTPUT=str(output), + ENGINE_CALLS=str(calls)) + env.pop("SCRATCHV_W5_BASE_IMAGE", None) + return env, calls, output + + +def run(env, *args): + return subprocess.run(["bash", str(SCRIPT), *args], env=env, + text=True, capture_output=True, timeout=10) + + +def recorded(path): + return [json.loads(line) for line in path.read_text().splitlines()] + + +def test_unavailable_engine_does_not_create_output(setup): + env, calls, output = setup + env["SCRATCHV_CONTAINER_ENGINE"] += "-absent" + result = run(env, "--help") + assert result.returncode == 127 + assert "unavailable" in result.stderr + assert not calls.exists() + assert not output.exists() + + +def test_unready_engine_does_not_build_or_create_output(setup): + env, calls, output = setup + env["ENGINE_INFO_EXIT"] = "1" + result = run(env) + assert result.returncode == 1 + assert "not ready" in result.stderr + assert recorded(calls) == [["info"]] + assert not output.exists() + + +def test_missing_assets_do_not_build_or_run(setup): + env, calls, output = setup + env["SCRATCHV_W5_ASSETS"] += "-absent" + result = run(env) + assert result.returncode == 2 + assert "Assets directory does not exist" in result.stderr + assert recorded(calls) == [["info"]] + assert not output.exists() + + +def test_build_failure_never_starts_generation(setup): + env, calls, output = setup + env["ENGINE_BUILD_EXIT"] = "7" + result = run(env, "--prompt", "The capital of France is") + assert result.returncode == 7 + assert [call[0] for call in recorded(calls)] == ["info", "build"] + + +def test_arguments_mounts_and_invoking_user_are_preserved(setup): + env, calls, output = setup + args = ["--model", "/assets/model.onnx", "--tokenizer-dir", "/assets/tokenizer", + "--prompt", "quotes '$HOME'; * stay literal", "--harts", "auto", + "--output-dir", "/output/second run"] + result = run(env, *args) + assert result.returncode == 0, result.stderr + invocations = recorded(calls) + assert [call[0] for call in invocations] == ["info", "build", "run"] + build = invocations[1] + assert build[build.index("--build-arg") + 1] == "BASE_IMAGE=ubuntu:24.04" + command = invocations[-1] + assert "--rm" in command and "--init" in command + assert command[command.index("--user") + 1] == f"{os.getuid()}:{os.getgid()}" + assert f"type=bind,src={env['SCRATCHV_W5_ASSETS']},dst=/assets,readonly" in command + assert f"type=bind,src={output},dst=/output" in command + assert command[-len(args):] == args + image_index = command.index("scratchv-w5:local") + assert command[image_index + 1:image_index + 3] == ["--output-dir", "/output/run"] + assert "SCRATCHV_ZIG_CACHE=/output/.zig-cache" in command + + +@pytest.mark.parametrize("value,expected", [ + ("", "ubuntu:24.04"), + ("public.ecr.aws/ubuntu/ubuntu:24.04", "public.ecr.aws/ubuntu/ubuntu:24.04"), + ("literal $(printf not-expanded); *", "literal $(printf not-expanded); *"), +]) +def test_base_image_override_is_one_literal_build_argument(setup, value, expected): + env, calls, output = setup + env["SCRATCHV_W5_BASE_IMAGE"] = value + result = run(env, "--help") + assert result.returncode == 0, result.stderr + invocations = recorded(calls) + assert [call[0] for call in invocations] == ["info", "build", "run"] + build = invocations[1] + assert build.count("--build-arg") == 1 + assert build[build.index("--build-arg") + 1] == "BASE_IMAGE=" + expected + assert not any(arg.startswith("BASE_IMAGE=") for arg in invocations[-1]) + + +def test_model_failure_exit_status_is_propagated(setup): + env, calls, output = setup + env["ENGINE_RUN_EXIT"] = "9" + assert run(env).returncode == 9 + + +def test_mount_delimiter_is_rejected_before_build(setup): + env, calls, output = setup + bad_assets = Path(env["SCRATCHV_W5_ASSETS"]).with_name("bad,assets") + bad_assets.mkdir() + env["SCRATCHV_W5_ASSETS"] = str(bad_assets) + result = run(env) + assert result.returncode == 2 + assert "commas" in result.stderr + assert recorded(calls) == [["info"]] + + +def test_container_copy_inputs_exist_and_do_not_include_assets(): + # Detect a missing packaged entry point without claiming an image was built. + dockerfile = ROOT / "deployment/w5/Dockerfile" + for line in dockerfile.read_text().splitlines(): + if not line.startswith("COPY "): + continue + for source in shlex.split(line)[1:-1]: + assert (ROOT / source).exists(), source + assert source not in {".", "assets", "output", ".venv", ".git"} + + +def test_packaged_cli_help_resolves_transitive_modules(tmp_path): + # Use only the image COPY inputs; no host model or generated ELF is copied. + # This catches missing probe/manifest dependencies before a Docker build. + dockerfile = ROOT / "deployment/w5/Dockerfile" + for line in dockerfile.read_text().splitlines(): + if not line.startswith("COPY "): + continue + parts = shlex.split(line) + for source in parts[1:-1]: + src = ROOT / source + target = tmp_path / source + target.parent.mkdir(parents=True, exist_ok=True) + if src.is_file(): + shutil.copyfile(src, target) + elif source == "probes": + for item in src.rglob("*"): + if item.is_file() and item.suffix in {".py", ".json"}: + dest = target / item.relative_to(src) + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(item, dest) + else: + shutil.copytree(src, target, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + env = dict(os.environ, PYTHONPATH=str(tmp_path), PYTHONDONTWRITEBYTECODE="1") + result = subprocess.run([sys.executable, str(tmp_path / "scripts/run_inference.py"), "--help"], + cwd=tmp_path, env=env, text=True, capture_output=True, timeout=30) + assert result.returncode == 0, result.stderr + assert "--reuse-build" in result.stdout + assert "--reference-suite" in result.stdout + + +@pytest.mark.parametrize("stage,signum,ignore_term", [ + ("info", signal.SIGTERM, False), + ("info", signal.SIGINT, False), + ("build", signal.SIGTERM, False), + ("build", signal.SIGINT, False), + ("build", signal.SIGTERM, True), +]) +def test_cancel_stage_reaps_client_and_stops_helpers_without_starting_model( + setup, tmp_path, stage, signum, ignore_term, +): + """A signal to Bash alone must reach the owned container-client group.""" + env, calls, output = setup + process_file = tmp_path / "stage-processes.json" + env.update(ENGINE_BLOCK_STAGE=stage, ENGINE_PROCESS_FILE=str(process_file)) + if ignore_term: + env["ENGINE_IGNORE_TERM"] = "1" + process = subprocess.Popen(["bash", str(SCRIPT)], env=env, start_new_session=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + owned = [] + try: + deadline = time.monotonic() + 5 + while not process_file.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert process_file.exists(), "container-client stage did not start" + owned = json.loads(process_file.read_text()) + assert os.getpgid(owned[0]) == owned[0] + process.send_signal(signum) + stdout, stderr = process.communicate(timeout=10) + assert process.returncode == 128 + signum, (stdout, stderr) + # The direct client must be reaped, not left a zombie owned by Bash. + with pytest.raises(ProcessLookupError): + os.kill(owned[0], 0) + # A helper can briefly await host PID 1's reap; it must not be running. + status = Path(f"/proc/{owned[1]}/stat") + if status.exists(): + assert status.read_text().rsplit(")", 1)[1].split()[0] == "Z" + assert [call[0] for call in recorded(calls)] == ( + ["info"] if stage == "info" else ["info", "build"] + ) + if stage == "info": + assert not output.exists() + finally: + # Ensure a failed regression cannot leave its own blocking substitutes. + for pgid in ([owned[0]] if owned else []) + [process.pid]: + try: + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) diff --git a/tests/test_w5_generation_reference.py b/tests/test_w5_generation_reference.py new file mode 100644 index 00000000..97ccc84b --- /dev/null +++ b/tests/test_w5_generation_reference.py @@ -0,0 +1,162 @@ +"""Small, model-free checks of the independent W5 reference oracle.""" + +from dataclasses import replace + +import numpy as np +import pytest + +from probes.w5_generation import run +from scratchv.runtime.llm_inputs import prepare_inputs + + +class Tokenizer: + model_vocab_size = 8 + pad_token_id = 0 + generation_eos_token_ids = (5, 6) + + def encode(self, text, *, add_special_tokens): + assert text == "raw prompt" and add_special_tokens is False + return [1, 2] + + def validate_token_ids(self, tokens): + if any(not 0 <= token < 7 for token in tokens): + raise ValueError("undecodable token ID") + return list(tokens) + + def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces): + assert clean_up_tokenization_spaces is False + pieces = {0: "", 1: "raw", 2: "prompt", 3: "Paris", 4: "London", + 5: "", 6: ""} + return "".join(pieces[token] for token in tokens + if not skip_special_tokens or token not in (0, 5, 6)) + + +class Session: + def __init__(self): + self.calls = [] + + def run(self, names, feed): + assert names == ["logits"] + # The final query exposes all valid keys, including any valid pad ID. + length = int(np.count_nonzero(feed["attention_mask"][0, 0, -1] == 0)) + self.calls.append(tuple(feed["input_ids"][0, :length])) + logits = np.zeros((1, 256, 8), dtype=np.float32) + logits[0, :, 7] = 100 # An incorrect padding-row selection cannot pass. + row = logits[0, length - 1] + row[:] = 0 + if length == 2: + row[3] = row[4] = 10 # The independent argmax must prefer lower ID. + else: + row[6] = 10 + return [logits] + + +def case(): + return {"name": "small", "prompt": "raw prompt", "max_new_tokens": 3, + "required_substring": "Paris"} + + +def frozen(): + return {**case(), "prompt_utf8_hex": "raw prompt".encode().hex(), + "prompt_token_ids": [1, 2], "generated_token_ids": [3, 6], + "generated_text": "Paris", "stop_reason": "eos"} + + +@pytest.mark.parametrize("key,replacement", [ + ("prompt", "different"), ("prompt_utf8_hex", "00"), + ("required_substring", "London"), ("max_new_tokens", 4), + ("prompt_token_ids", [2, 1]), ("generated_token_ids", [4, 6]), + ("generated_text", "London"), ("stop_reason", "max_new_tokens"), +]) +def test_frozen_comparison_rejects_each_contract_field_change(key, replacement): + expected = frozen() + actual = {**expected, key: replacement} + assert run.compare_frozen(actual, expected) == { + "passed": False, "different_fields": [key], + } + assert expected[key] != replacement + + +def test_frozen_comparison_ignores_measurement_fields(): + expected = frozen() + actual = {**expected, "wall_seconds": 2.0, "cpu_seconds": 3.0, + "process_peak_rss_kib": 100, "steps": [{"timing": 99}]} + assert run.compare_frozen(actual, expected) == { + "passed": True, "different_fields": [], + } + + +def test_run_case_checks_actual_greedy_rows_eos_and_host_append(): + session = Session() + result = run.run_case(case(), Tokenizer(), session) + assert session.calls == [(1, 2), (1, 2, 3)] + assert result["generated_token_ids"] == [3, 6] + assert result["generated_text"] == "Paris" + assert result["stop_reason"] == "eos" + assert result["semantic_passed"] and result["host_oracle_passed"] + assert [step["valid_length"] for step in result["steps"]] == [2, 3] + assert [step["token_id"] for step in result["steps"]] == [3, 6] + assert result["steps"][0]["top1_top2_margin"] == 0 + assert run.compare_frozen(result, frozen())["passed"] + + +def test_independent_input_oracle_rejects_wrong_host_append(monkeypatch): + def broken_generate(_prompt, *, tokenizer, forward, max_new_tokens): + forward(prepare_inputs([1, 2], pad_token_id=0, vocab_size=8)) + forward(prepare_inputs([1, 2, 4], pad_token_id=0, vocab_size=8)) + + monkeypatch.setattr(run, "generate", broken_generate) + session = Session() + with pytest.raises(ValueError, match="input_ids"): + run.run_case(case(), Tokenizer(), session) + # checked_inputs must fail before a second session call. + assert session.calls == [(1, 2)] + + +def test_independent_stop_oracle_rejects_extra_host_forward(monkeypatch): + original_generate = run.generate + + def broken_generate(prompt, *, tokenizer, forward, max_new_tokens): + result = original_generate(prompt, tokenizer=tokenizer, forward=forward, + max_new_tokens=max_new_tokens) + forward(prepare_inputs([1, 2, 3, 6], pad_token_id=0, vocab_size=8)) + return result + + monkeypatch.setattr(run, "generate", broken_generate) + session = Session() + with pytest.raises(AssertionError, match="after the independent stop"): + run.run_case(case(), Tokenizer(), session) + assert session.calls == [(1, 2), (1, 2, 3)] + + +@pytest.mark.parametrize("field,value", [ + ("prompt_token_ids", (2, 1)), ("generated_token_ids", (4, 6)), + ("generated_text", "London"), ("stop_reason", "max_new_tokens"), +]) +def test_independent_final_oracle_rejects_corrupted_host_result(monkeypatch, field, value): + original_generate = run.generate + + def broken_generate(*args, **kwargs): + result = original_generate(*args, **kwargs) + return replace(result, **{field: value}) + + monkeypatch.setattr(run, "generate", broken_generate) + with pytest.raises(AssertionError, match="Host"): + run.run_case(case(), Tokenizer(), Session()) + + +def test_reference_diagnostic_margin_handles_all_finite_fp32_logits(): + import json + + class ExtremeSession: + def run(self, names, feed): + logits = np.full((1, 256, 8), -np.finfo(np.float32).max, dtype=np.float32) + logits[0, 1, 3] = np.finfo(np.float32).max + return [logits] + + with np.errstate(over="raise"): + result = run.run_case({**case(), "max_new_tokens": 1}, Tokenizer(), ExtremeSession()) + assert result["generated_token_ids"] == [3] + assert result["generated_text"] == "Paris" + assert result["steps"][0]["top1_top2_margin"] == 2.0 * float(np.finfo(np.float32).max) + json.dumps(result, allow_nan=False) diff --git a/tests/test_w5_inference_cli.py b/tests/test_w5_inference_cli.py new file mode 100644 index 00000000..49fa90b6 --- /dev/null +++ b/tests/test_w5_inference_cli.py @@ -0,0 +1,757 @@ +"""W5 CLI orchestration boundaries without a model, compiler or QEMU.""" + +from importlib.util import module_from_spec, spec_from_file_location +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +from scratchv.runtime.conversation import Conversation +from scratchv.runtime.generation import GenerationCancelledError + + +@pytest.fixture +def cli(): + path = Path(__file__).resolve().parents[1] / "scripts/run_inference.py" + spec = spec_from_file_location("w5_inference_cli_tests", path) + module = module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Tokenizer: + model_vocab_size = 8 + tokenizer_vocab_size = 7 + pad_token_id = 0 + generation_eos_token_ids = (6,) + + def __init__(self): + self.prompts = [] + + def encode(self, text, *, add_special_tokens): + assert add_special_tokens is False + self.prompts.append(text) + return [1, 2] + + def validate_token_ids(self, tokens): + if any(type(token) is not int or not 0 <= token < 7 for token in tokens): + raise ValueError("unknown token") + return list(tokens) + + def decode(self, tokens, *, skip_special_tokens, clean_up_tokenization_spaces): + return "x" * sum(token == 3 for token in tokens) + + +def logits(token=3): + result = np.zeros((1, 256, 8), dtype=np.float32) + result[0, :, token] = 1 + return result + + +@pytest.fixture +def runtime(cli, monkeypatch): + tokenizer = Tokenizer() + executable = SimpleNamespace( + evidence={"matmul_policy": "sequential", "qemu_workspace_layout": "none"}, + toolchain=SimpleNamespace(cc="unused-cc", qemu="unused-qemu"), harts=1, + ) + state = SimpleNamespace(tokenizer=tokenizer, exe=executable, prepared=[], adapters=[], + saved=[], fma=[], failure=None, oracle=None, harts=[]) + + def prepare(args, out, harts): + state.prepared.append((args, out, harts)) + return executable + + class Adapter: + def __init__(self, exe, output, *, timeout): + self.exe, self.output, self.timeout = exe, output, timeout + self.vocab_size = tokenizer.model_vocab_size + self.runs, self.inputs = [], [] + state.adapters.append(self) + + def __call__(self, prepared): + self.inputs.append(prepared) + if state.failure is not None: + self.runs.append({"guest_completed": False}) + raise state.failure + self.runs.append({"guest_completed": True}) + return logits() + + def fma(*args, **kwargs): + state.fma.append((args, kwargs)) + return {"passed": True, "status": "PASS"} + + monkeypatch.setattr(cli.Qwen3Tokenizer, "from_directory", lambda path: tokenizer) + monkeypatch.setattr(cli, "prepare_executable", prepare) + monkeypatch.setattr(cli, "save_executable", lambda exe, out: state.saved.append((exe, out)) or "fixture-metadata-identity") + monkeypatch.setattr(cli, "QemuForward", Adapter) + monkeypatch.setattr(cli, "run_fma_probe", fma) + monkeypatch.setattr(cli, "oracle_session", lambda path: SimpleNamespace(run=lambda names, feed: [state.oracle])) + monkeypatch.setattr(cli, "resolve_harts", lambda requested: state.harts.append(requested) or (4 if requested == "auto" else requested)) + return state + + +def argv(tmp_path, *extra): + return ["--model", "small.onnx", "--tokenizer-dir", "tokenizer", "--output-dir", str(tmp_path / "run"), *extra] + + +def report(tmp_path): + return json.loads((tmp_path / "run/report.json").read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("requested,resolved", [(None, 1), ("2", 2), ("4", 4), ("auto", 4)]) +def test_raw_multitoken_one_build_one_adapter_and_harts_resolved_once(cli, runtime, tmp_path, requested, resolved): + options = [] if requested is None else ["--harts", requested] + assert cli.main(argv(tmp_path, "--prompt", " raw text ", "--max-new-tokens", "3", *options)) == 0 + value = report(tmp_path) + assert value["status"] == "PASS" and value["passed"] + assert value["full_model_gate"] == "NOT_RUN" + assert value["qemu_workspace_layout"] == "none" and value["mean_policy"] == "sequential" + assert value["nonlinear_policy"] == "libm" + assert value["executable_sha256"] == "fixture-metadata-identity" + assert value["harts"] == resolved + assert runtime.harts == ["auto" if requested == "auto" else int(requested or 1)] + assert len(runtime.prepared) == len(runtime.saved) == len(runtime.adapters) == 1 + assert len(runtime.fma) == 1 + assert runtime.adapters[0].exe is runtime.exe + assert [item.valid_length for item in runtime.adapters[0].inputs] == [2, 3, 4] + assert runtime.tokenizer.prompts == [" raw text "] + assert value["cases"][0]["generated_token_ids"] == [3, 3, 3] + assert value["qemu_forwards"] == 3 + + +def test_three_chat_turns_reuse_adapter_and_commit_ordered_history(cli, runtime, tmp_path, monkeypatch): + template = "{% for m in messages %}{{ m.role }}:{{ m.content }}|{% endfor %}assistant:" + chat = Conversation(runtime.tokenizer, template, system_prompt="brief") + monkeypatch.setattr(Conversation, "from_directory", lambda *a, **kw: chat) + assert cli.main(argv(tmp_path, "--mode", "chat", "--system-prompt", "brief", + "--chat-turn", "first", "--chat-turn", "second", "--chat-turn", "third", + "--max-new-tokens", "2")) == 0 + value = report(tmp_path) + assert [x["content"] for x in value["history"]] == ["brief", "first", "xx", "second", "xx", "third", "xx"] + assert runtime.tokenizer.prompts == [ + "system:brief|user:first|assistant:", + "system:brief|user:first|assistant:xx|user:second|assistant:", + "system:brief|user:first|assistant:xx|user:second|assistant:xx|user:third|assistant:", + ] + assert len(runtime.adapters) == len(runtime.prepared) == 1 + assert value["qemu_forwards"] == 6 and len(value["cases"]) == 3 + + +def test_build_only_never_executes_fma_or_forward_and_not_pass(cli, runtime, tmp_path): + assert cli.main(argv(tmp_path, "--build-only")) == 0 + value = report(tmp_path) + assert value["status"] == "BUILD_ONLY" and value["passed"] is False + assert value["full_model_gate"] == "NOT_RUN" and value["qemu_forwards"] == 0 + assert runtime.fma == [] and runtime.adapters[0].inputs == [] + + +@pytest.mark.parametrize("error,expected,status", [ + (GenerationCancelledError("cancelled"), 130, "CANCELLED"), + (KeyboardInterrupt(), 130, "CANCELLED"), + (TimeoutError("deadline expired"), 1, "FAIL"), + (RuntimeError("backend failed"), 1, "FAIL"), +]) +def test_cancel_failure_report_retains_attempt_without_claiming_success(cli, runtime, tmp_path, error, expected, status): + runtime.failure = error + assert cli.main(argv(tmp_path, "--prompt", "text", "--max-new-tokens", "2")) == expected + value = report(tmp_path) + assert value["status"] == status and value["passed"] is False + assert value["cases"] == [] and value["full_model_gate"] == "NOT_RUN" + assert value["forward_attempts"] == 1 and value["qemu_forwards"] == 0 + assert type(error).__name__ in value["error"] + + +def test_numeric_oracle_failure_is_reported_before_committing_tokens(cli, runtime, tmp_path): + runtime.oracle = logits() + runtime.oracle[0, -1, 0] = 0.1 + assert cli.main(argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1", "--verify-onnx", "small.onnx")) == 1 + value = report(tmp_path) + assert value["status"] == "FAIL" and not value["passed"] + assert value["numeric_steps"][0]["passed"] is False + assert value["cases"] == [] + + +def test_numeric_comparison_checks_padding_rows_and_equal_argmax(cli): + expected = logits() + bad_padding = expected.copy() + bad_padding[0, -1, 1] = cli.ATOL * 2 + assert cli.compare_step(bad_padding, expected, 2, 8)["passed"] is False + nonfinite = expected.copy() + nonfinite[0, -1, 0] = np.inf + with pytest.raises(ValueError): + cli.compare_step(nonfinite, expected, 2, 8) + expected[0, 1, 4] = 1 + cli.ATOL / 4 + actual = expected.copy() + actual[0, 1, 3], actual[0, 1, 4] = expected[0, 1, 4], expected[0, 1, 3] + comparison = cli.compare_step(actual, expected, 2, 8) + assert comparison["max_abs"] < cli.ATOL + assert comparison["qemu_token_id"] != comparison["ort_token_id"] + assert not comparison["passed"] + + +def matching_reuse_metadata(cli, monkeypatch, path): + path.parent.mkdir(parents=True) + path.write_text("{}", encoding="utf-8") + path.with_name("report.json").write_text(json.dumps({"executable_sha256": "matching"}), encoding="utf-8") + monkeypatch.setattr(cli, "sha256_file", lambda metadata: "matching") + + +@pytest.mark.parametrize("changes,harts", [ + ({"harts": 4}, 1), + ({"matmul_policy": "blocked_fma"}, 1), + ({"matmul_tile": "2x4"}, 1), + ({"matmul_rhs_layout": "column-panel"}, 1), + ({"qemu_workspace_layout": "two-buffer-pad-4k"}, 1), +]) +def test_reuse_rejects_changed_configuration_or_padding(cli, monkeypatch, tmp_path, changes, harts): + args = cli.parse_args(["--reuse-build", str(tmp_path / "old/executable.json"), + "--tokenizer-dir", "tokens", "--output-dir", str(tmp_path / "new"), + "--prompt", "text"]) + matching_reuse_metadata(cli, monkeypatch, args.reuse_build) + evidence = {"matmul_policy": "sequential", "matmul_tile": "1x4", "matmul_rhs_layout": "original", "qemu_workspace_layout": "none"} + evidence.update({k: v for k, v in changes.items() if k != "harts"}) + exe = SimpleNamespace(harts=changes.get("harts", 1), evidence=evidence) + monkeypatch.setattr(cli, "load_executable", lambda path: exe) + with pytest.raises(ValueError, match="differs"): + cli.prepare_executable(args, tmp_path, harts) + + +def test_matching_reuse_returns_same_executable_without_compile(cli, monkeypatch, tmp_path): + args = cli.parse_args(["--reuse-build", str(tmp_path / "old/executable.json"), + "--tokenizer-dir", "tokens", "--output-dir", str(tmp_path / "new"), + "--prompt", "text"]) + matching_reuse_metadata(cli, monkeypatch, args.reuse_build) + exe = SimpleNamespace(harts=1, evidence={"matmul_policy": "sequential"}) + monkeypatch.setattr(cli, "load_executable", lambda path: exe) + monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("must not rebuild")) + assert cli.prepare_executable(args, tmp_path, 1) is exe + + +@pytest.mark.parametrize("options", [ + ["--reference-suite"], + ["--reference-suite", "--verify-onnx", "model.onnx", "--max-new-tokens", "8"], + ["--reference-suite", "--verify-onnx", "model.onnx", "--mode", "chat"], + ["--chat-turn", "hello"], + ["--prompt", "hello", "--qemu-workspace-layout", "two-buffer-pad-4k"], +]) +def test_cli_rejects_mutated_reference_contract_or_implicit_chat_padding(cli, tmp_path, options): + with pytest.raises(SystemExit) as caught: + cli.parse_args(argv(tmp_path, *options)) + assert caught.value.code == 2 + + +def test_reference_session_refuses_small_graph_for_full_model_gate(cli, monkeypatch, tmp_path): + import onnx + from probes.w2_qwen3_parse import validation + fixture = json.loads((cli.ROOT / "probes/w5_generation/references.json").read_text(encoding="utf-8")) + monkeypatch.setattr(onnx, "load", lambda *a, **kw: object()) + monkeypatch.setattr(validation, "audit_graph_structure", lambda graph: {"passed": True, "layer_count": 2}) + with pytest.raises(ValueError, match="complete 28-layer"): + cli.reference_session(tmp_path / "onnx/model.onnx", fixture) + + +@pytest.mark.parametrize("audit,shape", [ + (None, (1, 256, 8)), + (None, (1, 256, 151936)), + ({"passed": True, "layer_count": 28}, (1, 256, 8)), + ({"passed": True, "layer_count": 2}, (1, 256, 151936)), + ({"passed": False, "layer_count": 28}, (1, 256, 151936)), +]) +def test_full_gate_requires_executable_audit_and_complete_output(cli, audit, shape): + evidence = {} if audit is None else {"w5_graph_audit": audit} + exe = SimpleNamespace(evidence=evidence, output=SimpleNamespace(shape=shape)) + with pytest.raises(ValueError, match="complete graph evidence"): + cli.require_full_executable(exe) + + +def test_full_gate_accepts_native_w5_complete_graph_audit(cli): + exe = SimpleNamespace(evidence={"w5_graph_audit": {"passed": True, "layer_count": 28}}, + output=SimpleNamespace(shape=(1, 256, 151936))) + assert cli.require_full_executable(exe) is None + + +def test_legacy_w4_evidence_survives_two_reuses_without_rechecking_identity(cli, monkeypatch, tmp_path): + from dataclasses import replace + from scratchv.runtime.riscv_external import ExternalExecutable + from scratchv.runtime.riscv_tensor import RiscVToolchain + from scratchv.ir import DataType + from scratchv.backend.tensor_c_codegen import TensorSpec + + legacy = tmp_path / "w4" + legacy.mkdir() + original = ExternalExecutable( + tmp_path / "model.elf", tmp_path / "weights", {}, {}, (), + TensorSpec("logits", DataType.FLOAT32, (1, 256, 151936)), (), + RiscVToolchain(("cc",), "qemu"), {"harts": 4, "matmul_tile": "2x4", + "matmul_rhs_layout": "column-panel", "qemu_workspace_layout": "none"}, harts=4, + ) + identity = cli.save_executable(original, legacy) + audit = {"passed": True, "layer_count": 28, "graph": "fixed Qwen3 graph"} + (legacy / "report.json").write_text(json.dumps({"executable_sha256": identity, + "matmul_policy": "blocked_fma", "graph_audit": audit}), encoding="utf-8") + checked = [] + hash_file = cli.sha256_file + monkeypatch.setattr(cli, "sha256_file", lambda path: checked.append(path) or hash_file(path)) + # Exercise the real metadata writer/identity link; lower-level ELF validation + # is covered by W4. No compiler, model or guest process is needed here. + monkeypatch.setattr(cli, "load_executable", lambda path: replace(original, + evidence=json.loads(path.read_text(encoding="utf-8"))["evidence"])) + monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("must not rebuild")) + current = legacy / "executable.json" + for name in ("w5-first", "w5-second"): + out = tmp_path / name + out.mkdir() + args = cli.parse_args(["--reuse-build", str(current), "--tokenizer-dir", "tokens", + "--output-dir", str(out), "--reference-suite", "--verify-onnx", "onnx/model.onnx", + "--harts", "4", "--matmul-policy", "blocked_fma", "--matmul-tile", "2x4", + "--matmul-rhs-layout", "column-panel"]) + exe = cli.prepare_executable(args, out, 4) + assert exe.evidence["matmul_policy"] == "blocked_fma" + assert exe.evidence["w5_graph_audit"] == audit + assert cli.require_full_executable(exe) is None + assert exe.elf_path == original.elf_path and exe.bundle_dir == original.bundle_dir + identity = cli.save_executable(exe, out) + # W5's report contains no legacy graph_audit/matmul_policy fallback. + (out / "report.json").write_text(json.dumps({"executable_sha256": identity}), encoding="utf-8") + current = out / "executable.json" + assert checked == [legacy / "executable.json", tmp_path / "w5-first/executable.json"] + assert "matmul_policy" not in original.evidence and "w5_graph_audit" not in original.evidence + + +def test_verified_legacy_report_cannot_overwrite_native_failed_audit(cli, monkeypatch, tmp_path): + from dataclasses import dataclass + + @dataclass + class Executable: + evidence: dict + harts: int = 1 + + metadata = tmp_path / "old/executable.json" + matching_reuse_metadata(cli, monkeypatch, metadata) + record = json.loads(metadata.with_name("report.json").read_text()) + record.update(matmul_policy="sequential", graph_audit={"passed": True, "layer_count": 28}) + metadata.with_name("report.json").write_text(json.dumps(record)) + exe = Executable({"w5_graph_audit": {"passed": False, "layer_count": 2}}) + monkeypatch.setattr(cli, "load_executable", lambda path: exe) + args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", + "--output-dir", str(tmp_path / "new"), "--prompt", "text"]) + reused = cli.prepare_executable(args, tmp_path, 1) + assert reused.evidence["matmul_policy"] == "sequential" + assert reused.evidence["w5_graph_audit"] == exe.evidence["w5_graph_audit"] + + +def test_reference_suite_rejects_small_executable_before_fma_or_oracle(cli, runtime, tmp_path): + runtime.exe.output = SimpleNamespace(shape=(1, 256, 8)) + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1 + value = report(tmp_path) + assert value["status"] == "FAIL" and value["full_model_gate"] == "NOT_RUN" + assert "complete graph evidence" in value["error"] + assert runtime.adapters == [] and runtime.fma == [] + + +def test_signal_during_fma_cannot_be_swallowed_by_exception_handler(cli, runtime, tmp_path, monkeypatch): + import signal + previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + swallowed = [] + + def fma(*args, **kwargs): + try: + signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None) + except Exception as exc: + swallowed.append(exc) + return {"passed": False, "status": "FAIL"} + pytest.fail("signal did not abort FMA") + + monkeypatch.setattr(cli, "run_fma_probe", fma) + assert cli.main(argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1")) == 130 + value = report(tmp_path) + assert value["status"] == "CANCELLED" and not value["passed"] + assert "KeyboardInterrupt" in value["error"] + assert value["stage"] == "fma_preflight" and value["qemu_forwards"] == 0 + assert not swallowed + assert all(signal.getsignal(sig) is handler for sig, handler in previous.items()) + + +@pytest.mark.parametrize("record", [None, {}, {"executable_sha256": "drifted"}]) +def test_reuse_identity_is_required_before_loading_or_compiling(cli, monkeypatch, tmp_path, record): + metadata = tmp_path / "old/executable.json" + metadata.parent.mkdir() + metadata.write_text("{}", encoding="utf-8") + if record is not None: + metadata.with_name("report.json").write_text(json.dumps(record), encoding="utf-8") + args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", + "--output-dir", str(tmp_path / "new"), "--prompt", "text"]) + monkeypatch.setattr(cli, "sha256_file", lambda path: "matching") + monkeypatch.setattr(cli, "load_executable", lambda *a: pytest.fail("identity must be checked before load")) + monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("identity failure must not compile")) + with pytest.raises(FileNotFoundError if record is None else ValueError): + cli.prepare_executable(args, tmp_path, 1) + + +@pytest.fixture +def reference_suite(cli, runtime, monkeypatch, tmp_path): + root = tmp_path / "fixture-root" + (root / "probes/w5_generation").mkdir(parents=True) + cases = [{"name": name, "prompt": name, "max_new_tokens": 2, + "prompt_token_ids": [1, 2], "generated_token_ids": [3, 3], + "generated_text": "xx", "stop_reason": "max_new_tokens", + "required_substring": "x"} for name in ("france", "germany", "japan")] + fixture = {"status": "frozen", "tokenizer": {"pad_token_id": 0, + "generation_eos_token_ids": [6], "model_vocab_size": 8, "tokenizer_vocab_size": 7}, + "cases": cases} + (root / "probes/w5_generation/references.json").write_text(json.dumps(fixture), encoding="utf-8") + monkeypatch.setattr(cli, "ROOT", root) + runtime.exe.evidence["w5_graph_audit"] = {"passed": True, "layer_count": 28} + runtime.exe.output = SimpleNamespace(shape=(1, 256, 151936)) + runtime.oracle = logits() + monkeypatch.setattr(cli, "reference_session", lambda *a: SimpleNamespace(run=lambda names, feed: [runtime.oracle])) + snapshots = [] + atomic = cli.atomic_text + + def save(path, text): + if path.name == "report.json": + snapshots.append(json.loads(text)) + atomic(path, text) + + monkeypatch.setattr(cli, "atomic_text", save) + return SimpleNamespace(fixture=fixture, snapshots=snapshots) + + +def test_full_suite_gate_records_running_then_pass(cli, runtime, reference_suite, tmp_path): + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 0 + value = report(tmp_path) + assert value["full_model_gate"] == "PASS" and value["status"] == "PASS" + assert len(value["numeric_steps"]) == 6 and len(value["cases"]) == 3 + assert all(row["reference_passed"] for row in value["cases"]) + assert all(row["numeric_step_count"] == row["forward_count"] == 2 for row in value["cases"]) + assert [(row["case"], row["step"], row["valid_length"]) for row in value["numeric_steps"]] == [ + (name, step, 2 + step) for name in ("france", "germany", "japan") for step in range(2)] + assert reference_suite.snapshots[0]["full_model_gate"] == "RUNNING" + assert reference_suite.snapshots[0]["qemu_forwards"] == 0 + + +@pytest.mark.parametrize("failure,code,gate", [ + (TimeoutError("expired"), 1, "FAIL"), + (GenerationCancelledError("cancelled"), 130, "CANCELLED"), + (KeyboardInterrupt(), 130, "CANCELLED"), +]) +def test_full_suite_running_error_has_terminal_gate(cli, runtime, reference_suite, tmp_path, failure, code, gate): + runtime.failure = failure + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == code + value = report(tmp_path) + assert value["full_model_gate"] == gate and value["status"] == gate + assert value["stage"] == "generation" and value["forward_attempts"] == 1 + assert value["cases"] == [] + assert reference_suite.snapshots[0]["full_model_gate"] == "RUNNING" + + +def test_full_suite_numeric_failure_marks_failed_not_not_run(cli, runtime, reference_suite, tmp_path): + runtime.oracle[0, -1, 0] = cli.ATOL * 2 + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1 + value = report(tmp_path) + assert value["full_model_gate"] == "FAIL" and value["status"] == "FAIL" + assert value["qemu_forwards"] == 1 and value["cases"] == [] + assert value["numeric_steps"][0]["passed"] is False + + +def test_full_suite_output_reference_failure_marks_failed(cli, runtime, reference_suite, tmp_path): + fixture = reference_suite.fixture + fixture["cases"][0]["generated_text"] = "wrong frozen text" + (cli.ROOT / "probes/w5_generation/references.json").write_text(json.dumps(fixture), encoding="utf-8") + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1 + value = report(tmp_path) + assert value["full_model_gate"] == "FAIL" and value["qemu_forwards"] == 2 + assert value["cases"][0]["reference_passed"] is False + assert all(row["passed"] for row in value["numeric_steps"]) + + +@pytest.mark.parametrize("stage,cancel", [ + ("prepare", False), ("fma_preflight", False), + ("fma_preflight", True), ("oracle_prepare", False), +]) +def test_full_suite_pre_generation_failure_remains_not_run(cli, runtime, reference_suite, tmp_path, monkeypatch, stage, cancel): + def fail(*args, **kwargs): + raise KeyboardInterrupt("cancel") if cancel else ValueError("setup failed") + + target = {"prepare": "prepare_executable", "fma_preflight": "run_fma_probe", + "oracle_prepare": "reference_session"}[stage] + monkeypatch.setattr(cli, target, fail) + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == (130 if cancel else 1) + value = report(tmp_path) + assert value["full_model_gate"] == "NOT_RUN" and value["stage"] == stage + assert value["status"] == ("CANCELLED" if cancel else "FAIL") + assert value["qemu_forwards"] == 0 and value["numeric_steps"] == [] + assert all(row["full_model_gate"] == "NOT_RUN" for row in reference_suite.snapshots) + + +@pytest.mark.parametrize("calls", [0, 1, 3]) +def test_full_suite_rejects_missing_or_extra_forward_evidence(cli, runtime, reference_suite, tmp_path, monkeypatch, calls): + from scratchv.runtime.generation import GenerationResult + from scratchv.runtime.llm_inputs import prepare_inputs + + def faulty_generate(prompt, *, forward, **kwargs): + history = [1, 2] + for _ in range(calls): + forward(prepare_inputs(history, pad_token_id=0, vocab_size=8)) + history.append(3) + return GenerationResult((1, 2), (3, 3), "xx", "max_new_tokens") + + monkeypatch.setattr(cli, "generate", faulty_generate) + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1 + value = report(tmp_path) + assert value["full_model_gate"] == "FAIL" and not value["passed"] + assert value["qemu_forwards"] == min(calls, 2) + + +@pytest.mark.parametrize("fault", ["wrong_length", "stale_history", "mask", "padding"]) +def test_full_suite_rejects_wrong_input_history_even_when_logits_match(cli, runtime, reference_suite, tmp_path, monkeypatch, fault): + from dataclasses import replace + from scratchv.runtime.generation import GenerationResult + from scratchv.runtime.llm_inputs import prepare_inputs + + def faulty_generate(prompt, *, forward, **kwargs): + prepared = prepare_inputs([1, 2], pad_token_id=0, vocab_size=8) + if fault == "wrong_length": + prepared = replace(prepared, valid_length=1) + elif fault == "stale_history": + prepared.input_ids[0, 1] = 4 + elif fault == "mask": + prepared.attention_mask[0, 0, 0, 1] = 0 + else: + prepared.input_ids[0, -1] = 5 + forward(prepared) + forward(prepare_inputs([1, 2, 3], pad_token_id=0, vocab_size=8)) + return GenerationResult((1, 2), (3, 3), "xx", "max_new_tokens") + + monkeypatch.setattr(cli, "generate", faulty_generate) + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1 + value = report(tmp_path) + assert value["full_model_gate"] == "FAIL" and value["qemu_forwards"] == 0 + + +@pytest.mark.parametrize("same_argmax", [False, True]) +def test_numeric_comparison_rejects_wrong_selected_row_independently(cli, monkeypatch, same_argmax): + expected = logits() + expected[0, 0, 3 if same_argmax else 4] = 2 + monkeypatch.setattr(cli, "last_valid_logits", lambda array, length, **kwargs: array[0, 0].copy()) + result = cli.compare_step(expected.copy(), expected, 2, 8) + assert result["max_abs"] == 0 and result["qemu_token_id"] == result["ort_token_id"] + assert result["selected_row_passed"] is False and result["passed"] is False + + +def test_full_suite_rejects_frozen_token_drift_with_equal_qemu_ort_logits(cli, runtime, reference_suite, tmp_path, monkeypatch): + runtime.oracle = logits(4) + original = cli.QemuForward + + class Adapter(original): + def __call__(self, prepared): + super().__call__(prepared) + return logits(4) + + monkeypatch.setattr(cli, "QemuForward", Adapter) + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == 1 + value = report(tmp_path) + row = value["numeric_steps"][0] + assert value["full_model_gate"] == "FAIL" and value["qemu_forwards"] == 1 + assert row["max_abs"] == 0 and row["selected_row_passed"] is True + assert row["qemu_token_id"] == row["ort_token_id"] == 4 + assert row["reference_token_id"] == 3 and row["reference_token_passed"] is False + assert row["case"] == "france" and row["step"] == 0 and row["valid_length"] == 2 + + +@pytest.mark.parametrize("error,code,status", [ + (TimeoutError("expired after first forward"), 1, "FAIL"), + (GenerationCancelledError("cancelled after first forward"), 130, "CANCELLED"), +]) +def test_full_suite_abort_retains_completed_step_evidence(cli, runtime, reference_suite, tmp_path, monkeypatch, error, code, status): + generate = cli.generate + + def abort_after_forward(prompt, *, forward, **kwargs): + def guarded(prepared): + forward(prepared) + raise error + return generate(prompt, forward=guarded, **kwargs) + + monkeypatch.setattr(cli, "generate", abort_after_forward) + assert cli.main(argv(tmp_path, "--reference-suite", "--verify-onnx", "onnx/model.onnx")) == code + value = report(tmp_path) + assert value["full_model_gate"] == status and value["status"] == status + assert value["cases"] == [] and value["qemu_forwards"] == 1 + assert value["numeric_steps"][0]["passed"] is True + assert value["numeric_steps"][0]["case"] == "france" + + +@pytest.mark.parametrize("requested,cpu_count,resolved", [ + (None, 8, 1), ("1", 8, 1), ("2", 8, 2), ("4", 1, 4), + ("auto", 1, 1), ("auto", 3, 2), ("auto", 12, 4), +]) +def test_cli_uses_real_hart_resolver_without_changing_default(cli, runtime, tmp_path, monkeypatch, requested, cpu_count, resolved): + from scratchv.runtime import riscv_harts + affinity_reads = [] + monkeypatch.setattr(riscv_harts.os, "sched_getaffinity", lambda pid: affinity_reads.append(pid) or set(range(cpu_count))) + monkeypatch.setattr(cli, "resolve_harts", riscv_harts.resolve_harts) + options = [] if requested is None else ["--harts", requested] + assert cli.main(argv(tmp_path, "--build-only", *options)) == 0 + value = report(tmp_path) + assert value["harts"] == resolved and runtime.prepared[0][2] == resolved + assert value["qemu_workspace_layout"] == "none" and value["qemu_forwards"] == 0 + assert affinity_reads == ([0] if requested == "auto" else []) + + +@pytest.mark.parametrize("built", [None, "sequential", "two-four-lane-last-axis"]) +@pytest.mark.parametrize("requested", ["sequential", "two-four-lane-last-axis"]) +def test_w5_reuse_mean_policy_requires_explicit_matching_build(cli, monkeypatch, tmp_path, built, requested): + metadata = tmp_path / "old/executable.json" + matching_reuse_metadata(cli, monkeypatch, metadata) + record = json.loads(metadata.with_name("report.json").read_text()) + if built is not None: + record["mean_policy"] = built + metadata.with_name("report.json").write_text(json.dumps(record)) + evidence = {"matmul_policy": "sequential"} + if built is not None: + evidence["mean_policy"] = built + exe = SimpleNamespace(harts=1, evidence=evidence) + monkeypatch.setattr(cli, "load_executable", lambda path: exe) + monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("reuse must not compile")) + args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", + "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--mean-policy", requested]) + if (built or "sequential") == requested: + assert cli.prepare_executable(args, tmp_path, 1) is exe + else: + with pytest.raises(ValueError, match="Mean policy"): + cli.prepare_executable(args, tmp_path, 1) + + +def test_w5_reuse_mean_evidence_and_sibling_report_must_agree(cli, monkeypatch, tmp_path): + metadata = tmp_path / "old/executable.json" + matching_reuse_metadata(cli, monkeypatch, metadata) + exe = SimpleNamespace(harts=1, evidence={"matmul_policy": "sequential", "mean_policy": "two-four-lane-last-axis"}) + monkeypatch.setattr(cli, "load_executable", lambda path: exe) + args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", + "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--mean-policy", "two-four-lane-last-axis"]) + with pytest.raises(ValueError, match="Mean policy"): + cli.prepare_executable(args, tmp_path, 1) + + +@pytest.mark.parametrize("policy", ["sequential", "two-four-lane-last-axis"]) +def test_w5_mean_policy_reaches_real_codegen_without_compiler(cli, monkeypatch, tmp_path, policy): + from scratchv.frontend import onnx_parser + from scratchv.ir.builder import IRBuilder + from scratchv.ir.types import Value + from scratchv.runtime.riscv_external import ExternalExecutable + from scratchv.runtime.riscv_tensor import RiscVToolchain + builder = IRBuilder() + x = Value("x", shape=(2, 8)) + builder.new_function("main", [x]); builder.new_block("entry"); builder.ret(builder.neg(x)) + monkeypatch.setattr(onnx_parser, "ONNXParser", lambda: SimpleNamespace(initializers={}, parse=lambda *a, **kw: builder.program)) + toolchain = RiscVToolchain(("unused-cc",), "unused-qemu") + monkeypatch.setattr(cli, "discover_toolchain", lambda *a: toolchain) + built = [] + + def build(artifact, bundle, out, tools, **kwargs): + built.append(artifact) + return ExternalExecutable(out / "model.elf", bundle, {}, {}, artifact.inputs, + artifact.output, artifact.external_weights, tools, {"mean_policy": artifact.mean_policy}) + + monkeypatch.setattr(cli, "build_external", build) + args = cli.parse_args(argv(tmp_path, "--build-only", "--mean-policy", policy)) + exe = cli.prepare_executable(args, tmp_path, 1) + assert len(built) == 1 and built[0].mean_policy == policy + assert exe.evidence["mean_policy"] == policy and exe.harts == 1 + assert built[0].matmul_policy == "sequential" and built[0].qemu_workspace_layout == "none" + + +def test_w5_mean_policy_is_explicit_in_report(cli, runtime, tmp_path): + assert cli.main(argv(tmp_path, "--build-only", "--mean-policy", "two-four-lane-last-axis")) == 0 + assert report(tmp_path)["mean_policy"] == "two-four-lane-last-axis" + assert runtime.prepared[0][0].mean_policy == "two-four-lane-last-axis" + + +@pytest.mark.parametrize("built", [None, "libm", "fp32-reference"]) +@pytest.mark.parametrize("requested", ["libm", "fp32-reference"]) +def test_w5_reuse_nonlinear_policy_requires_explicit_matching_build(cli, monkeypatch, tmp_path, built, requested): + metadata = tmp_path / "old/executable.json" + matching_reuse_metadata(cli, monkeypatch, metadata) + record = json.loads(metadata.with_name("report.json").read_text()) + if built is not None: + record["nonlinear_policy"] = built + metadata.with_name("report.json").write_text(json.dumps(record)) + evidence = {"matmul_policy": "sequential"} + if built is not None: + evidence["nonlinear_policy"] = built + exe = SimpleNamespace(harts=1, evidence=evidence) + monkeypatch.setattr(cli, "load_executable", lambda path: exe) + monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("reuse must not compile")) + args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", + "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--nonlinear-policy", requested]) + if (built or "libm") == requested: + assert cli.prepare_executable(args, tmp_path, 1) is exe + else: + with pytest.raises(ValueError, match="nonlinear policy"): + cli.prepare_executable(args, tmp_path, 1) + + +def test_w5_reuse_nonlinear_evidence_and_sibling_report_must_agree(cli, monkeypatch, tmp_path): + metadata = tmp_path / "old/executable.json" + matching_reuse_metadata(cli, monkeypatch, metadata) + exe = SimpleNamespace(harts=1, evidence={"matmul_policy": "sequential", "nonlinear_policy": "fp32-reference"}) + monkeypatch.setattr(cli, "load_executable", lambda path: exe) + args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", + "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--nonlinear-policy", "fp32-reference"]) + with pytest.raises(ValueError, match="nonlinear policy"): + cli.prepare_executable(args, tmp_path, 1) + + +@pytest.mark.parametrize("policy", ["libm", "fp32-reference"]) +def test_w5_nonlinear_policy_reaches_real_codegen_without_compiler(cli, monkeypatch, tmp_path, policy): + from scratchv.frontend import onnx_parser + from scratchv.ir.builder import IRBuilder + from scratchv.ir.types import Value + from scratchv.runtime.riscv_external import ExternalExecutable + from scratchv.runtime.riscv_tensor import RiscVToolchain + builder = IRBuilder() + x = Value("x", shape=(2, 8)) + builder.new_function("main", [x]); builder.new_block("entry"); builder.ret(builder.neg(x)) + monkeypatch.setattr(onnx_parser, "ONNXParser", lambda: SimpleNamespace(initializers={}, parse=lambda *a, **kw: builder.program)) + toolchain = RiscVToolchain(("unused-cc",), "unused-qemu") + monkeypatch.setattr(cli, "discover_toolchain", lambda *a: toolchain) + built = [] + + def build(artifact, bundle, out, tools, **kwargs): + built.append(artifact) + return ExternalExecutable(out / "model.elf", bundle, {}, {}, artifact.inputs, + artifact.output, artifact.external_weights, tools, {"nonlinear_policy": artifact.nonlinear_policy}) + + monkeypatch.setattr(cli, "build_external", build) + args = cli.parse_args(argv(tmp_path, "--build-only", "--nonlinear-policy", policy)) + exe = cli.prepare_executable(args, tmp_path, 1) + assert len(built) == 1 and built[0].nonlinear_policy == policy + assert exe.evidence["nonlinear_policy"] == policy and exe.harts == 1 + assert built[0].matmul_policy == "sequential" and built[0].qemu_workspace_layout == "none" + + +def test_w5_nonlinear_policy_is_explicit_in_report(cli, runtime, tmp_path): + assert cli.main(argv(tmp_path, "--build-only", "--nonlinear-policy", "fp32-reference")) == 0 + assert report(tmp_path)["nonlinear_policy"] == "fp32-reference" + assert runtime.prepared[0][0].nonlinear_policy == "fp32-reference" + + +@pytest.mark.parametrize("policy", ["unknown", "sequential", ""]) +def test_w5_unknown_nonlinear_policy_fails_before_output(cli, tmp_path, policy): + with pytest.raises(SystemExit) as exc: + cli.main(argv(tmp_path, "--build-only", "--nonlinear-policy", policy)) + assert exc.value.code == 2 and not (tmp_path / "run").exists() + + +def test_finite_extreme_logits_have_json_safe_diagnostic_margin(cli): + expected = np.full((1, 256, 8), -np.finfo(np.float32).max, dtype=np.float32) + expected[0, 1, 3] = np.finfo(np.float32).max + with np.errstate(over="raise"): + comparison = cli.compare_step(expected.copy(), expected, 2, 8) + assert comparison["passed"] and comparison["max_abs"] == 0.0 + assert comparison["qemu_token_id"] == comparison["ort_token_id"] == 3 + assert comparison["ort_top2_margin"] == 2.0 * float(np.finfo(np.float32).max) + json.dumps(comparison, allow_nan=False) From 8ad347477f7f5ba15b2da7c91178b3c6668456cc Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Fri, 23 Oct 2026 15:40:19 +0800 Subject: [PATCH 2/7] test(w5): handle LFS-only tokenizer checkout and W3 selection guards --- docs/llm-deploy-v1.0/W5/README.md | 28 ++++++++++++++++++++++++ tests/test_conversation.py | 36 ++++++++++++++++++++++++++++--- tests/test_w3_workflows.py | 16 +++++++++----- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/docs/llm-deploy-v1.0/W5/README.md b/docs/llm-deploy-v1.0/W5/README.md index b9285181..19d25f03 100644 --- a/docs/llm-deploy-v1.0/W5/README.md +++ b/docs/llm-deploy-v1.0/W5/README.md @@ -264,6 +264,34 @@ Docker 本轮未启动，WSL 配置未改。 3. **已知运行边界**：同步构建、传输和 ORT 调用按阶段检查预算，不承诺硬中断任意 ORT 调用。 KV cache、持久 guest/权重驻留、RVV 和量化属于后续阶段，不属于 W5 v1 未完成的出口项。 +## 发布与首次远端 CI 反馈（2026-10-23） + +代码、测试和报告已作为独立 [PR #115](https://github.com/ScratchV-Compiler/ScratchV/pull/115) +提交，初始实现提交为 259fc8e。四份报告链接已经验证可访问；没有合并。 + +首次 [PR CI](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/39917939519) +实际执行并通过 12 个 W5 相关 QEMU 小图必跑节点及 2 个官方三轮模板节点。 +W4 unit JUnit 为 1762 passed、4 skipped，四个跳过为三种未提供模型资产的优化器对照及 +Linux 上的 Windows junction 专用测试，不涉及上述 12 个必跑节点。完整 W5 门槛按普通 +PR 路由未选择，汇总明确为 w5_generation_accepted=false。 + +通用测试发现 8 个失败：6 个来自旧 W3 测试求值器不支持新增 W5 互斥表达式； +另 2 个来自通用 checkout 未下载 tokenizer 的 Git LFS 内容，测试误把指针当作 JSON。 +修复仅涉及测试：补齐 W3/W5 选择矩阵；默认可选资产缺失或为 LFS 指针时明确 skip， +显式指定 tokenizer 目录时仍必须有效，真正损坏的 JSON 不隐藏为 skip。 +专用 CI 的两项模板必跑门槛保持不变。修复的直接验证为会话/部署 63 passed、 +W3 工作流 54 passed、相关 W4/W5 入口边界 28 passed；均未运行完整模型。 +修复推送后的最终 CI 状态以 PR 验证摘要为准，不把首次失败改写为成功。 + +W5-only 专项在 fork 调度时被 GitHub 以 +HTTP 422: Actions has been disabled for this repository 拒绝，未创建 run。 +仓库设置显示 Actions enabled=true、workflow active，重新启用指定 workflow 后仅重试 +一次仍失败。该外部阻塞尚未解除；详见 +[GitHub 官方说明](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository)。 +需要仓库所有者向 GitHub Support 核查，不能通过本地 PASS 或普通 PR CI 替代完整专项。 +没有改 WSL、默认 hart/padding 或模型数值策略，也没有重复本地完整模型。 +用户已接受仓库自动审查，没有额外手动发起审查。 + ## P0/P1 已有验证（2026-10-23，未重复执行） | 验证 | 结果 | 覆盖边界 | diff --git a/tests/test_conversation.py b/tests/test_conversation.py index 786dc13c..88e9f229 100644 --- a/tests/test_conversation.py +++ b/tests/test_conversation.py @@ -239,14 +239,44 @@ def test_from_directory_rejects_missing_or_ambiguous_template(tmp_path, config): Conversation.from_directory(tmp_path) +def _require_official_tokenizer(root, *, required): + source = root / "tokenizer.json" + unavailable = not source.is_file() + if not unavailable: + with source.open("rb") as stream: + unavailable = stream.read(128).startswith(b"version https://git-lfs.github.com/spec/") + if unavailable: + message = "official local Qwen3 tokenizer assets are missing or still Git LFS pointers" + if required: + pytest.fail(message) + pytest.skip(message) + + +@pytest.mark.parametrize("required", [False, True]) +@pytest.mark.parametrize("content", [None, b"version https://git-lfs.github.com/spec/v1\n"]) +def test_official_tokenizer_missing_assets_skip_only_when_optional(tmp_path, required, content): + if content is not None: + (tmp_path / "tokenizer.json").write_bytes(content) + expected = pytest.fail.Exception if required else pytest.skip.Exception + with pytest.raises(expected, match="missing or still Git LFS"): + _require_official_tokenizer(tmp_path, required=required) + + +@pytest.mark.parametrize("content", [b'{"version": "1.0"}', b"corrupt json"]) +def test_official_tokenizer_nonpointer_contents_are_left_to_parser(tmp_path, content): + source = tmp_path / "tokenizer.json" + source.write_bytes(content) + _require_official_tokenizer(tmp_path, required=False) + assert source.read_bytes() == content + + @pytest.mark.parametrize("system_prompt", [None, "Give brief answers."]) def test_official_three_round_rendering_matches_transformers(monkeypatch, system_prompt): - """Runs when local assets exist; does not load weights or execute a model.""" + """Optional in generic tests; explicit assets and dedicated CI remain mandatory.""" pytest.importorskip("tokenizers") transformers = pytest.importorskip("transformers") root = Path(os.environ.get("SCRATCHV_QWEN3_TOKENIZER_DIR", Path(__file__).resolve().parents[1] / "assets/qwen3-0.6b/tokenizer")) - if not (root / "tokenizer.json").is_file(): - pytest.skip("official local Qwen3 tokenizer assets are not installed") + _require_official_tokenizer(root, required="SCRATCHV_QWEN3_TOKENIZER_DIR" in os.environ) official = transformers.AutoTokenizer.from_pretrained(str(root), local_files_only=True) chat = Conversation.from_directory(root, system_prompt=system_prompt) replies = iter(("Paris", "Berlin", "Tokyo")) diff --git a/tests/test_w3_workflows.py b/tests/test_w3_workflows.py index b27bdb33..b8306a8d 100644 --- a/tests/test_w3_workflows.py +++ b/tests/test_w3_workflows.py @@ -6,6 +6,7 @@ import ast import os from pathlib import Path +import re import shutil import subprocess @@ -46,17 +47,21 @@ def run_script(bash, script, tmp_path, **values): cwd=tmp_path, env=env, capture_output=True, text=True) -def evaluate_guard(expression, event, enabled, w3_validation=False): +def evaluate_guard(expression, event, enabled, w3_validation=False, w5_validation=False): """Evaluate only the tiny boolean subset used by the real job guards.""" value = expression.removeprefix("${{").removesuffix("}}").strip() value = value.replace("github.event_name", repr(event)) value = value.replace("vars.W3_NIGHTLY_ENABLED", repr(enabled)) value = value.replace("inputs.run_w3_validation", repr(w3_validation)) + value = value.replace("inputs.run_w5_validation", repr(w5_validation)) value = value.replace("always()", "True").replace("||", "or").replace("&&", "and") + value = re.sub(r"!(?!=)", "not ", value).strip() def visit(node): if isinstance(node, ast.Constant): return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not bool(visit(node.operand)) if isinstance(node, ast.BoolOp): values = [bool(visit(item)) for item in node.values] return all(values) if isinstance(node.op, ast.And) else any(values) @@ -121,17 +126,18 @@ def test_nightly_expensive_jobs_require_opt_in_or_explicit_dispatch(event, enabl @pytest.mark.parametrize("event", ["pull_request", "schedule", "workflow_dispatch"]) @pytest.mark.parametrize("selected", [False, True]) -def test_existing_manual_entry_runs_w3_only_when_explicitly_requested(event, selected): +@pytest.mark.parametrize("w5_selected", [False, True]) +def test_existing_manual_entry_runs_w3_only_when_explicitly_requested(event, selected, w5_selected): data = workflow("llm-deploy.yml") option = data["on"]["workflow_dispatch"]["inputs"]["run_w3_validation"] assert option["type"] == "boolean" and option["default"] == "false" job = data["jobs"]["w3-validation"] assert job["uses"] == "./.github/workflows/w3-nightly.yml" - assert evaluate_guard(job["if"], event, "", selected) is ( - event == "workflow_dispatch" and selected) + expected = event == "workflow_dispatch" and selected and not w5_selected + assert evaluate_guard(job["if"], event, "", selected, w5_selected) is expected # The called workflow inherits the caller event. A manual call must run # both complete gates even while the scheduled nightly opt-in is disabled. - if event == "workflow_dispatch" and selected: + if expected: assert all(evaluate_guard(item["if"], event, "") for item in workflow("w3-nightly.yml")["jobs"].values()) From 7676304f764415402f75e66e2cb4455689a36763 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Fri, 23 Oct 2026 16:44:12 +0800 Subject: [PATCH 3/7] fix(ci): bound AI review timeouts and document W5 handoff --- docs/llm-deploy-v1.0/W5/README.md | 106 +++-- docs/llm-deploy-v1.0/W5/deployment.md | 12 +- .../W5/independent-reproduction.md | 365 ++++++++++++++++++ docs/llm-deploy-v1.0/W5/qemu-generation.md | 6 +- scripts/ai_code_review.py | 57 ++- tests/test_ai_code_review.py | 122 +++++- 6 files changed, 620 insertions(+), 48 deletions(-) create mode 100644 docs/llm-deploy-v1.0/W5/independent-reproduction.md diff --git a/docs/llm-deploy-v1.0/W5/README.md b/docs/llm-deploy-v1.0/W5/README.md index 19d25f03..7cc25116 100644 --- a/docs/llm-deploy-v1.0/W5/README.md +++ b/docs/llm-deploy-v1.0/W5/README.md @@ -14,12 +14,20 @@ W5 的生成合同、Host 循环、QEMU 适配/命令行、三用例验收入口 [完整模型实跑报告](qemu-generation.md)。本轮干净容器的默认 1 hart、手动 4 hart、 auto 小图和真实超时/取消、Docker 构建取消检查已通过；完整 28 层 France 单 token 容器部署也已通过，实际输出 ` Paris`，完整 logits 最大误差为 **0.000172138214**。 -**W5 本地功能出口已满足**；W5 CI 已接入既有流程并通过本地相关检查，远端运行 **NOT_RUN**； -WSL 配置未改，所有 W5 改动仍未提交、未推送。 +**W5 本地功能出口已满足**；代码、测试和报告已发布到 [PR115](https://github.com/ScratchV-Compiler/ScratchV/pull/115)， +已有数值实跑和普通 PR CI 对应功能基线 `8ad347477f7f5ba15b2da7c91178b3c6668456cc`； +本次后续变更仅包含审查超时处理、相关测试和交接/进度文档，不改变生成实现。完整 W5 +专项因 fork 仓库 Actions 拒绝创建运行，仍为 **BLOCKED / NOT_RUN**。第二人独立复现 +说明已准备，尚待实际执行和签收；WSL 配置未改，PR 未合并。 使用入口：[QEMU 生成与本轮报告](qemu-generation.md)、[三轮历史](chat.md)、 [干净容器部署](deployment.md)。 +PR115 第二人验收使用[独立复现交接说明](independent-reproduction.md)：固定提交 +`8ad347477f7f5ba15b2da7c91178b3c6668456cc`、基础镜像和数值配置，提供从新 clone 到 +三例 24 步验收、失败清理及报告回传的完整命令。文档准备不代表第二人复现已完成， +须由独立执行人回传结果后签收。 + ## 固定生成合同 | 项目 | v1 约定 | @@ -143,16 +151,24 @@ Mean `sequential` 和非线性 `libm` 均未改变，不能把新策略的验收 ## 阶段与验收状态 -| 阶段/门槛 | 本轮状态与后续要求 | -|---|---| -| P0：合同与 3 条参考 | **PASS（本地）**；合同冻结，3/3 真实 ORT/Host 参考与独立 oracle 通过，参考 IDs/text/停止原因冻结 | -| P1：Host token 循环 | **PASS（本地）**；核心 55 项及下述直接相关验证通过 | -| P2：QEMU 生成适配/CLI | **PASS（本地小图）**；构建一次，同一 ELF/bundle 跨 token 和跨命令复用；1/4 hart 数值与中断清理通过 | -| `e2e:qemu-paris` | **PASS（本地、显式新策略 4 hart）**；France 完整 8 token 的数值、IDs、文本和停止原因通过，输出包含 Paris。历史旧配置首步 FAIL 的 `max_abs=0.003272057` 保留于实跑报告 | -| `e2e:qemu-prompts` | **PASS（本地、显式新策略 4 hart）**；3/3 用例、24/24 完整 QEMU 前向及逐步数值通过，IDs/text/停止原因与冻结参考一致。历史旧配置初始轮仅检查 3/24 步且均 FAIL，余下 21 步当时未执行 | -| `unit:multiturn` | **PASS（本地）**；40 项会话测试含官方模板/IDs 六份对照；raw 与显式 chat 分开，失败不提交历史 | -| `deploy:script` | **PASS（本地真实容器）**；默认 1/手动 4/auto 小图、真实 QEMU 超时/取消复用与 Docker 构建取消通过；完整 28 层 France 单 token 实际生成 ` Paris`，数值与进程清理通过，不重复宿主三例套件 | -| W5 CI 接入 | **本地实现与检查 PASS；远端 NOT_RUN**；提交前 811 项相关检查通过，最终 11 项重叠边界复查通过；14 个真实必跑节点复用既有证据，本轮未重新执行模型；未提交/推送 | +以下状态核对于 2026-10-23，作者本地实跑、远端 CI 和第二人签收分别记录。 + +| 阶段/门槛 | 当前证据与状态 | 剩余事项 | +|---|---|---| +| P0：合同与 3 条参考 | **PASS**；合同冻结，3/3 真实 ORT/Host 参考与独立 oracle 通过，IDs/text/停止原因冻结 | 无实现缺项 | +| P1：Host token 循环 | **PASS**；核心 55 项及相关边界验证通过 | 无实现缺项 | +| P2：QEMU 生成适配/CLI | **PASS**；同一 ELF/bundle 跨 token 和命令复用；1/4 hart 数值与中断清理通过 | 无实现缺项 | +| 默认/手动/auto hart | **PASS（小图）**；默认单核、手动 4、auto→4 的真实容器 CLI 通过，padding 关闭 | 不将小图或显式策略通过推广为默认算术完整模型通过 | +| `e2e:qemu-paris` | **PASS（作者本地、显式新策略 4 hart）**；France 完整 8 token 的数值、IDs、文本和停止原因通过，输出包含 Paris | 远端完整专项及第二人复现 | +| `e2e:qemu-prompts` | **PASS（作者本地、显式新策略 4 hart）**；3/3 用例、24/24 完整 QEMU 前向及逐步数值通过，与冻结参考一致 | 远端完整专项及第二人复现 | +| `unit:multiturn` | **PASS**；40 项会话测试，官方模板/IDs 六份对照；远端官方三轮模板 2/2 通过 | 不宣称已做完整模型三轮对话生成 | +| `deploy:script` | **PASS（作者真实容器）**；小图、超时/取消和清理通过；完整 28 层 France 单 token 实际生成 ` Paris` | 容器三例 24 步尚未执行，不能用单 token 代替 | +| 普通 PR CI | **PASS**；[39919353111](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/39919353111) 为 14 成功、7 个按路由跳过；W5 真实小图 12/12、模板 2/2 通过 | 不替代完整生成专项 | +| W5 完整专项 CI | **BLOCKED / NOT_RUN**；工作流已接入，fork Actions 以 HTTP 422 拒绝 dispatch，未创建 run | 恢复平台执行能力后运行容器三例 24 步 | +| 第二人独立复现 | **说明已准备，执行待办**；固定提交、环境、命令、验收和回传清单见[交接说明](independent-reproduction.md) | 同事独立构建、实跑并回传签收，作者或 CI 不能替代 | + +旧算术配置的历史失败保留于实跑报告：初始轮三例均在首步失败（仅执行 3/24 步）， +France 的 `max_abs=0.003272057`。显式新策略的 PASS 不改写该历史结果。 W4 完整单次前向的已有证据不替代 W5 token 循环；本页 W5 结果来自上述独立完整生成验收。 W4 独立复现、Nightly 和团队签收仍按 [W4 验收说明](../W4/validation.md)分别确认。 @@ -162,8 +178,8 @@ W4 独立复现、Nightly 和团队签收仍按 [W4 验收说明](../W4/validati gate 失败状态、旧 W4 构建审计传递、逐步验收证据及取消清理已补强。 本地功能出口已有完整证据：宿主三例 24 步、宿主独立前向，以及干净容器镜像、 小图、中断清理和完整 Paris 单 token 均已通过，见 [部署记录](deployment.md)。 -容器此次只补一次完整前向，不将其写成容器三例套件；CI 已完成本地接入和相关检查，远端 -**NOT_RUN**，本地功能完成不等于已经取得远端 Actions 或发布验收记录。 +容器此次只补一次完整前向，不将其写成容器三例套件；普通 PR CI 已通过，完整专项仍 +**BLOCKED / NOT_RUN**。本地功能完成不等于远端完整验收或第二人签收完成。 不把候选 token 一致或静态部署检查写成实跑 PASS。当前每 token 启动 QEMU，但复用构建； 持久 guest、权重驻留、KV cache 等不属于 W5 v1 必需实现。 @@ -191,10 +207,11 @@ GitHub 要求手动工作流存在于默认分支，见 审查时远端 main 的 `w4-full-numeric.yml` 尚无 `workflow_dispatch`，因此不能把新文件的 直接手动入口当作仅推送 W5 分支后必然可用；不要为了建立入口先合并未验收的 W5。 -下列命令仅供后续获得提交、推送和远端验证授权后使用，本轮没有执行；还需要仓库写权限： +下列入口已获授权并尝试调用；fork 仓库的 Actions 目前在 dispatch 阶段返回 HTTP 422， +未创建运行。解除该限制后才可重试；执行仍需仓库写权限： ```bash -gh workflow run llm-deploy.yml --ref codex/w5-generation \ +gh workflow run llm-deploy.yml --repo yuki-328/ScratchV --ref codex/w5-generation \ -f run_w5_validation=true -f run_w3_validation=false -f run_w4_validation=false ``` @@ -202,7 +219,7 @@ gh workflow run llm-deploy.yml --ref codex/w5-generation \ W5 完整 job/步骤/CLI 总预算分别为 **350 / 285 / 275 分钟**；镜像和固定资产落盘后才检查 **至少 4 CPU、10 GiB 可用 RAM、 10 GiB 剩余磁盘**，不足即明确失败，不通过缩小模型或跳过 token 凑齐门槛。 -远端 runner 是否实际满足资源和时限，仍须授权提交/推送后实跑确认。 +远端 runner 是否实际满足资源和时限，仍须在专项能够创建运行后实跑确认。 W4 unit 增加 **12 个真实 QEMU 必跑节点**（生成与取消 2、Mean 8、非线性 2）； [llm-deploy.yml](../../../.github/workflows/llm-deploy.yml) 在现有 tokenizer 环境补 **2 个 @@ -219,10 +236,13 @@ JUnit 缺失/skip/fail、资源短缺、退出码与汇总；其中严格报告 `output/w5-followup-20261023/qemu-reference-4hart/run/report.json` 离线检查通过。 本地 CI 检查记录为 `output/w5-ci-integration-20261023/validation.json`， 同目录保存测试日志、14 个收集节点 ID 以及 98 段内嵌 shell 的语法检查结果。 -这些都是本地实现/证据检查，**远端 W5 CI 仍为 NOT_RUN**，未提交、未推送。 +上述记录来自提交前的本地实现/证据检查，当时未提交、未推送；其后的普通 PR CI +已通过，完整专项仍为 **BLOCKED / NOT_RUN**，详见页末发布记录。 ## 提交前检查与收尾（2026-10-23） +本节保留提交前检查时点；后续提交、普通 CI 和专项阻塞状态见页末发布记录。 + 已核对 W5 全部待提交改动：生成合同与冻结参考、Host/会话循环、QEMU 适配与 CLI、 显式数值策略、运行器清理、容器打包、第三方声明、CI 路由和报告。 **本地可执行的 W5 工作已完成，未发现尚未处理的本地阻塞；远端 CI 验收仍待执行。** @@ -256,8 +276,9 @@ Docker 本轮未启动，WSL 配置未改。 剩余事项按性质区分： -1. **合入/发布前仍须取得远端 CI 证据**：后续获准推送后，使用上节 W5-only 入口； - runner 资源不足、失败或取消必须明确失败，不能用本地 PASS 替代。 +1. **当时仍待远端 CI 证据**：提交前计划获准推送后使用上节 W5-only 入口； + 后续普通 CI 已通过，完整专项被 Actions 平台阻塞。runner 资源不足、失败或取消 + 必须明确失败，不能用本地 PASS 替代。 2. **数值验收配置限制**：完整模型 PASS 仅属于已记录的显式参考算术组合； 原默认 Mean sequential/nonlinear libm 的历史完整 logits 超差仍保留，不能宣称默认路径也通过。 本轮没有修改算术，因此未重复完整模型。 @@ -281,14 +302,43 @@ PR 路由未选择，汇总明确为 w5_generation_accepted=false。 显式指定 tokenizer 目录时仍必须有效，真正损坏的 JSON 不隐藏为 skip。 专用 CI 的两项模板必跑门槛保持不变。修复的直接验证为会话/部署 63 passed、 W3 工作流 54 passed、相关 W4/W5 入口边界 28 passed；均未运行完整模型。 -修复推送后的最终 CI 状态以 PR 验证摘要为准，不把首次失败改写为成功。 - -W5-only 专项在 fork 调度时被 GitHub 以 -HTTP 422: Actions has been disabled for this repository 拒绝，未创建 run。 -仓库设置显示 Actions enabled=true、workflow active，重新启用指定 workflow 后仅重试 -一次仍失败。该外部阻塞尚未解除；详见 -[GitHub 官方说明](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository)。 -需要仓库所有者向 GitHub Support 核查，不能通过本地 PASS 或普通 PR CI 替代完整专项。 +修复提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 推送后， +[普通 PR CI 39919353111](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/39919353111) +已 **SUCCESS**：14 个 job 成功、7 个按路由跳过；通用测试 6107 passed、93 skipped， +模拟器 7 passed；W4 unit 1762 passed、4 skipped；W5 必跑小图 12/12 和官方模板 +2/2 均通过。此轮测试的是 PR 临时合并提交 `ce9bd0e438e361ca070fad95a95c4fe412ec751b`， +不把首次失败改写为成功，也不把未选择的完整生成 job 计作通过。 + +完整 W5 专项在 fork `yuki-328/ScratchV` 的 dispatch 返回 +`HTTP 422: Actions has been disabled for this repository.`，尚未创建运行。 +已核对仓库 Actions `enabled=true`、工作流 `active`；重新应用相同启用设置返回 +HTTP 204，随后 dispatch 仍为 422。现有证据只能确认平台拒绝创建运行，不能认定是 +模型、runner 资源或工作流中的测试失败；需仓库所有者处理 GitHub 侧限制，恢复后再验收。 + +另一个自动 AI Code Review 的 `review (1)`、`review (2)` 两个批次失败： +分别有 2、5 份文件各两次请求均出现 `TimeoutError`。这是外部 API 请求超时导致的 +审查覆盖不完整，不是上述数值或单测失败；检查时后续批次仍在运行/排队。 +本地[审查脚本](../../../scripts/ai_code_review.py)已补有界缓解：新计划改为每批 +5 个文件、180 秒 socket 超时，仍最多两次尝试；旧 10 文件计划保留 90 秒窗口和 +原分组。新增逐次脱敏耗时/结果记录，失败仍使门禁失败。 +[相关测试](../../../tests/test_ai_code_review.py) **71 passed**；未调用外部 AI API。 +180 秒不是整个请求的墙钟硬上限，最终仍受 40 分钟 job 限制。 + +这部分修复与本轮文档更新作为 PR115 后续变更一起交付；尚未部署到可信主干。 +工作流通过 `pull_request_target` 使用可信 base 的脚本，不能改为携密钥执行 PR head。 +生效顺序是：维护者完成正常审批，将修复通过正常合并纳入 `main`；此后新发生的 +PR 事件才会使用新脚本。仅推送 PR 分支或重新运行仍固定旧 base 的历史 run， +不应当作修复已经部署。当前失败记录保留，不改写为成功；本轮不合并，也不手动 +追加或重跑 AI 审查。现有日志尚不能区分服务排队、生成慢或网络等待， +不能声明外部服务已恢复。 + +第二人[独立复现说明](independent-reproduction.md)已准备，实际执行和签收仍待同事完成。 +当前证据位于 `output/w5-publish-20261023/publication.json`、`actions-recovery.json` 及 +`ci-final-artifacts/`；报告与代码已发布，PR 未合并。 + +[GitHub 官方说明](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository) +指出，仓库/账号可能处于独立的平台停用状态，修改普通开关无法恢复，需要联系 +GitHub Support。当前平台未说明具体原因；普通 PR CI 或本地 PASS 不替代完整专项。 没有改 WSL、默认 hart/padding 或模型数值策略，也没有重复本地完整模型。 用户已接受仓库自动审查，没有额外手动发起审查。 diff --git a/docs/llm-deploy-v1.0/W5/deployment.md b/docs/llm-deploy-v1.0/W5/deployment.md index 5742154a..403c5a8a 100644 --- a/docs/llm-deploy-v1.0/W5/deployment.md +++ b/docs/llm-deploy-v1.0/W5/deployment.md @@ -4,6 +4,10 @@ [一键脚本](../../../scripts/deploy_w5.sh) 的使用方式。 生成合同、固定参考和阶段门槛见 [W5 总览](README.md)。 +交给同事复现 PR115 时，使用[第二人独立复现说明](independent-reproduction.md)， +其中固定代码提交、基础镜像及三例完整验收参数，并列出环境差异处理、清理和回传清单。 +下文保留已有部署用法与作者实跑记录，不作为第二人已完成的证据。 + ## 部署范围 镜像使用 Ubuntu 24.04 / Python 3.12，安装发行版 `qemu-system-misc`，并直接复用 @@ -197,5 +201,9 @@ W5 CI 已复用既有流程接入；容器收尾时 753 项本地轻量检查通 后续提交前检查补齐 W5-only 手动入口和 job 余量，811 项相关检查通过，最终 11 项重叠 边界复查通过，未重新执行完整模型，见 [提交前收尾](README.md#提交前检查与收尾2026-10-23)。 具体模式、资源要求与证据边界见 [CI 接入说明](README.md#ci-接入与远端执行边界)。 -远端 **NOT_RUN**，未提交、未推送，不能将本地结果写成远端 CI 已通过。 -W5 本地功能出口与本轮容器收尾已完成；默认配置及 WSL 配置不变。 +上述容器收尾和提交前检查发生时尚未提交、未推送。随后 W5 已发布为 PR115， +提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 的普通 PR CI 已通过；完整 W5 +专项因 fork Actions 在 dispatch 阶段返回 HTTP 422，仍为 **BLOCKED / NOT_RUN**。 +完整容器三例 24 步和第二人独立复现尚待完成，不能将本页单 token 结果扩张为这些验收通过。 +最新状态见 [W5 阶段表](README.md#阶段与验收状态)。W5 本地功能出口与容器收尾已完成； +默认配置及 WSL 配置不变，PR 未合并。 diff --git a/docs/llm-deploy-v1.0/W5/independent-reproduction.md b/docs/llm-deploy-v1.0/W5/independent-reproduction.md new file mode 100644 index 00000000..5b1a69a3 --- /dev/null +++ b/docs/llm-deploy-v1.0/W5/independent-reproduction.md @@ -0,0 +1,365 @@ +# W5：PR115 第二人独立复现交接 + +本说明供同事在独立的 Ubuntu 环境中执行，针对 +[PR115](https://github.com/ScratchV-Compiler/ScratchV/pull/115) 的固定提交 +`8ad347477f7f5ba15b2da7c91178b3c6668456cc`，不跟随分支最新提交。 +该 SHA 是已有完整数值验收的功能基线；本次后续提交仅修改审查超时处理、相关测试和文档， +生成实现未变，因此复现仍固定此版本，不随分支移动。 +本文整理完成不表示第二人已复现；执行人回传下面的记录后才能签收。 +作者再次执行或 CI 通过不能代替第二人独立复现，见 +[Linux 交付约定](../LINUX_REPRODUCTION.md#4-执行与留存证据)。 + +## 复现范围与固定环境 + +使用新 clone、新容器构建和独立输出目录，只从固定源码及既有资产来源准备环境。 +不要复制作者的虚拟环境、`output/`、ELF、bundle 或报告。 +复用 [deploy_w5.sh](../../../scripts/deploy_w5.sh)，容器内独立构建一次 ELF/bundle； +三条用例的 **24 次前向共用同一构建**，每个 token 仍启动一次 QEMU。 +不先单独运行 Paris、不重录 ORT 参考，也不额外跑 W4 完整前向。 + +| 项目 | 本次约定 | +|---|---| +| 源码 | 上述完整 SHA；全程工作树干净，不修改文件以消除失败 | +| 主机 | Ubuntu 24.04 x86_64；Docker Engine/Buildx 已安装且当前账号可用 | +| 主机工具 | Bash、Git/Git LFS、Python 3.12、GNU time、coreutils（timeout）、util-linux（setsid）、tar | +| 基础镜像 | `public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca` | +| 容器 Python/依赖 | Python 3.12；依赖以固定提交的 `pyproject.toml[project.optional-dependencies].llm-deploy` 为准，不另安装一套推理环境 | +| 已有实跑版本 | Python 3.12.3、QEMU 8.2.2（Ubuntu 包 `1:8.2.2+ds-0ubuntu1.18`）、Zig 0.14.1、ORT 1.22.1、NumPy 2.2.6、Torch 2.7.1+cpu | +| 模型 | `Qwen/Qwen3-0.6B`；revision `c1899de289a04d12100db370d81485cdf75e47ca`；固定 FP32 ONNX、batch=1、容量 256、无 KV cache | +| 生成配置 | 4 hart；`blocked_fma / 2x4 / column-panel`；Mean `two-four-lane-last-axis`；非线性 `fp32-reference`；padding `none` | +| 资源 | 镜像和资产准备后，至少 4 个可用逻辑 CPU、10 GiB 可用内存、10 GiB 剩余磁盘；初始建议预留至少 25 GiB 磁盘 | +| 时间预算 | 预留数小时；单步 1800 s，CLI 总预算 16500 s，外层 280 分钟，超时后最多再等 30 s | + +基础镜像 digest 来自原容器实际拉取和构建记录，用于固定已有制品身份，不新增哈希流程。 +**它只固定基础镜像**：原 Dockerfile 的 apt 软件源和未逐项锁定的间接依赖仍可能更新。 +下面记录实际镜像 ID、系统包、依赖和工具版本，并确认 suite 前后镜像 ID 一致。 +若 QEMU 不再是 8.2.2、固定 Python 依赖不匹配，或系统包版本与上表不同， +先回传环境差异，由负责人确认后再运行完整套件；不要自行降级、改 Dockerfile 或宣称完全相同环境。 +原有 Docker 实跑使用 Engine 29.9.2 / Buildx 0.38.1；同事记录自己的版本，版本不同须注明。 +这些说明不要求修改 WSL/Windows 或安装宿主 QEMU。 + +**默认值不变**：默认单 hart，仍可手动选择 `1/2/4` 或 `auto`。 +此交接只显式验收上述 4 hart 数值策略；不把它写成默认算术配置已通过完整模型。 +`auto` 在构建时解析一次，不在 token 间调整。4 hart 也不意味着容器只获配 4 个宿主 CPU。 +W5 CLI 固定 padding 关闭，**没有** `--qemu-workspace-layout` 参数。 + +## 1. 新 clone 并固定提交 + +以下命令在同一个 Bash 会话按顺序执行，从有足够空间的父目录开始。 +目录名 `scratchv-w5-repro` 必须不存在；重试时改用新的目录名，保留失败记录。 + +```bash +set -euo pipefail +export W5_COMMIT=8ad347477f7f5ba15b2da7c91178b3c6668456cc +test ! -e scratchv-w5-repro +GIT_LFS_SKIP_SMUDGE=1 git clone --no-checkout --filter=blob:none \ + https://github.com/ScratchV-Compiler/ScratchV.git scratchv-w5-repro +cd scratchv-w5-repro +git fetch --no-tags origin refs/pull/115/head +GIT_LFS_SKIP_SMUDGE=1 git checkout --detach "$W5_COMMIT" +test "$(git rev-parse HEAD)" = "$W5_COMMIT" +test -z "$(git status --porcelain)" + +export W5_RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" +export SCRATCHV_CONTAINER_ENGINE=docker +export SCRATCHV_W5_IMAGE="scratchv-w5:independent-$W5_RUN_ID" +export SCRATCHV_W5_ASSETS="$PWD/output/w5-independent-assets" +export SCRATCHV_W5_OUTPUT="$PWD/output/w5-independent-$W5_RUN_ID" +export SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca +test ! -e "$SCRATCHV_W5_ASSETS" +test ! -e "$SCRATCHV_W5_OUTPUT" +mkdir -p "$SCRATCHV_W5_OUTPUT" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/start-utc.txt" +{ + git rev-parse HEAD + git status --porcelain + git remote -v +} > "$SCRATCHV_W5_OUTPUT/source.txt" +{ + cat /etc/os-release + uname -a + lscpu + free -b + df -B1 . + python3 --version + git --version + git lfs version + docker version + docker buildx version + docker info --format 'OS={{.OSType}} Arch={{.Architecture}} CPUs={{.NCPU}} Memory={{.MemTotal}}' + command -v setsid timeout tar + /usr/bin/time --version +} > "$SCRATCHV_W5_OUTPUT/runner-resources.txt" 2>&1 +docker info > /dev/null +test "$(uname -m)" = x86_64 +python3 -c 'import sys; assert sys.version_info[:2] == (3, 12)' +printf 'Run output: %s\n' "$SCRATCHV_W5_OUTPUT" +``` + +检查 `runner-resources.txt`：宿主应为 Ubuntu 24.04，Docker daemon 也是本机 Linux； +若使用远端 daemon、容器资源配额不足或没有引擎，先停止并回报，不用修改 WSL 来凑条件。 +挂载目录不能包含逗号。不要把私人凭据写入报告。 + +## 2. 构建同一镜像并记录实际环境 + +复用现有 Dockerfile；随后部署脚本会再检查构建并复用 Docker 缓存。 +基础镜像和构建参数不变，不使用作者构建好的镜像。 + +```bash +docker build --file deployment/w5/Dockerfile \ + --build-arg "BASE_IMAGE=$SCRATCHV_W5_BASE_IMAGE" \ + --tag "$SCRATCHV_W5_IMAGE" . \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/image-build.log" +docker image inspect "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/container-image.json" +docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/image-id-before.txt" +docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" -m pip freeze \ + > "$SCRATCHV_W5_OUTPUT/dependencies.txt" +docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" \ + -c 'import json; from probes.w1_qwen3_export.run import environment; print(json.dumps(environment(export=True), indent=2))' \ + > "$SCRATCHV_W5_OUTPUT/image-environment.json" +docker run --rm --entrypoint sh "$SCRATCHV_W5_IMAGE" -c \ + 'python --version; python -m ziglang version; qemu-system-riscv64 --version; dpkg-query -W python3.12 qemu-system-misc' \ + > "$SCRATCHV_W5_OUTPUT/image-tools.txt" +cat "$SCRATCHV_W5_OUTPUT/image-tools.txt" +``` + +人工核对 `image-tools.txt` 与环境表后再继续。版本有偏差时，保留日志并联系负责人； +部署脚本本身不锁 apt 补丁，也不会替你判定偏差是否可接受。 + +## 3. 独立获取固定资产 + +下载只调用现有 `acquire_model`，沿用 W1 manifest 中已有的下载/分片完整性检查。 +它只用 Python 标准库，不执行模型、不重新导出、不安装宿主推理依赖。 +Tokenizer 从同一提交的 Git LFS 获取。 + +```bash +git lfs install --local +git lfs pull --include="assets/qwen3-0.6b/tokenizer/*" --exclude="" \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/tokenizer-acquisition.log" +python3 - <<'PY' 2>&1 | tee "$SCRATCHV_W5_OUTPUT/model-acquisition.log" +import json, os, shutil +from pathlib import Path +from probes.w1_qwen3_export.run import acquire_model +assets = Path(os.environ["SCRATCHV_W5_ASSETS"]) +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +result = acquire_model(assets / "onnx") +shutil.copytree("assets/qwen3-0.6b/tokenizer", assets / "tokenizer") +(out / "acquisition.json").write_text(json.dumps(result, indent=2) + "\n") +PY +test -z "$(git status --porcelain)" +``` + +不要另跑参考采集脚本或添加 `--record`：suite 内部已经逐步对照当轮 ORT 与冻结参考， +无需再运行一遍完整 ORT 生成作为“预热”。 + +## 4. 先过三轮模板及资源预检 + +直接复用 CI 的两条官方模板测试，确认有/无 system prompt 的三轮历史拼接。 +这两条不运行完整模型，也不代表三轮完整 QEMU 聊天已验收。 + +```bash +docker run --rm --init --user "$(id -u):$(id -g)" --env HOME=/tmp \ + --env SCRATCHV_QWEN3_TOKENIZER_DIR=/assets/tokenizer \ + --mount "type=bind,src=$SCRATCHV_W5_ASSETS,dst=/assets,readonly" \ + --mount "type=bind,src=$SCRATCHV_W5_OUTPUT,dst=/output" \ + --mount "type=bind,src=$PWD/tests/test_conversation.py,dst=/tmp/test_conversation.py,readonly" \ + --entrypoint python "$SCRATCHV_W5_IMAGE" \ + -m pytest -q -c /dev/null -p no:cacheprovider \ + /tmp/test_conversation.py::test_official_three_round_rendering_matches_transformers \ + --junitxml /output/multiturn-tests.xml \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/multiturn-tests.log" +python3 - <<'PY' +import json, os, shutil, xml.etree.ElementTree as ET +from pathlib import Path +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +root = ET.parse(out / "multiturn-tests.xml").getroot() +for suffix in ("None", "Give brief answers."): + name = f"test_official_three_round_rendering_matches_transformers[{suffix}]" + rows = [node for node in root.iter("testcase") if node.get("name") == name] + if len(rows) != 1 or any(rows[0].find(tag) is not None for tag in ("skipped", "error", "failure")): + raise SystemExit(f"Official three-round template must execute and pass: {name}") +mem = {line.split(":")[0]: int(line.split()[1]) * 1024 + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith(("MemTotal:", "MemAvailable:"))} +evidence = {"stage": "after_image_and_assets", + "available_cpus": len(os.sched_getaffinity(0)), "memory": mem, + "disk_free_bytes": shutil.disk_usage(out).free, + "minimum_available_cpus": 4, + "minimum_available_memory_bytes": 10 * 1024**3, + "minimum_disk_free_bytes": 10 * 1024**3} +evidence["passed"] = (evidence["available_cpus"] >= 4 + and mem.get("MemAvailable", 0) >= 10 * 1024**3 + and evidence["disk_free_bytes"] >= 10 * 1024**3) +(out / "resource-preflight.json").write_text(json.dumps(evidence, indent=2) + "\n") +if not evidence["passed"]: + raise SystemExit("Insufficient W5 CPU/RAM/disk; generation NOT RUN.") +PY +``` + +预检检查主机可用资源；还需确认 Docker 没有更小的 CPU/内存配额。 +任一预检失败即停止，不运行 suite；回传已有日志，标记 `NOT_RUN`，不自动放宽门槛。 + +## 5. 一次运行三条完整用例，并始终留证 + +整块复制执行。`--reference-suite` 自带三条用例及每例 8-token 预算， +不要同时传 `--max-new-tokens`，不要传 `--reuse-build`。 +代码从固定模型重新构建一次，跨用例自动复用同一 ELF/bundle。 + +```bash +test "$(git rev-parse HEAD)" = "$W5_COMMIT" +test -z "$(git status --porcelain)" +test ! -e "$SCRATCHV_W5_OUTPUT/run" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-start-utc.txt" + +set +e +timeout --signal=INT --kill-after=30s 280m \ + /usr/bin/time -v -o "$SCRATCHV_W5_OUTPUT/process-time.txt" \ + bash scripts/deploy_w5.sh \ + --model /assets/onnx/model.onnx --tokenizer-dir /assets/tokenizer \ + --reference-suite --verify-onnx /assets/onnx/model.onnx \ + --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ + --matmul-rhs-layout column-panel \ + --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ + --timeout 1800 --total-timeout 16500 \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/runner.log" +pipeline_status=("${PIPESTATUS[@]}") +printf '%s\n' "${pipeline_status[0]}" > "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" +printf '%s\n' "${pipeline_status[1]}" > "$SCRATCHV_W5_OUTPUT/log-exit-code.txt" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-end-utc.txt" + +# 此唯一 tag 只供本次使用；不停止其他人的容器。 +{ + containers=$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE") + list_status=$? + printf 'list_exit_code=%s\n' "$list_status" + for container in $containers; do + docker stop --time 30 "$container" + printf 'stop_exit_code=%s\n' "$?" + done + docker ps --filter "ancestor=$SCRATCHV_W5_IMAGE" \ + --format '{{.ID}} {{.Status}}' + printf 'final_list_exit_code=%s\n' "$?" +} > "$SCRATCHV_W5_OUTPUT/container-cleanup.log" 2>&1 + +docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/image-id-after.txt" +cmp "$SCRATCHV_W5_OUTPUT/image-id-before.txt" \ + "$SCRATCHV_W5_OUTPUT/image-id-after.txt" +printf '%s\n' "$?" > "$SCRATCHV_W5_OUTPUT/image-match-exit-code.txt" + +python3 -B -X utf8 scripts/check_w5_ci.py \ + --report "$SCRATCHV_W5_OUTPUT/run/report.json" \ + --reference probes/w5_generation/references.json \ + --exit-code-file "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" \ + --output "$SCRATCHV_W5_OUTPUT/ci-gate.json" \ + > "$SCRATCHV_W5_OUTPUT/gate-check.log" 2>&1 +printf '%s\n' "$?" > "$SCRATCHV_W5_OUTPUT/gate-exit-code.txt" +cat "$SCRATCHV_W5_OUTPUT/gate-check.log" +{ + date -u +%FT%TZ + git rev-parse HEAD + git status --porcelain + free -b + df -B1 . +} > "$SCRATCHV_W5_OUTPUT/final-resources.txt" +set -e +printf 'Preserved output: %s\n' "$SCRATCHV_W5_OUTPUT" +``` + +这里为失败留证临时关闭 `errexit`，**脚本走到最后不等于 PASS**。 +即使 suite 非零退出，仍运行容器清理和已有报告检查器；报告缺失时 checker 应失败， +不能补造成功字段。`ci-gate.json` 在本地的 `commit` 可为 null；实际 SHA 由 +`source.txt` 和最终 Git 记录确认，不冒充 GitHub Actions 执行。 + +若手动 Ctrl-C 使整段 shell 中断，重新进入同一 clone，恢复上面输出目录和镜像变量， +执行本节从“此唯一 tag”注释之后的收尾段，再打包已有报告。不要直接重新跑模型。 +主机掉电、daemon 失联或强制终止可能阻止清理；如清理日志有非零码或最后仍列出容器， +标为未完成并回报，不视为通过。 + +## 6. 三条用例及共同验收标准 + +全部为 **raw completion**：按 prompt 原文 tokenize，不加 chat 模板，greedy， +每例新生成 8 token，停止原因均为 `max_new_tokens`。字符串包含**前导空格**。 + +| 用例 | 原始 prompt | 必须匹配的新生成文本 | +|---|---|---| +| France | `The capital of France is` | ` Paris. The capital of Italy is Rome` | +| Germany | `The capital of Germany is` | ` Berlin. The capital of France is Paris` | +| Japan | `The capital of Japan is` | ` Tokyo. The capital of the United States` | + +新生成 token IDs 也必须与[冻结参考](../../../probes/w5_generation/references.json)完全一致： + +- France：`[12095,13,576,6722,315,15344,374,21718]`。 +- Germany：`[19846,13,576,6722,315,9625,374,12095]`。 +- Japan：`[26194,13,576,6722,315,279,3639,4180]`。 + +只看见 Paris/Berlin/Tokyo 不够。`check_w5_ci.py` 必须退出 0， +`ci-gate.json` 和 `run/report.json` 都为 PASS，且具备： + +- **3/3 用例、24/24 次 QEMU 前向和数值对照**；prompt IDs、生成 IDs、文本和停止原因均匹配。 +- 每步比较完整 `256×151936=38,895,616` 个 logits，包括补齐行；均有限，严格 `max_abs < 1e-3`、`rtol=0`；QEMU、当轮 ORT、冻结参考逐 token 一致。 +- 28 层完整图审计通过；FMA 的 12 行 binary32 位模式和 FCSR 检查全部通过。 +- 所有步骤使用同一 ELF/bundle 的既有身份；所选策略完全匹配，padding `none`。 +- QEMU 正常完成、退出 0、已回收，无清理错误；套件退出码为 0。 + +此外人工确认：官方三轮模板 **2/2 PASS、0 skip**，环境/资源预检通过， +`log-exit-code.txt` 和 `image-match-exit-code.txt` 为 0，源码未改， +`container-cleanup.log` 中命令全部成功且最后没有容器行。 +任何失败、跳过或缺失证据都不能签为完整 PASS，不以放宽数值阈值或重录参考处理。 + +`container_reproduction=NOT_ASSERTED` 是内部 CLI 的既有字段，不要改成 PASS。 +容器构建/执行日志、真实退出码和完整 suite 的通过记录共同证明本次独立容器复现。 + +## 7. 回传材料与签收 + +在 `$SCRATCHV_W5_OUTPUT/handoff.txt` 填写以下内容（可用编辑器，不改项目源码）： + +| 字段 | 要填写的内容 | +|---|---| +| 执行人/日期 | 姓名或团队账号，UTC 开始/结束时间 | +| 独立性 | 独立 clone、容器构建和输出目录；是否复制作者产物（应为否） | +| 版本/环境 | 完整提交 SHA、主机/CPU/内存、Docker/Buildx、实际镜像 ID、所有已确认的环境偏差 | +| 结果 | PASS / FAIL / NOT_RUN；三例各自 token 数、文本/IDs 是否匹配、数值最大误差、模板 2/2 结果 | +| 资源与计时 | CLI 总耗时、各步 UART/进程耗时、可用 RSS 观测峰值和采样缺失情况；未采集字段写“未独立采集” | +| 清理/异常 | 容器是否清空、工作树是否干净，失败阶段、日志位置及是否有人为中断 | + +**计时口径**：`process-time.txt` 的 CPU/RSS 是 Docker 客户端，不是容器或 QEMU 的总 CPU/RSS。 +逐步 QEMU 耗时/RSS 使用 `run/steps/*/run.json` 中已有数据；若采样状态为 partial， +注明为观测值。现有链路没有独立采集容器总 CPU，就写“未独立采集”； +不要把作者的历史 CPU 数据填到同事报告，不据此宣称跨机器加速比。 + +在同一 Bash 会话执行以下命令，复用 CI 的报告扩展名白名单，并额外保留小型 UART 完成帧 +`uart.bin`；排除缓存和其他二进制大制品： + +```bash +python3 - <<'PY' +import os, tarfile +from pathlib import Path +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +allowed = {".json", ".xml", ".txt", ".log", ".stderr", ".stdout"} +archive = out.parent / (out.name + "-reports.tar.gz") +if archive.exists(): + raise SystemExit(f"Keep earlier evidence; archive already exists: {archive}") +with tarfile.open(archive, "w:gz") as bundle: + for path in sorted(out.rglob("*")): + relative = path.relative_to(out) + if (path.is_file() and not path.is_symlink() + and ".zig-cache" not in relative.parts + and (path.suffix in allowed or path.name == "uart.bin")): + bundle.add(path, arcname=str(Path(out.name) / relative), recursive=False) +print(archive) +PY +``` + +回传 `output/w5-independent--reports.tar.gz`，至少包含： + +- `handoff.txt`、`source.txt`、UTC 时间、资源预检、实际镜像/工具/依赖记录、下载记录。 +- `image-build.log`、`multiturn-tests.xml/.log`、`runner.log`、所有退出码、清理日志和 `ci-gate.json`。 +- `run/report.json`、`run/executable.json`、`run/build/` 中的 JSON， + `run/steps/step-0000…0023/run.json` 和 UART 完成帧 `uart.bin`/错误日志，以及 `run/fma-conformance/` 中的报告。 + +失败时也按同一方式回传已有文件，并说明缺失项。准备阶段失败则标记 suite `NOT_RUN`。 +无需传权重、ELF、原始 logits、Zig 缓存；保留本地原始产物直到签收，若需重试则新建目录， +不覆盖旧报告。本任务不要求执行人提交代码、发起 AI 审查、操作 Actions 开关或合并 PR。 diff --git a/docs/llm-deploy-v1.0/W5/qemu-generation.md b/docs/llm-deploy-v1.0/W5/qemu-generation.md index 01153225..f2e342d9 100644 --- a/docs/llm-deploy-v1.0/W5/qemu-generation.md +++ b/docs/llm-deploy-v1.0/W5/qemu-generation.md @@ -500,7 +500,9 @@ QEMU RSS 观测峰值 **5.398GiB**。RSS 有 2071 个有效样本，**1 次读 检查通过，14 个新增必跑节点仅收集核对、未重跑；已有真实 24 步报告的严格检查通过。 后续提交前检查与修复见 [收尾记录](README.md#提交前检查与收尾2026-10-23)。 模式和资源门槛见 [CI 接入说明](README.md#ci-接入与远端执行边界)； -未提交/推送，远端 **NOT_RUN**。 +上述记录属于提交前时点。随后 W5 已发布为 PR115，提交 `8ad3474` 的普通 PR CI +已通过；完整专项因 fork Actions 拒绝 dispatch，仍为 **BLOCKED / NOT_RUN**。 +第二人说明已准备、执行待办，详见 [当前阶段表](README.md#阶段与验收状态)。 证据相对仓库根：`output/w5-container-20261023/small-summary.json`、 `small-runs/real-qemu-tests.xml`（同目录）、`build-cancel/report.json`、 @@ -572,7 +574,7 @@ report 到 executable metadata 的身份校验，缺失或漂移先于加载/编 1. **默认算术配置的数值边界仍需保留**：显式组合策略已经完成 Host 和 QEMU 各三例 24/24 步验收；初始 `mean sequential / nonlinear libm` 配置的三例首步仍有完整 logits 超过 `1e-3` 的历史失败，不能把显式策略 PASS 推广为默认路径 PASS。默认值未改变，完整验收需显式选择本文配置。独立单次前向也已通过，并已单独记录，不能与 24 步套件混算。 2. **本地功能与远端门槛分开**：真实容器镜像、小图、中断清理及唯一追加的完整 France 单 token 已通过，W5 本地功能出口已满足；没有重复容器三例套件。引擎及 socket 已停止并禁用自启，临时权限已恢复，WSL 配置不变。 -3. **CI 本地实现与检查通过，远端 NOT_RUN**：提交前补齐现有 llm-deploy 的 W5-only 手动入口，811 项相关检查及最终 11 项重叠边界复查通过；保留 12 个真实 QEMU 小图、2 个官方模板及完整生成门槛。14 个真实节点本轮没有重新运行；远端资源和时限仍待授权提交/推送后验证，不把本地通过写成远端 Actions 已通过。详见 [提交前收尾](README.md#提交前检查与收尾2026-10-23)。 +3. **普通 CI 通过，完整专项仍阻塞**：PR115 提交 `8ad3474` 的普通 CI `39919353111` 已通过（14 成功、7 个按路由跳过），12 个 W5 真实 QEMU 小图和 2 个官方模板节点均通过。提交前的 811 项相关检查及最终 11 项重叠复查仍按原时点保留；完整专项因 fork Actions 在 dispatch 阶段返回 HTTP 422，未创建运行，故远端资源、时限和容器三例 24 步仍待实跑。第二人独立复现说明已准备，执行及签收待办。详见 [当前阶段表](README.md#阶段与验收状态)。 4. **同步边界**：QEMU UART/QMP 等待可响应取消并回收进程；构建/传输/可选 ORT 校验等同步工作仍在各自阶段边界检查总预算，Host deadline 不承诺硬中断任意 ORT 调用。SIGINT/SIGTERM 会经已有编译/QEMU清理路径退出，不返回成功的部分生成。 固定 256 长度、无 KV cache 意味着每个新 token 重算完整前向；当前实现复用构建， diff --git a/scripts/ai_code_review.py b/scripts/ai_code_review.py index a9189e93..5625e898 100644 --- a/scripts/ai_code_review.py +++ b/scripts/ai_code_review.py @@ -3,6 +3,7 @@ import argparse from collections import Counter +from functools import partial import json import os from pathlib import Path @@ -11,7 +12,8 @@ import urllib.request API_URL = "https://token.sensenova.cn/v1/chat/completions" -BATCH_SIZE = 10 +BATCH_SIZE = 5 +REQUEST_TIMEOUT_SECONDS = 180 MAX_RESPONSE_BYTES = 2 * 1024 * 1024 @@ -35,6 +37,14 @@ def patch_problem(file: dict) -> str | None: return None +def plan_batch_size(plan: dict) -> int: + # Persisted ten-file plans must retain their original file-to-batch mapping. + size = plan.get("batch_size", BATCH_SIZE) + if type(size) is not int or size not in (BATCH_SIZE, 10): + raise ValueError("Invalid review batch size; expected 5 or legacy 10.") + return size + + def make_plan(raw: dict) -> dict: files = raw.get("files", []) errors = list(raw.get("errors", [])) @@ -51,9 +61,14 @@ def make_plan(raw: dict) -> dict: errors.append("Fetched file count differs from PR changed_files (including the API 3000-file limit).") if not raw.get("head") or raw.get("head") != raw.get("head_after"): errors.append("PR head changed while fetching files, or its identity is missing.") - batches = max(1, (len(files) + BATCH_SIZE - 1) // BATCH_SIZE) + try: + batch_size = plan_batch_size(raw) + except ValueError as exc: + errors.append(str(exc)) + batch_size = BATCH_SIZE + batches = max(1, (len(files) + batch_size - 1) // batch_size) if batches > 256: - errors.append("PR exceeds the 256-job matrix limit at 10 files per batch; review requires a separately sized run.") + errors.append(f"PR exceeds the 256-job matrix limit at {batch_size} files per batch; review requires a separately sized run.") batches = 1 # Keep the failed plan valid for artifact publication; review jobs are skipped. return { "repository": raw.get("repository"), @@ -62,12 +77,12 @@ def make_plan(raw: dict) -> dict: "changed_files": raw.get("changed_files"), "files": files, "errors": errors, - "batch_size": BATCH_SIZE, + "batch_size": batch_size, "batches": batches, } -def request_review(payload: dict, key: str) -> tuple[str, int, str]: +def request_review(payload: dict, key: str, *, timeout_seconds: int = REQUEST_TIMEOUT_SECONDS) -> tuple[str, int, str]: """Return content, HTTP status and Retry-After without logging secret-bearing data.""" req = urllib.request.Request( API_URL, json.dumps(payload).encode(), @@ -75,7 +90,7 @@ def request_review(payload: dict, key: str) -> tuple[str, int, str]: method="POST", ) try: - response = urllib.request.urlopen(req, timeout=90) + response = urllib.request.urlopen(req, timeout=timeout_seconds) except urllib.error.HTTPError as exc: return "", exc.code, exc.headers.get("Retry-After", "") with response: @@ -95,7 +110,7 @@ def request_review(payload: dict, key: str) -> tuple[str, int, str]: def review_file(file: dict, prompt: str, key: str, request=request_review, sleep=time.sleep) -> dict: - record = {"filename": file["filename"], "status": "pending", "attempts": 0} + record = {"filename": file["filename"], "status": "pending", "attempts": 0, "request_attempts": []} problem = patch_problem(file) if problem: record.update(status="unreviewable", reason=problem) @@ -118,9 +133,13 @@ def review_file(file: dict, prompt: str, key: str, request=request_review, sleep record["attempts"] = attempt retry = True delay = 5 + started = time.monotonic() + outcome = "interrupted" try: content, status, retry_after = request(payload, key) + outcome = f"HTTP {status}" if status == 200: + record.pop("reason", None) record.update(status="reviewed", review=content) return record record["reason"] = f"HTTP {status}" @@ -131,7 +150,14 @@ def review_file(file: dict, prompt: str, key: str, request=request_review, sleep pass except (OSError, ValueError, KeyError, IndexError, TypeError) as exc: # Exception text or HTTP response bodies may echo credentials; record only the class. - record["reason"] = f"No usable API result ({type(exc).__name__})." + outcome = type(exc).__name__ + record["reason"] = f"No usable API result ({outcome})." + finally: + # Record only timing/status, never request/response bodies or exception text. + record["request_attempts"].append({ + "attempt": attempt, "outcome": outcome, + "elapsed_seconds": round(time.monotonic() - started, 6), + }) if attempt == 2 or not retry: break sleep(delay) @@ -145,7 +171,12 @@ def run_batch(plan: dict, batch: int, prompt: str, output: Path, key: str, raise ValueError("Cannot review an invalid or stale plan.") if not 0 <= batch < plan["batches"]: raise ValueError("Invalid batch index.") - files = plan["files"][batch * BATCH_SIZE:(batch + 1) * BATCH_SIZE] + batch_size = plan_batch_size(plan) + if request is request_review: + # Legacy ten-file plans keep their original window and batch time budget. + timeout_seconds = REQUEST_TIMEOUT_SECONDS if batch_size == BATCH_SIZE else 90 + request = partial(request_review, timeout_seconds=timeout_seconds) + files = plan["files"][batch * batch_size:(batch + 1) * batch_size] result = { "head": plan["head"], "batch": batch, "files": [{"filename": file["filename"], "status": "pending", "attempts": 0} for file in files], @@ -165,6 +196,12 @@ def summarize(plan: dict, batches: list[dict], current_head: str | None = None) if current_head is not None and current_head != plan.get("head"): errors.append("PR head changed after planning; results are stale.") expected = [file["filename"] for file in plan["files"]] + try: + batch_size = plan_batch_size(plan) + except ValueError as exc: + errors.append(str(exc)) + batches = [] + batch_size = BATCH_SIZE records: dict[str, dict] = {} for batch in batches: if not isinstance(batch, dict) or not isinstance(batch.get("files"), list): @@ -177,7 +214,7 @@ def summarize(plan: dict, batches: list[dict], current_head: str | None = None) if type(number) is not int or not 0 <= number < plan["batches"]: errors.append("Invalid batch identity.") continue - allowed = set(expected[number * BATCH_SIZE:(number + 1) * BATCH_SIZE]) + allowed = set(expected[number * batch_size:(number + 1) * batch_size]) for record in batch.get("files", []): name = record.get("filename") if isinstance(record, dict) else None if not isinstance(name, str) or name not in allowed: diff --git a/tests/test_ai_code_review.py b/tests/test_ai_code_review.py index 66c86a5a..cee75e4d 100644 --- a/tests/test_ai_code_review.py +++ b/tests/test_ai_code_review.py @@ -192,11 +192,11 @@ def interrupted(*_): def test_missing_job_and_pending_record_fail_accounting(): - p = plan(11) - batch = reviewed_batch(names=tuple(f"{i}.py" for i in range(9))) - batch["files"].append({"filename": "9.py", "status": "pending"}) + p = plan(6) + batch = reviewed_batch(names=tuple(f"{i}.py" for i in range(4))) + batch["files"].append({"filename": "4.py", "status": "pending"}) summary = review.summarize(p, [batch]) - assert summary["counts"] == {"reviewed": 9, "failed": 2, "unreviewable": 0} + assert summary["counts"] == {"reviewed": 4, "failed": 2, "unreviewable": 0} assert not summary["complete"] @@ -261,9 +261,10 @@ def test_failed_fetch_still_writes_plan_and_matrix(tmp_path, monkeypatch): assert json.loads(output.read_text().split("=", 1)[1]) == {"include": [{"batch": 0}]} def test_large_pr_reports_matrix_limit_without_generating_invalid_matrix(): - p = plan(2561) + assert plan(1280)["batches"] == 256 + p = plan(1281) assert p["errors"] and p["batches"] == 1 - assert "256-job" in p["errors"][0] + assert "256-job" in p["errors"][0] and "5 files per batch" in p["errors"][0] @pytest.mark.parametrize("invalid", [[], None, {"head": "abc", "batch": 0, "files": [None]}, @@ -306,3 +307,112 @@ def request(*_): summary = review.summarize(plan(2), [result]) assert summary["counts"] == {"failed": 1, "reviewed": 1, "unreviewable": 0} assert not summary["complete"] + + +def test_longer_request_window_uses_smaller_batches_with_job_margin(monkeypatch): + calls = [] + class Response: + status = 200 + def __enter__(self): return self + def __exit__(self, *_): return None + def read(self, _): + return b'{"choices":[{"finish_reason":"stop","message":{"content":"No issue."}}]}' + def urlopen(request, *, timeout): + calls.append(timeout) + return Response() + monkeypatch.setattr(review.urllib.request, "urlopen", urlopen) + assert review.request_review({}, "test-key") == ("No issue.", 200, "") + assert calls == [180] + p = plan(47) + assert p["batch_size"] == 5 and p["batches"] == 10 + # Nominal socket timeout windows plus maximal backoff leave setup/upload margin. + nominal_seconds = p["batch_size"] * (2 * calls[0] + 30) + p["batch_size"] - 1 + job_seconds = int(workflow()["jobs"]["review"]["timeout-minutes"]) * 60 + assert nominal_seconds <= job_seconds - 300 + + +@pytest.mark.parametrize("first", [(429, "3"), (503, ""), TimeoutError("private API key/body")]) +def test_each_request_records_sanitized_duration_and_outcome(monkeypatch, first): + ticks = iter((10.0, 12.25, 30.0, 33.5)) + monkeypatch.setattr(review.time, "monotonic", lambda: next(ticks)) + calls = [] + def request(*_): + calls.append(1) + if len(calls) == 1: + if isinstance(first, Exception): + raise first + return "private API body", *first + return ok() + record = review.review_file(file(), "private prompt", "private-key", request=request, sleep=lambda _: None) + outcome = "TimeoutError" if isinstance(first, Exception) else f"HTTP {first[0]}" + assert record["request_attempts"] == [ + {"attempt": 1, "outcome": outcome, "elapsed_seconds": 2.25}, + {"attempt": 2, "outcome": "HTTP 200", "elapsed_seconds": 3.5}, + ] + assert record["status"] == "reviewed" and "reason" not in record + assert "private" not in json.dumps(record) + + +def test_terminal_timeouts_retain_both_sanitized_attempts(monkeypatch): + ticks = iter((0.0, 180.0, 185.0, 365.0)) + monkeypatch.setattr(review.time, "monotonic", lambda: next(ticks)) + def request(*_): + raise TimeoutError("private API key/body") + record = review.review_file(file(), "prompt", "test-key", request=request, sleep=lambda _: None) + assert record["status"] == "failed" and record["attempts"] == 2 + assert record["request_attempts"] == [ + {"attempt": 1, "outcome": "TimeoutError", "elapsed_seconds": 180.0}, + {"attempt": 2, "outcome": "TimeoutError", "elapsed_seconds": 180.0}, + ] + assert "private" not in json.dumps(record) + + +def test_saved_ten_file_plan_keeps_run_and_cli_summary_boundaries(tmp_path): + old = plan(11) + old.update(batch_size=10, batches=2) + download = tmp_path / "download" + review.write_json(download / "plan.json", old) + batches = [review.run_batch(old, number, "prompt", download / f"batch-{number}.json", "test-key", + request=ok, sleep=lambda _: None) for number in range(2)] + assert [[r["filename"] for r in batch["files"]] for batch in batches] == [ + [f"{i}.py" for i in range(10)], ["10.py"], + ] + assert review.summarize(old, batches)["complete"] + out = tmp_path / "summary" + assert review.main(["summarize", "--directory", str(download), "--output", str(out), + "--run-url", "https://example.test/run"]) == 0 + result = json.loads((out / "summary.json").read_text()) + assert result["complete"] and result["counts"] == {"reviewed": 11, "failed": 0, "unreviewable": 0} + assert review.main(["gate", "--summary", str(out / "summary.json")]) == 0 + + +@pytest.mark.parametrize("invalid", [0, -1, 6, 11, True, "10", None]) +def test_invalid_saved_batch_size_fails_closed(tmp_path, invalid): + saved = plan(1) + saved["batch_size"] = invalid + with pytest.raises(ValueError, match="batch size"): + review.run_batch(saved, 0, "prompt", tmp_path / "batch.json", "test-key", request=ok) + summary = review.summarize(saved, [reviewed_batch()]) + assert not summary["complete"] and summary["counts"]["failed"] == 1 + rebuilt = review.make_plan({**saved, "head_after": saved["head"]}) + assert rebuilt["errors"] and not review.summarize(rebuilt, [reviewed_batch()])["complete"] + + +@pytest.mark.parametrize("batch_size,timeout_seconds", [(5, 180), (10, 90)]) +def test_real_adapter_keeps_each_plan_timeout_budget(tmp_path, monkeypatch, batch_size, timeout_seconds): + calls = [] + class Response: + status = 200 + def __enter__(self): return self + def __exit__(self, *_): return None + def read(self, _): + return b'{"choices":[{"finish_reason":"stop","message":{"content":"No issue."}}]}' + def urlopen(request, *, timeout): + calls.append(timeout) + return Response() + monkeypatch.setattr(review.urllib.request, "urlopen", urlopen) + saved = plan(1) + saved["batch_size"] = batch_size + result = review.run_batch(saved, 0, "prompt", tmp_path / "batch.json", "test-key") + assert result["files"][0]["status"] == "reviewed" + assert calls == [timeout_seconds] From 1b1bb56e043d59496b68eda5bba30526a5117d8f Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Sun, 25 Oct 2026 14:50:44 +0800 Subject: [PATCH 4/7] fix(w5): preserve report errors and verify Docker cancellation Preserve primary errors and signal exit codes when report persistence fails. Verify real Docker SIGINT/SIGTERM cleanup and executable reuse on 1/4-hart small graphs. Consolidate W5 development documentation and reproduction instructions. --- docs/llm-deploy-v1.0/W5/README.md | 797 ++++++++++-------- docs/llm-deploy-v1.0/W5/chat.md | 27 - docs/llm-deploy-v1.0/W5/deployment.md | 209 ----- .../W5/independent-reproduction.md | 365 -------- docs/llm-deploy-v1.0/W5/qemu-generation.md | 581 ------------- ...00\345\217\221\350\256\241\345\210\222.md" | 2 +- scripts/run_inference.py | 227 ++--- tests/llm-deploy/w5/README.md | 62 ++ tests/llm-deploy/w5/conftest.py | 19 + .../llm-deploy/w5/test_docker_cancellation.py | 264 ++++++ .../llm-deploy/w5/test_report_persistence.py | 268 ++++++ 11 files changed, 1199 insertions(+), 1622 deletions(-) delete mode 100644 docs/llm-deploy-v1.0/W5/chat.md delete mode 100644 docs/llm-deploy-v1.0/W5/deployment.md delete mode 100644 docs/llm-deploy-v1.0/W5/independent-reproduction.md delete mode 100644 docs/llm-deploy-v1.0/W5/qemu-generation.md create mode 100644 tests/llm-deploy/w5/README.md create mode 100644 tests/llm-deploy/w5/conftest.py create mode 100644 tests/llm-deploy/w5/test_docker_cancellation.py create mode 100644 tests/llm-deploy/w5/test_report_persistence.py diff --git a/docs/llm-deploy-v1.0/W5/README.md b/docs/llm-deploy-v1.0/W5/README.md index 7cc25116..51d9af8b 100644 --- a/docs/llm-deploy-v1.0/W5/README.md +++ b/docs/llm-deploy-v1.0/W5/README.md @@ -1,357 +1,486 @@ -# W5：Host 生成合同与端到端验收 - -合同版本：`w5-generation-v1`，冻结日期：2026-10-23。W5 从 PR104 已合入的 -`main`（`61bded377097900f4c61323269fd6aed41f333d5`）开始，独立分支为 -`codex/w5-generation`。阶段目标和责任分工沿用[开发计划](../开发计划.md)。 - -W5 的生成合同、Host 循环、QEMU 适配/命令行、三用例验收入口、三轮历史和容器部署文件 -现已实现。P0/P1 的冻结 ORT 参考与上轮 1/4 hart 小图记录保留。初始旧算术配置的 -4 hart 完整模型三例均在首步完整 logits 检查失败，这些历史结果仍完整保留。 -后续显式启用 Mean 与非线性参考策略后，Host 三例预检与正式 4 hart QEMU 套件均已 -**24/24 步、3/3 用例通过**。QEMU 每步完整 logits、冻结 token IDs、最终文本和停止 -原因均通过，全部步骤最大绝对误差为 **0.000303983688**，保持严格 `<1e-3` 门槛。 -通过结果限定于下述显式配置，未将默认算术路径标为通过，详见 -[完整模型实跑报告](qemu-generation.md)。本轮干净容器的默认 1 hart、手动 4 hart、 -auto 小图和真实超时/取消、Docker 构建取消检查已通过；完整 28 层 France 单 token -容器部署也已通过，实际输出 ` Paris`，完整 logits 最大误差为 **0.000172138214**。 -**W5 本地功能出口已满足**；代码、测试和报告已发布到 [PR115](https://github.com/ScratchV-Compiler/ScratchV/pull/115)， -已有数值实跑和普通 PR CI 对应功能基线 `8ad347477f7f5ba15b2da7c91178b3c6668456cc`； -本次后续变更仅包含审查超时处理、相关测试和交接/进度文档，不改变生成实现。完整 W5 -专项因 fork 仓库 Actions 拒绝创建运行，仍为 **BLOCKED / NOT_RUN**。第二人独立复现 -说明已准备，尚待实际执行和签收；WSL 配置未改，PR 未合并。 - -使用入口：[QEMU 生成与本轮报告](qemu-generation.md)、[三轮历史](chat.md)、 -[干净容器部署](deployment.md)。 - -PR115 第二人验收使用[独立复现交接说明](independent-reproduction.md)：固定提交 -`8ad347477f7f5ba15b2da7c91178b3c6668456cc`、基础镜像和数值配置，提供从新 clone 到 -三例 24 步验收、失败清理及报告回传的完整命令。文档准备不代表第二人复现已完成， -须由独立执行人回传结果后签收。 - -## 固定生成合同 - -| 项目 | v1 约定 | +# W5：端到端文本生成 + +## 本分支增加了什么 + +W5 将 W4 的单次 QEMU 前向接成文本生成：Host 负责 tokenizer、逐 token 输入更新、 +greedy 选择与停止，QEMU 执行前向。命令行支持原始文本补全、三条固定参考用例和 +多轮文本对话，Linux 一键部署脚本复用同一个入口。 + +- [Host 生成循环](../../../scratchv/runtime/generation.py)规定输入、输出和停止行为， + 每步只接收一个新 token，保留异常、超时和取消信息。 +- [QEMU 适配器](../../../scratchv/runtime/qemu_generation.py)跨 token、跨 prompt + 复用同一 ELF 和权重 bundle，也支持已有构建的显式复用。 +- [多轮对话](../../../scratchv/runtime/conversation.py)使用本地官方聊天模板， + 成功后一起保存 user/assistant 消息，失败时保持原历史。 +- [命令行](../../../scripts/run_inference.py)输出逐步数值与进程记录； + [部署脚本](../../../scripts/deploy_w5.sh)和 + [Dockerfile](../../../deployment/w5/Dockerfile)提供干净 Linux 容器运行方式。 + +**默认配置保持不变**：单 hart、MatMul `sequential / 1x4 / original`、Mean `sequential`、 +非线性 `libm`、padding `none`。可手动指定 `--harts 1/2/4` 或 `--harts auto`； +auto 按进程可用 CPU 在构建或启动时选择 1/2/4，上限 4，不在 token 间动态调整。 +完整模型通过的记录使用下文显式参考策略，不代表默认算术策略也通过同一数值要求。 +W5 没有 `--qemu-workspace-layout` 参数，padding 始终关闭。 + +## 整体怎么实现 + +**生成接口与数值行为。** `generate(prompt, *, tokenizer, forward, max_new_tokens, +cancelled=None, timeout_seconds=None)` 返回不可变的 `GenerationResult`，包含 prompt +IDs、新生成 IDs、文本和 `stop_reason`。固定模型是 FP32 `Qwen/Qwen3-0.6B`， +batch=1、序列容量 256，无 KV cache。 + +| 项目 | 行为 | |---|---| -| 模型 | `Qwen/Qwen3-0.6B`，revision `c1899de289a04d12100db370d81485cdf75e47ca` | -| 资产身份 | 沿用 [W1 manifest](../../../probes/w1_qwen3_export/manifest.json) 中固定 ONNX、权重和 tokenizer；不另建校验体系 | -| 计算范围 | FP32、batch=1、固定序列容量 256、无 KV cache；每个新 token 重新执行完整前向 | -| 输入模式 | raw completion：使用 prompt 原文，不插入 BOS/EOS、系统提示或 chat 模板；tokenizer 按既有 NFC 规范化 | -| 输入张量 | INT64 `input_ids[1,256]`，右侧使用固定 `pad_token_id` 补齐；FP32 `attention_mask[1,1,256,256]`，沿用 W2 causal/key-padding 语义 | -| 有效长度 | 由实际 token 数确定；prompt 中出现 pad/EOS ID 不改变长度，不据此停止 | -| 前向输出 | 完整 FP32 `logits[1,256,151936]`；包括未使用 padding 行在内，所有值都必须有限 | -| 选择规则 | 取 `valid_length - 1` 行，直接 greedy argmax；平局选较小 ID，不额外计算 Softmax | -| 可解码范围 | 选中 ID 必须属于固定 tokenizer 的真实 ID 集合；不能仅以小于模型词表宽度为准 | -| 追加规则 | 每次前向最多追加一个合法 token；不可解码 ID 报错，不屏蔽、不替换、不重采样 | -| 停止顺序 | 新生成 EOS → 总长度达到 256 → 已生成数量达到 `max_new_tokens` | -| EOS | 使用固定 tokenizer 的 `generation_eos_token_ids`：`151645` 和 `151643`；只有新生成 token 触发 EOS | -| 输出文本 | 仅解码新生成 IDs，`skip_special_tokens=True`、`clean_up_tokenization_spaces=False`；完整新生成 IDs 是验证依据，EOS ID 保留在其中 | -| 返回数据 | prompt IDs、新生成 IDs、生成文本和停止原因组成不可变结果；无隐式会话历史 | - -生成核心复用 [W2 输入与 greedy 辅助函数](../../../scratchv/runtime/llm_inputs.py) -和[本地 tokenizer](../../../scratchv/runtime/qwen3_tokenizer.py)。模型词表为 151936， -固定 tokenizer 实际包含 151669 个 ID；这两者不能混用。主循环不调整数值误差门槛， -也不会只检查用于 argmax 的那一行而漏掉其他输出中的 NaN/Inf。 - -`max_new_tokens` 是上限，不预留或扩展容量。prompt 非空且不超过 256 个 token； -空 prompt、超长 prompt、非法 token、负数/布尔预算明确报错，不静默截断。 -0 预算不会调用前向；初始长度已为 256 时也不调用前向，按容量原因返回。 -若一次追加同时触发 EOS、容量和预算，按上表顺序报告原因。生成 EOS 仍计入预算。 - -## Host 调用及失败语义 - -[Host 核心](../../../scratchv/runtime/generation.py) 的公开入口为： - -```python -generate(prompt, *, tokenizer, forward, max_new_tokens, - cancelled=None, timeout_seconds=None) -> GenerationResult +| 原始文本模式 | 直接编码 prompt，不自动插入 BOS、EOS 或聊天模板；沿用 tokenizer 的 NFC 归一化 | +| 输入 | INT64 `input_ids[1,256]`；FP32 `attention_mask[1,1,256,256]`，包含 causal 与 padding mask | +| 输出 | FP32 `logits[1,256,151936]`；全部元素必须有限，包括补齐行 | +| 选词 | 取 `valid_length-1` 行的 greedy argmax，同值选较小 ID；不额外做 Softmax | +| 有效词表 | tokenizer 为 151669，模型输出宽度为 151936；选中不可解码 ID 时失败，不过滤后重选 | +| 停止 | 对新 token 按 EOS、容量、预算顺序判断；EOS IDs 为 `151645`、`151643` | +| 输入与预算 | prompt 非空且不超过 256，不隐式截断；零预算或初始上下文已满时不调用 backend | +| 异常与中断 | 不返回看似成功的部分生成；QEMU 适配器负责运行中的超时、取消和进程回收 | + +每步准备当前 token 输入和 mask、运行前向并取下一个 token。复用的是构建产物； +每 token 仍启动一个 QEMU 进程并装载权重。编译、FMA 预检和 ORT session 初始化 +不按 token 重做；完成后沿用 UART 完成帧及 QMP stop、dump、quit。已有构建元数据 +检查继续约束 ELF/bundle 身份与所选配置。 + +`--verify-onnx` 为相同输入运行 ORT 并比较完整输出，不以 ORT 替代 QEMU 生成。 +要求所有值有限、`max_abs < 1e-3`、`rtol=0`、greedy ID 相同，并记录 top-2 margin。 +`--reference-suite` 进一步比较冻结 prompt IDs、逐 token IDs、文本与停止原因； +它不会自动选择算术策略，也不接受另传 `--max-new-tokens`。完整三例显式配置为： + +```text +--harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 +--matmul-rhs-layout column-panel +--mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference ``` -它接受一个 `PreparedInputs -> np.ndarray` 的同步 `forward` 回调。 -回调可接 ORT 或已实现的 `QemuForward` 适配器；核心不编译模型、不管理 QEMU，也不为每个 -新 token 重建 tokenizer。每一步从本次生成的 token 历史重新准备输入，复用后端输出 -缓冲区不会改变已经选出的 token；不同调用不共享生成状态。 +**多轮文本。** `Conversation.from_directory(tokenizer_directory, system_prompt=None)` +加载同一快照的 `tokenizer_config.json` 官方 `chat_template`，通过 Jinja2 不可变沙箱 +渲染，固定 `add_generation_prompt=True`、`enable_thinking=False`。生产路径使用 +tokenizers/Jinja2，不依赖 Transformers/Torch。仅支持文本 system/user/assistant， +system 只在构造时提供；每轮重新渲染完整历史并编码，不截断旧消息，不缓存 KV。 + +`turn(...)` 只有成功且至少生成一个 token，才同时保存 user 和 assistant；异常、取消、 +超时、零预算或没有生成空间时历史不变。EOS-only 可以保存合法的空 assistant 回复。 +`history` 是不可变快照，`reset()` 清除已完成轮次但保留初始 system。历史保存文本， +不承诺下一轮 token IDs 直接拼接；超过 256-token 容量会在前向前失败。单个会话拒绝 +并发或重入 turn/reset。原始三例不使用聊天模板。 + +**部署与报告。** 容器只打包运行所需源码和依赖，资产只读挂载到 `/assets`，构建及报告 +写入 `/output`，以调用者 UID/GID 运行。默认镜像 `scratchv-w5:local`、基础镜像 +`ubuntu:24.04`；可显式设置 `SCRATCHV_CONTAINER_ENGINE`、`SCRATCHV_W5_IMAGE`、 +`SCRATCHV_W5_BASE_IMAGE`、`SCRATCHV_W5_ASSETS` 和 `SCRATCHV_W5_OUTPUT`。 +路径可含空格，挂载路径不能含逗号。脚本检查引擎和资产后构建、运行，不安装引擎， +不下载模型，不更改 WSL 或 Windows。依赖来自 +[pyproject.toml](../../../pyproject.toml) 的 `llm-deploy`，不另维护一份版本锁。 + +部署预检和构建客户端各有独立进程组，SIGINT/SIGTERM 时转发终止并等待清理，必要时 +5 秒后强制结束，返回 130/143。容器运行阶段保留 `exec ... run --init` 信号转发， +QEMU 适配器负责其子进程回收。CLI 最终保存报告失败时向 stderr 说明保存错误， +保留已有生成失败或取消的退出码 1/130/143；原本成功或仅构建成功时改为退出 1。 +数值、参考文本等原始错误不会被保存错误覆盖，也不打印误导性的成功报告路径。 +运行中的 SIGINT/SIGTERM 均记录为取消，分别保留退出码 130/143。 + +## 如何复现 + +**本地已有环境。** 从仓库根执行，激活项目 Python 3.12 环境并按 +[Linux 开发说明](../LINUX_REPRODUCTION.md)安装依赖。已有固定资产时，原始补全命令为： -失败行为属于合同： +```bash +python scripts/run_inference.py \ + --model assets/qwen3-0.6b/onnx/model.onnx \ + --tokenizer-dir assets/qwen3-0.6b/tokenizer \ + --prompt 'The capital of France is' --max-new-tokens 1 \ + --verify-onnx assets/qwen3-0.6b/onnx/model.onnx \ + --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ + --matmul-rhs-layout column-panel \ + --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ + --timeout 1800 --total-timeout 3600 --output-dir output/w5-paris-new +``` + +这会执行一次完整 28 层模型。只构建可加 `--build-only`，其 `BUILD_ONLY` 不表示 +完成生成。复用时以 `--reuse-build output/w5-paris-new/executable.json` 替换 +`--model ...`，仍传相同 tokenizer 和所有原有算术/hart 配置，并使用新的输出目录。 +元数据可能含绝对路径，不直接把宿主构建搬进容器复用；跨环境从模型重新构建。 + +聊天模式将上例 `--prompt ...` 换为下列参数，其他模型、参考策略及输出目录参数保留： + +```text +--mode chat --chat-turn 'Hello.' --chat-turn 'What is 1+1?' +--chat-turn 'Reply with one word.' --system-prompt 'Give brief answers.' +``` + +这会按顺序运行三轮，前两轮回答也计入后续 256-token 容量。模板一致性检查不等于 +完整模型三轮生成已实跑。相关测试命令集中在 +[W5 测试说明](../../../tests/llm-deploy/w5/README.md)。 + +**固定版本的独立容器复现。** 以下步骤在同一 Bash 会话依次执行，使用独立的 +Ubuntu 24.04 x86_64 环境，不复制作者的虚拟环境、`output/`、ELF、bundle 或报告。 +三例共用一次新构建、执行 24 次前向，不先另跑 Paris，也不重新录制 ORT 参考。 +固定提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 是已完成完整数值实跑的功能基线； +该提交用于复现既有完整数值结果，不含本轮报告保存及 SIGTERM 分类修复；这些行为 +使用当前分支的相关小图验证，不将旧完整结果标为本轮重跑。 + +| 条件 | 固定内容或要求 | +|---|---| +| 主机工具 | Bash、Git/Git LFS、Python 3.12、Docker Engine/Buildx、GNU time、timeout、setsid、tar | +| 模型 | `Qwen/Qwen3-0.6B`，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 ONNX 及同版本 tokenizer | +| 基础镜像 | `public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca` | +| 原实跑工具 | Python 3.12.3、QEMU 8.2.2（Ubuntu `1:8.2.2+ds-0ubuntu1.18`）、Zig 0.14.1、ORT 1.22.1、NumPy 2.2.6、Torch 2.7.1+cpu | +| 原容器引擎 | Docker Engine 29.9.2、Buildx 0.38.1 | +| 准备后的资源 | 至少 4 个可用逻辑 CPU、10 GiB 可用内存、10 GiB 剩余磁盘；初始建议预留 25 GiB 磁盘 | +| 时间预算 | 单步 1800 s，总预算 16500 s，外层 280 分钟，强制清理再留 30 s | + +镜像 digest 只固定基础镜像；apt 与未逐项锁定的间接依赖仍可能变化。记录实际镜像、 +工具和依赖版本，先核对差异是否影响结果比较，不默认为相同环境。这里沿用原资产 +身份检查，不增加源码、输出或报告哈希。保留部署前后镜像 ID 作为运行记录， +不要求两者必须相等：Buildx 的缓存构建仍可能更新 provenance/attestation 元数据， +使 manifest ID 改变。ID 变化本身不足以说明运行内容改变，也不代表版本差异可忽略； +继续记录固定源码、模型和实际工具/依赖版本，复用既有 ELF/bundle 检查。 + +**1. 新 clone 并固定提交。** + +以下命令在同一个 Bash 会话按顺序执行，从有足够空间的父目录开始。 +目录名 `scratchv-w5-repro` 必须不存在；重试时改用新的目录名，保留失败记录。 + +```bash +set -euo pipefail +export W5_COMMIT=8ad347477f7f5ba15b2da7c91178b3c6668456cc +test ! -e scratchv-w5-repro +GIT_LFS_SKIP_SMUDGE=1 git clone --no-checkout --filter=blob:none \ + https://github.com/ScratchV-Compiler/ScratchV.git scratchv-w5-repro +cd scratchv-w5-repro +git fetch --no-tags origin refs/pull/115/head +GIT_LFS_SKIP_SMUDGE=1 git checkout --detach "$W5_COMMIT" +test "$(git rev-parse HEAD)" = "$W5_COMMIT" +test -z "$(git status --porcelain)" + +export W5_RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" +export SCRATCHV_CONTAINER_ENGINE=docker +export SCRATCHV_W5_IMAGE="scratchv-w5:independent-$W5_RUN_ID" +export SCRATCHV_W5_ASSETS="$PWD/output/w5-independent-assets" +export SCRATCHV_W5_OUTPUT="$PWD/output/w5-independent-$W5_RUN_ID" +export SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca +test ! -e "$SCRATCHV_W5_ASSETS" +test ! -e "$SCRATCHV_W5_OUTPUT" +mkdir -p "$SCRATCHV_W5_OUTPUT" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/start-utc.txt" +{ + git rev-parse HEAD + git status --porcelain + git remote -v +} > "$SCRATCHV_W5_OUTPUT/source.txt" +{ + cat /etc/os-release + uname -a + lscpu + free -b + df -B1 . + python3 --version + git --version + git lfs version + docker version + docker buildx version + docker info --format 'OS={{.OSType}} Arch={{.Architecture}} CPUs={{.NCPU}} Memory={{.MemTotal}}' + command -v setsid timeout tar + /usr/bin/time --version +} > "$SCRATCHV_W5_OUTPUT/runner-resources.txt" 2>&1 +docker info > /dev/null +test "$(uname -m)" = x86_64 +python3 -c 'import sys; assert sys.version_info[:2] == (3, 12)' +printf 'Run output: %s\n' "$SCRATCHV_W5_OUTPUT" +``` + +检查 `runner-resources.txt`：宿主应为 Ubuntu 24.04，Docker daemon 也是本机 Linux； +若使用远端 daemon、容器资源配额不足或没有引擎，先停止并回报，不用修改 WSL 来凑条件。 +挂载目录不能包含逗号。不要把私人凭据写入报告。 + +**2. 构建镜像并记录实际环境。** + +复用现有 Dockerfile；随后部署脚本会再检查构建并复用 Docker 缓存。 +基础镜像和构建参数不变，不使用作者构建好的镜像。 -- 后端原始异常向调用方传播；非法形状、dtype、非有限输出及不可解码 ID 都使本次生成失败。 -- 取消抛 `GenerationCancelledError`（`RuntimeError` 子类），超时抛 `TimeoutError`，不返回看似成功的部分结果。下一次调用从新的 prompt 开始，不能继承失败调用的 token。 -- `cancelled` 是可选取消检查回调；检查发生在同步前向前后及返回前。它不强制中断正在阻塞的前向。 -- `timeout_seconds` 是可选有限非负秒数，布尔值不合法；0 表示立即超时，即使 token 预算为 0。 -- 超时从输入与配置验证完成后开始，以单调时钟作协作式检查；取消与超时同时触发时先报告取消。正在运行的同步后端由后端自身负责限时、取消和进程清理；Host 合同不声称能终止任意阻塞回调。 +```bash +docker build --file deployment/w5/Dockerfile \ + --build-arg "BASE_IMAGE=$SCRATCHV_W5_BASE_IMAGE" \ + --tag "$SCRATCHV_W5_IMAGE" . \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/image-build.log" +docker image inspect "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/container-image.json" +docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/image-id-before.txt" +docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" -m pip freeze \ + > "$SCRATCHV_W5_OUTPUT/dependencies.txt" +docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" \ + -c 'import json; from probes.w1_qwen3_export.run import environment; print(json.dumps(environment(export=True), indent=2))' \ + > "$SCRATCHV_W5_OUTPUT/image-environment.json" +docker run --rm --entrypoint sh "$SCRATCHV_W5_IMAGE" -c \ + 'python --version; python -m ziglang version; qemu-system-riscv64 --version; dpkg-query -W python3.12 qemu-system-misc' \ + > "$SCRATCHV_W5_OUTPUT/image-tools.txt" +cat "$SCRATCHV_W5_OUTPUT/image-tools.txt" +``` + +核对 `image-tools.txt` 与上表；环境差异应保留在报告中，部署脚本本身不锁 apt 补丁。 + +**3. 独立获取固定资产。** + +下载只调用现有 `acquire_model`，沿用 W1 manifest 中已有的下载/分片完整性检查。 +它只用 Python 标准库，不执行模型、不重新导出、不安装宿主推理依赖。 +Tokenizer 从同一提交的 Git LFS 获取。 + +```bash +git lfs install --local +git lfs pull --include="assets/qwen3-0.6b/tokenizer/*" --exclude="" \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/tokenizer-acquisition.log" +python3 - <<'PY' 2>&1 | tee "$SCRATCHV_W5_OUTPUT/model-acquisition.log" +import json, os, shutil +from pathlib import Path +from probes.w1_qwen3_export.run import acquire_model +assets = Path(os.environ["SCRATCHV_W5_ASSETS"]) +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +result = acquire_model(assets / "onnx") +shutil.copytree("assets/qwen3-0.6b/tokenizer", assets / "tokenizer") +(out / "acquisition.json").write_text(json.dumps(result, indent=2) + "\n") +PY +test -z "$(git status --porcelain)" +``` -本轮不会为每个 token 新建线程来模拟可中断执行，也不会吞掉失败后继续选 token。 -QEMU 适配已复用现有运行器，在 UART/QMP 等待中响应取消，并验证超时/取消后的进程及进程组回收。失败运行按既有策略保留 transport 诊断副本。 +不要另跑参考采集脚本或添加 `--record`：suite 内部已经逐步对照当轮 ORT 与冻结参考， +无需再运行一遍完整 ORT 生成作为“预热”。 -## 3 条固定参考用例 +**4. 资源检查。** 镜像和资产准备后核对可用资源，Docker 配额不能低于要求。 -以下三条原文和预算在 ORT 执行前选定；Paris 原文与 W5 硬门槛保持一致。 +```bash +python3 - <<'PY' +import json, os, shutil +from pathlib import Path +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +mem = {line.split(":")[0]: int(line.split()[1]) * 1024 + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith(("MemTotal:", "MemAvailable:"))} +evidence = {"stage": "after_image_and_assets", + "available_cpus": len(os.sched_getaffinity(0)), "memory": mem, + "disk_free_bytes": shutil.disk_usage(out).free, + "minimum_available_cpus": 4, + "minimum_available_memory_bytes": 10 * 1024**3, + "minimum_disk_free_bytes": 10 * 1024**3} +evidence["passed"] = (evidence["available_cpus"] >= 4 + and mem.get("MemAvailable", 0) >= 10 * 1024**3 + and evidence["disk_free_bytes"] >= 10 * 1024**3) +(out / "resource-preflight.json").write_text(json.dumps(evidence, indent=2) + "\n") +if not evidence["passed"]: + raise SystemExit("Insufficient W5 CPU/RAM/disk; generation NOT RUN.") +PY +``` -| 用例 | prompt 原文 | `max_new_tokens` | 预先约定的内容检查 | -|---|---|---:|---| -| `france` | `The capital of France is` | 8 | 新生成文本包含 `Paris` | -| `germany` | `The capital of Germany is` | 8 | 新生成文本包含 `Berlin` | -| `japan` | `The capital of Japan is` | 8 | 新生成文本包含 `Tokyo` | +资源不足时保留记录，不运行完整生成。 -参考必须记录 prompt IDs、新生成 IDs、文本、EOS/停止原因和实际 ORT 版本； -不能只填写上述预期词。未来 QEMU 对照优先比较逐 token ID；若出现分歧，保留相同 -输入的数值误差和 top-1/top-2 margin，定位原因，不用“语义合理”覆盖数值失败。 -原始全量 logits 是运行中验证对象，无需将每一步约 148 MiB 输出都加入 Git。 +**5. 一次运行三条完整用例并保存报告。** -[固定参考文件](../../../probes/w5_generation/references.json)保存三条输入及冻结后的 -精确输出，检查入口为 [run.py](../../../probes/w5_generation/run.py)。在仓库根目录， -激活已按 [Linux 复现说明](../LINUX_REPRODUCTION.md)准备的环境后执行： +整块复制执行。`--reference-suite` 自带三条用例及每例 8-token 预算， +不要同时传 `--max-new-tokens`，不要传 `--reuse-build`。 +代码从固定模型重新构建一次，跨用例自动复用同一 ELF/bundle。 ```bash -python probes/w5_generation/run.py \ - --assets assets/qwen3-0.6b \ - --output-dir output/w5-generation-reference +test "$(git rev-parse HEAD)" = "$W5_COMMIT" +test -z "$(git status --porcelain)" +test ! -e "$SCRATCHV_W5_OUTPUT/run" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-start-utc.txt" + +set +e +timeout --signal=INT --kill-after=30s 280m \ + /usr/bin/time -v -o "$SCRATCHV_W5_OUTPUT/process-time.txt" \ + bash scripts/deploy_w5.sh \ + --model /assets/onnx/model.onnx --tokenizer-dir /assets/tokenizer \ + --reference-suite --verify-onnx /assets/onnx/model.onnx \ + --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ + --matmul-rhs-layout column-panel \ + --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ + --timeout 1800 --total-timeout 16500 \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/runner.log" +pipeline_status=("${PIPESTATUS[@]}") +printf '%s\n' "${pipeline_status[0]}" > "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" +printf '%s\n' "${pipeline_status[1]}" > "$SCRATCHV_W5_OUTPUT/log-exit-code.txt" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-end-utc.txt" + +# 此唯一 tag 只供本次使用；不停止其他人的容器。 +{ + containers=$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE") + list_status=$? + printf 'list_exit_code=%s\n' "$list_status" + for container in $containers; do + docker stop --time 30 "$container" + printf 'stop_exit_code=%s\n' "$?" + done + docker ps --filter "ancestor=$SCRATCHV_W5_IMAGE" \ + --format '{{.ID}} {{.Status}}' + printf 'final_list_exit_code=%s\n' "$?" +} > "$SCRATCHV_W5_OUTPUT/container-cleanup.log" 2>&1 + +docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/image-id-after.txt" +python3 -B -X utf8 scripts/check_w5_ci.py \ + --report "$SCRATCHV_W5_OUTPUT/run/report.json" \ + --reference probes/w5_generation/references.json \ + --exit-code-file "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" \ + --output "$SCRATCHV_W5_OUTPUT/result-check.json" \ + > "$SCRATCHV_W5_OUTPUT/result-check.log" 2>&1 +printf '%s\n' "$?" > "$SCRATCHV_W5_OUTPUT/check-exit-code.txt" +cat "$SCRATCHV_W5_OUTPUT/result-check.log" +{ + date -u +%FT%TZ + git rev-parse HEAD + git status --porcelain + free -b + df -B1 . +} > "$SCRATCHV_W5_OUTPUT/final-resources.txt" +set -e +printf 'Preserved output: %s\n' "$SCRATCHV_W5_OUTPUT" ``` -这条命令使用固定资产和同一 ORT CPU session 验证三条参考，并在输出目录保存 -`reference-run.json`；不会启动 QEMU。默认检查已冻结参考，不自动覆盖它。 -初次采集需要显式 `--record`，并按预先固定的内容标准确认结果后才冻结；不能为消除 -失败任意重录参考。QEMU 用户入口已实现为 `scripts/run_inference.py`；与本节 ORT 参考工具用途不同，见 [QEMU 说明](qemu-generation.md)。 -2026-10-23 使用同一个 ORT CPU session 实跑，三条均完成 8 个 token，停止原因为 -`max_new_tokens`。独立逐步输入/greedy/停止 oracle 与 Host 核心一致，三个预先固定的 -内容检查均通过，精确 token IDs 已冻结到上述 JSON。 - -| 用例 | 实际生成的新文本（保留前导空格） | 耗时 | CPU 时间 | -|---|---|---:|---:| -| `france` | ` Paris. The capital of Italy is Rome` | 44.672 s | 44.649 s | -| `germany` | ` Berlin. The capital of France is Paris` | 42.910 s | 42.894 s | -| `japan` | ` Tokyo. The capital of the United States` | 42.406 s | 42.394 s | - -全轮耗时 **130.676 s**，CPU 时间 **130.611 s**；进程峰值 RSS **5.002 GiB** -（`ru_maxrss` 为整个进程累计峰值，不能当作每例独立分配或增量）。模型 session -加载耗时 0.307 s；这些是 ORT 参考实测，不是 QEMU 生成速度。 - -P0/P1 阶段只执行一次真实 ORT 采集并在每步使用独立 oracle 验证，随后冻结参考，完成离线 -字段一致性检查；未重复执行“冻结后第二次 ORT 回放”。精简公开记录、逐 token -margin 与环境位于 JSON 的 `baseline` / `cases`，详细本地产物为 -`output/w5-generation-reference/reference-run.json`。原始续写可以继续列举其他事实， -并因 8-token 预算停止；这三条只验证约定的 raw completion 内容，不宣称 chat 或多轮能力。 - -## 保留的执行配置 - -既有 RV64 路径保持默认 **1 hart**，显式接受 **1 / 2 / 4 / auto**。 -`auto` 只在构建/启动时按进程可用 CPU 解析一次，上限为 4，不在 token 之间动态改数。 -复用 ELF 的 hart 数必须与构建记录一致。Host 生成核心不覆盖这套配置，也不会把 -ORT 的 CPU 线程数误当成 guest hart 数。 - -QEMU 工作区布局保持 `qemu_workspace_layout=none`，即 padding 关闭。 -均值默认仍为 `sequential`；显式 `--mean-policy two-four-lane-last-axis` -可用于数值对照，但单独启用尚不能让完整三例通过，见续作报告。 -非线性路径默认仍为 `libm`；显式 `--nonlinear-policy fp32-reference` 选择已有参考的 -Sin/Cos、Sigmoid、Softmax 算术顺序，不改变 hart 选择，也不要求目标具有 x86 SIMD。 -本轮完整 QEMU 三例通过的配置为 `4 hart / blocked_fma / 2x4 / column-panel`，同时 -显式启用 `--mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference`， -padding 为 `none`；24 步复用同一 ELF/bundle。默认 `1 hart / sequential / 1x4 / original`、 -Mean `sequential` 和非线性 `libm` 均未改变，不能把新策略的验收结果归给旧算术配置。 -复用时必须显式匹配构建策略；实验 padding、RVV、KV cache、动态长度和量化不纳入本轮。 - -## 阶段与验收状态 - -以下状态核对于 2026-10-23，作者本地实跑、远端 CI 和第二人签收分别记录。 - -| 阶段/门槛 | 当前证据与状态 | 剩余事项 | +这里为失败留证临时关闭 `errexit`，**脚本走到最后不等于 PASS**。 +即使 suite 非零退出，仍运行容器清理和已有报告检查器；报告缺失时 checker 应失败， +不能补造成功字段。`result-check.json` 在本地的 `commit` 可为 null；实际 SHA 由 +`source.txt` 和最终 Git 记录确认。 + +若手动 Ctrl-C 使整段 shell 中断，重新进入同一 clone，恢复上面输出目录和镜像变量， +执行本节从“此唯一 tag”注释之后的收尾段，再打包已有报告。不要直接重新跑模型。 +主机掉电、daemon 失联或强制终止可能阻止清理；如清理日志有非零码或最后仍列出容器， +标为未完成并回报，不视为通过。 + +**三条用例与结果判定。** 全部为 raw completion、greedy、8 个新 token， +停止原因均为 `max_new_tokens`。文本含前导空格；只出现城市名不足以判定一致， +需同时核对[冻结参考](../../../probes/w5_generation/references.json)的 IDs 和文本： + +| 原始 prompt | 新生成 IDs | 新生成文本 | |---|---|---| -| P0：合同与 3 条参考 | **PASS**；合同冻结，3/3 真实 ORT/Host 参考与独立 oracle 通过，IDs/text/停止原因冻结 | 无实现缺项 | -| P1：Host token 循环 | **PASS**；核心 55 项及相关边界验证通过 | 无实现缺项 | -| P2：QEMU 生成适配/CLI | **PASS**；同一 ELF/bundle 跨 token 和命令复用；1/4 hart 数值与中断清理通过 | 无实现缺项 | -| 默认/手动/auto hart | **PASS（小图）**；默认单核、手动 4、auto→4 的真实容器 CLI 通过，padding 关闭 | 不将小图或显式策略通过推广为默认算术完整模型通过 | -| `e2e:qemu-paris` | **PASS（作者本地、显式新策略 4 hart）**；France 完整 8 token 的数值、IDs、文本和停止原因通过，输出包含 Paris | 远端完整专项及第二人复现 | -| `e2e:qemu-prompts` | **PASS（作者本地、显式新策略 4 hart）**；3/3 用例、24/24 完整 QEMU 前向及逐步数值通过，与冻结参考一致 | 远端完整专项及第二人复现 | -| `unit:multiturn` | **PASS**；40 项会话测试，官方模板/IDs 六份对照；远端官方三轮模板 2/2 通过 | 不宣称已做完整模型三轮对话生成 | -| `deploy:script` | **PASS（作者真实容器）**；小图、超时/取消和清理通过；完整 28 层 France 单 token 实际生成 ` Paris` | 容器三例 24 步尚未执行，不能用单 token 代替 | -| 普通 PR CI | **PASS**；[39919353111](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/39919353111) 为 14 成功、7 个按路由跳过；W5 真实小图 12/12、模板 2/2 通过 | 不替代完整生成专项 | -| W5 完整专项 CI | **BLOCKED / NOT_RUN**；工作流已接入，fork Actions 以 HTTP 422 拒绝 dispatch，未创建 run | 恢复平台执行能力后运行容器三例 24 步 | -| 第二人独立复现 | **说明已准备，执行待办**；固定提交、环境、命令、验收和回传清单见[交接说明](independent-reproduction.md) | 同事独立构建、实跑并回传签收，作者或 CI 不能替代 | - -旧算术配置的历史失败保留于实跑报告：初始轮三例均在首步失败（仅执行 3/24 步）， -France 的 `max_abs=0.003272057`。显式新策略的 PASS 不改写该历史结果。 - -W4 完整单次前向的已有证据不替代 W5 token 循环；本页 W5 结果来自上述独立完整生成验收。 -W4 独立复现、Nightly 和团队签收仍按 [W4 验收说明](../W4/validation.md)分别确认。 - -完整 QEMU 三例验收已通过，原始结果位于 -`output/w5-followup-20261023/qemu-reference-4hart/run/report.json`。 -gate 失败状态、旧 W4 构建审计传递、逐步验收证据及取消清理已补强。 -本地功能出口已有完整证据：宿主三例 24 步、宿主独立前向，以及干净容器镜像、 -小图、中断清理和完整 Paris 单 token 均已通过，见 [部署记录](deployment.md)。 -容器此次只补一次完整前向，不将其写成容器三例套件；普通 PR CI 已通过，完整专项仍 -**BLOCKED / NOT_RUN**。本地功能完成不等于远端完整验收或第二人签收完成。 -不把候选 token 一致或静态部署检查写成实跑 PASS。当前每 token 启动 QEMU，但复用构建； -持久 guest、权重驻留、KV cache 等不属于 W5 v1 必需实现。 - - -## CI 接入与远端执行边界 - -W5 复用 [w4-full-numeric.yml](../../../.github/workflows/w4-full-numeric.yml)，没有再建立 -一套并行的完整模型工作流。手动 `workflow_dispatch` 的 `mode` 为互斥选择： - -| mode | 执行范围 | -|---|---| -| `small`（默认） | 相关单测和真实小图，不运行完整模型 | -| `w4-forward` | 原 W4 完整单次前向 | -| `w5-generation` | 原 `deploy_w5.sh` 在真实容器中运行三例 3×8 token，再校验完整报告 | +| `The capital of France is` | `[12095,13,576,6722,315,15344,374,21718]` | ` Paris. The capital of Italy is Rome` | +| `The capital of Germany is` | `[19846,13,576,6722,315,9625,374,12095]` | ` Berlin. The capital of France is Paris` | +| `The capital of Japan is` | `[26194,13,576,6722,315,279,3639,4180]` | ` Tokyo. The capital of the United States` | -已有 `workflow_call` 的 `run_full/case` 接口保留，新增 `run_w5=false`，不自动开启生成。 -W5 选择与 W4 完整前向冲突，或在非手动事件请求 W5 时，先于模型执行明确失败。 +成功要求：三例、24 次前向全部完成，prompt IDs/新 IDs/文本/停止原因匹配；每步全部 +`256×151936=38,895,616` 个 logits 有限且严格 `max_abs < 1e-3`；28 层完整图检查、 +FMA 12 行 binary32 位模式与 FCSR 检查通过；同一 ELF/bundle 与所选策略匹配、 +padding `none`；每个 QEMU 正常退出并回收，没有清理错误。套件、日志管道、报告 +检查器退出码均为 0，最终无本次容器残留，源码没有改动。 +`container_reproduction=NOT_ASSERTED` 是内部 CLI 不会自行宣称容器复现的字段， +不能手改；容器身份由外层构建、运行和退出记录说明。 -**合并前的手动入口**使用 main 上已经存在的 -[llm-deploy.yml](../../../.github/workflows/llm-deploy.yml)，显式传 -`run_w5_validation=true`；该模式跳过旧 W1/W2 完整任务与 W3/W4 完整执行， -只调用同一提交的 W5 验证。W5 与 W3/W4 同时勾选会直接失败，不同时跑多套模型。 -GitHub 要求手动工作流存在于默认分支，见 -[官方手动运行说明](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manually-run-a-workflow)。 -审查时远端 main 的 `w4-full-numeric.yml` 尚无 `workflow_dispatch`，因此不能把新文件的 -直接手动入口当作仅推送 W5 分支后必然可用；不要为了建立入口先合并未验收的 W5。 +**回传报告。** 附 `handoff.txt` 说明执行人、UTC 时间、完整 SHA、独立 clone/构建、 +环境差异、三例 IDs/文本/误差、计时与缺失采样、清理结果以及失败阶段，保留： -下列入口已获授权并尝试调用；fork 仓库的 Actions 目前在 dispatch 阶段返回 HTTP 422， -未创建运行。解除该限制后才可重试；执行仍需仓库写权限： +- 来源、主机资源、镜像/依赖/工具、下载记录、所有退出码与构建/运行/清理日志。 +- `run/report.json`、`result-check.json`、`run/executable.json`、`run/build/` 的 JSON。 +- `run/steps/step-0000…0023/run.json`、UART 完成帧 `uart.bin`、错误日志及 + `run/fma-conformance/` 的报告。 + +不传权重、ELF、原始 logits 或 Zig 缓存；失败时也回传已有报告并标明缺失项。 +原始产物保留至复现结果确认；重试使用新目录。沿用原小文件筛选方式打包： ```bash -gh workflow run llm-deploy.yml --repo yuki-328/ScratchV --ref codex/w5-generation \ - -f run_w5_validation=true -f run_w3_validation=false -f run_w4_validation=false +python3 - <<'PY' +import os, tarfile +from pathlib import Path +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +allowed = {".json", ".xml", ".txt", ".log", ".stderr", ".stdout"} +archive = out.parent / (out.name + "-reports.tar.gz") +if archive.exists(): + raise SystemExit(f"Keep earlier evidence; archive already exists: {archive}") +with tarfile.open(archive, "w:gz") as bundle: + for path in sorted(out.rglob("*")): + relative = path.relative_to(out) + if (path.is_file() and not path.is_symlink() + and ".zig-cache" not in relative.parts + and (path.suffix in allowed or path.name == "uart.bin")): + bundle.add(path, arcname=str(Path(out.name) / relative), recursive=False) +print(archive) +PY ``` -新 `w4-full-numeric.yml` 手动入口进入默认分支后，也可使用上表的 `mode`。 -W5 完整 job/步骤/CLI 总预算分别为 -**350 / 285 / 275 分钟**；镜像和固定资产落盘后才检查 **至少 4 CPU、10 GiB 可用 RAM、 -10 GiB 剩余磁盘**，不足即明确失败，不通过缩小模型或跳过 token 凑齐门槛。 -远端 runner 是否实际满足资源和时限，仍须在专项能够创建运行后实跑确认。 - -W4 unit 增加 **12 个真实 QEMU 必跑节点**（生成与取消 2、Mean 8、非线性 2）； -[llm-deploy.yml](../../../.github/workflows/llm-deploy.yml) 在现有 tokenizer 环境补 **2 个 -官方三轮模板节点**。W5-only 跳过普通 llm-deploy job 时，使用已经构建的同一 -W5 镜像、只读 tokenizer 与测试文件执行这两个模板节点，并保留必跑 JUnit 检查； -不会因此再安装一套依赖或重复生成。普通 W5 单测继续由既有通用 CI 承担。 -最终汇总区分未选择的 `NOT_RUN` 与请求后失败/取消/skip，不能把缺失证据作为通过。 - -上一轮 CI 接入的 **753 项轻量检查通过（19.64s）**，覆盖工作流结构、旧 W4 接口、路径路由、 -JUnit 缺失/skip/fail、资源短缺、退出码与汇总；其中严格报告 gate 为 44 项。 -上述 **14 个必跑节点在 CI 接入阶段仅 collect-only 核对 ID**，该阶段没有再次执行这些节点或完整模型； -已有真实容器生成/取消 2 节点通过，Mean/非线性沿用此前真实小图证据，不把收集当实跑。 -[严格报告检查器](../../../scripts/check_w5_ci.py) 对已有真实三例 24 步报告 -`output/w5-followup-20261023/qemu-reference-4hart/run/report.json` 离线检查通过。 -本地 CI 检查记录为 `output/w5-ci-integration-20261023/validation.json`， -同目录保存测试日志、14 个收集节点 ID 以及 98 段内嵌 shell 的语法检查结果。 -上述记录来自提交前的本地实现/证据检查，当时未提交、未推送；其后的普通 PR CI -已通过，完整专项仍为 **BLOCKED / NOT_RUN**，详见页末发布记录。 - -## 提交前检查与收尾（2026-10-23） - -本节保留提交前检查时点；后续提交、普通 CI 和专项阻塞状态见页末发布记录。 - -已核对 W5 全部待提交改动：生成合同与冻结参考、Host/会话循环、QEMU 适配与 CLI、 -显式数值策略、运行器清理、容器打包、第三方声明、CI 路由和报告。 -**本地可执行的 W5 工作已完成，未发现尚未处理的本地阻塞；远端 CI 验收仍待执行。** -本轮没有重新运行完整模型或 QEMU 小图，没有暂存、提交或推送。 - -本轮修复和澄清： - -- 两个有限 FP32 logits 的诊断差值原先可能溢出为 Infinity，导致严格 JSON 报告写入失败。 - CLI 和 Host 参考工具现先转为 Python float 再相减；只改变诊断计算，不改变模型、 - argmax、数值容差、冻结 IDs/text 或每行计算顺序。新增正负 FP32 最大值边界测试。 -- 补齐合并前可用的 W5-only 手动 CI 入口：复用已注册的 llm-deploy，拒绝与 W3/W4 - 同时选择，保留真实小图、官方三轮模板和完整生成门槛。job 预算增加到 350 分钟， - 给准备、清理和报告上传留余量，生成步骤与 CLI 的时限未放宽。 -- 修正测量口径：宿主/进程树的 21,032 次采样无读取错误，但 24 步中有 17 步的 - 独立 QEMU RSS 各缺一次读取；RSS 峰值是观测值。容器运行器记录的 430.481 秒 - 不包含前置输入打包/写入和 ELF/bundle 校验，原始测量值没有改动。 - -| 本轮必要验证 | 结果与边界 | -|---|---| -| CLI 与 Host 参考工具 | 107 passed，0 skip；包含两处溢出复现及严格 JSON 边界 | -| 数值策略、external 运行器及 W4 契约 | 280 passed，0 skip；默认值、错误传播、上下文/工作区复用和取消清理 | -| CI、部署与打包相关检查 | 811 passed，0 skip；最终 11 项边界复查与此集合重叠，不重复累计 | -| 工作流与脚本静态检查 | actionlint 1.7.12 检查三份工作流、Bash 语法及 diff 检查通过；未运行外部 ShellCheck/Pyflakes | -| 既有三例 24 步报告 | 严格 CI 检查器离线核验 PASS；未重新推理 | - -本轮证据在工作树下的 `output/w5-precommit-20261023/`： -`generation-audit.json`、`numeric-audit.json`、`ci-audit.json` 及对应 JUnit。 -前后轮测试集合存在交叉，不将数量累加为独立覆盖总数。 -默认 1 hart、手动 1/2/4 与启动时 auto 保持正常，W5 padding 强制为 none； -Docker 本轮未启动，WSL 配置未改。 - -剩余事项按性质区分： - -1. **当时仍待远端 CI 证据**：提交前计划获准推送后使用上节 W5-only 入口； - 后续普通 CI 已通过，完整专项被 Actions 平台阻塞。runner 资源不足、失败或取消 - 必须明确失败，不能用本地 PASS 替代。 -2. **数值验收配置限制**：完整模型 PASS 仅属于已记录的显式参考算术组合； - 原默认 Mean sequential/nonlinear libm 的历史完整 logits 超差仍保留，不能宣称默认路径也通过。 - 本轮没有修改算术，因此未重复完整模型。 -3. **已知运行边界**：同步构建、传输和 ORT 调用按阶段检查预算，不承诺硬中断任意 ORT 调用。 - KV cache、持久 guest/权重驻留、RVV 和量化属于后续阶段，不属于 W5 v1 未完成的出口项。 - -## 发布与首次远端 CI 反馈（2026-10-23） - -代码、测试和报告已作为独立 [PR #115](https://github.com/ScratchV-Compiler/ScratchV/pull/115) -提交，初始实现提交为 259fc8e。四份报告链接已经验证可访问；没有合并。 - -首次 [PR CI](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/39917939519) -实际执行并通过 12 个 W5 相关 QEMU 小图必跑节点及 2 个官方三轮模板节点。 -W4 unit JUnit 为 1762 passed、4 skipped，四个跳过为三种未提供模型资产的优化器对照及 -Linux 上的 Windows junction 专用测试，不涉及上述 12 个必跑节点。完整 W5 门槛按普通 -PR 路由未选择，汇总明确为 w5_generation_accepted=false。 - -通用测试发现 8 个失败：6 个来自旧 W3 测试求值器不支持新增 W5 互斥表达式； -另 2 个来自通用 checkout 未下载 tokenizer 的 Git LFS 内容，测试误把指针当作 JSON。 -修复仅涉及测试：补齐 W3/W5 选择矩阵；默认可选资产缺失或为 LFS 指针时明确 skip， -显式指定 tokenizer 目录时仍必须有效，真正损坏的 JSON 不隐藏为 skip。 -专用 CI 的两项模板必跑门槛保持不变。修复的直接验证为会话/部署 63 passed、 -W3 工作流 54 passed、相关 W4/W5 入口边界 28 passed；均未运行完整模型。 -修复提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 推送后， -[普通 PR CI 39919353111](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/39919353111) -已 **SUCCESS**：14 个 job 成功、7 个按路由跳过；通用测试 6107 passed、93 skipped， -模拟器 7 passed；W4 unit 1762 passed、4 skipped；W5 必跑小图 12/12 和官方模板 -2/2 均通过。此轮测试的是 PR 临时合并提交 `ce9bd0e438e361ca070fad95a95c4fe412ec751b`， -不把首次失败改写为成功，也不把未选择的完整生成 job 计作通过。 - -完整 W5 专项在 fork `yuki-328/ScratchV` 的 dispatch 返回 -`HTTP 422: Actions has been disabled for this repository.`，尚未创建运行。 -已核对仓库 Actions `enabled=true`、工作流 `active`；重新应用相同启用设置返回 -HTTP 204，随后 dispatch 仍为 422。现有证据只能确认平台拒绝创建运行，不能认定是 -模型、runner 资源或工作流中的测试失败；需仓库所有者处理 GitHub 侧限制，恢复后再验收。 - -另一个自动 AI Code Review 的 `review (1)`、`review (2)` 两个批次失败： -分别有 2、5 份文件各两次请求均出现 `TimeoutError`。这是外部 API 请求超时导致的 -审查覆盖不完整，不是上述数值或单测失败；检查时后续批次仍在运行/排队。 -本地[审查脚本](../../../scripts/ai_code_review.py)已补有界缓解：新计划改为每批 -5 个文件、180 秒 socket 超时，仍最多两次尝试；旧 10 文件计划保留 90 秒窗口和 -原分组。新增逐次脱敏耗时/结果记录，失败仍使门禁失败。 -[相关测试](../../../tests/test_ai_code_review.py) **71 passed**；未调用外部 AI API。 -180 秒不是整个请求的墙钟硬上限，最终仍受 40 分钟 job 限制。 - -这部分修复与本轮文档更新作为 PR115 后续变更一起交付；尚未部署到可信主干。 -工作流通过 `pull_request_target` 使用可信 base 的脚本，不能改为携密钥执行 PR head。 -生效顺序是：维护者完成正常审批，将修复通过正常合并纳入 `main`；此后新发生的 -PR 事件才会使用新脚本。仅推送 PR 分支或重新运行仍固定旧 base 的历史 run， -不应当作修复已经部署。当前失败记录保留，不改写为成功；本轮不合并，也不手动 -追加或重跑 AI 审查。现有日志尚不能区分服务排队、生成慢或网络等待， -不能声明外部服务已恢复。 - -第二人[独立复现说明](independent-reproduction.md)已准备，实际执行和签收仍待同事完成。 -当前证据位于 `output/w5-publish-20261023/publication.json`、`actions-recovery.json` 及 -`ci-final-artifacts/`；报告与代码已发布，PR 未合并。 - -[GitHub 官方说明](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/enabling-features-for-your-repository/managing-github-actions-settings-for-a-repository) -指出，仓库/账号可能处于独立的平台停用状态，修改普通开关无法恢复，需要联系 -GitHub Support。当前平台未说明具体原因；普通 PR CI 或本地 PASS 不替代完整专项。 -没有改 WSL、默认 hart/padding 或模型数值策略，也没有重复本地完整模型。 -用户已接受仓库自动审查，没有额外手动发起审查。 - -## P0/P1 已有验证（2026-10-23，未重复执行） - -| 验证 | 结果 | 覆盖边界 | -|---|---:|---| -| [Host 生成核心](../../../tests/test_generation.py) | 55 passed | 多步输入与结果、0/负数/布尔预算、EOS/容量优先级、非法输出/ID、第二步错误、取消/超时、不同上下文及输出缓冲区复用 | -| [参考工具](../../../tests/test_w5_generation_reference.py) | 16 passed | 独立逐步输入/greedy/停止 oracle、冻结字段差异、语义检查和错误识别 | -| [输入辅助函数](../../../tests/test_llm_inputs.py) / [tokenizer](../../../tests/test_qwen3_tokenizer.py) | 148 passed | 既有固定 ABI、mask、词表、解码和严格边界保持正常 | -| [hart 选择辅助函数](../../../tests/test_riscv_external_harts.py) | 28 passed | 默认/手动/auto、CPU 可用性回退、非法计数和栈规划；11 项不相关测试未选中 | -| [W4 CLI 配置边界](../../../tests/test_w4_full_probe_contract.py) | 10 passed | mock 构建覆盖默认单核、手动 2/4、auto 一次解析及复用 hart 匹配；未启动 QEMU | - -P0/P1 共 **257 passed**；本轮新增的适配、CLI、多轮、部署及相关运行器检查见 [续作报告](qemu-generation.md)。不将前后轮重叠测试简单累加。 -这组结果验证新 Host 核心及已有配置边界，不代表真实 QEMU 的多 token 端到端门槛通过。 -文档检查确认本页本地链接和索引入口可解析，未增加 AI 审查。 +`process-time.txt` 的 CPU/RSS 属于 Docker 客户端，不是容器或 QEMU 总量。逐步耗时、 +RSS 使用 `run/steps/*/run.json`，partial 采样明确注明；未采集的 CPU 写“未独立采集”， +不填作者历史数据，也不据不同机器的记录计算加速比。 + +## 实际复现结果 + +以下完整模型数字来自 **2026-10-23 的历史实跑**，其功能基线为上述 `8ad3474`。 +本轮只补异常处理与文档相关验证，没有重新执行完整模型。完整模型均为 +**28 层 Qwen3-0.6B**，使用显式 `4 hart / blocked_fma / 2x4 / column-panel / +two-four-lane-last-axis / fp32-reference / padding none`。 + +**宿主 QEMU 三条固定生成。** 三例各 8 步，24 步的文本、IDs 与停止原因全部符合上表， +每步完整 logits 均满足严格误差要求，全部 QEMU 进程和进程组已回收。 + +| 用例 | 生成调用耗时（s） | QEMU CPU（s） | Host CPU（s） | 最大绝对误差 | +|---|---:|---:|---:|---:| +| France | 3375.024 | 11517.789 | 167.657 | 0.000303983688 | +| Germany | 3668.852 | 12574.174 | 190.673 | 0.000262737274 | +| Japan | 3489.655 | 11958.881 | 178.759 | 0.000197887421 | + +整次测量 10540.190 s（约 2 小时 55 分 40 秒），包括准备、ORT 和数值比较。 +24 步 UART 耗时合计 9990.355 s，平均 **416.265 s/token**；QEMU 完整进程合计 +9998.091 s，CPU 合计 36050.844 s，平均约 360.58%（100% 表示一个逻辑 CPU）。 +Host 总 CPU 540.717 s；表中生成阶段合计 537.089 s。 +RSS 观测峰值 QEMU 5.406 GiB、Host 4.254 GiB、同时进程合计 7.644 GiB。 +17/24 步的采样各有一次读取失败，属于 partial，不能当作无缺失峰值。 + +**独立单次完整前向与干净容器 Paris。** 两次都只生成 `[12095]`、文本 ` Paris`， +完整 logits 最大误差同为 `0.000172138214`。容器内重新构建一次，没有重复三例套件； +155,582,464 字节完整输出与宿主单次前向逐字节相同。 + +| 测量项 | 宿主单次前向 | 容器单次前向 | +|---|---:|---:| +| 全过程耗时（s） | 448.263 | 547.454 | +| UART 耗时（s） | 418.954 | 415.058 | +| QEMU 完整进程耗时（s） | 419.383 | 415.478 | +| QEMU CPU（s） | 1498.851 | 未独立采集 | +| QEMU RSS 观测峰值（GiB） | 5.398 | 5.398 | + +宿主生成调用耗时 441.033 s，Host 总 CPU 25.332 s（生成期间 21.604 s）， +Host/同时进程 RSS 观测峰值分别 4.119/7.210 GiB。容器总耗时含新构建、FMA 和 ORT； +运行器记录 430.481 s 包括权重传输、QEMU、后检查及清理，不含更早的输入打包、写入和 +ELF/bundle 检查。容器 RSS 有 2071 个有效样本、1 次读取失败，同样为 partial。 + +UART 耗时包含 QEMU 启动和权重装载，不是纯算子时间；完整进程另含 stop/dump/quit。 +宿主单次前向启动了新 QEMU，但未清空宿主页缓存，不宣称冷缓存性能。 +容器单 prompt 的 `full_model_gate=NOT_RUN` 表示未运行三例，不表示未运行完整模型。 + +**小图与多轮行为。** 历史真实容器默认单 hart、手动 4 hart、auto 解析为 4 的小图均 +生成 IDs `[2,3,4,6]`、文本 `2 3 4`，EOS 停止，逐步完整 logits 对 ORT 的最大误差为 0， +padding `none`。不同上下文和中断后的再次生成可复用同一配置 ELF/bundle； +历史 QEMU 超时/取消及 Docker 构建 SIGINT/SIGTERM 后未遗留目标进程。 +官方聊天模板的有/无 system 三轮渲染和 token 序列与 Transformers 一致；这些回复由 +模板验证预设，未表示完整模型三轮聊天实跑。 + +**2026-10-25 局部修复验证。** 基于 `e93233143d2323de7de6888ff7d73bdbf50784c3` +加入本轮报告保存与取消处理修复后，注入权限、磁盘满、I/O 和序列化失败，CLI 均保留原始生成错误 +或取消退出码；原本成功的运行在最终报告无法保存时返回失败，不误报成功。后续调用 +仍可正常保存。本轮没有重新执行完整模型，上方完整数值与性能数据仍属于历史基线。 + +真实 Docker 小图在默认 1 hart 和手动 4 hart 下，正常生成均得到 `[2,3,4,6]`、 +文本 `2 3 4`，逐步 ORT 最大误差为 0。向 Docker 客户端发送 SIGINT/SIGTERM 后， +报告均为 `CANCELLED`，退出码分别为 130/143；宿主 PID 被回收、QEMU 进程组为空、 +`--rm` 容器消失，0.3 秒后复查仍无残留。中断检查以临时 `-S` 包装器控制真实 guest +的暂停时机，不修改生产 harness；正常生成和恢复使用实际小图前向。 + +两种信号之后均复用原 ELF/bundle 恢复生成，IDs、文本及全部逐步 logits 仍与基线一致； +4 hart 的恢复还实测 `auto` 解析为 4，两组 padding 均为 `none`。这些结果说明小图 +取消、清理和复用正常，不扩大为完整模型重跑或完整多轮聊天结果。 + +原始产物为本地 `output/` 内容，不提交权重、ELF 和逐步 logits： + +- `output/w5-followup-20261023/qemu-reference-4hart/run/report.json`： + 完整三例；同目录 `steps.json`、`numeric-steps.json`、`resources.json` 为计时与资源细节。 +- `output/w5-followup-20261023/qemu-final-single-forward-4hart/`：宿主独立单次前向。 +- `output/w5-container-20261023/paris/run/report.json`：容器完整 Paris 单 token。 +- `output/w5-container-20261023/small-summary.json`、`small-runs/` 与 `build-cancel/`： + 容器小图、构建取消和进程清理记录。 +- `output/w5-hardening-20261025/docker-final/test_external_signals_reap_gue{0,1}/docker-cancellation-summary.json`： + 本轮 1/4 hart 小图外部取消、进程清理和恢复结果。 + +完整三例的独立容器运行以及其他执行人的独立复现记录仍未取得；不能把宿主三例、 +容器单 token 或本轮局部验证合计成这些结果。现有实现仍是无 KV cache、逐 token +重启 QEMU 与装载权重；常驻运行、多轮完整模型语义和其他算术策略的完整结果未在 +上述实跑范围内。 diff --git a/docs/llm-deploy-v1.0/W5/chat.md b/docs/llm-deploy-v1.0/W5/chat.md deleted file mode 100644 index 397e0fcb..00000000 --- a/docs/llm-deploy-v1.0/W5/chat.md +++ /dev/null @@ -1,27 +0,0 @@ -# W5 Host 三轮文本对话 - -状态：Host 历史管理和官方模板一致性已实现；这里的验证不加载模型权重，不代表完整 QEMU 多轮语义验收。 - -## 接口和语义 - -`Conversation.from_directory(tokenizer_directory, system_prompt=None)` 从同一固定本地 snapshot 加载 tokenizer 与 `tokenizer_config.json` 中的官方 `chat_template`。生产运行只需 `tokenizers` 和 `Jinja2`，不依赖 Transformers 或 Torch。模板通过 Jinja2 的不可变沙箱渲染，固定 `add_generation_prompt=True`、`enable_thinking=False`，不手写替代 Qwen3 的模板。 - -`turn(user_text, forward=backend, max_new_tokens=..., cancelled=..., timeout_seconds=...)` 复用 Host `generate`，返回同一 `GenerationResult`。仅支持文本 system/user/assistant：system 只能在构造时提供，后续 user/assistant 顺序由成功的 turn 维护，不支持工具、图片或其他角色。`history` 是不可变快照；`reset()` 仅清除已完成回合，保留初始 system 文本。 - -每轮将完整历史重新套用官方模板并编码，不保留 KV cache，不暗中删除早期消息，也不截断到 256 tokens。超过容量会在调用 backend 前报错。原始完成模式的三条参考用例保持 raw prompt，不自动套用聊天模板。 - -一次 turn 只有成功返回且至少生成一个 token，才同时提交 user 和 assistant 消息。异常、取消、超时、零 token 预算，以及恰好占满 256-token 上下文且没有生成空间的情况，均不修改历史。EOS-only 回复可作为合法空 assistant 回复提交。预算或容量中止后的回复保留返回的文本，`stop_reason` 明确记录其终止原因。历史存储文本；下轮重新编码，未承诺历史 token IDs 原样拼接。 - -超时与取消沿用 Host 生成合同。该层的协作检查不能中断正在阻塞的 backend；QEMU adapter 负责运行中的取消、超时以及进程清理。一个 Conversation 实例不能并发或重入使用，活动 turn 中拒绝另一个 turn 或 reset。 - -## 相关验证 - -```bash -python -m pytest tests/test_conversation.py -q -``` - -固定本地 tokenizer 存在时，测试还会使用 `AutoTokenizer.from_pretrained(..., local_files_only=True)` 独立核对官方 Transformers 的 `apply_chat_template`，无网络下载、无模型前向。可通过 `SCRATCHV_QWEN3_TOKENIZER_DIR` 指定本地快照，默认是仓库 `assets/qwen3-0.6b/tokenizer`。 - -2026-10-23：40 项通过，无跳过。覆盖三轮精确历史顺序、非 thinking 格式、两会话隔离、reset、不可变快照、角色/文本校验、零预算、EOS-only、容量边界、异常原样传播、取消/超时回滚以及重入保护。其中官方 tokenizer 对照覆盖有/无 system 的各三轮，共六份渲染文本和 token 序列；assistant 文本在此模板测试中为预设值，未作完整模型生成。 - -默认单核、手动/自动 hart 选择及 padding 策略由已有 backend 保持，本模块不改变它们。 diff --git a/docs/llm-deploy-v1.0/W5/deployment.md b/docs/llm-deploy-v1.0/W5/deployment.md deleted file mode 100644 index 403c5a8a..00000000 --- a/docs/llm-deploy-v1.0/W5/deployment.md +++ /dev/null @@ -1,209 +0,0 @@ -# W5：Linux 容器部署入口 - -本页说明 [Dockerfile](../../../deployment/w5/Dockerfile) 和 -[一键脚本](../../../scripts/deploy_w5.sh) 的使用方式。 -生成合同、固定参考和阶段门槛见 [W5 总览](README.md)。 - -交给同事复现 PR115 时，使用[第二人独立复现说明](independent-reproduction.md)， -其中固定代码提交、基础镜像及三例完整验收参数，并列出环境差异处理、清理和回传清单。 -下文保留已有部署用法与作者实跑记录，不作为第二人已完成的证据。 - -## 部署范围 - -镜像使用 Ubuntu 24.04 / Python 3.12，安装发行版 `qemu-system-misc`，并直接复用 -`python -m pip install -e ".[llm-deploy]"` 固定的依赖，其中包括 Zig 0.14.1。 -不再维护第二份 Python/Zig 版本列表。镜像中的 QEMU 由 Ubuntu 软件源提供； -正式验收须记录实际版本，不能将它的耗时与另一 QEMU 版本的历史记录直接比较。 - -基础镜像默认仍为 `ubuntu:24.04`。Docker Hub 不可达时，可显式设置 -`SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu:24.04`，选择 Canonical 发布的 -[Amazon ECR Public Ubuntu 镜像](https://gallery.ecr.aws/ubuntu/ubuntu)； -来源说明见 [Ubuntu 官方容器文档](https://ubuntu.com/docs/oci-registries/)。 -脚本将该值作为一个完整的 `--build-arg BASE_IMAGE=...` 参数传入 Dockerfile； -未设置或为空时保留默认值，不自动切换来源、不重标记 `ubuntu` 镜像，也不修改引擎或代理配置。 -例如，在下方部署命令前增加： - -```bash -SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu:24.04 \ - bash scripts/deploy_w5.sh --help -``` - -此例仍会通过原脚本构建镜像，再运行 CLI 帮助；实际生成时追加下方原有参数即可。 - -[构建上下文白名单](../../../deployment/w5/Dockerfile.dockerignore) 仅包含源码、安装 -元数据和命令行入口所需文件，模型权重、tokenizer、虚拟环境、ELF、Git 和历史输出 -均不随镜像构建上下文发送。模型资产在运行时只读挂载到 `/assets`，构建和报告写入 -可写挂载 `/output`。脚本按调用者的 Linux UID/GID 运行，避免产生 root 所有的报告。 - -脚本只做容器预检、构建与启动，所有生成参数原样传给已有 CLI，不再实现一套 -采样、token 循环、模型下载或校验流程。未准备资产时明确失败，不自动下载权重。 -Linux 主机需要 `setsid`（通常由 util-linux 提供）。引擎预检和镜像构建客户端在各自 -进程组内运行；脚本收到 SIGINT/SIGTERM 时会向客户端和辅助进程转发终止、等待清理， -5 秒后对仍存在的进程组强制结束，并以 130/143 返回，不继续启动模型。 -容器启动后保留 `exec ... run --init` 的信号转发路径。本轮已额外验证真实 Docker -构建取消时客户端进程组和引擎侧 RUN 进程均消失，实测范围和证据见下文。 -Docker 是本页的部署目标；替换容器 CLI 前需确认其支持 Dockerfile 专用 ignore 文件， -本轮未验证 Podman 兼容性。 - -## 小图部署验证 - -先在正常 Linux 开发环境准备一个满足生成 ABI 的小图及对应本地 tokenizer，放在同一 -资产目录下：`model.onnx`、`tokenizer/`；若 ONNX 使用外置数据，数据文件也必须位于 -该挂载目录中。小图使用与正式生成相同的固定长度输入和 logits 接口，但不能当作 -Qwen3 的语言能力或 Paris 门槛证据。 - -本轮已有真实 CLI 小图资产 `output/w5-cli-small-20261023/model.onnx` 和 -`output/w5-cli-small-20261023/tokenizer/`，可在同一工作树中复用。先复制到统一的 -只读资产挂载目录，再执行下面的容器命令；本轮真实容器结果见页末： - -```bash -mkdir output/w5-small-assets -cp -a output/w5-cli-small-20261023/model.onnx \ - output/w5-cli-small-20261023/tokenizer output/w5-small-assets/ - -SCRATCHV_W5_ASSETS="$PWD/output/w5-small-assets" \ -SCRATCHV_W5_OUTPUT="$PWD/output/w5-container-smoke" \ - bash scripts/deploy_w5.sh \ - --model /assets/model.onnx \ - --tokenizer-dir /assets/tokenizer \ - --prompt 1 --max-new-tokens 8 \ - --matmul-policy blocked_fma \ - --harts 1 --timeout 30 --total-timeout 120 \ - --verify-onnx /assets/model.onnx -``` - -2026-10-23，以上资产已通过宿主 Linux CLI 的真实 QEMU 对照：1 和 4 hart 均 -**PASS**，各运行 4 次前向，生成 IDs 同为 `[2, 3, 4, 6]`，文本为 `2 3 4`，由 EOS -停止；每步进行 ORT 数值/greedy 对照。使用显式 `blocked_fma / 1x4 / original`、 -padding `none`。证据位于 `output/w5-cli-final-20261023/hart1/report.json` 和 -`output/w5-cli-final-20261023/hart4/report.json`，这两份是本地诊断产物，不提交 Git。 -上述两份记录是**宿主 CLI 小图通过**，本轮新增的真实容器证据另列于页末。宿主完整模型的初始旧算术 -配置曾使三例均在首步完整 logits 检查失败；该历史结果保留于 [实跑报告](qemu-generation.md)。 -随后显式启用 Mean 与非线性参考策略的正式 4 hart QEMU 套件已完成 **3/3 用例、24/24 步通过**， -数值、冻结 IDs、最终文本与停止原因均满足合同。通过配置对应下方完整验收示例， -默认单核、Mean `sequential`、非线性 `libm` 与 padding `none` 不变。 -这些宿主结果不替代容器证据；本轮容器小图和清理、完整 Paris 单 token 均已有独立实跑通过记录。 - -脚本默认将 CLI 输出写入 `/output/run`，该子目录必须尚不存在；重复验证时显式 -增加 `--output-dir /output/run-h4` 等新目录，并传 `--harts 4`。不重复使用已有输出目录， -也不自动删除旧报告。宿主输出目录可以包含空格，挂载路径不可包含逗号。 - -没有指定 `--harts` 时仍为单核，手动 `1/2/4` 和 `auto` 原样交给 CLI。`auto` 在构建时 -解析一次，不在 token 之间改变 ELF 的 hart 数。padding 始终关闭。运行器每个 token -可以重新启动 QEMU，但复用同一轮构建得到的 ELF 和权重 bundle，不按 token 重新编译。 - -`--reuse-build` 仅复用相同容器路径下的已保存构建。构建元数据可能包含绝对路径， -不要直接把宿主生成的 ELF/bundle 元数据拿到 `/output` 下宣称可跨环境复用；跨环境 -从 `--model` 重新构建。在容器内先构建、后复用时，保持挂载目标和构建路径一致。 - -## 容器完整模型最短生成命令（本轮已通过） - -固定真实模型资产沿用 W1 manifest 和 W5 三条参考；目录结构是 -`onnx/model.onnx`、外置权重和 `tokenizer/`。本轮仅补 France 的一个真实生成 token， -不在容器重复已经通过的宿主三例 24 步： - -```bash -SCRATCHV_W5_ASSETS="$PWD/assets/qwen3-0.6b" \ -SCRATCHV_W5_OUTPUT="$PWD/output/w5-container-acceptance" \ -SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu:24.04 \ - bash scripts/deploy_w5.sh \ - --model /assets/onnx/model.onnx \ - --tokenizer-dir /assets/tokenizer \ - --prompt 'The capital of France is' --max-new-tokens 1 \ - --verify-onnx /assets/onnx/model.onnx --harts 4 \ - --matmul-policy blocked_fma --matmul-tile 2x4 \ - --matmul-rhs-layout column-panel \ - --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ - --timeout 1800 --total-timeout 3600 -``` - -该命令执行一遍完整 28 层模型，验证容器内实际生成的 ` Paris`，不会运行 3×8 token。 -本轮实测产物位于 `output/w5-container-20261023/paris/`，已成功生成 ` Paris`，进程退出码为 `0`。 -单 prompt 命令的 `full_model_gate=NOT_RUN` 表示没有再次运行三例 suite； -不能把它误读为没执行完整模型，也不能将单次成功计为容器三例通过。 -宿主三例和独立单次前向记录继续保留于 [实跑报告](qemu-generation.md)。 - -可设置 `SCRATCHV_CONTAINER_ENGINE` 为 Docker 可执行文件， -`SCRATCHV_W5_IMAGE` 为本地镜像标签；默认分别为 `docker` 和 `scratchv-w5:local`。 -一键脚本会先检查引擎存在且可用，再构建镜像；不会安装引擎、修改 WSL 或 Windows。 - -## 本轮真实容器验证（2026-10-23） - -早期检查时环境没有容器引擎，历史静态检查不冒充容器验收。本轮已使用 **Docker -Engine 29.9.2 / Buildx 0.38.1** 真正构建镜像并运行容器。引擎仅手动启动； -验证结束后 `docker.service / docker.socket / containerd.service` 均已 -**inactive + disabled**，停止前无运行容器。临时 socket 权限恢复为 `root:docker 0660`， -未新增永久组成员，WSL 配置原文保持不变；已安装引擎保留，不开启自启。 - -Docker Hub 的认证请求出现 TLS EOF，IPv4/TLS 1.2 复查仍失败；本轮因此显式设置 -`SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu:24.04`，使用上方 Canonical -官方来源。默认仍为 `ubuntu:24.04`，没有重标记该镜像或修改代理配置。 -基础镜像和部署镜像实际拉取/构建成功；小图镜像内 Python **3.12.3**、Zig **0.14.1**、 -QEMU **8.2.2（Ubuntu 包 1:8.2.2+ds-0ubuntu1.18）**、ORT **1.22.1**、 -NumPy **2.2.6**，Torch 为 **2.7.1+cpu**。 - -| 真实容器检查 | 结果 | -|---|---| -| 默认单 hart CLI 小图 | PASS；4 次前向，IDs `[2,3,4,6]`，文本 `2 3 4`，EOS 停止 | -| 手动 4 hart CLI 小图 | PASS；同上，完整 logits 逐位 ORT 一致 | -| `auto` 复用 4 hart 构建 | PASS；解析为 4，4 次前向，IDs/text/EOS 一致 | -| 三组小图数值与布局 | 全部 `max_abs=0`，padding `none` | -| 真实 QEMU 1/4 hart 多 token、超时、取消、复用 | 2 项测试 PASS、0 skip；每个配置 8 次 guest 启动（6 次成功、2 次预期中断），16 次进程/进程组均回收 | -| Docker 实际构建 SIGINT/SIGTERM | 2/2 PASS；退出码 130/143，客户端组与引擎侧 RUN 进程均消失，未启动模型 | -| 完整 28 层 France 单 token 生成 | **PASS**；IDs `[12095]`、文本 ` Paris`，预算停止，完整 logits `max_abs=0.000172138214 < 0.001`；28 层图审计、FMA、QEMU 回收均通过 | - -QEMU 清理测试直接复用已有 `test_real_qemu_multitoken_reuse_numerics_timeout_and_cancel`， -不同上下文生成 `[4]`，超时/取消后再次生成 `[6]`，均使用同一配置内的 ELF/bundle。 -真实 Docker 构建取消使用原部署脚本的逐字节相同副本和独立小型诊断上下文， -在实际引擎侧 RUN 进程已运行后分别发 SIGINT/SIGTERM；两次约 **0.117s / 0.115s** -观察到清理完成，延后复查无残留。它验证实际 Docker 构建生命周期，不运行模型或下载资产。 - -之前的 Bash 语法及 **17 项**隔离/打包测试继续有效，包含默认基础镜像、空值回退、 -ECR 与字面量参数边界及客户端取消；本轮真实引擎结果另列，不把替身检查重复累计成真实容器测试。 -完整 Paris 验收由实际 Docker 调用独立执行：容器内从只读 ONNX 资产重新构建 ELF/bundle， -显式使用 `4 hart / blocked_fma / 2x4 / column-panel / mean two-four-lane-last-axis / -nonlinear fp32-reference / padding none`，仅生成 1 token。输入 prompt IDs 为 -`[785,6722,315,9625,374]`，输出 `[12095]`、` Paris`，停止为 `max_new_tokens`。 -全部 **38,895,616** 个 logits 对当轮 ORT 的最大误差为 **0.000172138214**，没有跳过补齐行。 - -| 容器 Paris 实测范围 | 结果 | -|---|---:| -| CLI 总耗时（含独立重新构建、FMA、ORT 和核验） | 547.454 s | -| UART 前向耗时（含 QEMU 启动和装载） | 415.058 s | -| QEMU 完整进程耗时（另含 stop/dump/quit） | 415.478 s | -| 运行器记录耗时（权重传输、QEMU、后检查及清理） | 430.481 s | -| QEMU RSS 观测峰值 | 5.398 GiB | -| 模型 QEMU 前向次数 / 成功回收 | 1 / 1 | - -运行器记录耗时不含计时起点之前的输入打包、输入文件写入及 ELF/bundle 校验，不能当作整个 adapter 调用耗时。 - -RSS 来自 0.2s 周期采样，共 2071 个有效样本；有 **1 次 RSS 读取失败**， -记录为 `partial`，上述值是观测峰值，不声称完整无缺失采样。本次没有独立采集 -QEMU/Host CPU 总时间，不用 Docker 客户端时间或宿主前向记录冒充容器 CPU。 -完整 **155,582,464 字节**输出已直接与既有宿主独立完整前向比较，**逐字节一致**， -没有新增哈希检查。 -正式 CLI 保留 `full_model_gate=NOT_RUN` 和 `container_reproduction=NOT_ASSERTED`： -单 prompt 没有重跑三例 gate，而容器身份不由内部 CLI 自行宣称。外层真实 Docker 构建、 -容器调用、退出码与完整结果共同构成 **deploy:script PASS（本地真实容器）** 的证据； -宿主三例 24 步继续单独记录，不扩张为容器三例通过。 - -本地证据路径相对仓库根： - -- `output/w5-container-20261023/engine.json`、`image.json`：引擎和镜像内实际工具版本。 -- 同目录 `default-h1.log`、`base-pull.log`、`default-h1-ecr.log`：Docker Hub 失败、官方 ECR 拉取与实际构建记录。 -- 同目录 `small-summary.json` 与 `small-runs/{default-h1,manual-h4,auto-reuse}/report.json`：三组真实 CLI 小图。 -- 同目录 `small-runs/real-qemu-tests.xml`、`real-qemu-tests.log`，以及 `small-runs/real-qemu-tests/` 下两份 `generation-check.json`：真实 QEMU 数值、中断及复用。 -- 同目录 `build-cancel/report.json`、`build-cancel/delayed-recheck.json`：Docker 实际构建取消与延后清理检查。 -- 同目录 `paris.log`、`paris-summary.json`、`paris/run/report.json` 和 `paris/run/steps/step-0000/run.json`：本轮唯一完整模型容器生成、图审计/FMA/数值与回收证据。 -- 同目录 `services-final.json`：容器清空、服务 inactive/disabled、临时 socket 权限恢复与 WSL 配置未变。 - -W5 CI 已复用既有流程接入；容器收尾时 753 项本地轻量检查通过，新增必跑节点仅收集核对。 -后续提交前检查补齐 W5-only 手动入口和 job 余量，811 项相关检查通过，最终 11 项重叠 -边界复查通过，未重新执行完整模型，见 [提交前收尾](README.md#提交前检查与收尾2026-10-23)。 -具体模式、资源要求与证据边界见 [CI 接入说明](README.md#ci-接入与远端执行边界)。 -上述容器收尾和提交前检查发生时尚未提交、未推送。随后 W5 已发布为 PR115， -提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 的普通 PR CI 已通过；完整 W5 -专项因 fork Actions 在 dispatch 阶段返回 HTTP 422，仍为 **BLOCKED / NOT_RUN**。 -完整容器三例 24 步和第二人独立复现尚待完成，不能将本页单 token 结果扩张为这些验收通过。 -最新状态见 [W5 阶段表](README.md#阶段与验收状态)。W5 本地功能出口与容器收尾已完成； -默认配置及 WSL 配置不变，PR 未合并。 diff --git a/docs/llm-deploy-v1.0/W5/independent-reproduction.md b/docs/llm-deploy-v1.0/W5/independent-reproduction.md deleted file mode 100644 index 5b1a69a3..00000000 --- a/docs/llm-deploy-v1.0/W5/independent-reproduction.md +++ /dev/null @@ -1,365 +0,0 @@ -# W5：PR115 第二人独立复现交接 - -本说明供同事在独立的 Ubuntu 环境中执行，针对 -[PR115](https://github.com/ScratchV-Compiler/ScratchV/pull/115) 的固定提交 -`8ad347477f7f5ba15b2da7c91178b3c6668456cc`，不跟随分支最新提交。 -该 SHA 是已有完整数值验收的功能基线；本次后续提交仅修改审查超时处理、相关测试和文档， -生成实现未变，因此复现仍固定此版本，不随分支移动。 -本文整理完成不表示第二人已复现；执行人回传下面的记录后才能签收。 -作者再次执行或 CI 通过不能代替第二人独立复现，见 -[Linux 交付约定](../LINUX_REPRODUCTION.md#4-执行与留存证据)。 - -## 复现范围与固定环境 - -使用新 clone、新容器构建和独立输出目录，只从固定源码及既有资产来源准备环境。 -不要复制作者的虚拟环境、`output/`、ELF、bundle 或报告。 -复用 [deploy_w5.sh](../../../scripts/deploy_w5.sh)，容器内独立构建一次 ELF/bundle； -三条用例的 **24 次前向共用同一构建**，每个 token 仍启动一次 QEMU。 -不先单独运行 Paris、不重录 ORT 参考，也不额外跑 W4 完整前向。 - -| 项目 | 本次约定 | -|---|---| -| 源码 | 上述完整 SHA；全程工作树干净，不修改文件以消除失败 | -| 主机 | Ubuntu 24.04 x86_64；Docker Engine/Buildx 已安装且当前账号可用 | -| 主机工具 | Bash、Git/Git LFS、Python 3.12、GNU time、coreutils（timeout）、util-linux（setsid）、tar | -| 基础镜像 | `public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca` | -| 容器 Python/依赖 | Python 3.12；依赖以固定提交的 `pyproject.toml[project.optional-dependencies].llm-deploy` 为准，不另安装一套推理环境 | -| 已有实跑版本 | Python 3.12.3、QEMU 8.2.2（Ubuntu 包 `1:8.2.2+ds-0ubuntu1.18`）、Zig 0.14.1、ORT 1.22.1、NumPy 2.2.6、Torch 2.7.1+cpu | -| 模型 | `Qwen/Qwen3-0.6B`；revision `c1899de289a04d12100db370d81485cdf75e47ca`；固定 FP32 ONNX、batch=1、容量 256、无 KV cache | -| 生成配置 | 4 hart；`blocked_fma / 2x4 / column-panel`；Mean `two-four-lane-last-axis`；非线性 `fp32-reference`；padding `none` | -| 资源 | 镜像和资产准备后，至少 4 个可用逻辑 CPU、10 GiB 可用内存、10 GiB 剩余磁盘；初始建议预留至少 25 GiB 磁盘 | -| 时间预算 | 预留数小时；单步 1800 s，CLI 总预算 16500 s，外层 280 分钟，超时后最多再等 30 s | - -基础镜像 digest 来自原容器实际拉取和构建记录，用于固定已有制品身份，不新增哈希流程。 -**它只固定基础镜像**：原 Dockerfile 的 apt 软件源和未逐项锁定的间接依赖仍可能更新。 -下面记录实际镜像 ID、系统包、依赖和工具版本，并确认 suite 前后镜像 ID 一致。 -若 QEMU 不再是 8.2.2、固定 Python 依赖不匹配，或系统包版本与上表不同， -先回传环境差异，由负责人确认后再运行完整套件；不要自行降级、改 Dockerfile 或宣称完全相同环境。 -原有 Docker 实跑使用 Engine 29.9.2 / Buildx 0.38.1；同事记录自己的版本，版本不同须注明。 -这些说明不要求修改 WSL/Windows 或安装宿主 QEMU。 - -**默认值不变**：默认单 hart，仍可手动选择 `1/2/4` 或 `auto`。 -此交接只显式验收上述 4 hart 数值策略；不把它写成默认算术配置已通过完整模型。 -`auto` 在构建时解析一次，不在 token 间调整。4 hart 也不意味着容器只获配 4 个宿主 CPU。 -W5 CLI 固定 padding 关闭，**没有** `--qemu-workspace-layout` 参数。 - -## 1. 新 clone 并固定提交 - -以下命令在同一个 Bash 会话按顺序执行，从有足够空间的父目录开始。 -目录名 `scratchv-w5-repro` 必须不存在；重试时改用新的目录名，保留失败记录。 - -```bash -set -euo pipefail -export W5_COMMIT=8ad347477f7f5ba15b2da7c91178b3c6668456cc -test ! -e scratchv-w5-repro -GIT_LFS_SKIP_SMUDGE=1 git clone --no-checkout --filter=blob:none \ - https://github.com/ScratchV-Compiler/ScratchV.git scratchv-w5-repro -cd scratchv-w5-repro -git fetch --no-tags origin refs/pull/115/head -GIT_LFS_SKIP_SMUDGE=1 git checkout --detach "$W5_COMMIT" -test "$(git rev-parse HEAD)" = "$W5_COMMIT" -test -z "$(git status --porcelain)" - -export W5_RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" -export SCRATCHV_CONTAINER_ENGINE=docker -export SCRATCHV_W5_IMAGE="scratchv-w5:independent-$W5_RUN_ID" -export SCRATCHV_W5_ASSETS="$PWD/output/w5-independent-assets" -export SCRATCHV_W5_OUTPUT="$PWD/output/w5-independent-$W5_RUN_ID" -export SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca -test ! -e "$SCRATCHV_W5_ASSETS" -test ! -e "$SCRATCHV_W5_OUTPUT" -mkdir -p "$SCRATCHV_W5_OUTPUT" -date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/start-utc.txt" -{ - git rev-parse HEAD - git status --porcelain - git remote -v -} > "$SCRATCHV_W5_OUTPUT/source.txt" -{ - cat /etc/os-release - uname -a - lscpu - free -b - df -B1 . - python3 --version - git --version - git lfs version - docker version - docker buildx version - docker info --format 'OS={{.OSType}} Arch={{.Architecture}} CPUs={{.NCPU}} Memory={{.MemTotal}}' - command -v setsid timeout tar - /usr/bin/time --version -} > "$SCRATCHV_W5_OUTPUT/runner-resources.txt" 2>&1 -docker info > /dev/null -test "$(uname -m)" = x86_64 -python3 -c 'import sys; assert sys.version_info[:2] == (3, 12)' -printf 'Run output: %s\n' "$SCRATCHV_W5_OUTPUT" -``` - -检查 `runner-resources.txt`：宿主应为 Ubuntu 24.04，Docker daemon 也是本机 Linux； -若使用远端 daemon、容器资源配额不足或没有引擎，先停止并回报，不用修改 WSL 来凑条件。 -挂载目录不能包含逗号。不要把私人凭据写入报告。 - -## 2. 构建同一镜像并记录实际环境 - -复用现有 Dockerfile；随后部署脚本会再检查构建并复用 Docker 缓存。 -基础镜像和构建参数不变，不使用作者构建好的镜像。 - -```bash -docker build --file deployment/w5/Dockerfile \ - --build-arg "BASE_IMAGE=$SCRATCHV_W5_BASE_IMAGE" \ - --tag "$SCRATCHV_W5_IMAGE" . \ - 2>&1 | tee "$SCRATCHV_W5_OUTPUT/image-build.log" -docker image inspect "$SCRATCHV_W5_IMAGE" \ - > "$SCRATCHV_W5_OUTPUT/container-image.json" -docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ - > "$SCRATCHV_W5_OUTPUT/image-id-before.txt" -docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" -m pip freeze \ - > "$SCRATCHV_W5_OUTPUT/dependencies.txt" -docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" \ - -c 'import json; from probes.w1_qwen3_export.run import environment; print(json.dumps(environment(export=True), indent=2))' \ - > "$SCRATCHV_W5_OUTPUT/image-environment.json" -docker run --rm --entrypoint sh "$SCRATCHV_W5_IMAGE" -c \ - 'python --version; python -m ziglang version; qemu-system-riscv64 --version; dpkg-query -W python3.12 qemu-system-misc' \ - > "$SCRATCHV_W5_OUTPUT/image-tools.txt" -cat "$SCRATCHV_W5_OUTPUT/image-tools.txt" -``` - -人工核对 `image-tools.txt` 与环境表后再继续。版本有偏差时，保留日志并联系负责人； -部署脚本本身不锁 apt 补丁，也不会替你判定偏差是否可接受。 - -## 3. 独立获取固定资产 - -下载只调用现有 `acquire_model`，沿用 W1 manifest 中已有的下载/分片完整性检查。 -它只用 Python 标准库，不执行模型、不重新导出、不安装宿主推理依赖。 -Tokenizer 从同一提交的 Git LFS 获取。 - -```bash -git lfs install --local -git lfs pull --include="assets/qwen3-0.6b/tokenizer/*" --exclude="" \ - 2>&1 | tee "$SCRATCHV_W5_OUTPUT/tokenizer-acquisition.log" -python3 - <<'PY' 2>&1 | tee "$SCRATCHV_W5_OUTPUT/model-acquisition.log" -import json, os, shutil -from pathlib import Path -from probes.w1_qwen3_export.run import acquire_model -assets = Path(os.environ["SCRATCHV_W5_ASSETS"]) -out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) -result = acquire_model(assets / "onnx") -shutil.copytree("assets/qwen3-0.6b/tokenizer", assets / "tokenizer") -(out / "acquisition.json").write_text(json.dumps(result, indent=2) + "\n") -PY -test -z "$(git status --porcelain)" -``` - -不要另跑参考采集脚本或添加 `--record`：suite 内部已经逐步对照当轮 ORT 与冻结参考， -无需再运行一遍完整 ORT 生成作为“预热”。 - -## 4. 先过三轮模板及资源预检 - -直接复用 CI 的两条官方模板测试，确认有/无 system prompt 的三轮历史拼接。 -这两条不运行完整模型，也不代表三轮完整 QEMU 聊天已验收。 - -```bash -docker run --rm --init --user "$(id -u):$(id -g)" --env HOME=/tmp \ - --env SCRATCHV_QWEN3_TOKENIZER_DIR=/assets/tokenizer \ - --mount "type=bind,src=$SCRATCHV_W5_ASSETS,dst=/assets,readonly" \ - --mount "type=bind,src=$SCRATCHV_W5_OUTPUT,dst=/output" \ - --mount "type=bind,src=$PWD/tests/test_conversation.py,dst=/tmp/test_conversation.py,readonly" \ - --entrypoint python "$SCRATCHV_W5_IMAGE" \ - -m pytest -q -c /dev/null -p no:cacheprovider \ - /tmp/test_conversation.py::test_official_three_round_rendering_matches_transformers \ - --junitxml /output/multiturn-tests.xml \ - 2>&1 | tee "$SCRATCHV_W5_OUTPUT/multiturn-tests.log" -python3 - <<'PY' -import json, os, shutil, xml.etree.ElementTree as ET -from pathlib import Path -out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) -root = ET.parse(out / "multiturn-tests.xml").getroot() -for suffix in ("None", "Give brief answers."): - name = f"test_official_three_round_rendering_matches_transformers[{suffix}]" - rows = [node for node in root.iter("testcase") if node.get("name") == name] - if len(rows) != 1 or any(rows[0].find(tag) is not None for tag in ("skipped", "error", "failure")): - raise SystemExit(f"Official three-round template must execute and pass: {name}") -mem = {line.split(":")[0]: int(line.split()[1]) * 1024 - for line in Path("/proc/meminfo").read_text().splitlines() - if line.startswith(("MemTotal:", "MemAvailable:"))} -evidence = {"stage": "after_image_and_assets", - "available_cpus": len(os.sched_getaffinity(0)), "memory": mem, - "disk_free_bytes": shutil.disk_usage(out).free, - "minimum_available_cpus": 4, - "minimum_available_memory_bytes": 10 * 1024**3, - "minimum_disk_free_bytes": 10 * 1024**3} -evidence["passed"] = (evidence["available_cpus"] >= 4 - and mem.get("MemAvailable", 0) >= 10 * 1024**3 - and evidence["disk_free_bytes"] >= 10 * 1024**3) -(out / "resource-preflight.json").write_text(json.dumps(evidence, indent=2) + "\n") -if not evidence["passed"]: - raise SystemExit("Insufficient W5 CPU/RAM/disk; generation NOT RUN.") -PY -``` - -预检检查主机可用资源；还需确认 Docker 没有更小的 CPU/内存配额。 -任一预检失败即停止，不运行 suite；回传已有日志，标记 `NOT_RUN`，不自动放宽门槛。 - -## 5. 一次运行三条完整用例，并始终留证 - -整块复制执行。`--reference-suite` 自带三条用例及每例 8-token 预算， -不要同时传 `--max-new-tokens`，不要传 `--reuse-build`。 -代码从固定模型重新构建一次，跨用例自动复用同一 ELF/bundle。 - -```bash -test "$(git rev-parse HEAD)" = "$W5_COMMIT" -test -z "$(git status --porcelain)" -test ! -e "$SCRATCHV_W5_OUTPUT/run" -date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-start-utc.txt" - -set +e -timeout --signal=INT --kill-after=30s 280m \ - /usr/bin/time -v -o "$SCRATCHV_W5_OUTPUT/process-time.txt" \ - bash scripts/deploy_w5.sh \ - --model /assets/onnx/model.onnx --tokenizer-dir /assets/tokenizer \ - --reference-suite --verify-onnx /assets/onnx/model.onnx \ - --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ - --matmul-rhs-layout column-panel \ - --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ - --timeout 1800 --total-timeout 16500 \ - 2>&1 | tee "$SCRATCHV_W5_OUTPUT/runner.log" -pipeline_status=("${PIPESTATUS[@]}") -printf '%s\n' "${pipeline_status[0]}" > "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" -printf '%s\n' "${pipeline_status[1]}" > "$SCRATCHV_W5_OUTPUT/log-exit-code.txt" -date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-end-utc.txt" - -# 此唯一 tag 只供本次使用；不停止其他人的容器。 -{ - containers=$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE") - list_status=$? - printf 'list_exit_code=%s\n' "$list_status" - for container in $containers; do - docker stop --time 30 "$container" - printf 'stop_exit_code=%s\n' "$?" - done - docker ps --filter "ancestor=$SCRATCHV_W5_IMAGE" \ - --format '{{.ID}} {{.Status}}' - printf 'final_list_exit_code=%s\n' "$?" -} > "$SCRATCHV_W5_OUTPUT/container-cleanup.log" 2>&1 - -docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ - > "$SCRATCHV_W5_OUTPUT/image-id-after.txt" -cmp "$SCRATCHV_W5_OUTPUT/image-id-before.txt" \ - "$SCRATCHV_W5_OUTPUT/image-id-after.txt" -printf '%s\n' "$?" > "$SCRATCHV_W5_OUTPUT/image-match-exit-code.txt" - -python3 -B -X utf8 scripts/check_w5_ci.py \ - --report "$SCRATCHV_W5_OUTPUT/run/report.json" \ - --reference probes/w5_generation/references.json \ - --exit-code-file "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" \ - --output "$SCRATCHV_W5_OUTPUT/ci-gate.json" \ - > "$SCRATCHV_W5_OUTPUT/gate-check.log" 2>&1 -printf '%s\n' "$?" > "$SCRATCHV_W5_OUTPUT/gate-exit-code.txt" -cat "$SCRATCHV_W5_OUTPUT/gate-check.log" -{ - date -u +%FT%TZ - git rev-parse HEAD - git status --porcelain - free -b - df -B1 . -} > "$SCRATCHV_W5_OUTPUT/final-resources.txt" -set -e -printf 'Preserved output: %s\n' "$SCRATCHV_W5_OUTPUT" -``` - -这里为失败留证临时关闭 `errexit`，**脚本走到最后不等于 PASS**。 -即使 suite 非零退出，仍运行容器清理和已有报告检查器；报告缺失时 checker 应失败， -不能补造成功字段。`ci-gate.json` 在本地的 `commit` 可为 null；实际 SHA 由 -`source.txt` 和最终 Git 记录确认，不冒充 GitHub Actions 执行。 - -若手动 Ctrl-C 使整段 shell 中断，重新进入同一 clone，恢复上面输出目录和镜像变量， -执行本节从“此唯一 tag”注释之后的收尾段，再打包已有报告。不要直接重新跑模型。 -主机掉电、daemon 失联或强制终止可能阻止清理；如清理日志有非零码或最后仍列出容器， -标为未完成并回报，不视为通过。 - -## 6. 三条用例及共同验收标准 - -全部为 **raw completion**：按 prompt 原文 tokenize，不加 chat 模板，greedy， -每例新生成 8 token，停止原因均为 `max_new_tokens`。字符串包含**前导空格**。 - -| 用例 | 原始 prompt | 必须匹配的新生成文本 | -|---|---|---| -| France | `The capital of France is` | ` Paris. The capital of Italy is Rome` | -| Germany | `The capital of Germany is` | ` Berlin. The capital of France is Paris` | -| Japan | `The capital of Japan is` | ` Tokyo. The capital of the United States` | - -新生成 token IDs 也必须与[冻结参考](../../../probes/w5_generation/references.json)完全一致： - -- France：`[12095,13,576,6722,315,15344,374,21718]`。 -- Germany：`[19846,13,576,6722,315,9625,374,12095]`。 -- Japan：`[26194,13,576,6722,315,279,3639,4180]`。 - -只看见 Paris/Berlin/Tokyo 不够。`check_w5_ci.py` 必须退出 0， -`ci-gate.json` 和 `run/report.json` 都为 PASS，且具备： - -- **3/3 用例、24/24 次 QEMU 前向和数值对照**；prompt IDs、生成 IDs、文本和停止原因均匹配。 -- 每步比较完整 `256×151936=38,895,616` 个 logits，包括补齐行；均有限，严格 `max_abs < 1e-3`、`rtol=0`；QEMU、当轮 ORT、冻结参考逐 token 一致。 -- 28 层完整图审计通过；FMA 的 12 行 binary32 位模式和 FCSR 检查全部通过。 -- 所有步骤使用同一 ELF/bundle 的既有身份；所选策略完全匹配，padding `none`。 -- QEMU 正常完成、退出 0、已回收，无清理错误；套件退出码为 0。 - -此外人工确认：官方三轮模板 **2/2 PASS、0 skip**，环境/资源预检通过， -`log-exit-code.txt` 和 `image-match-exit-code.txt` 为 0，源码未改， -`container-cleanup.log` 中命令全部成功且最后没有容器行。 -任何失败、跳过或缺失证据都不能签为完整 PASS，不以放宽数值阈值或重录参考处理。 - -`container_reproduction=NOT_ASSERTED` 是内部 CLI 的既有字段，不要改成 PASS。 -容器构建/执行日志、真实退出码和完整 suite 的通过记录共同证明本次独立容器复现。 - -## 7. 回传材料与签收 - -在 `$SCRATCHV_W5_OUTPUT/handoff.txt` 填写以下内容（可用编辑器，不改项目源码）： - -| 字段 | 要填写的内容 | -|---|---| -| 执行人/日期 | 姓名或团队账号，UTC 开始/结束时间 | -| 独立性 | 独立 clone、容器构建和输出目录；是否复制作者产物（应为否） | -| 版本/环境 | 完整提交 SHA、主机/CPU/内存、Docker/Buildx、实际镜像 ID、所有已确认的环境偏差 | -| 结果 | PASS / FAIL / NOT_RUN；三例各自 token 数、文本/IDs 是否匹配、数值最大误差、模板 2/2 结果 | -| 资源与计时 | CLI 总耗时、各步 UART/进程耗时、可用 RSS 观测峰值和采样缺失情况；未采集字段写“未独立采集” | -| 清理/异常 | 容器是否清空、工作树是否干净，失败阶段、日志位置及是否有人为中断 | - -**计时口径**：`process-time.txt` 的 CPU/RSS 是 Docker 客户端，不是容器或 QEMU 的总 CPU/RSS。 -逐步 QEMU 耗时/RSS 使用 `run/steps/*/run.json` 中已有数据；若采样状态为 partial， -注明为观测值。现有链路没有独立采集容器总 CPU，就写“未独立采集”； -不要把作者的历史 CPU 数据填到同事报告，不据此宣称跨机器加速比。 - -在同一 Bash 会话执行以下命令，复用 CI 的报告扩展名白名单，并额外保留小型 UART 完成帧 -`uart.bin`；排除缓存和其他二进制大制品： - -```bash -python3 - <<'PY' -import os, tarfile -from pathlib import Path -out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) -allowed = {".json", ".xml", ".txt", ".log", ".stderr", ".stdout"} -archive = out.parent / (out.name + "-reports.tar.gz") -if archive.exists(): - raise SystemExit(f"Keep earlier evidence; archive already exists: {archive}") -with tarfile.open(archive, "w:gz") as bundle: - for path in sorted(out.rglob("*")): - relative = path.relative_to(out) - if (path.is_file() and not path.is_symlink() - and ".zig-cache" not in relative.parts - and (path.suffix in allowed or path.name == "uart.bin")): - bundle.add(path, arcname=str(Path(out.name) / relative), recursive=False) -print(archive) -PY -``` - -回传 `output/w5-independent--reports.tar.gz`，至少包含： - -- `handoff.txt`、`source.txt`、UTC 时间、资源预检、实际镜像/工具/依赖记录、下载记录。 -- `image-build.log`、`multiturn-tests.xml/.log`、`runner.log`、所有退出码、清理日志和 `ci-gate.json`。 -- `run/report.json`、`run/executable.json`、`run/build/` 中的 JSON， - `run/steps/step-0000…0023/run.json` 和 UART 完成帧 `uart.bin`/错误日志，以及 `run/fma-conformance/` 中的报告。 - -失败时也按同一方式回传已有文件，并说明缺失项。准备阶段失败则标记 suite `NOT_RUN`。 -无需传权重、ELF、原始 logits、Zig 缓存；保留本地原始产物直到签收，若需重试则新建目录， -不覆盖旧报告。本任务不要求执行人提交代码、发起 AI 审查、操作 Actions 开关或合并 PR。 diff --git a/docs/llm-deploy-v1.0/W5/qemu-generation.md b/docs/llm-deploy-v1.0/W5/qemu-generation.md deleted file mode 100644 index f2e342d9..00000000 --- a/docs/llm-deploy-v1.0/W5/qemu-generation.md +++ /dev/null @@ -1,581 +0,0 @@ -# W5 QEMU 生成入口与续作报告 - -日期：2026-10-23。分支 `codex/w5-generation`，基于已合并 PR104 的 `61bded3`。 -所有变更仍在独立工作树内，未暂存、提交、推送。WSL 配置未修改；W5 CI 已接入既有流程并通过本地相关检查，远端尚未运行。 -生成合同及参考用例见 [总览](README.md)。 - -当前进展：初始旧算术配置的三例首步 FAIL 作为历史记录保留；后续显式 Mean 与非线性 -参考策略已完成 Host 与正式 4 hart QEMU **各 24/24 步、3/3 用例通过**，未放宽数值门槛。 -对应套件 CLI 的 `status` 和 `full_model_gate` 均为 `PASS`。额外的独立完整模型单次前向 -也已通过，完整输出与套件 France 首步逐字节一致；它单独记录，不计入这 24 步验收。 -下文旧失败表不代表新策略的最终状态。 - -## 实现范围 - -- [命令行入口](../../../scripts/run_inference.py)：从 ONNX 构建一次，或通过 `--reuse-build` 载入经过现有 ELF/权重/布局/生成源码检查的 `executable.json`。 -- [QemuForward](../../../scratchv/runtime/qemu_generation.py)：同一 ELF/bundle 跨 token、不同上下文复用；每次只更新输入并运行现有 QEMU stop/dump/quit 流程，完整输出后按有效位置 greedy。不会每 token 重新编译、写权重 bundle、运行 FMA 预检或准备 ORT session。 -- [运行器](../../../scratchv/runtime/riscv_external.py)：保留 UART 事件通知、QMP 停止/转储/退出；增加等待期间取消检查，任何异常均经过原有进程清理。50ms 是有取消回调时的最长检查间隔，UART/QMP 数据到达会立即唤醒，不额外给完成计时加 50ms。 -- [三轮历史](chat.md)：官方本地模板、显式 non-thinking、成功后提交 user/assistant；错误、取消、超时不污染后续历史。 -- [部署文件](deployment.md)：Ubuntu24.04/Python3.12 镜像、固定依赖与薄启动脚本。真实容器小图、QEMU 中断复用、Docker 构建取消及完整 Paris 单 token 均已通过。 - -保持默认 `1 hart / sequential / 1x4 / original / padding none`。 -Mean 默认仍为 `sequential`，非线性策略默认仍为 `libm`。 -手动 `--harts 1/2/4` 和 `auto` 均保留，auto 在构建/启动时只解析一次。 -复用 ELF 必须显式匹配其 hart、算术策略、tile 和 RHS 布局；不能因请求 auto 或省略参数 -悄悄换成另一种编译配置。1 与 4 hart 各有自己的编译产物，每种配置内部复用同一个 ELF/bundle。 - -## 命令行用法 - -在已准备 W4 固定环境的仓库根目录使用以下参数。这里以符合 ABI 的小图为例： - -```bash -python scripts/run_inference.py \ - --model output/w5-cli-small-20261023/model.onnx \ - --tokenizer-dir output/w5-cli-small-20261023/tokenizer \ - --prompt 1 --max-new-tokens 8 --harts 4 \ - --matmul-policy blocked_fma \ - --verify-onnx output/w5-cli-small-20261023/model.onnx \ - --output-dir output/w5-small-h4 --timeout 30 --total-timeout 120 - -python scripts/run_inference.py \ - --reuse-build output/w5-small-h4/executable.json \ - --tokenizer-dir output/w5-cli-small-20261023/tokenizer \ - --prompt 2 --max-new-tokens 8 --harts 4 \ - --matmul-policy blocked_fma \ - --verify-onnx output/w5-cli-small-20261023/model.onnx \ - --output-dir output/w5-small-h4-reuse -``` - -输出目录必须为新目录。小图资产由本轮测试准备，属于本地 `output`，不是预训练模型。 -单独准备构建可用 `--build-only`，其状态为 `BUILD_ONLY`，不会伪装成生成 PASS。 - -`--mode chat --chat-turn 'first question' --chat-turn 'second question' --chat-turn 'third question'` -可在一条命令内进行三轮，另可传 `--system-prompt`；三轮共用同一个 backend/构建。 -默认 `--prompt` 是 raw completion，不隐式插入聊天模板。 - -`--verify-onnx` 在同一个 ORT CPU session 上比较每步相同输入的完整 logits，并要求 -最大绝对误差严格小于已有 W4 的 `1e-3`，greedy ID 也必须一致；保留 top-2 margin。 -上轮小图全部逐位一致，实际 `max_abs=0`，并非仅靠误差阈值通过。 -ORT 是可选诊断 oracle，不参与生成输出的替代计算。QEMU 仍执行全部客体算术。 - -真实三用例入口为 `--reference-suite --verify-onnx <固定onnx/model.onnx>`，使用冻结的 -三个 prompt、预算和参考；同时要求可执行文件的完整图证据、ORT 的完整 28 层结构 -以及固定资产接口。缺失图证据或只有小图时拒绝给出完整模型 PASS。W4 旧产物沿用 -已有 metadata 身份链接，未新增每 token 的指纹或溯源体系。 -完整模型验收示例显式传入 `--mean-policy two-four-lane-last-axis` 和 -`--nonlinear-policy fp32-reference`，配合 `blocked_fma / 2x4 / column-panel / padding none`。 -`--reference-suite` 不会自动启用这两项；复用该配置的 ELF 时也必须显式匹配。 -上轮未执行完整模型；本轮实跑进展见下节。完整命令见 [部署说明](deployment.md)。 - -## 初始旧算术配置的完整模型 4 hart 验收（2026-10-23，历史结果） - -**初始旧配置结果为 FAIL：三条固定用例均在首步完整 logits 数值检查失败，未完成 3×8 token 生成。** -France 首步使正式 `--reference-suite` 按合同退出。随后用同一入口分别运行 Germany、 -Japan，并在 output 计量包装中执行相同冻结参考逐步检查；两例也在首步失败。 -共完成 **3 次完整前向**，其余 **21 步没有执行**，没有可报告的完整生成文本或停止原因。 -没有跳过 padding 行、放宽阈值或将候选 token 一致解释为生成验收通过。 - -三次均复用同一份既有的完整 Qwen3-0.6B **28 层** ELF 和权重 bundle，显式配置为 -`4 hart / blocked_fma / 2x4 / column-panel / padding none`。未重新编译模型，默认单核、 -手动/auto 选择、CI、WSL 配置不变。该初始验收阶段只新增 output 诊断产物和更新文档，未改正式实现；后续显式策略实现与验证见下文。 -每个命令沿用一次小型 FMA 预检，不计入模型前向；由于失败后独立运行剩余用例，ORT -session 在各命令分别初始化，未声称三个命令共用同一个 session 或驻留 QEMU 进程。 - -### 三条用例实测结果 - -检查每步完整 `[1,256,151936]` 的 **38,895,616** 个 FP32 logits： -与同输入的本轮单线程 ORT 比较，要求 `max_abs < 1e-3`、`rtol=0`，greedy ID 必须一致。 -冻结 JSON 没有保存全部 logits，因此另核对冻结 ID、有效长度和选中 logit； -最终文本、全部新 IDs 和停止原因属于用例末尾检查，本次因首步失败未到达。 - -下表均为**首步完整前向**，不是每例完整 8-token 生成的成本： - -| 固定用例 | UART 前向耗时 s | QEMU CPU s | QEMU RSS 观测峰值 GiB | 完整 logits max_abs | 数值结果 | -|---|---:|---:|---:|---:|---| -| France | 432.607 | 1492.427 | 5.393 | 0.003272057 | FAIL | -| Germany | 400.287 | 1435.645 | 5.401 | 0.001770496 | FAIL | -| Japan | 399.097 | 1428.588 | 5.399 | 0.001197100 | FAIL | - -三例首步候选分别为 **Paris / Berlin / Tokyo**，QEMU、当轮 ORT 和冻结 ID 一致。 -QEMU 选中 logit 与冻结值的误差分别为 `1.335144e-5 / 1.716614e-5 / 1.144409e-5`； -当轮 ORT 选中 logit 与冻结值均精确一致。这不改变完整 logits 已失败的结论。 - -| 用例 | Host 生成调用耗时 s | Host CPU s | QEMU 完整进程耗时 s | QEMU 平均 CPU | Host RSS 峰值 GiB | 同时树 RSS 峰值 GiB | -|---|---:|---:|---:|---:|---:|---:| -| France | 453.789 | 20.722 | 432.969 | 344.7% | 4.068 | 6.699 | -| Germany | 421.424 | 20.245 | 400.650 | 358.3% | 4.100 | 8.372 | -| Japan | 422.524 | 22.998 | 399.460 | 357.6% | 4.099 | 6.399 | - -- UART 前向时间从 QEMU 启动到完成帧验证，包含启动和权重装载；完整进程时间另含停止、转储和退出。 -- QEMU CPU 为每步 `RUSAGE_CHILDREN` user+system 差值，该窗口唯一子进程为 QEMU；平均 CPU 以完整进程墙钟为分母，100% 表示一个逻辑 CPU。 -- Host 生成调用时间另含 ORT、输入/权重传输和数值检查。Host CPU 与 QEMU CPU 分列；采样线程开销计入 Host。 -- QEMU RSS 沿用运行器 0.2s 采样，Host/同时进程树 RSS 为 0.5s 采样，均为观测峰值。树 RSS 求和可能重复共享页，不能将独立的 Host/QEMU 峰值相加当作同时峰值。 -- 三次命令含准备/FMA 预检分别耗时 458.113s、428.556s、430.072s。全部子进程 CPU 还包含 FMA 编译/运行，不能全称模型 CPU。 - -三次 QEMU 均 `qemu_reaped=true`，最终检查 PID 和 POSIX 进程组均不存在，成功 transport -副本已清理。数值失败在正常回收之后发生。采样中 perf 与 MONOTONIC_RAW、UTC 与 perf -的最大差均小于 **0.000362s**，未观察到历史跳时。本轮不计算加速比，也不将这些失败首步 -外推为完整生成速度。 - -Germany 的命令准备/Host 阶段与一次 France 离线 ORT 诊断重叠约 10 秒；该诊断在首次 -采样到 Germany 模型 QEMU 前约 7.4 秒已结束。没有证据显示模型 QEMU 窗口重叠， -但 Germany 全程数据仍不作为无干扰性能基线。诊断只执行一次，未重跑任何 QEMU 前向。 - -### France 错误定位：有效行通过，全输出失败 - -使用保留的 QEMU 输出和一次同输入 ORT 离线重放进行定位。实际 token IDs 和 attention -mask 与冻结合同的 `prepare_inputs` 逐位相同；ORT 仍为 intra/inter 1 线程、 -`ORT_DISABLE_ALL`、关闭 CPU memory arena。 - -| 输出范围 | 最大绝对误差 | 大于等于 1e-3 的元素数 | -|---|---:|---:| -| 有效 query 行 0–4 | 0.000104904 | 0 | -| 用于选 token 的 row 4 | 0.0000572205 | 0 | -| padding query 行 5–255 | 0.003272057 | 20,641 | - -超阈值只出现在 padding query 行 **23、101、250**，分别有 555、6、20,080 个元素。 -最大坐标 `[0,250,45743]`：QEMU `-5.272073746`，ORT `-5.275345802`， -相对误差约 `0.0620%`。全部 256 行 argmax 一致，选取行 top-2 margin 为 `3.161660194`。 -Germany/Japan 本轮未追加逐行定位，不能直接将 France 的行分布推广到另两例。 - -这里的 padding 指**序列补齐位置**，与本轮保持关闭的 **4KiB 工作区 padding 选项**不是同一件事。 -既有 mask 允许这些 query 关注前面的有效 key；不能直接改成全遮挡以规避误差。 -目前只确定误差位置，尚未确定首次产生/放大误差的算子，也未证明是否与 4 hart 并行有关。 -后续补做的同配置 1/4 hart、同源码 Host 和逐层诊断见下节。 - -### 产物及发现的记录问题 - -本地路径相对当前 W5 工作树： - -- `output/w5-full-reference-4hart-20261023/summary.json`：三例联合汇总，明确 FAIL、3/24 前向和 21 步未执行。 -- `output/w5-full-reference-4hart-20261023/`：France；`output/w5-full-reference-4hart-20261023-germany/`、`output/w5-full-reference-4hart-20261023-japan/`：其余两例。 -- 每个目录的 `run/report.json`、`steps.json`、`cases.json`、`numeric-steps.json`：原始 CLI 状态、逐步资源与数值结果；`resources.json`、`resources.jsonl` 为资源汇总/采样。 -- 每个目录的 `run/steps/step-0000/output.bin`：完整 QEMU 输出；France 另有 `step-0000-error-localization.json`、`step-0000-ort-logits.npy`。 -- `output/run_w5_full_reference.py`、`output/diagnose_w5_step0_error.py`：计量/离线定位工具。没有修改原始失败报告或冻结参考。 - -首次完整验收确认了两个 CLI 证据记录问题；原始报告保持原样，下节续作已修复这两项： - -1. 正式 suite 失败后 `status=FAIL`，但 `full_model_gate` 仍保留初始化的 `NOT_RUN`。联合汇总根据实际前向和数值失败记为 FAIL，不能将该字段误读为本轮没运行完整模型。 -2. 从旧 W4 构建载入的完整图审计只用于当次检查，没有带入新保存的 W5 metadata/report。对新副本离线调用 `require_full_executable` 会报缺少完整图证据；原始 W4 metadata/report 仍可复用，本轮三例均使用原始路径，因此该问题没有改变此次计算结果。 - -## 持续完善与数值定位（2026-10-23） - -本轮从北京时间 **2026-10-23 09:35:57**（UTC 01:35:57）开始，持续处理 W5 的剩余问题。 -截至最终输出重复性和进程清理核查的 **13:41:27**（UTC 05:41:27），已持续 **4 小时 5 分 30 秒**； -按分钟约为 09:35–13:41、4 小时 6 分钟。下文保留诊断过程和最终验收结果。 -冻结合同和 `max_abs < 1e-3` 不变，没有忽略 padding query 行或使用参考输出替代 QEMU。 -下列修改均未提交、未推送，CI、WSL、默认单核及手动/auto 选择不变。 - -### 已修复的边界与验收证据 - -- 完整生成开始时将 `full_model_gate` 从 `NOT_RUN` 改为 `RUNNING`，失败/取消分别记为 - `FAIL`/`CANCELLED`。准备失败和 build-only 保留 `NOT_RUN`，不会声称运行了模型。 -- 通过已有身份检查载入旧 W4 构建后，把其完整图审计保存到 W5 元数据；真实 28 层构建 - 连续转换、复用两次成功（均为 build-only，没有新前向）。 -- 参考 suite 将冻结历史、实际前向次数、每步完整 logits 对照、独立选择行和冻结 token - 逐步绑定；缺步、额外步、错误历史/长度/mask/padding 或选错行不得只凭最终文本一致通过。 -- QEMU 失败时，损坏、非对象、不可读或含非有限数的诊断 JSON 不再覆盖原始超时/取消异常。 - 后续 backend 复用和会话历史回滚保持正常。 -- 部署脚本对 info/build 客户端按进程组转发取消并回收，避免脚本退出后编译客户端残留。 - 守护进程侧的构建取消仍需真实容器引擎验证。 - -本轮 CLI **66 项**相关单测通过；适配器边界及跨层历史/缓冲区复用 **27 项**检查通过（2 项真实 QEMU 测试未重跑）； -部署脚本 **14 项**检查通过。以上是各自相关验证，不与历史 216 项重复累加。 -真实 CLI 另复用既有小图构建，分别完成默认 1 hart、手动 4 hart、auto→4 的两 token -调用：三组均产生 `[2,3]`、按预算停止，6 次前向完整输出与 ORT 逐位一致，全部 QEMU 已回收。 -小图仅证明接口/边界，不用于性能加速结论。 - -产物：`output/w5-followup-20261023/cli-evidence-roundtrip/summary.json`、 -`output/w5-followup-20261023/cli-regression/summary.json`。 - -### 三例错误分布补齐 - -复用三例原始 QEMU 输出；Germany/Japan 各进行一次相同输入 ORT 离线对照，没有重跑 QEMU。 - -| 用例 | 有效行最大误差 | 序列补齐行最大误差 | 超阈值 query 行 | 全 256 行 argmax | -|---|---:|---:|---|---| -| France | 0.000104904 | 0.003272057 | 23、101、250 | 全部一致 | -| Germany | 0.000104904 | 0.001770496 | 59、69、121、130 | 全部一致 | -| Japan | 0.000104904 | 0.001197100 | 23、55、58、122、167、177 | 全部一致 | - -各例输入 IDs/mask 均与固定合同逐位相同。对应各例原目录下新增 -`step-0000-error-localization.json` 和 `step-0000-ort-logits.npy`；首步失败记录没有被覆盖。 -有效行通过仍不代表完整 logits 门槛通过。 - -### 同源码与逐层证据 - -相同 `2x4 / column-panel` 生成 C、权重和输入在 Host 顺序执行，完整 logits 与 ORT -最大误差为 **0.002274513**，与先前 4 hart QEMU 的最大差为 **0.001323700**。 -因此存在独立于多 hart 调度的数值差异；Host 与 RV64 的数学库、目标架构不同，不能 -直接把这两个差值归给某一种原因。Host 该次前向 20.819s、CPU 20.814s、RSS 峰值约 3.168GiB， -只作为诊断记录，不替代 QEMU 验收。 - -在同源码 C 增加 embedding、28 层输出、最终归一化共 30 个观测点，额外缓存 30MiB； -原工作区和 bundle 不变。Host 观测版与其原始 logits 逐位一致；ORT 观测版与已保存的 -原始 ORT logits 也逐位一致，排除了这次增加观测点改变最终结果的影响。 - -embedding 完全一致，第一层补齐行最大误差约 `6.68e-6`。row 250 在第 10–13 层输出的 -误差依次为 `0.000189 → 0.000610 → 0.001075 → 0.001394`，提示这一段对上游误差敏感。 -第 2 层有较早超过中间诊断阈值的有效行，但对应约 6543 的激活，仅相差少量 FP32 ULP， -不能把“首次过阈值”直接当根因。随后用同一中间输入拆分第 12 层，区分局部计算与累积，结果见下文。 - -产物：`output/w5-followup-20261023/host-same-source/summary.json`、 -`output/w5-followup-20261023/checkpoints/comparison.json`。 -同配置 1 hart QEMU 完整前向已结束，与 4 hart 的 **38,895,616 个 FP32 logits 逐位一致**， -相对 ORT 的最大误差同为 `0.003272057`；这个输入上的失败不由并行引入。 -单 hart QEMU 进程耗时 **1320.992s**、CPU **1318.380s**、RSS 观测峰值 **5.320GiB**， -平均约 99.8% CPU；进程及进程组已回收。该运行与 Host 诊断编译/执行有重叠， -仅记录实测成本，不用它和历史 4 hart 时间计算严格加速比。 -产物：`output/w5-followup-20261023/qemu-1hart/summary.json`。 - -进一步把同源 C 编译为静态 x86_64-musl，确认无动态装载器，CPU feature 集合与原 Host -编译一致；所得全部 logits 与 QEMU 也逐位一致。其前向 21.622s、CPU 21.497s、RSS 约 -2.951GiB，是本机 Host 诊断而非 QEMU 加速后的时间。它与 glibc Host 存在差别， -说明数学库路径值得继续拆分，但不能据此认定某一个 libm 函数已是唯一根因。 -产物：`output/w5-followup-20261023/host-musl-same-source/summary.json`。 - -同一 ORT 第 11 层输出送入第 12 层切片时，切片 ORT 与原整图该层逐位一致， -Host 开/关观测点也逐位一致；该层独立注入的 padding 最大误差仅 `1.72e-5`、row250 -仅 `2.15e-6`，远低于整图该行累计 `0.001075`。这表明主要存在上游差异和敏感放大， -不能把第 12 层指认为单一错误算子。后续平方、均值及非线性路径的完整 Host 单变量对照见下文。 -产物：`output/w5-followup-20261023/layer12-same-input-v5/report.json`。 - -### 可选 Mean 策略与多 token 收益边界 - -已实现 `--mean-policy sequential|two-four-lane-last-axis`，默认仍为 `sequential`。 -新策略只改变 FP32 末轴 ReduceMean 的求和顺序，对齐既有 W3 的 `mean_last` 参考； -它不启用 SIMD 指令，也不改变 MatMul、Softmax 或 Pow。构建 evidence、运行报告和 -W4/W5 复用检查都携带该字段。旧 ELF 缺字段只能按旧顺序解释，新策略必须重新构建， -不得给旧产物改标签。helper 保留原有 MPL 许可证与第三方声明。 - -| 受控 France 首步 Host 候选 | 对 ORT 全 logits max_abs | 判断 | -|---|---:|---| -| 原顺序、静态 musl | 0.003272057 | FAIL，且与旧 QEMU 逐位一致 | -| 仅 Pow 精确平方 | 0.003275394 | FAIL；局部平方更准确不保证整图误差减小，未纳入正式修复 | -| 仅末轴 Mean 固定求和顺序 | 0.000959873 | 首步低于阈值，余量较窄 | - -Mean 的 4 组真实算子输入和合成边界共 158 个诊断样本与既有 FP32 参考逐位一致。 -正式实现新增 20 项 Host 测试并通过原有 5 项相关默认归约回归；真实 1/4 hart 小图 -8 项通过，覆盖宽 13/128/1024、不同上下文、符号零、溢出报错后同构建复用。 -追加的错误码/进程组断言两项通过，属于这 8 项的复测，不重复累加。 -CLI/运行器合同相关三文件合计 271 项通过，替代本轮早先对应子集的计数。 - -真实新 CLI 对带 ReduceMean 的小图完成默认单核、手动 4、auto→4 的两 token 对照， -完整 logits 逐位一致；从新策略构建复用时若省略策略、退回默认，则在启动任何前向前拒绝。 -这些结果只说明新选项与复用边界正确，不等于完整模型已验收。 - -随后复用同一静态 musl 候选、bundle 和一个 ORT session，按冻结历史进行三例多 token -Host 预检。**3/3 用例仍失败**，共检查 7 步、4 步通过，每例遇到严格数值失败立即停止： - -| 用例 | 已通过 step（零基） | 停止 step | 停止步 full max_abs | -|---|---|---:|---:| -| France | 0、1 | 2 | 0.002007961 | -| Germany | 0 | 1 | 0.001288414 | -| Japan | 0 | 1 | 0.001218975 | - -7 步中 France 首步复用先前已核对输入的成功输出，新增 6 次 Host 前向；没有重跑 QEMU。 -所有已检查步的有效选择行和冻结 ID 一致，但不能绕过完整 logits 门槛继续生成或宣布通过。 -因此此 Mean 选项只能作为显式的算术对照策略，单独启用尚未解除 W5 阻塞。 -下一步以 France step 2 的同一输入分别对照三角函数、Sigmoid、Softmax，避免将多个改动 -同时加入后无法归因。正式默认与 Softmax 行顺序当前都未改变。 - -资源口径：该多 token 诊断从持有 ORT 的父进程启动 native driver，Linux `ru_maxrss` -可能带入父进程高水位，不能当作 driver 独立峰值；原始 stdout 保留并加注，后续采用 -独立轻量父进程或 `/proc` 观测。其 CPU/墙钟和数值判断不受这个 RSS 字段问题影响。 - -产物均相对当前 W5 工作树： - -- `output/w5-followup-20261023/host-mean-candidate/summary.json` -- `output/w5-followup-20261023/host-pow2-candidate/summary.json` -- `output/w5-followup-20261023/mean-host-reference-suite/report.json` -- `output/w5-followup-20261023/mean-rv64-tests/`、`mean-rv64-overflow-final/` -- `output/w5-followup-20261023/mean-cli/summary.json` - -### 非线性算子单变量对照（Host 诊断） - -Mean-only 在 France step02（7 个有效 token）失败后，使用同一保存输入、原 bundle 和 -静态 musl driver，分别只替换 Sin/Cos、Sigmoid 或 Softmax 的参考算术。每个候选的 -原语先对照现有 FP32 reference；完整前向没有改变冻结参考或 `max_abs < 1e-3` 门槛。 - -| 显式诊断候选 | France step02 完整 logits max_abs | 耗时 / CPU 时间 | driver 峰值 RSS | -|---|---:|---:|---:| -| Mean-only（前述基线） | 0.002007961 | 见原逐步记录 | 不使用继承的高水位作独立峰值 | -| Mean + Sin/Cos 参考路径 | 0.000732422 | 21.330 / 21.327 s | 3,258,296 KiB | -| Mean + Sigmoid 参考路径 | 0.000707626 | 19.876 / 19.874 s | 3,258,368 KiB | -| Mean + Softmax 参考路径 | 0.000941038 | 20.056 / 20.048 s | 3,258,232 KiB | - -三种单变量候选都让这一 Host 步骤通过，选中 ID 均为冻结参考的 `576`；这不代表 -整套生成或 QEMU 已通过。继续运行 Mean + Sin/Cos 三例预检时，France 首步又出现 -`max_abs=0.0010201931`，已按原门槛停止并保留失败结果，证明不能从单步外推收益。 -上述单变量候选保留在 `output`。后续已将组合路径接为显式选项，见下节；默认实现未改变。参考路径使用标量 C；Softmax -的 16-lane/ldexp 描述固定的 FP32 运算顺序，不要求 RISC-V 目标具有 AVX512 指令。 -Sin/Cos 仅在已验证区间使用参考多项式，区间外保留原 libm 回退。 - -原语和完整单步证据： - -- `output/w5-followup-20261023/trig-candidate/primitive-validation.json` -- `output/w5-followup-20261023/host-mean-trig-france-step02/summary.json` -- `output/w5-followup-20261023/sigmoid-diagnostic/micro-summary.json` -- `output/w5-followup-20261023/host-mean-sigmoid-step02/summary.json` -- `output/w5-followup-20261023/softmax-candidate/validation.json` -- `output/w5-followup-20261023/host-mean-softmax-candidate/step02-summary.json` -- `output/w5-followup-20261023/mean-trig-host-reference-suite/report.json`（多 token 收益边界） - -### 显式非线性策略与正式入口验证 - -正式后端新增 `nonlinear_policy="libm"|"fp32-reference"`，W4/W5 CLI 对应 -`--nonlinear-policy`。默认仍为 `libm`；参考路径需显式选择，Mean 另由 -`--mean-policy two-four-lane-last-axis` 控制。原始 MatMul/Pow/独立 Exp 和默认 hart -均不随选项改变。旧 ELF 缺字段按 `libm` 解释，策略不匹配在运行前拒绝；构建、运行 -和复用报告记录实际策略，不增加新的哈希链。许可证与范围见 -[运行合同](../W4/runtime-contract.md)。 - -- 新后端直接检查 **21 passed**，相关既有 Softmax/Mean 检查 **48 passed**； - 六份修改前默认 C 源码快照逐字节相同，包含 inline/external、回调与 Mean 配置组合。 -- runtime/W4/W5 三份相关 contract 文件 **328 passed**，覆盖新策略、旧元数据、 - 双向复用不匹配、默认/手动/auto 及 Mean 组合；本组包含前述重叠检查，不重复累计。 -- 正式生成代码的真实 RV64 混合图 **2 passed**（1/4 hart）：含 Sin/Cos/Sigmoid、 - 末轴与非末轴 Softmax、Mean，两个上下文全部与固定参考逐位一致；中途发生有限输入 - 差值溢出时，仍传播 guest `status 2` 并回收进程，原 ELF/bundle 随后可正常复用。 -- 真实 CLI 的默认 1 hart、手动 4 hart、auto→4 hart 各完成两步 `[2,3]`， - 共 6 次前向全部与 ORT 逐位一致，QEMU 进程及进程组均已消失。 -- 此前 output-only Softmax 诊断另验证窄/宽行、非末轴、回调上下文与工作区恢复； - 注入零和/NaN 归一化错误后调用方输出不污染。它不计为完整模型验收。 - -正式验证产物为 `output/w5-followup-20261023/backend-nonlinear-validation.json`、 -`nonlinear-rv64-tests/` 和 `nonlinear-cli/summary.json`。 - -组合 Host 三例预检已完成 **24/24 步、3/3 用例通过**；其中 France 首步仅按同一 -驱动及逐字节相同输入复用一次，其余 23 次新执行。每步检查完整 logits、所选行及 -冻结 ID，三条最终文本和停止原因完全一致，未改变误差门槛或过滤 padding 行。 - -| Host 诊断用例 | 全 logits 最大误差 | 有效行最大误差 | 8 次前向耗时 / CPU 时间 | -|---|---:|---:|---:| -| France | 0.000303984 | 0.000008583 | 158.853 / 158.753 s | -| Germany | 0.000262737 | 0.000020981 | 159.154 / 159.040 s | -| Japan | 0.000197887 | 0.000012875 | 158.336 / 158.239 s | - -上表 France 含已完成首步的原始测量。预检这次实际总耗时 577.597 s（包含 ORT 与 -对照，且不重复执行已复用首步），不能与上表求和混为同一计时口径。独立驱动通过 -`/proc` 观测的峰值 RSS 最大 3,258,564 KiB，ORT 父进程累计峰值 5,290,672 KiB。 -原 `getrusage` 子进程高水位继承父进程值的问题已明确标注，不用它冒充独立 driver 峰值。 - -证据为 `output/w5-followup-20261023/mean-trig-sigmoid-softmax-host-reference-suite/report.json`。 -上述数据属于完整 Host 数值预检，**不能替代 QEMU 验收**。后续使用正式 codegen 的新 -4 hart ELF、复用原 bundle 完成了三例 24 步 QEMU 验证；其独立实测结果见下节。 - -### 正式组合策略 QEMU 完整生成（3/3 用例、24/24 步通过） - -正式 codegen 已构建新的完整 Qwen3-0.6B **28 层** ELF；三条冻结用例复用同一 -ELF 和原权重 bundle，逐 token 只更新输入并启动 QEMU。显式配置为 -`4 hart / blocked_fma / 2x4 / column-panel / mean two-four-lane-last-axis / nonlinear fp32-reference / padding none`。 -没有将 Host 输出替代 QEMU 计算，也没有修改默认单核、手动/auto hart 选择或冻结参考。 -24 份运行记录中的既有 ELF/bundle 身份字段完全一致，没有新增哈希检查。 - -三例各完整生成 8 token，**3/3 用例、24/24 步数值及冻结参考检查通过**，命令退出码为 -`0`，正式 CLI 的 `status=PASS / full_model_gate=PASS / qemu_forwards=24`。 -每步都比较完整 `[1,256,151936]` 的 **38,895,616** 个 FP32 logits，严格要求 -`max_abs < 1e-3`、`rtol=0`，没有跳过序列补齐行。全套最大误差为 -`0.000303983688`；这表示容差一致，不表示全部 logits 逐位相等。 - -| 固定用例 | 已完成步数 | 生成调用墙钟 s | QEMU CPU s | Host CPU s | 全 logits 最大误差 | 当前状态 | -|---|---:|---:|---:|---:|---:|---| -| France | 8/8 | 3375.024 | 11517.789 | 167.657 | 0.000303983688 | 整例通过 | -| Germany | 8/8 | 3668.852 | 12574.174 | 190.673 | 0.000262737274 | 整例通过 | -| Japan | 8/8 | 3489.655 | 11958.881 | 178.759 | 0.000197887421 | 整例通过 | - -三例新 token IDs、完整文本和停止原因均与冻结参考一致，停止原因均为 `max_new_tokens`： - -| 用例 | 新 token IDs | 生成文本 | -|---|---|---| -| France | `[12095, 13, 576, 6722, 315, 15344, 374, 21718]` | ` Paris. The capital of Italy is Rome` | -| Germany | `[19846, 13, 576, 6722, 315, 9625, 374, 12095]` | ` Berlin. The capital of France is Paris` | -| Japan | `[26194, 13, 576, 6722, 315, 279, 3639, 4180]` | ` Tokyo. The capital of the United States` | - -各例最大完整 logits 误差分别位于本例 step 3、3、1；冻结所选 token 的 logit 最大误差 -分别为 `1.907349e-6 / 5.722046e-6 / 5.722046e-6`。全部 24 个不同 QEMU 进程 -均记录 `qemu_reaped=true`；套件结束后只读核查这 24 个 PID 和 POSIX 进程组均已消失。 - -| 每例 8 步资源口径 | France | Germany | Japan | -|---|---:|---:|---:| -| UART 前向耗时合计 / 每步均值 | 3197.261 s / 399.658 s | 3480.225 s / 435.028 s | 3312.868 s / 414.109 s | -| 单步 UART 前向耗时范围 | 395.824–407.598 s | 423.936–453.608 s | 406.769–428.952 s | -| QEMU 完整进程耗时合计 | 3199.695 s | 3482.999 s | 3315.397 s | -| QEMU 平均 CPU 占用（100% 为一个逻辑 CPU） | 359.97% | 361.02% | 360.71% | -| QEMU RSS 观测峰值 | 5.406 GiB | 5.406 GiB | 5.406 GiB | -| Host RSS 观测峰值 | 4.254 GiB | 4.102 GiB | 4.098 GiB | -| 同时进程树 RSS 观测峰值 | 7.565 GiB | 7.644 GiB | 7.411 GiB | - -整次测量命令墙钟 **10540.190s（约 2 小时 55 分 40 秒）**，包含准备、ORT 与核验； -24 步 UART 前向合计 **9990.355s**、平均 **416.265s/token**，完整 QEMU 进程合计 -**9998.091s**。模型 QEMU CPU 合计 **36050.844s**，平均 CPU 占用 **360.58%**； -Host CPU 全程 **540.717s**，其中三例生成阶段合计 **537.089s**。 -所有子进程 CPU 全程为 **36050.974s**，另含准备/FMA 检查，不能全部算作模型 CPU。 - -计量口径沿用前文：UART 时间包括 QEMU 启动和权重装载，完整进程时间另含 -stop/dump/quit；平均 CPU 用 QEMU CPU 总时间除以完整进程耗时之和。 -生成调用墙钟和此表 Host CPU 包含本例 ORT 与数值核验，逐步 adapter 的 Host CPU -仅覆盖 adapter 调用，两者不能混用。QEMU RSS 为 0.2s 采样，Host/同时进程树为 -0.5s 采样；全程 Host 与同时树峰值分别为 **4.254 / 7.644 GiB**。 -不能将逐步峰值相加，也不能将独立 Host/QEMU 峰值相加当作同时峰值；树 RSS 可能重复共享页。 -Host/同时进程树采样器全程 21,032 次采样、0 次错误。独立的 QEMU RSS 采样中， -24 步有 **17 步为 `partial`，各有 1 次读取错误**；5.406 GiB 仍只是观测峰值， -不能据此声称 QEMU 内存采样完全无缺失。`perf−MONOTONIC_RAW` 与 `UTC−perf` 的 -最大绝对漂移分别为 **0.000773s / 0.000504s**，未观察到历史跳时。 -本次不使用旧计时异常或不同算术策略的记录计算加速比。 - -本地产物路径相对当前 W5 工作树： - -- `output/w5-followup-20261023/formal-reference-4hart/executable.json`：本次正式 ELF 的复用入口。 -- `output/w5-followup-20261023/formal-reference-4hart/report.json`：构建与原 bundle/布局匹配记录；该构建报告本身不代表生成验收通过。 -- `../w4-harts-short-20261021/optimization-completion-20261022/integration-full-4hart/run/weights/`：继续复用的原权重 bundle。 -- `output/w5-followup-20261023/qemu-reference-4hart/metadata.json`：实际命令、环境与测量配置。 -- 同目录 `cases.json`、`steps.json`、`numeric-steps.json`：三例结果、24 步资源及完整 logits/冻结参考对照。 -- 同目录 `resources.jsonl` 和 `resources.json`：全程资源采样与最终汇总。该次命令使用原计量包装，不要求后来单次前向包装新增的 `measurement_passed` 字段；终态以 CLI、3 例输出和 24 步数值证据共同确认。 -- 同目录 `run/report.json` 及 `run/steps/step-0000/run.json` 至 `step-0023/run.json`：CLI 最终状态与逐步运行证据。 - -### 独立完整模型单次前向（已通过） - -三例通过后，另用相同正式 4 hart ELF/bundle、France 固定 prompt 完成独立的 **1 token / -1 次完整前向**计量，沿用相同算术策略和 `padding none`。命令退出码 `0`，生成 -`[12095]`、文本 ` Paris`，按 `max_new_tokens` 停止，与冻结首 token、文本前缀及预算停止规则一致。 -选中 logit 为 `17.498851776123047`，与当轮 ORT 和冻结值完全一致；完整 -**38,895,616** 个 logits 对 ORT 的 `max_abs=0.000172138214 < 0.001`。 - -| 单次完整前向指标 | 实测 | -|---|---:| -| UART 完成耗时 | 418.954 s，约 6 分 59 秒 | -| QEMU 完整进程耗时 | 419.383 s | -| QEMU CPU 时间 / 平均 CPU 占用 | 1498.851 s / 357.39% | -| 生成调用墙钟 / Host CPU | 441.033 s / 21.604 s | -| 整次测量墙钟 / Host CPU（含准备） | 448.263 s / 25.332 s | -| QEMU RSS 观测峰值 | 5.398 GiB | -| Host RSS 观测峰值 | 4.119 GiB | -| 同时进程树 RSS 观测峰值 | 7.210 GiB | - -本次启动新 QEMU 进程，但**没有清空系统文件缓存，因此不称为冷启动测量**。 -完整 **155,582,464 字节**输出与先前三例套件 France 首步逐字节一致。 -已有 ELF/bundle 身份也相同，没有新增哈希；最终确认套件与本次共 **25 个 QEMU PID -及其进程组全部消失**。单次采样 0 次错误，`perf−MONOTONIC_RAW` 与 `UTC−perf` -最大绝对漂移为 **0.000160s / 0.000144s**。 - -此命令范围为 `full-model single forward`，没有再次执行三例参考 suite，故正式 CLI -`full_model_gate=NOT_RUN` 是正常范围标记，**不是没有运行完整模型**。 -计量结果 `measurement_passed=true`，`full_reference_suite_evaluated=false`； -三例 gate 的 `PASS` 来自前述独立的 24 步验收，不能把这一次重复前向计为第 25 个 suite 步骤。 - -本地产物目录：`output/w5-followup-20261023/qemu-final-single-forward-4hart/`。 -其中 `steps.json`、`cases.json`、`numeric-steps.json`、`resources.json` 分别记录单步、 -生成调用、数值和资源结果；`run/report.json` 保留正式 CLI 范围。 -`repeat-and-cleanup-check.json` 记录同 ELF/bundle、完整输出逐字节一致和 25 个进程/进程组清理核查。 - -### 真实容器部署续验（本地通过） - -本轮已用 Docker Engine 29.9.2 / Buildx 0.38.1 真正构建并启动干净镜像。 -Docker Hub 认证 TLS EOF 后显式选择 Canonical 官方 ECR Ubuntu 24.04，默认基础镜像 -仍为 `ubuntu:24.04`。引擎仅手动启动，验证后 Docker 服务、socket 和 containerd -均为 **inactive + disabled**；停止前无运行容器，临时权限已恢复，WSL 配置未变。 -工具版本、完整命令与证据见 [部署记录](deployment.md)。 - -默认 1 hart、手动 4 hart、`auto→4` 的容器 CLI 小图各 4 次前向均通过， -IDs `[2,3,4,6]`、EOS 停止，完整 logits 与 ORT 逐位一致，padding `none`。 -已有真实 QEMU 超时/取消及失败后复用测试在镜像内 **2/2 通过、0 skip**， -1/4 hart 共 16 次 guest 启动均完成进程/进程组回收。实际 Docker 构建 -SIGINT/SIGTERM 另 **2/2 通过**，客户端组及引擎侧 RUN 进程均消失，延后复查无残留。 - -本轮只追加一次容器内完整 28 层 France 单 token 前向，现已 **PASS、退出码 0**： -输出 `[12095]`、文本 ` Paris`、预算停止，完整 **38,895,616** 个 logits 的 -`max_abs=0.000172138214 < 0.001`；图审计确认 28 层，FMA 与 QEMU 回收通过。 -容器从只读模型资产独立重新构建，CLI 总耗时 **547.454s**（含构建/FMA/ORT/核验）， -UART 前向 **415.058s**、QEMU 完整进程 **415.478s**、运行器记录耗时 **430.481s**， -后者含权重传输、QEMU、后检查与清理，不含计时起点之前的输入打包/写入及 ELF/bundle 校验。 -QEMU RSS 观测峰值 **5.398GiB**。RSS 有 2071 个有效样本，**1 次读取失败**， -可用性为 `partial`；本次未独立测量 CPU 总时间，不借用宿主或 Docker 客户端数据。 -容器输出 **155,582,464 字节**与既有宿主独立完整前向直接比较逐字节一致，未新增哈希。 - -内部 CLI 的 `full_model_gate=NOT_RUN / container_reproduction=NOT_ASSERTED` 保留： -单 prompt 不重跑三例 gate，实际容器身份由外层 Docker 构建/运行和退出证据确认。 -它与既有宿主三例 24 步及宿主独立前向分别记录，构成 **deploy:script 本地真实容器 PASS**， -不声称容器三例通过。**W5 本地功能出口已满足**；容器收尾时 CI 接入的 753 项轻量 -检查通过，14 个新增必跑节点仅收集核对、未重跑；已有真实 24 步报告的严格检查通过。 -后续提交前检查与修复见 [收尾记录](README.md#提交前检查与收尾2026-10-23)。 -模式和资源门槛见 [CI 接入说明](README.md#ci-接入与远端执行边界)； -上述记录属于提交前时点。随后 W5 已发布为 PR115，提交 `8ad3474` 的普通 PR CI -已通过；完整专项因 fork Actions 拒绝 dispatch，仍为 **BLOCKED / NOT_RUN**。 -第二人说明已准备、执行待办，详见 [当前阶段表](README.md#阶段与验收状态)。 - -证据相对仓库根：`output/w5-container-20261023/small-summary.json`、 -`small-runs/real-qemu-tests.xml`（同目录）、`build-cancel/report.json`、 -`build-cancel/delayed-recheck.json` 及 `paris.log`、`paris-summary.json`、`paris/run/report.json`、`services-final.json`。 - -## 上轮小图数值与清理验证 - -适配器使用真实 Gather + MatMul 小图：输入 `[1,256]` token IDs，输出完整 `[1,256,8]` -FP32 logits，随历史输入改变。合成图声明完整 mask ABI 但不消费 mask；mask 的精确 -causal/key-padding 语义由已有 P1 测试验证，不能将该小图称为 Transformer 语言验收。 - -| 对照 | 1 hart | 4 hart | -|---|---|---| -| 同一 ELF/bundle 的多 token | `[2,3,4,6]`，EOS 停止 | 相同 | -| 独立上下文与预算停止 | `[4]`，`max_new_tokens` | 相同 | -| 每次成功前向的完整输出 | 2048 元素与 ORT 逐位相等 | 相同 | -| 正在运行的 WFI 测试 guest 超时 | 抛 TimeoutError；进程和进程组已回收 | 相同 | -| 正在运行时取消 | 抛 GenerationCancelledError；进程和进程组已回收 | 相同 | -| 中断后复用 backend/ELF | 再生成 `[6]`，EOS | 相同 | -| 独立真实启动数 | 8 次：6 成功、1 超时、1 取消 | 相同 | - -这组 16 次真实启动全部核对 PID 与 POSIX 进程组消失，无清理错误。 -失败运行按原有策略各保留 8192 字节 transport 诊断副本；这是故意保留的文件，不是 -仍在运行的编译/QEMU 进程。成功运行的 transport 副本已清理。 - -另外从真实 CLI（本地小 tokenizer，并非 Python mock）完成四次命令： - -| CLI 调用 | 新 token IDs | 前向次数 | 完整输出 max_abs | 结果 | -|---|---|---:|---:|---| -| 1 hart 首次构建，prompt `1` | `[2,3,4,6]` | 4 | 0 | PASS | -| 4 hart 首次构建，prompt `1` | `[2,3,4,6]` | 4 | 0 | PASS | -| 1 hart 复用构建，prompt `2` | `[3,4,6]` | 3 | 0 | PASS | -| 4 hart 复用构建，prompt `2` | `[3,4,6]` | 3 | 0 | PASS | - -表中引用最终入口的复测产物；早期小图记录保留，不与最终矩阵重复累计。 -跨命令的 ELF 和 bundle 路径确实相同，所有 QEMU 均回收。复用前沿用已有 -report 到 executable metadata 的身份校验，缺失或漂移先于加载/编译失败。每条生成命令仅在准备阶段 -做一次独立的小型 FMA 预检；它不算模型 token 前向。默认配置另做真实小图 build-only， -未启动 QEMU，记录默认 1 hart、sequential 和 padding none。 - -这些约 20ms 的 guest 是正确性冒烟。未采集 CPU 总时间，0.2s 的 RSS 周期只捕捉到 -启动样本，不能据此宣称加速或峰值内存改善；未使用历史异常计时或完整模型速度作推算。 - -## 上轮相关检查与产物 - -| 检查 | 通过数量 | -|---|---:| -| 新 QEMU 适配/取消/QMP 边界与真实 1/4 hart 小图 | 18 | -| 既有 external runtime 与 W4 契约相关回归 | 112 | -| 三轮历史及官方模板文本/token IDs 对照 | 40 | -| CLI 参数、复用、数值、错误/信号与完整图门槛 | 37 | -| 部署脚本/打包/失败路径隔离检查 | 9 | - -上轮共 **216 项相关检查通过**；与 P0/P1 的旧记录分开列出，不将重叠项重复累计。 -另完成 12 个 Python 文件语法检查、部署脚本 Bash 语法检查、6 份相关文档的 -53 个本地链接检查（含 33 个 Markdown/HTML 链接），未引入重复文档名称； -最终四份 CLI 报告与汇总、构建复用路径及进程回收记录离线核对一致。 -上轮未运行完整模型；初始完整验收阶段仅进行上述完整验收和必要离线定位，未重跑这组 216 项单测或全量回归。后续显式策略的相关检查另列于持续完善部分；未追加 AI 审查。CI/WSL 未改，代码未提交推送。 - -本地诊断路径（相对当前 W5 工作树）： - -- `output/w5-qemu-generation-summary.json`：适配器 1/4 hart 汇总。 -- `output/w5-qemu-generation-real/`：真实逐步输出、超时/取消和回收证据。 -- `output/w5-cli-final-20261023/summary.json`：四次 CLI 构建/复用对照。 -- `output/w5-cli-final-20261023/{hart1,hart4,hart1-reuse,hart4-reuse}/report.json`：逐 token ORT/QEMU 记录。 -- `output/w5-cli-small-20261023/default-build-only/report.json`：默认配置构建检查。 - -## 剩余问题和明确未执行项 - -1. **默认算术配置的数值边界仍需保留**：显式组合策略已经完成 Host 和 QEMU 各三例 24/24 步验收；初始 `mean sequential / nonlinear libm` 配置的三例首步仍有完整 logits 超过 `1e-3` 的历史失败，不能把显式策略 PASS 推广为默认路径 PASS。默认值未改变，完整验收需显式选择本文配置。独立单次前向也已通过，并已单独记录，不能与 24 步套件混算。 -2. **本地功能与远端门槛分开**：真实容器镜像、小图、中断清理及唯一追加的完整 France 单 token 已通过，W5 本地功能出口已满足；没有重复容器三例套件。引擎及 socket 已停止并禁用自启，临时权限已恢复，WSL 配置不变。 -3. **普通 CI 通过，完整专项仍阻塞**：PR115 提交 `8ad3474` 的普通 CI `39919353111` 已通过（14 成功、7 个按路由跳过），12 个 W5 真实 QEMU 小图和 2 个官方模板节点均通过。提交前的 811 项相关检查及最终 11 项重叠复查仍按原时点保留；完整专项因 fork Actions 在 dispatch 阶段返回 HTTP 422，未创建运行，故远端资源、时限和容器三例 24 步仍待实跑。第二人独立复现说明已准备，执行及签收待办。详见 [当前阶段表](README.md#阶段与验收状态)。 -4. **同步边界**：QEMU UART/QMP 等待可响应取消并回收进程；构建/传输/可选 ORT 校验等同步工作仍在各自阶段边界检查总预算，Host deadline 不承诺硬中断任意 ORT 调用。SIGINT/SIGTERM 会经已有编译/QEMU清理路径退出，不返回成功的部分生成。 - -固定 256 长度、无 KV cache 意味着每个新 token 重算完整前向；当前实现复用构建， -不宣称 QEMU 进程或权重驻留。持久 guest、KV/RVV/量化等属于后续优化，不是本轮漏做的生成链路。 diff --git "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" index fd2c1914..4827c38e 100644 --- "a/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" +++ "b/docs/llm-deploy-v1.0/\345\274\200\345\217\221\350\256\241\345\210\222.md" @@ -157,7 +157,7 @@ Host 端循环： | Nightly 完整门禁 | 每日夜间 | 完整 ONNX、完整 IR 数值、QEMU 单次前向、端到端 | 2–4 h | 阻塞里程碑 | | Weekly/Release 门禁 | 每周五 / tag | 多轮、文档复现、性能回归、交付物 | 4–8 h | 阻塞发布 | -上表是分层目标，不代表所有未来 job 已交付。当前 `.github/workflows/llm-deploy.yml` 使用 GitHub 托管 Ubuntu runner：PR 小模型任务和手动/定时完整 ONNX 任务各有 60 分钟 job 上限，QEMU 小模型步骤另有超时。W3 已提供 `w3-preparation.yml`、`w3-full-numeric.yml` 和显式启用的 `w3-nightly.yml`，并可通过既有 `llm-deploy.yml` 手动设置 `run_w3_validation=true`，从同一提交调用 W3 编排。作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 通过同提交两套 W3 gate 及汇总，完整 raw 已下载验真、audit 通过，定时 schedule 未启用。通用 PR CI 全绿不能替代这些专项记录。W4 的 PR 小图回归由统一 CI 调用 `w4-full-numeric.yml`；`llm-deploy.yml` 手动设置 `run_w4_validation=true` 才调用完整前向，另有默认关闭的 `w4-nightly.yml`；所有 Nightly 入口需仓库变量 `W4_NIGHTLY_ENABLED=true`。配置和本地检查不代表已取得远端成功记录，完整门槛、独立复现与团队验收分别记录。W5 生成门禁已经实现，并以显式参考算术策略完成本地完整 QEMU 三例 24/24 步验收，详见 [W5 状态与证据](W5/README.md)；W5 CI 已接入既有工作流，并补齐合并前 W5-only 手动入口；全部待提交改动的本地检查与修复已完成，详见 [W5 提交前收尾](W5/README.md#提交前检查与收尾2026-10-23)。远端尚未提交运行，不能与本地 PASS 混同。实际工具为 Zig 0.14.1、`qemu-system-riscv64`（裸机）和固定 Python 依赖；旧独立测试仍使用 `qemu-riscv32`，不能互换。更大内存的自托管 runner 与缓存策略仍按资源计划推进；W5 容器封装已经实现，真实容器小图、中断清理和完整 Paris 单 token 均已通过，本地功能出口已满足，见 [部署记录](W5/deployment.md)。这些记录不替代远端 CI 或正式发布验收。 +上表是分层目标，不代表所有未来 job 已交付。当前 `.github/workflows/llm-deploy.yml` 使用 GitHub 托管 Ubuntu runner：PR 小模型任务和手动/定时完整 ONNX 任务各有 60 分钟 job 上限，QEMU 小模型步骤另有超时。W3 已提供 `w3-preparation.yml`、`w3-full-numeric.yml` 和显式启用的 `w3-nightly.yml`，并可通过既有 `llm-deploy.yml` 手动设置 `run_w3_validation=true`，从同一提交调用 W3 编排。作者[第四轮 Linux 手动编排](https://github.com/yuki-328/ScratchV/actions/runs/37299235103)已在 `1c49e8acbcd9174491b2d0c2a5a0072178ae5f9a` 通过同提交两套 W3 gate 及汇总，完整 raw 已下载验真、audit 通过，定时 schedule 未启用。通用 PR CI 全绿不能替代这些专项记录。W4 的 PR 小图回归由统一 CI 调用 `w4-full-numeric.yml`；`llm-deploy.yml` 手动设置 `run_w4_validation=true` 才调用完整前向，另有默认关闭的 `w4-nightly.yml`；所有 Nightly 入口需仓库变量 `W4_NIGHTLY_ENABLED=true`。配置和本地检查不代表已取得远端成功记录，完整门槛、独立复现与团队验收分别记录。W5 已实现 Host token 循环、QEMU 生成适配、多轮文本和容器部署；显式参考算术策略的宿主完整三例 24 步及容器 Paris 单 token 已有历史实跑，功能、复现命令和结果集中在 [W5 开发说明](W5/README.md)、[独立复现步骤](W5/README.md#如何复现)与[实际复现结果](W5/README.md#实际复现结果)。完整三例的独立容器运行与其他执行人的独立复现仍待取得，不由现有局部结果替代。实际工具为 Zig 0.14.1、`qemu-system-riscv64`（裸机）和固定 Python 依赖；旧独立测试仍使用 `qemu-riscv32`，不能互换。工作流配置与运行范围见 [CI 说明](../CI.md)，开发文档不维护随运行变化的远端状态。 ### 4.2 硬门槛 CI Job diff --git a/scripts/run_inference.py b/scripts/run_inference.py index 2dab02e7..b3323980 100644 --- a/scripts/run_inference.py +++ b/scripts/run_inference.py @@ -214,6 +214,7 @@ def main(argv=None): started = time.perf_counter() adapter = None previous = {} + exit_code = 1 def interrupted(signum, frame): raise KeyboardInterrupt(f"generation interrupted by signal {signum}") @@ -242,126 +243,142 @@ def save(): report["build"] = exe.evidence if args.build_only: report.update(status="BUILD_ONLY", stage="build_complete") - return 0 - report["stage"] = "fma_preflight" - fma = run_fma_probe(out / "fma-conformance", cc=exe.toolchain.cc, - qemu=exe.toolchain.qemu, timeout=min(args.build_timeout, 120)) - report["fma_conformance"] = fma - if not fma.get("passed") or fma.get("status") != "PASS": - raise ValueError("Actual RV64 execution toolchain failed FMA conformance") - fixture = None - if args.reference_suite: - report["stage"] = "reference_contract" - fixture = json.loads((ROOT / "probes/w5_generation/references.json").read_text(encoding="utf-8")) - meta = {"pad_token_id": tokenizer.pad_token_id, - "generation_eos_token_ids": list(tokenizer.generation_eos_token_ids), - "model_vocab_size": tokenizer.model_vocab_size, - "tokenizer_vocab_size": tokenizer.tokenizer_vocab_size} - if fixture["status"] != "frozen" or fixture["tokenizer"] != meta or len(fixture["cases"]) != 3: - raise ValueError("Reference suite/tokenizer differs from the frozen contract") - cases = fixture["cases"] else: - texts = args.chat_turn or [args.prompt] - cases = [{"name": f"turn-{i}", "prompt": text, - "max_new_tokens": 8 if args.max_new_tokens is None else args.max_new_tokens} - for i, text in enumerate(texts)] - report["stage"] = "oracle_prepare" - session = (reference_session(args.verify_onnx, fixture) if fixture is not None else - oracle_session(args.verify_onnx) if args.verify_onnx else None) - conversation = None - if args.mode == "chat": - report["stage"] = "conversation_prepare" - from scratchv.runtime.conversation import Conversation - conversation = Conversation.from_directory(args.tokenizer_dir, system_prompt=args.system_prompt) - deadline = time.monotonic() + args.total_timeout - adapter.deadline = deadline - report["stage"] = "generation" - if fixture is not None: - report["full_model_gate"] = "RUNNING" - save() - - def forward(prepared): - step = len(report["numeric_steps"]) - numeric_start - if fixture is not None: - if step >= len(case["generated_token_ids"]): - raise ValueError("Host called forward after the frozen reference ended") - feed = checked_inputs(reference_history, prepared, tokenizer.pad_token_id) + report["stage"] = "fma_preflight" + fma = run_fma_probe(out / "fma-conformance", cc=exe.toolchain.cc, + qemu=exe.toolchain.qemu, timeout=min(args.build_timeout, 120)) + report["fma_conformance"] = fma + if not fma.get("passed") or fma.get("status") != "PASS": + raise ValueError("Actual RV64 execution toolchain failed FMA conformance") + fixture = None + if args.reference_suite: + report["stage"] = "reference_contract" + fixture = json.loads((ROOT / "probes/w5_generation/references.json").read_text(encoding="utf-8")) + meta = {"pad_token_id": tokenizer.pad_token_id, + "generation_eos_token_ids": list(tokenizer.generation_eos_token_ids), + "model_vocab_size": tokenizer.model_vocab_size, + "tokenizer_vocab_size": tokenizer.tokenizer_vocab_size} + if fixture["status"] != "frozen" or fixture["tokenizer"] != meta or len(fixture["cases"]) != 3: + raise ValueError("Reference suite/tokenizer differs from the frozen contract") + cases = fixture["cases"] else: - feed = prepared.as_feed() - expected = session.run(["logits"], feed)[0] if session is not None else None - actual = adapter(prepared) - if expected is not None: - comparison = compare_step(actual, expected, prepared.valid_length, adapter.vocab_size) - comparison.update(case=case["name"], step=step, valid_length=prepared.valid_length) - if fixture is not None: - # Select independently from frozen history, never from a - # caller-declared row or the generation sampling helper. - position = len(reference_history) - 1 - qemu_id = int(np.argmax(actual[0, position])) - ort_id = int(np.argmax(expected[0, position])) - frozen_id = case["generated_token_ids"][step] - comparison["reference_token_id"] = frozen_id - comparison["selected_row_passed"] = ( - comparison["selected_row_passed"] and comparison["qemu_token_id"] == qemu_id - and comparison["ort_token_id"] == ort_id) - comparison["reference_token_passed"] = qemu_id == ort_id == frozen_id - comparison["passed"] = (comparison["passed"] - and comparison["selected_row_passed"] and comparison["reference_token_passed"]) - report["numeric_steps"].append(comparison) - if not comparison["passed"]: - save() - raise ValueError("QEMU/ORT full logits, selected row or frozen token differ; see numeric_steps") + texts = args.chat_turn or [args.prompt] + cases = [{"name": f"turn-{i}", "prompt": text, + "max_new_tokens": 8 if args.max_new_tokens is None else args.max_new_tokens} + for i, text in enumerate(texts)] + report["stage"] = "oracle_prepare" + session = (reference_session(args.verify_onnx, fixture) if fixture is not None else + oracle_session(args.verify_onnx) if args.verify_onnx else None) + conversation = None + if args.mode == "chat": + report["stage"] = "conversation_prepare" + from scratchv.runtime.conversation import Conversation + conversation = Conversation.from_directory(args.tokenizer_dir, system_prompt=args.system_prompt) + deadline = time.monotonic() + args.total_timeout + adapter.deadline = deadline + report["stage"] = "generation" + if fixture is not None: + report["full_model_gate"] = "RUNNING" + save() + + def forward(prepared): + step = len(report["numeric_steps"]) - numeric_start if fixture is not None: - reference_history.append(frozen_id) - return actual + if step >= len(case["generated_token_ids"]): + raise ValueError("Host called forward after the frozen reference ended") + feed = checked_inputs(reference_history, prepared, tokenizer.pad_token_id) + else: + feed = prepared.as_feed() + expected = session.run(["logits"], feed)[0] if session is not None else None + actual = adapter(prepared) + if expected is not None: + comparison = compare_step(actual, expected, prepared.valid_length, adapter.vocab_size) + comparison.update(case=case["name"], step=step, valid_length=prepared.valid_length) + if fixture is not None: + # Select independently from frozen history, never from a + # caller-declared row or the generation sampling helper. + position = len(reference_history) - 1 + qemu_id = int(np.argmax(actual[0, position])) + ort_id = int(np.argmax(expected[0, position])) + frozen_id = case["generated_token_ids"][step] + comparison["reference_token_id"] = frozen_id + comparison["selected_row_passed"] = ( + comparison["selected_row_passed"] and comparison["qemu_token_id"] == qemu_id + and comparison["ort_token_id"] == ort_id) + comparison["reference_token_passed"] = qemu_id == ort_id == frozen_id + comparison["passed"] = (comparison["passed"] + and comparison["selected_row_passed"] and comparison["reference_token_passed"]) + report["numeric_steps"].append(comparison) + if not comparison["passed"]: + # The outer handler saves the failure after preserving its cause. + raise ValueError("QEMU/ORT full logits, selected row or frozen token differ; see numeric_steps") + if fixture is not None: + reference_history.append(frozen_id) + return actual - for case in cases: - numeric_start, run_start = len(report["numeric_steps"]), len(adapter.runs) - reference_history = list(case["prompt_token_ids"]) if fixture is not None else None - remaining = max(0.0, deadline - time.monotonic()) - kwargs = {"forward": forward, "max_new_tokens": case["max_new_tokens"], - "timeout_seconds": remaining} - result = (conversation.turn(case["prompt"], **kwargs) if conversation else - generate(case["prompt"], tokenizer=tokenizer, **kwargs)) - row = {"name": case["name"], "prompt": case["prompt"], - "max_new_tokens": case["max_new_tokens"], **asdict(result)} + for case in cases: + numeric_start, run_start = len(report["numeric_steps"]), len(adapter.runs) + reference_history = list(case["prompt_token_ids"]) if fixture is not None else None + remaining = max(0.0, deadline - time.monotonic()) + kwargs = {"forward": forward, "max_new_tokens": case["max_new_tokens"], + "timeout_seconds": remaining} + result = (conversation.turn(case["prompt"], **kwargs) if conversation else + generate(case["prompt"], tokenizer=tokenizer, **kwargs)) + row = {"name": case["name"], "prompt": case["prompt"], + "max_new_tokens": case["max_new_tokens"], **asdict(result)} + if fixture is not None: + row["numeric_step_count"] = len(report["numeric_steps"]) - numeric_start + row["forward_count"] = len(adapter.runs) - run_start + row["reference_passed"] = ( + row["numeric_step_count"] == row["forward_count"] == len(case["generated_token_ids"]) + and list(result.prompt_token_ids) == case["prompt_token_ids"] + and list(result.generated_token_ids) == case["generated_token_ids"] + and result.generated_text == case["generated_text"] + and result.stop_reason == case["stop_reason"] + and case["required_substring"] in result.generated_text) + report["cases"].append(row) + if fixture is not None and not row["reference_passed"]: + raise ValueError(f"Frozen reference mismatch: {case['name']}") + save() + print(result.generated_text, flush=True) + if conversation: + report["history"] = [asdict(message) for message in conversation.history] if fixture is not None: - row["numeric_step_count"] = len(report["numeric_steps"]) - numeric_start - row["forward_count"] = len(adapter.runs) - run_start - row["reference_passed"] = ( - row["numeric_step_count"] == row["forward_count"] == len(case["generated_token_ids"]) - and list(result.prompt_token_ids) == case["prompt_token_ids"] - and list(result.generated_token_ids) == case["generated_token_ids"] - and result.generated_text == case["generated_text"] - and result.stop_reason == case["stop_reason"] - and case["required_substring"] in result.generated_text) - report["cases"].append(row) - save() - if fixture is not None and not row["reference_passed"]: - raise ValueError(f"Frozen reference mismatch: {case['name']}") - print(result.generated_text, flush=True) - if conversation: - report["history"] = [asdict(message) for message in conversation.history] - if fixture is not None: - report["full_model_gate"] = "PASS" - report.update(passed=True, status="PASS", stage="complete") - return 0 - except (GenerationCancelledError, KeyboardInterrupt) as exc: + report["full_model_gate"] = "PASS" + report.update(passed=True, status="PASS", stage="complete") + exit_code = 0 + except (GenerationCancelledError, KeyboardInterrupt, SystemExit) as exc: + # The runner's temporary signal handler exits with 128 + signal. + # Other SystemExit exceptions retain their original propagation. + if isinstance(exc, SystemExit) and exc.code not in (130, 143): + raise report.update(status="CANCELLED", error=f"{type(exc).__name__}: {exc}") if report["full_model_gate"] == "RUNNING": report["full_model_gate"] = "CANCELLED" - return 130 + exit_code = exc.code if isinstance(exc, SystemExit) else 130 + print(report["error"], file=sys.stderr, flush=True) except Exception as exc: report.update(error=f"{type(exc).__name__}: {exc}") if report["full_model_gate"] == "RUNNING": report["full_model_gate"] = "FAIL" - return 1 + print(report["error"], file=sys.stderr, flush=True) finally: for sig, handler in previous.items(): signal.signal(sig, handler) - save() - gc.collect() - print(f"{report['status']}: {out / 'report.json'}", flush=True) + try: + save() + except Exception as exc: + # A secondary persistence error must not hide generation/cancellation. + # Even successful generation or build-only needs a usable final report. + exit_code = exit_code or 1 + print(f"Report save failed: {type(exc).__name__}: {exc}; " + f"{out / 'report.json'} was not updated and may be missing or stale.", + file=sys.stderr, flush=True) + else: + print(f"{report['status']}: {out / 'report.json'}", flush=True) + finally: + gc.collect() + return exit_code if __name__ == "__main__": diff --git a/tests/llm-deploy/w5/README.md b/tests/llm-deploy/w5/README.md new file mode 100644 index 00000000..4344ff5b --- /dev/null +++ b/tests/llm-deploy/w5/README.md @@ -0,0 +1,62 @@ +# W5 报告保存与容器取消测试 + +本目录验证生成 CLI 的报告保存边界，以及真实 Docker 容器中的外部取消和进程清理。只使用小图，不下载或运行完整 Qwen3 模型。功能、实现和实跑说明见 [W5 开发说明](../../../docs/llm-deploy-v1.0/W5/README.md)。 + +## 环境与约定 + +在仓库根目录执行命令，使用已安装 `.[llm-deploy]` 的 Ubuntu 24.04 / Python 3.12 虚拟环境。`conftest.py` 复用既有 CLI fixtures；Docker 测试复用 `tests/test_qemu_generation.py` 的小图生成器和原 `scripts/deploy_w5.sh`。 + +真实容器测试要求本机 Linux Docker Engine 正常运行、当前用户有访问权限，且能够构建 `deployment/w5/Dockerfile`；模型和输出通过临时目录挂载。宿主不需要额外安装 QEMU，镜像提供它及编译工具链。可通过 `SCRATCHV_W5_BASE_IMAGE` 指定可访问的 Ubuntu 24.04 基础镜像，默认使用 `public.ecr.aws/ubuntu/ubuntu:24.04`。首次构建需要下载依赖，后续复用 Docker 缓存。 + +测试不会安装或启动 Docker，也不修改 WSL 配置。每次使用独立镜像标签，仅清理自己启动的客户端、容器及标签。中断诊断使用临时 QEMU 启动包装器的 `-S` 暂停真实 guest，不修改生产 harness。默认单核、手动 4 hart 和 padding 关闭均保持；4 hart 的恢复阶段在容器可用 CPU 自动选择为 4 时使用 `--harts auto`。 + +## 运行全部本目录测试 + +显式启用真实容器检查；缺少引擎或权限时失败，不会将缺失条件当作通过： + +```bash +SCRATCHV_W5_DOCKER_TESTS=1 python -B -m pytest tests/llm-deploy/w5 -q -p no:cacheprovider +``` + +未设置该变量时，报告保存测试照常执行，真实 Docker 两项会跳过；这不能作为容器清理验收结果。若要保留小图、日志与 JSON，可追加 `--basetemp=output/<新的空目录>`，注意 pytest 会清理指定的 basetemp，不要指向已有证据目录。 + +## 分项运行 + +| 文件 | 覆盖内容 | 通过标准 | +| --- | --- | --- | +| `test_report_persistence.py` | 权限、磁盘满、I/O/序列化失败；生成、超时、取消、数值/参考文本错误；中间保存恢复与下一次调用 | 原错误和退出码保留；保存失败不误报 PASS；信号处理恢复、清理执行；后续可正常保存 | +| `test_docker_cancellation.py` | 默认 1 / 手动 4 hart 的真实多 token、Docker 客户端 SIGINT/SIGTERM、同 ELF/bundle 恢复 | 正常输出 `[2,3,4,6]` / `2 3 4`，四步 ORT 误差为 0；SIGINT/SIGTERM 分别返回 130/143、报告 CANCELLED；容器、宿主 PID 和 QEMU 进程组消失；恢复结果一致 | + +仅报告保存边界： + +```bash +python -B -m pytest tests/llm-deploy/w5/test_report_persistence.py -q -p no:cacheprovider +``` + +仅真实容器取消与恢复： + +```bash +SCRATCHV_W5_DOCKER_TESTS=1 python -B -m pytest tests/llm-deploy/w5/test_docker_cancellation.py -q -p no:cacheprovider +``` + +Docker 测试在每项临时目录写入 `baseline.log`、`sigint.log`、`sigterm.log`、相应恢复日志和 `docker-cancellation-summary.json`；`deployment/<运行名>/report.json` 保留逐步数值及 QEMU 清理状态。清理断言来自独立的宿主进程和容器检查，失败后的兜底清理不能使测试变为通过。 + +既有 CLI 与部署编排回归仍位于原位置，可按影响范围单独执行： + +```bash +python -B -m pytest tests/test_w5_inference_cli.py -q -p no:cacheprovider +``` + +```bash +python -B -m pytest tests/test_w5_deployment.py -q -p no:cacheprovider +``` + +后者覆盖引擎不可用、构建失败、参数透传，以及 info/build 阶段的取消清理；使用假引擎，不替代本目录的真实 Docker run 阶段检查。 + +原多轮文档中的模板对照入口保留如下，使用本地官方 tokenizer，不运行完整模型： + +```bash +SCRATCHV_QWEN3_TOKENIZER_DIR=assets/qwen3-0.6b/tokenizer python -B -m pytest tests/test_conversation.py -q -p no:cacheprovider +``` + +它对照有/无 system 的三轮官方模板渲染和 token 序列；回复由测试预设，不能当作完整模型三轮聊天结果。依赖本地官方 tokenizer 及 Transformers。显式指定 tokenizer 目录时，缺资产会失败；未指定且默认位置缺资产时，对应官方资产测试会跳过。 diff --git a/tests/llm-deploy/w5/conftest.py b/tests/llm-deploy/w5/conftest.py new file mode 100644 index 00000000..22417cf8 --- /dev/null +++ b/tests/llm-deploy/w5/conftest.py @@ -0,0 +1,19 @@ +"""Reuse the existing W5 CLI fixtures for focused persistence checks.""" +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest + +_path = Path(__file__).resolve().parents[2] / "test_w5_inference_cli.py" +_spec = spec_from_file_location("w5_existing_cli_fixtures", _path) +_existing = module_from_spec(_spec) +_spec.loader.exec_module(_existing) + +cli = _existing.cli +runtime = _existing.runtime +reference_suite = _existing.reference_suite + + +@pytest.fixture +def cli_helpers(): + return _existing diff --git a/tests/llm-deploy/w5/test_docker_cancellation.py b/tests/llm-deploy/w5/test_docker_cancellation.py new file mode 100644 index 00000000..d0897e26 --- /dev/null +++ b/tests/llm-deploy/w5/test_docker_cancellation.py @@ -0,0 +1,264 @@ +"""Opt-in real Docker signal propagation, guest cleanup, and executable reuse.""" +import json +import os +from pathlib import Path +import runpy +import shutil +import signal +import subprocess +import time +import uuid + +import pytest + + +ROOT = Path(__file__).resolve().parents[3] +pytestmark = pytest.mark.skipif( + os.environ.get("SCRATCHV_W5_DOCKER_TESTS") != "1", + reason="set SCRATCHV_W5_DOCKER_TESTS=1 for real Docker small-graph checks", +) + + +def _docker(*args, check=True): + return subprocess.run( + ["docker", *args], text=True, capture_output=True, timeout=30, check=check, + ) + + +@pytest.fixture(scope="module") +def docker_assets(tmp_path_factory): + # Explicit opt-in must fail on an unavailable engine, never silently skip. + assert shutil.which("docker"), "Docker is required with SCRATCHV_W5_DOCKER_TESTS=1" + assert Path("/proc/self/stat").exists(), "These checks require a local Linux Docker engine" + ready = _docker("info", "--format", "{{.ServerVersion}}", check=False) + assert ready.returncode == 0, ready.stderr + assets = tmp_path_factory.mktemp("docker-small-assets") + helper = runpy.run_path(str(ROOT / "tests/test_qemu_generation.py")) + helper["make_small_onnx"](assets / "model.onnx") + + from tokenizers import AddedToken, Tokenizer, models, pre_tokenizers + tokenizer_dir = assets / "tokenizer" + tokenizer_dir.mkdir() + tokenizer = Tokenizer(models.WordLevel({str(i): i for i in range(8)}, unk_token="5")) + tokenizer.pre_tokenizer = pre_tokenizers.WhitespaceSplit() + tokenizer.add_special_tokens([ + AddedToken(value, special=True, normalized=False) for value in ("0", "6") + ]) + tokenizer.save(str(tokenizer_dir / "tokenizer.json")) + added = { + str(token_id): { + name: getattr(token, name) + for name in ("content", "lstrip", "rstrip", "single_word", "normalized", "special") + } + for token_id, token in tokenizer.get_added_tokens_decoder().items() + } + metadata = { + "tokenizer_config.json": { + "tokenizer_class": "Qwen2Tokenizer", "pad_token": "0", "eos_token": "6", + "unk_token": None, "bos_token": None, "added_tokens_decoder": added, + "additional_special_tokens": [], + "chat_template": "{% for message in messages %}{{ message['content'] }} {% endfor %}", + }, + "generation_config.json": {"pad_token_id": 0, "eos_token_id": [6]}, + "config.json": {"model_type": "qwen3", "vocab_size": 8}, + } + for name, value in metadata.items(): + (tokenizer_dir / name).write_text(json.dumps(value), encoding="utf-8") + + # Pause only actual generation guests. Tool discovery and FMA preflight run + # normally; no fake QEMU or production harness change is involved. + wrapper = assets / "qemu-control" + wrapper.write_text( + "#!/usr/bin/env python\n" + "import os, sys\n" + "args = sys.argv[1:]\n" + "if '/steps/' in os.getcwd() and os.path.exists('/assets/park'):\n" + " args += ['-S']\n" + "os.execv('/usr/bin/qemu-system-riscv64', ['qemu-system-riscv64', *args])\n", + encoding="utf-8", + ) + wrapper.chmod(0o755) + image = f"scratchv-w5-cancellation:{uuid.uuid4().hex[:12]}" + yield assets, image + # Only this fixture's unique image is eligible for cleanup. + _docker("image", "rm", image, check=False) + + +def _owned_containers(image, output): + found = [] + listing = _docker("ps", "-aq", "--filter", f"ancestor={image}") + for cid in listing.stdout.split(): + result = _docker("inspect", cid, check=False) + if result.returncode: + continue + record = json.loads(result.stdout)[0] + if any(m.get("Source") == str(output.resolve()) and m.get("Destination") == "/output" + for m in record["Mounts"]): + found.append(cid) + return found + + +def _process_identity(pid): + try: + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + status = Path(f"/proc/{pid}/status").read_text() + except FileNotFoundError: + return None + nspid = next((line.split()[1:] for line in status.splitlines() + if line.startswith("NSpid:")), [str(pid)]) + return {"pid": int(pid), "pgid": int(fields[2]), "starttime": fields[19], + "container_pid": int(nspid[-1])} + + +def _wait_parked(client, image, output): + deadline = time.monotonic() + 600 # Includes the deployment script's image build. + while time.monotonic() < deadline: + assert client.poll() is None, f"Deployment exited before paused guest: {client.returncode}" + for cid in _owned_containers(image, output): + top = _docker("top", cid, "-eo", "pid,pgid,comm,args", check=False) + if top.returncode: + continue + rows = [line.split(None, 3) for line in top.stdout.splitlines()[1:]] + guests = [row for row in rows if len(row) == 4 + and "qemu-system-riscv64" in row[3] and "-S" in row[3].split()] + if guests: + processes = [_process_identity(row[0]) for row in rows] + processes = [item for item in processes if item is not None] + guest = _process_identity(guests[0][0]) + assert guest is not None, "Paused guest disappeared unexpectedly" + return cid, processes, guest + time.sleep(0.2) + pytest.fail("Timed out waiting for real QEMU -S inside the deployment container") + + +def _assert_reaped(cid, processes, guest, evidence): + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + live = [item for item in processes if _process_identity(item["pid"]) == item] + present = _docker("inspect", cid, check=False).returncode == 0 + if not live and not present: + break + time.sleep(0.1) + assert not live, f"Container host processes survived cancellation: {live}" + assert not present, f"--rm container survived cancellation: {cid}" + time.sleep(0.3) + assert all(_process_identity(item["pid"]) != item for item in processes) + assert _docker("inspect", cid, check=False).returncode != 0 + # Inspect the actual guest process group independently of report assertions. + group_members = [] + for entry in Path("/proc").iterdir(): + if entry.name.isdigit(): + identity = _process_identity(entry.name) + if identity is not None and identity["pgid"] == guest["pgid"]: + group_members.append(identity) + assert not group_members, f"QEMU process group still has members: {group_members}" + evidence.update(container_removed=True, host_processes_reaped=True, + qemu_group_empty=True, delayed_recheck_seconds=0.3) + + +def _report(output, name): + return json.loads((output / name / "report.json").read_text(encoding="utf-8")) + + +def _assert_pass(record, harts): + assert record["status"] == "PASS" and record["passed"] is True + assert record["harts"] == harts and record["qemu_workspace_layout"] == "none" + assert record["qemu_forwards"] == 4 + assert record["cases"][0]["generated_token_ids"] == [2, 3, 4, 6] + assert record["cases"][0]["generated_text"] == "2 3 4" + assert len(record["numeric_steps"]) == 4 + assert all(step["passed"] and step["max_abs"] == 0 for step in record["numeric_steps"]) + assert all(run["qemu_reaped"] and run["guest_completed"] for run in record["runs"]) + + +@pytest.mark.parametrize("harts", [1, 4], ids=["default-single", "manual-four"]) +def test_external_signals_reap_guest_and_reuse_build(tmp_path, docker_assets, harts): + assets, image = docker_assets + output = tmp_path / "deployment" + output.mkdir() + env = dict(os.environ, SCRATCHV_W5_ASSETS=str(assets), SCRATCHV_W5_OUTPUT=str(output), + SCRATCHV_W5_IMAGE=image, + SCRATCHV_W5_BASE_IMAGE=os.environ.get( + "SCRATCHV_W5_BASE_IMAGE", "public.ecr.aws/ubuntu/ubuntu:24.04")) + env["SCRATCHV_CONTAINER_ENGINE"] = "docker" + common = ["--tokenizer-dir", "/assets/tokenizer", "--prompt", "1", + "--max-new-tokens", "8", "--verify-onnx", "/assets/model.onnx", + "--qemu", "/assets/qemu-control", "--matmul-policy", "blocked_fma", + "--timeout", "90", "--total-timeout", "180", "--build-timeout", "180"] + # The single-hart baseline really exercises the CLI default. + hart_args = [] if harts == 1 else ["--harts", "4"] + active = None + evidence = {"harts": harts, "image": image, "cancellations": []} + + def launch(name, source, selection): + log = (tmp_path / f"{name}.log").open("w", encoding="utf-8") + try: + return subprocess.Popen( + ["bash", str(ROOT / "scripts/deploy_w5.sh"), *source, *common, *selection, + "--output-dir", f"/output/{name}"], + cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True, + ) + finally: + log.close() + + try: + active = launch("baseline", ["--model", "/assets/model.onnx"], hart_args) + assert active.wait(timeout=600) == 0, (tmp_path / "baseline.log").read_text()[-6000:] + baseline = _report(output, "baseline") + _assert_pass(baseline, harts) + original = json.loads((output / "baseline/executable.json").read_text()) + evidence["baseline"] = baseline + reuse = ["--reuse-build", "/output/baseline/executable.json"] + auto = int(_docker("run", "--rm", "--entrypoint", "python", image, "-c", + "from scratchv.runtime.riscv_harts import resolve_harts; " + "print(resolve_harts('auto'))").stdout.strip()) + evidence["available_auto_harts"] = auto + recovery_args = ["--harts", "auto"] if harts == 4 and auto == 4 else hart_args + + for sig in (signal.SIGINT, signal.SIGTERM): + name = sig.name.lower() + (assets / "park").touch() + active = launch(name, reuse, hart_args) + cid, processes, guest = _wait_parked(active, image, output) + cancellation = {"signal": sig.name, "client_pid": active.pid, "container_id": cid, + "host_processes": processes, "qemu_host_process": guest} + evidence["cancellations"].append(cancellation) + # Exercise Docker's real signal proxy. killpg would bypass this chain. + active.send_signal(sig) + cancellation["exit_code"] = active.wait(timeout=30) + assert cancellation["exit_code"] == 128 + int(sig), ( + tmp_path / f"{name}.log").read_text()[-6000:] + _assert_reaped(cid, processes, guest, cancellation) + cancelled = _report(output, name) + cancellation["report"] = cancelled + assert cancelled["status"] == "CANCELLED" and not cancelled["passed"] + assert cancelled["harts"] == harts and cancelled["qemu_workspace_layout"] == "none" + assert cancelled["qemu_forwards"] == 0 and len(cancelled["runs"]) == 1 + run = cancelled["runs"][0] + assert run["qemu_reaped"] and not run["guest_completed"] + assert run["qemu_pid"] == guest["container_pid"] + (assets / "park").unlink() + + recovery_name = f"{name}-recovery" + active = launch(recovery_name, reuse, recovery_args) + assert active.wait(timeout=600) == 0, (tmp_path / f"{recovery_name}.log").read_text()[-6000:] + recovered = _report(output, recovery_name) + _assert_pass(recovered, harts) + assert recovered["cases"][0]["generated_token_ids"] == baseline["cases"][0]["generated_token_ids"] + saved = json.loads((output / recovery_name / "executable.json").read_text()) + assert all(saved[key] == original[key] for key in ("elf", "bundle_dir", "qemu", "harts")) + assert recovered["harts_requested"] == ("auto" if recovery_args == ["--harts", "auto"] else harts) + cancellation["recovery"] = recovered + evidence["passed"] = True + finally: + (assets / "park").unlink(missing_ok=True) + # Failure cleanup cannot satisfy the assertions above, and targets only + # our unique image plus output mount and our separately started client. + for cid in _owned_containers(image, output): + _docker("rm", "-f", cid, check=False) + if active is not None and active.poll() is None: + os.killpg(active.pid, signal.SIGKILL) + active.wait(timeout=15) + (tmp_path / "docker-cancellation-summary.json").write_text( + json.dumps(evidence, indent=2) + "\n", encoding="utf-8", + ) diff --git a/tests/llm-deploy/w5/test_report_persistence.py b/tests/llm-deploy/w5/test_report_persistence.py new file mode 100644 index 00000000..cc49cfbc --- /dev/null +++ b/tests/llm-deploy/w5/test_report_persistence.py @@ -0,0 +1,268 @@ +"""Report I/O failures retain the primary outcome; no model or QEMU runs.""" +import errno +import json +import signal + +import pytest + +from scratchv.runtime.generation import GenerationCancelledError + + +@pytest.mark.parametrize("error", [ + PermissionError(errno.EACCES, "report directory is read-only"), + OSError(errno.ENOSPC, "report disk is full"), + OSError(errno.EIO, "report device failed"), +], ids=["permission", "disk-full", "io"]) +@pytest.mark.parametrize("build_only", [False, True], ids=["generation", "build-only"]) +def test_final_write_failure_never_reports_success( + cli, runtime, cli_helpers, tmp_path, monkeypatch, capsys, error, build_only, +): + atomic = cli.atomic_text + attempted = [] + collected = [] + previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + + def write(path, text): + value = json.loads(text) + attempted.append(value) + if value["status"] in {"PASS", "BUILD_ONLY"}: + raise error + atomic(path, text) + + monkeypatch.setattr(cli, "atomic_text", write) + monkeypatch.setattr(cli.gc, "collect", lambda: collected.append(True)) + options = ["--build-only"] if build_only else ["--prompt", "text", "--max-new-tokens", "1"] + assert cli.main(cli_helpers.argv(tmp_path, *options)) == 1 + output = capsys.readouterr() + assert "Report save failed:" in output.err and str(error) in output.err + assert "missing or stale" in output.err + assert "PASS:" not in output.out and "BUILD_ONLY:" not in output.out + assert attempted[-1]["harts"] == 1 and attempted[-1]["qemu_workspace_layout"] == "none" + assert collected == [True] + assert all(signal.getsignal(sig) is handler for sig, handler in previous.items()) + if build_only: + assert not runtime.fma and not runtime.adapters[0].runs + assert not (tmp_path / "run/report.json").exists() + else: + # The older atomic checkpoint cannot be mistaken for final success. + saved = cli_helpers.report(tmp_path) + assert not saved["passed"] and saved["status"] != "PASS" + + +@pytest.mark.parametrize("failure,code", [ + (RuntimeError("primary generation failure"), 1), + (TimeoutError("primary forward timed out"), 1), + (GenerationCancelledError("primary cancellation"), 130), + (KeyboardInterrupt("primary interrupt"), 130), + (SystemExit(130), 130), + (SystemExit(143), 143), +], ids=["generation", "timeout", "cancel", "interrupt", "runner-sigint", "runner-sigterm"]) +@pytest.mark.parametrize("save_error", [ + PermissionError(errno.EACCES, "secondary save permission failure"), + OSError(errno.ENOSPC, "secondary save disk full"), +], ids=["permission", "disk-full"]) +def test_secondary_write_failure_keeps_primary_error_and_exit( + cli, runtime, cli_helpers, tmp_path, monkeypatch, capsys, failure, code, save_error, +): + runtime.failure = failure + snapshots = [] + collected = [] + previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + + def write(path, text): + snapshots.append(json.loads(text)) + raise save_error + + monkeypatch.setattr(cli, "atomic_text", write) + monkeypatch.setattr(cli.gc, "collect", lambda: collected.append(True)) + assert cli.main(cli_helpers.argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1")) == code + output = capsys.readouterr() + assert str(failure) in output.err and type(failure).__name__ in output.err + assert str(save_error) in output.err and "Report save failed:" in output.err + assert "report.json" not in output.out + assert snapshots[-1]["error"] == f"{type(failure).__name__}: {failure}" + assert snapshots[-1]["status"] == ("CANCELLED" if code in (130, 143) else "FAIL") + assert snapshots[-1]["forward_attempts"] == 1 and not snapshots[-1]["passed"] + assert collected == [True] + assert all(signal.getsignal(sig) is handler for sig, handler in previous.items()) + + +def test_intermediate_write_failure_is_saved_as_failure_when_storage_recovers( + cli, runtime, cli_helpers, tmp_path, monkeypatch, capsys, +): + atomic = cli.atomic_text + attempts = [] + + def write(path, text): + attempts.append(json.loads(text)) + if len(attempts) == 1: + raise OSError(errno.ENOSPC, "checkpoint disk full") + atomic(path, text) + + monkeypatch.setattr(cli, "atomic_text", write) + assert cli.main(cli_helpers.argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1")) == 1 + saved = cli_helpers.report(tmp_path) + assert len(attempts) == 2 + assert saved["status"] == "FAIL" and not saved["passed"] + assert saved["qemu_forwards"] == 1 and len(saved["cases"]) == 1 + assert "checkpoint disk full" in saved["error"] + assert "checkpoint disk full" in capsys.readouterr().err + + +def test_suite_initial_checkpoint_failure_prevents_generation( + cli, runtime, reference_suite, cli_helpers, tmp_path, monkeypatch, +): + atomic = cli.atomic_text + attempts = [] + + def write(path, text): + attempts.append(json.loads(text)) + if len(attempts) == 1: + raise PermissionError(errno.EACCES, "cannot save RUNNING") + atomic(path, text) + + monkeypatch.setattr(cli, "atomic_text", write) + assert cli.main(cli_helpers.argv( + tmp_path, "--reference-suite", "--verify-onnx", "model.onnx", + )) == 1 + saved = cli_helpers.report(tmp_path) + assert saved["full_model_gate"] == "FAIL" and saved["qemu_forwards"] == 0 + assert saved["numeric_steps"] == [] and saved["cases"] == [] + assert "cannot save RUNNING" in saved["error"] + + +def test_numeric_failure_survives_report_write_failure( + cli, runtime, reference_suite, cli_helpers, tmp_path, monkeypatch, capsys, +): + runtime.oracle = cli_helpers.logits(4) + atomic = cli.atomic_text + snapshots = [] + + def write(path, text): + value = json.loads(text) + snapshots.append(value) + if value["full_model_gate"] == "FAIL": + raise OSError(errno.ENOSPC, "numeric-report disk full") + atomic(path, text) + + monkeypatch.setattr(cli, "atomic_text", write) + assert cli.main(cli_helpers.argv( + tmp_path, "--reference-suite", "--verify-onnx", "model.onnx", + )) == 1 + output = capsys.readouterr() + assert "QEMU/ORT full logits, selected row or frozen token differ" in output.err + assert "numeric-report disk full" in output.err + assert snapshots[-1]["full_model_gate"] == "FAIL" + assert not snapshots[-1]["numeric_steps"][0]["passed"] + assert "QEMU/ORT" in snapshots[-1]["error"] + assert cli_helpers.report(tmp_path)["full_model_gate"] == "RUNNING" + assert "PASS:" not in output.out + + +def test_reference_text_failure_survives_report_write_failure( + cli, runtime, reference_suite, cli_helpers, tmp_path, monkeypatch, capsys, +): + reference_suite.fixture["cases"][0]["generated_text"] = "not-the-result" + fixture_path = cli.ROOT / "probes/w5_generation/references.json" + fixture_path.write_text(json.dumps(reference_suite.fixture)) + atomic = cli.atomic_text + snapshots = [] + + def write(path, text): + value = json.loads(text) + snapshots.append(value) + if value["cases"]: + raise PermissionError(errno.EACCES, "reference-report denied") + atomic(path, text) + + monkeypatch.setattr(cli, "atomic_text", write) + assert cli.main(cli_helpers.argv( + tmp_path, "--reference-suite", "--verify-onnx", "model.onnx", + )) == 1 + output = capsys.readouterr() + assert "Frozen reference mismatch: france" in output.err + assert "reference-report denied" in output.err + assert snapshots[-1]["error"] == "ValueError: Frozen reference mismatch: france" + assert snapshots[-1]["full_model_gate"] == "FAIL" + assert snapshots[-1]["numeric_steps"][0]["passed"] + assert not snapshots[-1]["cases"][0]["reference_passed"] + + +def test_serialization_failure_returns_failure_without_false_saved_message( + cli, runtime, cli_helpers, tmp_path, capsys, +): + runtime.exe.evidence["invalid_measurement"] = float("nan") + assert cli.main(cli_helpers.argv(tmp_path, "--build-only")) == 1 + output = capsys.readouterr() + assert "Report save failed: ValueError:" in output.err + assert "BUILD_ONLY:" not in output.out + assert not (tmp_path / "run/report.json").exists() + + +def test_next_invocation_can_save_after_failed_final_report( + cli, runtime, cli_helpers, tmp_path, monkeypatch, +): + atomic = cli.atomic_text + + def deny(path, text): + raise PermissionError(errno.EACCES, "first output unavailable") + + monkeypatch.setattr(cli, "atomic_text", deny) + assert cli.main(cli_helpers.argv(tmp_path, "--build-only")) == 1 + monkeypatch.setattr(cli, "atomic_text", atomic) + next_root = tmp_path / "next" + assert cli.main(cli_helpers.argv(next_root, "--prompt", "text", "--max-new-tokens", "1")) == 0 + saved = cli_helpers.report(next_root) + assert saved["status"] == "PASS" and saved["passed"] + assert saved["qemu_forwards"] == 1 and "error" not in saved + assert saved["harts"] == 1 and saved["qemu_workspace_layout"] == "none" + + +@pytest.mark.parametrize("code", [130, 143], ids=["sigint", "sigterm"]) +@pytest.mark.parametrize("suite", [False, True], ids=["prompt", "reference-suite"]) +def test_runner_signal_exit_is_saved_as_cancellation( + cli, runtime, reference_suite, cli_helpers, tmp_path, monkeypatch, capsys, code, suite, +): + runtime.failure = SystemExit(code) + collected = [] + previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + monkeypatch.setattr(cli.gc, "collect", lambda: collected.append(True)) + options = (["--reference-suite", "--verify-onnx", "model.onnx"] if suite else + ["--prompt", "text", "--max-new-tokens", "1"]) + assert cli.main(cli_helpers.argv(tmp_path, *options)) == code + saved = cli_helpers.report(tmp_path) + assert saved["status"] == "CANCELLED" and not saved["passed"] + assert saved["error"] == f"SystemExit: {code}" + assert saved["full_model_gate"] == ("CANCELLED" if suite else "NOT_RUN") + assert saved["forward_attempts"] == 1 and saved["qemu_forwards"] == 0 + assert f"SystemExit: {code}" in capsys.readouterr().err + assert collected == [True] + assert all(signal.getsignal(sig) is handler for sig, handler in previous.items()) + + +@pytest.mark.parametrize("code", [0, 2, "143"], ids=["success", "other-error", "message"]) +@pytest.mark.parametrize("save_fails", [False, True], ids=["saved", "save-fails"]) +def test_non_signal_system_exit_propagates_after_cleanup( + cli, runtime, cli_helpers, tmp_path, monkeypatch, capsys, code, save_fails, +): + failure = SystemExit(code) + runtime.failure = failure + collected = [] + previous = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + monkeypatch.setattr(cli.gc, "collect", lambda: collected.append(True)) + if save_fails: + def deny(path, text): + raise OSError(errno.ENOSPC, "unrelated exit report disk full") + monkeypatch.setattr(cli, "atomic_text", deny) + with pytest.raises(SystemExit) as caught: + cli.main(cli_helpers.argv(tmp_path, "--prompt", "text", "--max-new-tokens", "1")) + assert caught.value is failure and caught.value.code == code + output = capsys.readouterr() + assert "CANCELLED" not in output.out + if save_fails: + assert "Report save failed:" in output.err + assert "unrelated exit report disk full" in output.err + else: + assert cli_helpers.report(tmp_path)["status"] == "FAIL" + assert collected == [True] + assert all(signal.getsignal(sig) is handler for sig, handler in previous.items()) From e8c94a71becfae96c4171674e332ed1239335c4d Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Sun, 25 Oct 2026 15:30:46 +0800 Subject: [PATCH 5/7] fix(w5): fail generation checks when log capture fails --- .github/workflows/w4-full-numeric.yml | 12 +++++-- docs/llm-deploy-v1.0/W5/README.md | 6 ++++ scripts/check_w5_ci.py | 5 +++ tests/llm-deploy/w5/README.md | 15 ++++++++ tests/test_w4_workflows.py | 49 +++++++++++++++++++++++---- tests/test_w5_ci.py | 30 +++++++++++++--- 6 files changed, 104 insertions(+), 13 deletions(-) diff --git a/.github/workflows/w4-full-numeric.yml b/.github/workflows/w4-full-numeric.yml index 5aa47017..e3f067f3 100644 --- a/.github/workflows/w4-full-numeric.yml +++ b/.github/workflows/w4-full-numeric.yml @@ -570,10 +570,15 @@ jobs: --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ --timeout 1800 --total-timeout 16500 \ 2>&1 | tee output/w5-ci/runner.log - status=${PIPESTATUS[0]} + pipeline_status=("${PIPESTATUS[@]}") set -e - printf '%s\n' "$status" > output/w5-ci/runner-exit-code.txt - exit "$status" + printf '%s\n' "${pipeline_status[0]}" > output/w5-ci/runner-exit-code.txt + printf '%s\n' "${pipeline_status[1]}" > output/w5-ci/log-exit-code.txt + # Preserve generation failures (including timeout/cancellation) first. + if (( pipeline_status[0] != 0 )); then + exit "${pipeline_status[0]}" + fi + exit "${pipeline_status[1]}" - name: Reap any container left by an interrupted Docker client if: always() timeout-minutes: 2 @@ -592,6 +597,7 @@ jobs: python -B -X utf8 scripts/check_w5_ci.py \ --report output/w5-ci/run/report.json \ --exit-code-file output/w5-ci/runner-exit-code.txt \ + --log-exit-code-file output/w5-ci/log-exit-code.txt \ --output output/w5-ci/ci-gate.json - name: Record final W5 resource and retention scope if: always() diff --git a/docs/llm-deploy-v1.0/W5/README.md b/docs/llm-deploy-v1.0/W5/README.md index 51d9af8b..724288d1 100644 --- a/docs/llm-deploy-v1.0/W5/README.md +++ b/docs/llm-deploy-v1.0/W5/README.md @@ -83,6 +83,12 @@ QEMU 适配器负责其子进程回收。CLI 最终保存报告失败时向 stde 数值、参考文本等原始错误不会被保存错误覆盖，也不打印误导性的成功报告路径。 运行中的 SIGINT/SIGTERM 均记录为取消，分别保留退出码 130/143。 +本分支的报告检查器支持 `--log-exit-code-file`：指定后要求生成与日志退出码均为 0， +日志状态缺失、不可读或内容无效也返回 FAIL。外层 Bash 分别保存 +`runner-exit-code.txt` 和 `log-exit-code.txt`；生成失败优先保留原退出码， +生成成功而 `tee` 失败时返回日志退出码。下方固定 `8ad3474` 的历史复现命令保留原版 +参数，仍须按其成功要求单独核对日志退出码，不向旧检查器传入新参数。 + ## 如何复现 **本地已有环境。** 从仓库根执行，激活项目 Python 3.12 环境并按 diff --git a/scripts/check_w5_ci.py b/scripts/check_w5_ci.py index ba14f742..425b768e 100644 --- a/scripts/check_w5_ci.py +++ b/scripts/check_w5_ci.py @@ -106,6 +106,8 @@ def main(argv=None): cli.add_argument("--report", type=Path, required=True) cli.add_argument("--reference", type=Path, default=ROOT / "probes/w5_generation/references.json") cli.add_argument("--exit-code-file", type=Path, required=True) + cli.add_argument("--log-exit-code-file", type=Path, + help="Also require the tee log capture to have exited successfully") cli.add_argument("--output", type=Path, required=True) args = cli.parse_args(argv) result = {"status": "FAIL", "passed": False, "scope": "Three frozen cases, 24 full QEMU forwards", @@ -113,6 +115,9 @@ def main(argv=None): "defaults_changed": False, "raw_logits_uploaded": False} try: require(args.exit_code_file.read_text().strip() == "0", "Generation process did not exit successfully") + if args.log_exit_code_file is not None: + require(args.log_exit_code_file.read_text().strip() == "0", + "Log capture process did not exit successfully") report = json.loads(args.report.read_text(encoding="utf-8")) validate_report(report, json.loads(args.reference.read_text(encoding="utf-8"))) result.update(status="PASS", passed=True, numeric_steps=24, cases=3, diff --git a/tests/llm-deploy/w5/README.md b/tests/llm-deploy/w5/README.md index 4344ff5b..83c393c0 100644 --- a/tests/llm-deploy/w5/README.md +++ b/tests/llm-deploy/w5/README.md @@ -53,6 +53,21 @@ python -B -m pytest tests/test_w5_deployment.py -q -p no:cacheprovider 后者覆盖引擎不可用、构建失败、参数透传，以及 info/build 阶段的取消清理；使用假引擎，不替代本目录的真实 Docker run 阶段检查。 +W5 专项的日志退出状态沿用现有工作流测试，直接执行 YAML 中的 Bash，并以短小假生成器 +替代模型。需要 Bash、GNU tee、`/dev/full` 和 PyYAML，不需要 Docker 或模型资产： +生成或日志任一失败必须失败，分别保留退出码；状态文件本身写失败也不能成功。 + +```bash +python -B -m pytest tests/test_w4_workflows.py -k 'w5_deployment_pipeline or w5_manual_job' -q -p no:cacheprovider +``` + +报告检查器的无模型测试覆盖有效、非零、缺失、不可读和无效的日志状态；即使生成报告 +为 PASS，也不能覆盖日志失败。旧版未传日志检查参数的调用仍可用： + +```bash +python -B -m pytest tests/test_w5_ci.py -q -p no:cacheprovider +``` + 原多轮文档中的模板对照入口保留如下，使用本地官方 tokenizer，不运行完整模型： ```bash diff --git a/tests/test_w4_workflows.py b/tests/test_w4_workflows.py index 505af10f..18f2db58 100644 --- a/tests/test_w4_workflows.py +++ b/tests/test_w4_workflows.py @@ -676,24 +676,61 @@ def test_w5_manual_job_reuses_deployment_and_preserves_bounded_full_scope(): gate = step("Require complete W5 generation evidence") assert gate["if"] == "always()" and "scripts/check_w5_ci.py" in gate["run"] assert "--exit-code-file output/w5-ci/runner-exit-code.txt" in gate["run"] + assert "--log-exit-code-file output/w5-ci/log-exit-code.txt" in gate["run"] assert all("continue-on-error" not in item for item in job["steps"]) @pytest.mark.parametrize("exit_code", [0, 1, 124, 130, 143]) -def test_w5_deployment_pipeline_preserves_actual_exit(bash, tmp_path, exit_code): - (tmp_path / "output/w5-ci").mkdir(parents=True) +@pytest.mark.parametrize("log_failure", [False, True], ids=["log-ok", "log-full"]) +def test_w5_deployment_pipeline_preserves_actual_exit(bash, tmp_path, exit_code, log_failure): + if log_failure and not Path("/dev/full").exists(): + pytest.skip("Real tee write-failure check requires /dev/full") + output = tmp_path / "output/w5-ci" + output.mkdir(parents=True) + if log_failure: + (output / "runner.log").symlink_to("/dev/full") script = step("Run the frozen W5 suite through the deployment script")["run"] - wrapper = 'timeout() { printf "%s\\n" "$@" > "$ARG_LOG"; return "$MOCK_EXIT"; }\n' + # Replace only the expensive producer; execute the actual workflow and real tee. + wrapper = ('timeout() { printf "%s\\n" "$@" > "$ARG_LOG"; ' + 'printf "generation stdout\\n"; printf "generation stderr\\n" >&2; ' + 'return "$MOCK_EXIT"; }\n') capture = tmp_path / "arguments.txt" result = subprocess.run([bash, "--noprofile", "--norc", "-c", wrapper + script], cwd=tmp_path, - env=dict(os.environ, ARG_LOG=str(capture), MOCK_EXIT=str(exit_code)), capture_output=True, text=True) - assert result.returncode == exit_code, result.stderr - assert int((tmp_path / "output/w5-ci/runner-exit-code.txt").read_text()) == exit_code + env=dict(os.environ, ARG_LOG=str(capture), MOCK_EXIT=str(exit_code)), + capture_output=True, text=True, timeout=10) + assert (result.returncode != 0) is (log_failure or exit_code != 0), result.stderr + log_exit = int((output / "log-exit-code.txt").read_text()) + assert (log_exit != 0) is log_failure + assert result.returncode == (exit_code or log_exit), result.stderr + assert int((output / "runner-exit-code.txt").read_text()) == exit_code + assert "generation stdout" in result.stdout and "generation stderr" in result.stdout + if log_failure: + assert "runner.log" in result.stderr + else: + assert (output / "runner.log").read_text() == "generation stdout\ngeneration stderr\n" args = capture.read_text().splitlines() assert "scripts/deploy_w5.sh" in args and "--reference-suite" in args assert args[args.index("--total-timeout") + 1] == "16500" +@pytest.mark.parametrize("status_file", ["runner-exit-code.txt", "log-exit-code.txt"]) +def test_w5_deployment_pipeline_rejects_status_write_failure(bash, tmp_path, status_file): + if not Path("/dev/full").exists(): + pytest.skip("Real status write-failure check requires /dev/full") + output = tmp_path / "output/w5-ci" + output.mkdir(parents=True) + (output / status_file).symlink_to("/dev/full") + script = step("Run the frozen W5 suite through the deployment script")["run"] + wrapper = 'timeout() { printf "generation completed\\n"; return 0; }\n' + result = subprocess.run([bash, "--noprofile", "--norc", "-c", wrapper + script], + cwd=tmp_path, capture_output=True, text=True, timeout=10) + assert result.returncode != 0, result.stderr + assert (output / "runner.log").read_text() == "generation completed\n" + assert result.stderr # The status write, not generation or tee, must fail. + if status_file == "log-exit-code.txt": + assert (output / "runner-exit-code.txt").read_text() == "0\n" + + @pytest.mark.parametrize("cpus,ram,disk,passed", [ (4, 10, 10, True), (8, 16, 30, True), (3, 16, 30, False), (4, 9, 30, False), (4, 16, 9, False)]) diff --git a/tests/test_w5_ci.py b/tests/test_w5_ci.py index 115f8c28..c6adf267 100644 --- a/tests/test_w5_ci.py +++ b/tests/test_w5_ci.py @@ -188,8 +188,17 @@ def test_fma_summary_cannot_mask_incomplete_or_failing_rows(gate, report, refere gate.validate_report(report, reference) -@pytest.mark.parametrize("exit_text,expected", [("0\n", 0), ("130\n", 1), ("", 1)]) -def test_cli_checks_real_exit_code_and_writes_status(gate, report, tmp_path, monkeypatch, exit_text, expected): +@pytest.mark.parametrize("exit_text", ["0\n", "130\n", ""]) +@pytest.mark.parametrize("log_text", [ + pytest.param(None, id="legacy-no-log-check"), + pytest.param("0\n", id="log-success"), + pytest.param("1\n", id="log-failure"), + pytest.param("", id="log-empty"), + pytest.param("invalid\n", id="log-invalid"), + pytest.param("missing", id="log-missing"), + pytest.param("directory", id="log-unreadable"), +]) +def test_cli_checks_real_exit_code_and_writes_status(gate, report, tmp_path, monkeypatch, exit_text, log_text): report_path = tmp_path / "report.json" report_path.write_text(json.dumps(report)) exit_path = tmp_path / "exit-code.txt" @@ -197,13 +206,26 @@ def test_cli_checks_real_exit_code_and_writes_status(gate, report, tmp_path, mon output = tmp_path / "gate.json" summary = tmp_path / "summary.md" monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) - code = gate.main(["--report", str(report_path), "--exit-code-file", str(exit_path), - "--output", str(output)]) + argv = ["--report", str(report_path), "--exit-code-file", str(exit_path), + "--output", str(output)] + if log_text is not None: + log_path = tmp_path / "log-exit-code.txt" + if log_text == "directory": + log_path.mkdir() + elif log_text != "missing": + log_path.write_text(log_text) + argv += ["--log-exit-code-file", str(log_path)] + expected = 0 if exit_text == "0\n" and log_text in (None, "0\n") else 1 + code = gate.main(argv) assert code == expected result = json.loads(output.read_text()) assert result["status"] == ("PASS" if expected == 0 else "FAIL") assert result["passed"] is (expected == 0) assert result["status"] in summary.read_text() + if exit_text == "0\n" and log_text in ("1\n", "", "invalid\n"): + assert "Log capture process" in result["error"] + if expected: + assert "24/24 numeric steps" not in summary.read_text() def test_missing_run_report_produces_failure_artifact(gate, tmp_path): From 3782bf80f2b49188b69ea0a52fb250bd5dd482d0 Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Fri, 30 Oct 2026 20:56:27 +0800 Subject: [PATCH 6/7] fix(w5): preserve core naming compatibility and publish validation docs Use cores consistently while retaining legacy CLI, Python and metadata aliases. Cover argparse attributes, dataclass serialization/replacement and conflicting configuration values without changing default single-core execution. Publish the expanded W5 generation/report explanations and historical summary. Separate reused builds, program elapsed time and container compilation time. Validation: 591 relevant W4/W5 interface and small-QEMU tests, plus 40 instruction-counter tests after merging main. No full model or Docker rerun. --- .github/workflows/w4-full-numeric.yml | 2 +- README.md | 1 + docs/INDEX.md | 2 +- docs/llm-deploy-v1.0/W4/README.md | 2 +- .../full-hart-comparison-2026-10-22.json | 46 +- .../W4/softsim-optimization-2026-10-22.md | 34 +- docs/llm-deploy-v1.0/W4/validation.md | 64 +- docs/llm-deploy-v1.0/W5/README.md | 655 +++++++++++--- .../W5/reports/validation-summary.json | 826 ++++++++++++++++++ probes/w4_qwen3_full/diagnostic.py | 25 +- probes/w4_qwen3_full/run.py | 83 +- probes/w5_generation/README.md | 3 +- scratchv/backend/tensor_c_codegen.py | 27 +- scratchv/runtime/riscv_external.py | 77 +- scratchv/runtime/riscv_harts.py | 118 ++- scratchv/runtime/weight_bundle.py | 12 +- scripts/check_w5_ci.py | 19 +- scripts/run_inference.py | 29 +- tests/llm-deploy/w5/README.md | 65 +- .../llm-deploy/w5/test_docker_cancellation.py | 2 +- tests/test_qemu_generation.py | 2 +- tests/test_riscv_external_harts.py | 186 +++- tests/test_tensor_c_external.py | 2 +- tests/test_w4_diagnostic.py | 20 +- tests/test_w4_full_probe_contract.py | 82 +- tests/test_w5_ci.py | 35 + tests/test_w5_deployment.py | 5 +- tests/test_w5_inference_cli.py | 148 +++- 28 files changed, 2188 insertions(+), 384 deletions(-) create mode 100644 docs/llm-deploy-v1.0/W5/reports/validation-summary.json diff --git a/.github/workflows/w4-full-numeric.yml b/.github/workflows/w4-full-numeric.yml index e3f067f3..c6c95fb1 100644 --- a/.github/workflows/w4-full-numeric.yml +++ b/.github/workflows/w4-full-numeric.yml @@ -609,7 +609,7 @@ jobs: "reports_retention_days": 30, "raw_logits_uploaded": false, "weights_uploaded": false, - "scope": "Three frozen raw-completion prompts, eight greedy tokens each; one ELF/bundle, 4 hart, explicit Mean and nonlinear reference policies, padding off.", + "scope": "Three frozen raw-completion prompts, eight greedy tokens each; one ELF/bundle, 4 core, explicit Mean and nonlinear reference policies, padding off.", "container_scope": "The original deployment script exit, CLI gate and workflow job together establish container acceptance. CLI container_reproduction=NOT_ASSERTED is preserved.", "timing_scope": "GNU time measures the Docker client, not container CPU/RSS; per-forward QEMU elapsed/RSS evidence is in run.json. This CI run is not a calibrated speedup benchmark.", "failure_scope": "Normal failures retain reports. Runner loss, hard job timeout or cancellation can prevent cleanup and artifact upload.", diff --git a/README.md b/README.md index 7bf23241..77c4920a 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ ONNX 模型 (.onnx) |------|------| | [📘 课程首页](https://scratchv-compiler.github.io/ScratchV/docs/index.html) | 交互式课程（推荐新手） | | [📖 文档导航](docs/INDEX.md) | 全部 Markdown 文档索引 | +| [W5 功能、验证与性能数据](docs/llm-deploy-v1.0/W5/README.md#maintainer-overview) | 本次改动、完整生成实测、CI/benchmark 数据入口及未完成项 | | [🏗️ 架构总览](docs/reference/ARCHITECTURE.md) | ONNX→RISC-V 双路径详解 | | [📊 性能仪表盘](https://scratchv-compiler.github.io/ScratchV/dashboard.html) | LLVM vs ScratchV 对比 | | [课题 06 测试套件](docs/topics/06-性能基准套件/06-性能测试套件使用说明.md) | TinyFive 正确性验证、Benchmark 与回归报告 | diff --git a/docs/INDEX.md b/docs/INDEX.md index cda8f449..35625cc3 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -44,7 +44,7 @@ | [developer_guide.md](reference/developer_guide.md) | 开发者指南：代码规范、测试、CI | | [optimization_guide.md](reference/optimization_guide.md) | 优化指南：从哪里入手、怎么验证 | | [verification.md](reference/verification.md) | 验证方法：如何确认编译结果正确 | -| [W5 生成合同与当前进度](llm-deploy-v1.0/W5/README.md) | Host token 循环、固定参考、完整生成/容器证据与 CI 验收边界 | +| [W5 功能、验证与性能数据](llm-deploy-v1.0/W5/README.md#maintainer-overview) | 改动范围、数值/耗时、CI/benchmark、复现及未验证范围 | | [CODING_STANDARDS.md](reference/CODING_STANDARDS.md) | 代码风格规范 | | [CONTRIBUTING.md](../CONTRIBUTING.md) | 贡献指南 | diff --git a/docs/llm-deploy-v1.0/W4/README.md b/docs/llm-deploy-v1.0/W4/README.md index 5f90c55f..3271661a 100644 --- a/docs/llm-deploy-v1.0/W4/README.md +++ b/docs/llm-deploy-v1.0/W4/README.md @@ -19,7 +19,7 @@ W3 [#96](https://github.com/ScratchV-Compiler/ScratchV/pull/96) 已于 2026-10-1 | [复现步骤](reproduction.md) | 固定依赖与模型、完整运行、诊断和原始数组复核 | | [验收与 CI](validation.md) | 分层门槛、测试入口、默认关闭的 Nightly、证据边界 | | [阶段衔接与后续准备](evolution.md) | W1–W4 接口对应、W5 前置工作、未来 MLIR 转换边界 | -| [软仿优化实测报告](softsim-optimization-2026-10-22.md) | 多 hart、微内核、布局与运行器优化；完整模型对照和适用边界 | +| [软仿优化实测报告](softsim-optimization-2026-10-22.md) | 多核、微内核、布局与运行器优化；完整模型对照和适用边界 | ## 验收标准 diff --git a/docs/llm-deploy-v1.0/W4/reports/full-hart-comparison-2026-10-22.json b/docs/llm-deploy-v1.0/W4/reports/full-hart-comparison-2026-10-22.json index 3fb5ac5f..bfad256c 100644 --- a/docs/llm-deploy-v1.0/W4/reports/full-hart-comparison-2026-10-22.json +++ b/docs/llm-deploy-v1.0/W4/reports/full-hart-comparison-2026-10-22.json @@ -1,26 +1,26 @@ { "status": "PASS", "passed": true, - "scope": "One ordered pair of complete full_seed_0 runs, harts 4 then 1. No old timing-anomaly baseline; not a repeated-run median or stability estimate.", - "baseline": "1 hart, blocked_fma, 2x4, column-panel, padding none", - "candidate": "4 harts, otherwise identical policy and fixed input", + "scope": "One ordered pair of complete full_seed_0 runs, 4 core then 1 core. No old timing-anomaly baseline; not a repeated-run median or stability estimate.", + "baseline": "1 core, blocked_fma, 2x4, column-panel, padding none", + "candidate": "4 core, otherwise identical policy and fixed input", "identity_checks": [ - "1 hart retained QEMU report", - "1 hart audit case", - "1 hart audit numeric", - "1 hart FMA actual tools", - "1 hart codegen/build panels", - "1 hart affinity unchanged", - "1 hart measured forward wall", - "1 hart runtime weights", - "4 hart retained QEMU report", - "4 hart audit case", - "4 hart audit numeric", - "4 hart FMA actual tools", - "4 hart codegen/build panels", - "4 hart affinity unchanged", - "4 hart measured forward wall", - "4 hart runtime weights", + "1 core retained QEMU report", + "1 core audit case", + "1 core audit numeric", + "1 core FMA actual tools", + "1 core codegen/build panels", + "1 core affinity unchanged", + "1 core measured forward wall", + "1 core runtime weights", + "4 core retained QEMU report", + "4 core audit case", + "4 core audit numeric", + "4 core FMA actual tools", + "4 core codegen/build panels", + "4 core affinity unchanged", + "4 core measured forward wall", + "4 core runtime weights", "model_files", "source_sha256", "execution_tools", @@ -315,17 +315,17 @@ "max_abs": 0.0 }, "limitations": [ - "One pair, ordered 4 harts then 1 hart; no repeatability/confidence estimate. Host load, temperature and filesystem-cache state were not held identically fixed.", + "One pair, ordered 4 core then 1 core; no repeatability/confidence estimate. Host load, temperature and filesystem-cache state were not held identically fixed.", "Forward wall is QEMU launch through validated UART completion, not guest-only kernel time.", "Whole-process CPU includes launch, dump and shutdown; compare against QEMU wall, not the forward wall or the approximate 0.5-second CPU sample.", "RSS is a sampled QEMU process peak, separate from guest reserved RAM/workspace. Partial availability, sample counts and errors are retained for both runs.", "Clock summaries span each entire wrapper. Raw/monotonic/UTC sampling skew or clock-frequency discipline is not automatically evidence of a time jump.", - "ELFs, hart compiler definitions, per-hart stack reservations and parallel runtime can differ by design. Dynamic output paths, TCP ports and run timestamps are not identities.", - "Generic Linux QEMU thread names cannot reliably identify individual hart threads.", + "ELFs, core compiler definitions, per-core stack reservations and parallel runtime can differ by design. Dynamic output paths, TCP ports and run timestamps are not identities.", + "Generic Linux QEMU thread names cannot reliably identify individual simulated CPU execution threads.", "Existing report identities and freshly completed audit records are reused; no new hashes." ], "publication": { - "kind": "Publication copy of retained full-hart comparison; not a new model run", + "kind": "Publication copy of retained core-count comparison; not a new model run", "date": "2026-10-22", "source_result": "output/w4-harts-short-20261021/optimization-completion-20261022/full-hart-comparison/result.json", "paths": "Paths under output/ are local retained-artifact indexes relative to the ScratchV development root, not files included in Git.", diff --git a/docs/llm-deploy-v1.0/W4/softsim-optimization-2026-10-22.md b/docs/llm-deploy-v1.0/W4/softsim-optimization-2026-10-22.md index 5f1ae62c..dd2f7cc0 100644 --- a/docs/llm-deploy-v1.0/W4/softsim-optimization-2026-10-22.md +++ b/docs/llm-deploy-v1.0/W4/softsim-optimization-2026-10-22.md @@ -1,12 +1,14 @@ # QEMU 软仿优化报告（2026-10-22） +本文的 core 指 QEMU 模拟的 CPU 核，不表示宿主的物理核心。当前命令使用 `--cores`，旧 `--harts` 仍兼容；历史数据和文件路径保持原样。 + 本报告汇总 PR104 已有的软仿优化实测与取舍。固定完整模型用例为 Qwen3-0.6B、FP32、batch=1、L=256、28 层、无 KV cache。测试在 4 个逻辑 CPU 的 WSL2/Linux 环境下进行；目标为 RV64GC/LP64D、QEMU virt 裸机。它不代表真实 RISC-V 或 NPU 性能。 -同配置完整模型单/四 hart 各执行一次，前向从 **1384.65 s 降到 410.53 s，3.373×**；CPU 总时间增加 **5.46%**，38,895,616 个输出逐位一致。小图的微内核、布局及等待优化收益分别统计，不能与该倍数相乘。 +同配置完整模型分别以 1 core 和 4 core 各执行一次，前向从 **1384.65 s 降到 410.53 s，3.373×**；CPU 总时间增加 **5.46%**，38,895,616 个输出逐位一致。小图的微内核、布局及等待优化收益分别统计，不能与该倍数相乘。 ## 默认值与可选功能 -默认值保持 `1 hart / sequential / 1x4 / original / padding none`。手动选择 `--harts 1/2/4` 或 `--harts auto`；auto 只在启动/构建时按进程可用 CPU 选择 1/2/4，上限 4，不是运行中的动态调度，也不处理 cgroup CPU 带宽配额。 +默认值保持 `1 core / sequential / 1x4 / original / padding none`。手动选择 `--cores 1/2/4` 或 `--cores auto`；auto 只在启动/构建时按进程可用 CPU 选择 1/2/4，上限 4，不是运行中的动态调度，也不处理 cgroup CPU 带宽配额。 | 方向 | 已完成内容 | 适用边界 | |---|---|---| @@ -19,11 +21,11 @@ | 未默认启用 | 两缓冲 `two-buffer-pad-4k` 实验选项 | QEMU 专用、增加 4096 B，部分形状退化；下方主要性能对照均 padding none | | 未采用 | 本轮 RVV 候选 | 正确性通过但 QEMU 实测负收益，不纳入正式实现 | -## 完整模型：同配置 1/4 hart +## 完整模型：同配置 1/4 core -两边固定 `full_seed_0 / blocked_fma / 2x4 / column-panel / padding none`，按 4 hart、1 hart 的顺序各运行一次。编译器 Zig 0.14.1，QEMU 8.2.2（Ubuntu 包 1:8.2.2+ds-0ubuntu1.18），数学库为 Zig 自带 musl。 +两边固定 `full_seed_0 / blocked_fma / 2x4 / column-panel / padding none`，按 4 core、1 core 的顺序各运行一次。编译器 Zig 0.14.1，QEMU 8.2.2（Ubuntu 包 1:8.2.2+ds-0ubuntu1.18），数学库为 Zig 自带 musl。 -| 指标 | 1 hart | 4 hart | +| 指标 | 1 core | 4 core | |---|---:|---:| | 前向耗时 | 1384.65 s | 410.53 s | | QEMU 生命周期耗时 | 1385.97 s | 410.94 s | @@ -33,9 +35,9 @@ | 对 ORT 最大绝对误差 | 8.58306884765625e-05 | 8.58306884765625e-05 | | 对照输出逐位一致 | 是 | 是 | -并行加速为 **3.372844654×**，前向耗时减少 **70.35%**；CPU 成本为单 hart 的 **1.05455 倍**。全部 logits 满足严格 `max_abs < 0.001`、`rtol=0`，12 项 FMA 检查及保留数组审计通过。 +并行加速为 **3.372844654×**，前向耗时减少 **70.35%**；CPU 成本为单核的 **1.05455 倍**。全部 logits 满足严格 `max_abs < 0.001`、`rtol=0`，12 项 FMA 检查及保留数组审计通过。 -完整数值、测量范围、采样缺失和来源索引见 [1/4 hart 对照数据](reports/full-hart-comparison-2026-10-22.json)。 +完整数值、测量范围、采样缺失和来源索引见 [1/4 core 对照数据](reports/full-hart-comparison-2026-10-22.json)。 测量口径： @@ -44,13 +46,13 @@ - RSS 是进程观测峰值，不是 guest 预留 RAM；两侧分别有 3 次、1 次读取失败，均标为 partial，内存结论只称基本持平。 - 没有重复运行统计或置信区间；宿主负载、温度与文件缓存状态并未严格一致。没有使用早期跳时异常记录计算本次加速比。 - 执行时 Git 基线为 `af5768cbe49976896f88f10f8f0cc6ec08160f30`，工作树含未提交候选改动；不能仅凭该 commit 重建完整候选。两侧生产源码摘要比较通过，已有快照身份各自保留在对照 JSON 中。运行早于最终高 FD、Host C 清理等修复，不能视作当前 PR HEAD 的完整重测。 -- 完整模型本次没有可靠的 QMP hart/线程映射，不能将其线程采样精确标为主 hart 与各 worker。 +- 完整模型本次没有可靠的 QMP core/线程映射，不能将其线程采样精确标为主核与各 worker。 ## 小图：微内核与布局 下表采用 2026-10-22 UART 事件计时后的同批对照中位数；每配置 3 次新进程运行，慢轮未剔除，padding 均关闭。微内核共 54 次、正式布局共 24 次性能运行，输出均逐位一致。 -| 对照 | 1 hart：前向耗时 | 4 hart：前向耗时 | +| 对照 | 1 core：前向耗时 | 4 core：前向耗时 | |---|---:|---:| | Dense：1x4 → 2x4 | 5.8311 → 3.6155 s（1.613×） | 1.7681 → 1.1188 s（1.580×） | | Dense：1x4 → 1x8 | 5.8311 → 3.6519 s（1.597×） | 1.7681 → 1.1211 s（1.577×） | @@ -59,7 +61,7 @@ 正式布局的 direct_dense 为 LHS `[256,1024]`、RHS `[1024,1024]`；batch_transpose_wide 为 LHS `[2,64,1024]`、存储 RHS `[2048,1024]`，转置后参与乘法。后者工作区由 9,437,184 → 1,048,576 bytes，减少 8 MiB。phase3 报告未直接记录 Dense 形状，本表仅按原报告用例名引用，不用旧阶段形状替代它。 -Dense 的 1x4 → 2x4 对照中，CPU 中位数分别从 5.8332 → 3.6183 s、6.8124 → 4.2183 s。Attention 的 4 hart 2x4 为 0.999×，说明更大微内核不保证所有形状受益。 +Dense 的 1x4 → 2x4 对照中，CPU 中位数分别从 5.8332 → 3.6183 s、6.8124 → 4.2183 s。Attention 的 4 core 2x4 为 0.999×，说明更大微内核不保证所有形状受益。 布局测试保留原布局对照与容量/重叠/溢出/错误恢复/不同输入及工作区复用检查。完整模型实际采用 196 个 panel，逻辑与物理权重均为 1,761,607,680 bytes；共享 embedding/lm_head 与动态 RHS 保留原布局。ELF 必须与对应物理布局的权重 bundle 配套。布局收益没有新的 SoftMMU 采样支撑，不能精确拆成硬件缓存、TLB 或地址生成各自贡献。 @@ -67,18 +69,18 @@ Dense 的 1x4 → 2x4 对照中，CPU 中位数分别从 5.8332 → 3.6183 s、6 这些是不同阶段的诊断或实现对照，不能组合成整体收益。早期 WFI/回调复用及 padding 对照仍使用约 50 ms 完成轮询，须与后续事件计时区分。 -- **空闲等待：** 在 4 hart、64 次串行 Softmax 的历史对照中，WFI 使 CPU 9.780 → 2.125 s（每配置仅 2 次）。它主要减少空转；后续 Softmax 已按独立行并行，此数字不是当前完整模型收益。 -- **回调复用：** 64 个同形状回调变为 1 个；末轴回调机器码 17,920 → 280 B，非末轴 19,456 → 304 B，工作区仍为 2 MiB。单 hart text 未变，不宣称单核代码体积收益。 -- **有限性检查融合：** 后续事件复测的 256×1024、4 hart 中位前向 0.676418 → 0.649905 s（−3.92%），CPU 2.281089 → 2.300152 s（+0.84%）。独立线程采样显示 worker 非运行时间减少，但该区间同时包含调度等影响，不等于纯 WFI/屏障等待。 +- **空闲等待：** 在 4 core、64 次串行 Softmax 的历史对照中，WFI 使 CPU 9.780 → 2.125 s（每配置仅 2 次）。它主要减少空转；后续 Softmax 已按独立行并行，此数字不是当前完整模型收益。 +- **回调复用：** 64 个同形状回调变为 1 个；末轴回调机器码 17,920 → 280 B，非末轴 19,456 → 304 B，工作区仍为 2 MiB。单核 text 未变，不宣称单核代码体积收益。 +- **有限性检查融合：** 后续事件复测的 256×1024、4 core 中位前向 0.676418 → 0.649905 s（−3.92%），CPU 2.281089 → 2.300152 s（+0.84%）。独立线程采样显示 worker 非运行时间减少，但该区间同时包含调度等影响，不等于纯 WFI/屏障等待。 - **UART 事件计时：** 观测后唤醒中位延迟 25.367 → 0.098 ms，减少轮询量化；这是宿主完成检测改进，不代表 guest 计算加速。 -- **QMP 尾延迟：** nodelay 对照中 stop 中位耗时由 42.617 → 0.621 ms（1 hart）、42.227 → 0.760 ms（4 hart）。STOP 事件已先到达，服务端 NODELAY 与独立客户端 QUICKACK 对照强烈支持 TCP Nagle/ACK 交互造成回复尾延迟；没有抓包，不声称逐包证明，也不把这约 42 ms 计为模型计算收益。 +- **QMP 尾延迟：** nodelay 对照中 stop 中位耗时由 42.617 → 0.621 ms（1 core）、42.227 → 0.760 ms（4 core）。STOP 事件已先到达，服务端 NODELAY 与独立客户端 QUICKACK 对照强烈支持 TCP Nagle/ACK 交互造成回复尾延迟；没有抓包，不声称逐包证明，也不把这约 42 ms 计为模型计算收益。 - 正式运行器已采用 UART 事件与 QMP nodelay；高 FD 边界及 Host C 超时/取消进程清理随后补齐相关回归，没有借此重跑完整模型。 ## 退化用例与剩余问题 -- **Padding 有明确反例：** 256×1024 的实验中位耗时下降，但 255×1024 的 1/4 hart 耗时分别增加约 22.22%/23.14%；四 hart 收益组也保留慢运行。因此仅作默认关闭的 QEMU 两缓冲选项，不作为泛用硬件/NPU 缓存策略。 +- **Padding 有明确反例：** 256×1024 的实验中位耗时下降，但 255×1024 的 1/4 core 耗时分别增加约 22.22%/23.14%；4 core 收益组也保留慢运行。因此仅作默认关闭的 QEMU 两缓冲选项，不作为泛用硬件/NPU 缓存策略。 - **RVV 暂不采用：** 43 次边界正确性与 54 次性能运行逐位一致；本次 VLEN128/e32m1 一行四列候选，耗时为预选强标量方案的 2.31–2.92 倍，CPU 为 2.61–2.98 倍。标量与 RVV 的 tile/布局并非完全一致，这是实用候选比较，不是孤立的 ISA 因果试验，也不代表真实硬件或全部 RVV 实现。 -- 4 hart 剩余波动和额外 CPU 尚未完全分解为宿主调度、频率、硬件缓存、QEMU 浮点、SoftMMU/TB 等因素。独立采样的比例不能换算为无采样运行的 CPU 秒数。 +- 4 core 剩余波动和额外 CPU 尚未完全分解为宿主调度、频率、硬件缓存、QEMU 浮点、SoftMMU/TB 等因素。独立采样的比例不能换算为无采样运行的 CPU 秒数。 - 原始数组、ELF、权重、perf 采样与源码快照仍只保存在本地，公开摘要不等同于独立重建执行证据。E1/E3/E5、Nightly、团队验收和真实 RISC-V/NPU 部署另行完成。 - W5 token 循环、文本生成和会话部署不属于本报告的完成声明。 diff --git a/docs/llm-deploy-v1.0/W4/validation.md b/docs/llm-deploy-v1.0/W4/validation.md index 0301d733..7b4a1590 100644 --- a/docs/llm-deploy-v1.0/W4/validation.md +++ b/docs/llm-deploy-v1.0/W4/validation.md @@ -1,6 +1,8 @@ # W4 分层验证与 CI -## 日常最小验证与多 hart +## 日常最小验证与多核 + +本文的 core 指 QEMU 模拟的 CPU 核，不表示宿主的物理核心。当前命令参数使用 `--cores`，Python 参数与属性使用 `cores`。 先按改动选择直接相关的测试；文档和配置改动只做对应检查。矩阵生成或并行运行时 改动可先运行以下小图回归（按[复现步骤](reproduction.md)设置 Python、编译器和 QEMU）： @@ -11,19 +13,33 @@ python -B -X utf8 -m pytest -q \ tests/test_riscv_external_harts.py tests/test_w4_fma_cross_backend.py ``` -外置运行时 `build_external(..., harts=1)`、完整探测及检查点诊断入口的 -`--harts {1,2,4,auto}` 均默认单核，手动 1/2/4 保持原值。`--harts auto` -（或 `build_external(..., harts="auto")`）在构建前按当前进程可用 CPU 数向下选择： -1 个选 1 hart，2–3 个选 2 hart，4 个及以上选 4 hart。Linux 优先使用 CPU affinity； +外置运行时 `build_external(..., cores=1)`、完整探测及检查点诊断入口的 +`--cores {1,2,4,auto}` 均默认单核，手动 1/2/4 保持原值。`--cores auto` +（或 `build_external(..., cores="auto")`）在构建前按当前进程可用 CPU 数向下选择： +1 个选 1 core，2–3 个选 2 core，4 个及以上选 4 core。Linux 优先使用 CPU affinity； 不可用时回退进程/系统 CPU 数，无法确定时保守选 1。此口径不估计瞬时空闲率或 cgroup CPU 带宽配额，也不受 `OMP_NUM_THREADS` 等 BLAS 线程设置影响。 +旧 `--harts`、Python 的 `harts` 参数和属性仍兼容。CLI 解析结果的 `args.harts` +是 `args.cores` 的可读写别名，修改任一属性都会同步；默认值、手动值和 `auto` +在 `parse_args()` 返回时仍保持原字符串类型。命令行、普通构造和运行时 API 中， +新旧名称同时传入时必须一致，否则在构建或运行前报错。新构建与报告以 `cores` 为主要字段，同时保留等值的 +`harts` 兼容字段；有请求值的报告同样保留 `cores_requested` 与 `harts_requested`。 +读取旧记录时接受仅有 `harts` 的格式；两种核数字段均缺失时按原规则视为单核， +两种字段不一致或值无效时拒绝复用。历史报告和 ELF/bundle 不因改名而改写。 + +`dataclasses.asdict(executable)` 同时输出整数 `cores` 和等值的旧 `harts` 字段， +不输出私有核数字段；正式 `executable.json` 的保存和读取方式保持不变。 +`dataclasses.replace(executable, cores=2)` 和旧 `harts=2` 写法都会同步两个字段。 +`replace()` 会自动补入原值，因此一边为原值、另一边为新值时按新值更新； +两边都是不同于原值的新值且互不相等时拒绝。建议一次只用一个名称更新核数。 + 选择只发生在启动/构建时，运行中不切换；CLI 会打印自动选择结果，报告用 -`harts_requested` 记录请求、`harts` 记录实际整数。当前数学策略的默认值仍是 +`cores_requested` 记录请求、`cores` 记录实际整数。当前数学策略的默认值仍是 `sequential`；需要矩阵并行时必须同时显式使用 `--matmul-policy blocked_fma`。 MatMul 的多核路径只用于 `blocked_fma` 中足够大的矩阵，按独立输出块分工； 每个元素的 K 顺序和 K=128 分组保持不变，算子结束后才复用工作区。 -`sequential` 下的 MatMul 仍串行。每个 hart 独立使用 1 MiB 栈。 +`sequential` 下的 MatMul 仍串行。每个 core 独立使用 1 MiB 栈。 外置运行器在 Linux 上用 inotify 与 pidfd 等待已有 UART 完成帧，启动前注册目录监视， 不再固定每 50ms 轮询；平台或内核不支持时保留轮询回退。`run.json` 的 `timing_mode` @@ -32,14 +48,14 @@ MatMul 的多核路径只用于 `blocked_fma` 中足够大的矩阵，按独立 QMP 服务端连接启用 `nodelay=on`，减少 STOP 事件与命令回复之间的小包等待；仍等待 匹配回复，并在停止后复核 UART 帧，再转储输出、检查退出和清理资源。 -外置 Softmax 在实际 hart 数大于 1、总元素数至少为 32768，且独立行数不少于 -hart 数时可并行；它不依赖 `matmul_policy`，因此显式选择多 hart 后，即使 MatMul -策略为 `sequential`，符合条件的 Softmax 也会并行。默认仍为单 hart、`sequential`。 +外置 Softmax 在实际 core 数大于 1、总元素数至少为 32768，且独立行数不少于 +core 数时可并行；它不依赖 `matmul_policy`，因此显式选择多核后，即使 MatMul +策略为 `sequential`，符合条件的 Softmax 也会并行。默认仍为单核、`sequential`。 独立行对应固定的 `(outer, inner)` 坐标，每行完整保留原有求最大值、逐项 `expf` 与求和、逐项归一化顺序，不拆分行内归约。输出有限性检查紧跟每个 FP32 归一化值 -的写回，不再由主 hart 在算子结束后重复扫描整个输出；检查本身及输入、权重、其它 +的写回，不再由主核在算子结束后重复扫描整个输出；检查本身及输入、权重、其它 算子的校验仍保留。工作线程通过原子状态回传数值错误，`sv_parallel_for` 等全部回调 -完成后，主 hart 检查错误状态，再进行后续工作区复用。inline 存储、单 hart、小图 +完成后，主核检查错误状态，再进行后续工作区复用。inline 存储、单核、小图 及独立行数不足时仍走串行路径，并在同样的逐行写回位置检查输出。 32768 元素只是保守的启用门槛，尚未校准为各形状的最佳阈值；继续复用已有门铃/WFI 同步和 `expf` 页对齐。此代码生成改动需要重新构建 ELF，复用旧 ELF 不会加入 @@ -48,11 +64,11 @@ Softmax 并行或写回时的有限性检查融合。 同一次生成中，输入形状、dtype 和规范化 axis 相同的外置并行 Softmax 复用回调代码。 只共享函数与结构体定义；每次算子调用仍使用独立的输入、输出指针和初始化为零的 错误状态，完成同步后才检查状态和复用工作区。此复用不改变单核默认、手动/自动 -hart 选择、并行门槛或行内数值顺序；也不代表真实硬件缓存或完整模型的等比例收益。 +core 选择、并行门槛或行内数值顺序；也不代表真实硬件缓存或完整模型的等比例收益。 代码生成 API 可显式比较 `TensorCCodegen(..., constant_storage="external", matmul_policy="blocked_fma", matmul_tile="2x4")` 或 `matmul_tile="1x8"`。 -`matmul_tile` 默认仍为 `"1x4"`；候选仅适用于外置 `blocked_fma`，不改变 hart 的 +`matmul_tile` 默认仍为 `"1x4"`；候选仅适用于外置 `blocked_fma`，不改变 core 的 默认、手动或自动选择方式。`2x4` 复用两行之间的 B 加载，`1x8` 复用八列之间的 A 加载；每个输出仍按原 K 顺序执行显式 FP32 FMA，保留 128 项分组和块末加法。 尾行、尾列有边界保护，不使用 RVV 或 fast-math；面板布局由下述独立实验选项控制。 @@ -66,7 +82,7 @@ A 加载；每个输出仍按原 K 顺序执行显式 FP32 FMA，保留 128 项 `matmul_rhs_layout="column-panel"`；默认 `original` 保留原布局。该选项只打包二维 常量 RHS，允许 LHS 带批维；每个面板的列宽沿用 `matmul_tile`，物理布局为 `[ceil(N/NR), K, NR]`。它复用相邻列的权重访存，仍按递增 K 顺序执行 FMA，保留 -128 项分组与块末加法，不改变默认单核或手动、自动 hart 选择。 +128 项分组与块末加法，不改变默认单核或手动、自动 core 选择。 只有常量的全部消费者都把它用作 MatMul RHS 时才选择面板。独占常量经过合法二维 转置后仅供 MatMul RHS 使用时，在局部复制的 IR 容器中折叠该转置，交由原有生命周期 @@ -82,19 +98,19 @@ A 加载；每个输出仍按原 K 顺序执行显式 FP32 FMA，保留 128 项 `matmul_rhs_layout` 与 `matmul_rhs_panels` 在报告和构建证据中记录逻辑形状、物理形状、 原权重、折叠的转置及视图/复制方式；复用同时校验布局和面板记录。旧记录缺字段只兼容 `original`，面板构建必须重新生成；选择相同选项不意味着可以绕过已有源码、权重和 -构建身份校验。小图收益需与同 hart、同微内核、同计时方式的原布局比较，常量转置链 +构建身份校验。小图收益需与相同 core 数量、同微内核、同计时方式的原布局比较，常量转置链 还包含省去转置复制的收益，不能把总差值全部归为缓存效果。 外置 ELF 构建将频繁调用的 `expf` 代码段起始地址按 4096 字节页对齐，避免当前 实现因生成图代码长度变化而跨页、引入额外的 QEMU TCG 开销。这里只调整链接布局， -不改变 `expf` 的数值实现、默认单核或手动/自动 hart 选择。应用此修复必须重新构建 +不改变 `expf` 的数值实现、默认单核或手动/自动 core 选择。应用此修复必须重新构建 ELF；`--reuse-build` 复用旧 ELF 不会应用新布局。收益应通过相同条件的小图对照确认， 不能直接外推到完整模型。 QEMU 两缓冲工作区可显式尝试实验选项 `TensorCCodegen(..., constant_storage="external", qemu_workspace_layout="two-buffer-pad-4k")`。 完整探测及检查点 CLI 对应 `--qemu-workspace-layout {none,two-buffer-pad-4k}`， -默认 `none` 完整保留原布局；该选项不改变默认单核或手动、自动 hart 选择。 +默认 `none` 完整保留原布局；该选项不改变默认单核或手动、自动 core 选择。 仅当原工作区的所有分配可归入恰好两个固定、相邻、互不重叠的区间时，实验布局才在 两者之间插入 4096 字节，工作区需求也增加 4096 字节。其他图或容量不足会明确报错， 不会静默忽略选项；完整 Qwen 图和检查点不保证满足两缓冲条件。 @@ -107,8 +123,8 @@ QEMU 两缓冲工作区可显式尝试实验选项 与地址变化仍存在，应独立评估，不能把它当作 NPU 内核、硬件缓存或通用内存布局优化。 以下只是参数示例，不是要求运行完整模型： -`--qemu-workspace-layout two-buffer-pad-4k --harts 4`。 -先用相同输入的 1、4 hart 小图检查位级输出一致性、容量不足处理、耗时、累计 CPU 时间、 +`--qemu-workspace-layout two-buffer-pad-4k --cores 4`。 +先用相同输入的 1、4 core 小图检查位级输出一致性、容量不足处理、耗时、累计 CPU 时间、 RSS 和 SoftMMU 样本，覆盖收益与退化形状。报告的 `qemu_workspace_layout_requested` 保留请求，`codegen.qemu_workspace_layout` 和构建证据记录所用布局；构建证据的 `qemu_workspace_buffers` 记录缓冲区位置。修改布局须重新生成、构建 ELF；复用时请求 @@ -116,7 +132,7 @@ RSS 和 SoftMMU 样本，覆盖收益与退化形状。报告的 `qemu_workspace 多核任务、就绪和完成等待使用 QEMU virt 的 MSIP 软件中断门铃与 WFI。 等待端先清门铃，再检查任务代次、完成状态和错误；发送端先发布状态再通知， -避免通知先到时丢失唤醒。空闲 worker 和等待完成的主 hart 可休眠，默认单核路径 +避免通知先到时丢失唤醒。空闲 worker 和等待完成的主核可休眠，默认单核路径 不启用此机制。启动阶段极短的 BSS 就绪轮询保留；其他板型需适配中断控制器。 应同时比较同一候选、相同小图的耗时、累计 CPU 时间和输出位模式。 @@ -128,9 +144,9 @@ QEMU 小图，必要时再用 `diagnostic --checkpoint layer_0.output`。检查 `COMPLETE` 仅表示执行完成，还须查看 `diagnostic_threshold_passed`。完整模型与 原数组 audit 留到候选稳定、明确需要完整验收时运行，不作为每次编辑的默认步骤。 -同一候选重复执行可复用已有 `--reuse-build`；改变源码、数学策略或 hart 数必须重新 +同一候选重复执行可复用已有 `--reuse-build`；改变源码、数学策略或 core 数必须重新 构建。`auto` 也必须与当前探测后选择的数量一致，复用数量不符时明确报错； -不会修改旧 ELF 的 hart 数。旧的未记录 hart 数的构建按单核处理，不能直接以多核运行。 +不会修改旧 ELF 的 core 数。旧的未记录 core 数的构建按单核处理，不能直接以多核运行。 ## 按风险阅读实现和测试 @@ -227,7 +243,7 @@ step 200 分钟、job 240 分钟。正常失败上传已有报告与日志；宿 复现者 / 日期： 源码：候选完整 SHA、工作树是否干净；若为未提交快照，附基线 SHA、源码清单和快照 SHA256： 环境：宿主 OS/架构、Python/依赖、Zig/QEMU 实际版本及二进制 SHA256（见主报告）： -执行：完整命令、case、MatMul 策略、hart 数（默认 1）；是否独立重新构建并执行 QEMU： +执行：完整命令、case、MatMul 策略、core 数（默认 1）；是否独立重新构建并执行 QEMU： 单测：通过/失败/跳过数，tests.xml 位置；每项相关跳过原因： 完整门槛：build:riscv-full / smoke:qemu-full / numeric:qemu-full-qwen3： 数值：elements_compared、max_abs、atol、rtol；FMA 精确检查和离线 audit 结果： diff --git a/docs/llm-deploy-v1.0/W5/README.md b/docs/llm-deploy-v1.0/W5/README.md index 724288d1..e1f4050b 100644 --- a/docs/llm-deploy-v1.0/W5/README.md +++ b/docs/llm-deploy-v1.0/W5/README.md @@ -2,60 +2,163 @@ ## 本分支增加了什么 -W5 将 W4 的单次 QEMU 前向接成文本生成：Host 负责 tokenizer、逐 token 输入更新、 -greedy 选择与停止，QEMU 执行前向。命令行支持原始文本补全、三条固定参考用例和 -多轮文本对话，Linux 一键部署脚本复用同一个入口。 - -- [Host 生成循环](../../../scratchv/runtime/generation.py)规定输入、输出和停止行为， - 每步只接收一个新 token，保留异常、超时和取消信息。 -- [QEMU 适配器](../../../scratchv/runtime/qemu_generation.py)跨 token、跨 prompt - 复用同一 ELF 和权重 bundle，也支持已有构建的显式复用。 -- [多轮对话](../../../scratchv/runtime/conversation.py)使用本地官方聊天模板， - 成功后一起保存 user/assistant 消息，失败时保持原历史。 -- [命令行](../../../scripts/run_inference.py)输出逐步数值与进程记录； - [部署脚本](../../../scripts/deploy_w5.sh)和 - [Dockerfile](../../../deployment/w5/Dockerfile)提供干净 Linux 容器运行方式。 - -**默认配置保持不变**：单 hart、MatMul `sequential / 1x4 / original`、Mean `sequential`、 -非线性 `libm`、padding `none`。可手动指定 `--harts 1/2/4` 或 `--harts auto`； +W5 将 W4 的单次 QEMU 前向接成文本生成：本机 Python 准备输入、选择 token、控制循环并校验， +QEMU 中的 RISC-V 程序执行模型。支持原始文本补全、三条固定参考用例及 +[官方模板多轮文本](../../../scratchv/runtime/conversation.py)； +[命令行入口](../../../scripts/run_inference.py)、 +[部署脚本](../../../scripts/deploy_w5.sh)与[Dockerfile](../../../deployment/w5/Dockerfile) +使用同一条生成流程。 + + + +**先看已经取得的结果。** 2026-10-23 的历史实跑中，完整 28 层 Qwen3-0.6B 使用显式 +4 core 配置，三条用例各生成 8 个新 token，**24/24 步与冻结参考一致**； +最大绝对误差 **0.000303984**，每条端到端生成与校验约 **56～61 分钟**。 +具体文字、计时范围和模型配置见[实际复现结果](#实际复现结果)。 +这些是 `8ad3474` 功能基线的历史数据，不是后续命名和兼容修复的新测量。 + +**当前限制。** 无 KV cache，每个新 token 都执行一次完整前向并启动新的 QEMU。 +固定 8-token 预算不代表自然回答完毕；完整模型三轮聊天、三例独立容器运行及第二人复现 +尚无本报告可用的实跑记录，见[已验证范围与剩余问题](#remaining-scope)。 + +| 想了解什么 | 阅读入口 | +|---|---| +| 输入如何流动、Python 和 QEMU 各做什么 | [实现说明](#整体怎么实现)、[逐步流程表](#generation-flow) | +| 实际生成了什么、运行多快、如何判定正确 | [运行配置](#run-configuration)、[三条结果](#fixed-case-results)、[耗时表](#full-generation-results)、[误差判定](#max-absolute-error) | +| 数据与历史 CI 在哪里 | [报告入口](#report-locations)、[CI/benchmark 快照](#ci-and-benchmark) | +| 如何自己执行并核对 | [如何复现](#如何复现)，命令默认折叠 | + +**默认配置保持不变**：单核、MatMul `sequential / 1x4 / original`、Mean `sequential`、 +非线性 `libm`、padding `none`。可手动指定 `--cores 1/2/4` 或 `--cores auto`； auto 按进程可用 CPU 在构建或启动时选择 1/2/4，上限 4，不在 token 间动态调整。 完整模型通过的记录使用下文显式参考策略，不代表默认算术策略也通过同一数值要求。 W5 没有 `--qemu-workspace-layout` 参数，padding 始终关闭。 -## 整体怎么实现 +旧名称与 Python 接口的兼容细节见[兼容说明](#core-compatibility)。 + + + +
+展开 core/hart 名称与旧接口兼容说明 + +本文的 core 指 QEMU 模拟的 CPU 核。 +[hart（hardware thread）](https://docs.riscv.org/reference/debug/introduction.html) 是 RISC-V 的正式术语，指硬件线程； +本项目当前 QEMU 配置中每个模拟核对应一个 hart，因此改用更直观的 core 提高接口可读性， +执行方式不变，旧名称仍兼容；在其他硬件上，hart 与物理核不一定一一对应。 +当前命令使用 `--cores`，Python 参数和属性使用 `cores`。 + +旧 `--harts`、Python 的 `harts` 参数和属性仍兼容。CLI 解析结果的 `args.harts` +是 `args.cores` 的可读写别名，修改任一属性都会同步；默认值、手动值和 `auto` +在 `parse_args()` 返回时仍保持原字符串类型。命令行、普通构造和运行时 API 中， +新旧名称同时传入时必须一致，否则在构建或运行前报错。新报告用 `cores_requested` 记录请求、`cores` 记录实际数量， +并保留等值的 `harts_requested`、`harts` 兼容字段。旧构建记录只有 `harts` 时仍可读取； +两种核数字段均缺失时按原规则视为单核，两种字段不一致或值无效时拒绝复用。 +历史报告不改写，复用已有 ELF/bundle 仍须通过原有配置和身份检查。 + +`dataclasses.asdict(executable)` 同时输出整数 `cores` 和等值的旧 `harts` 字段， +不输出私有核数字段；正式 `executable.json` 的保存和读取方式保持不变。 +`dataclasses.replace(executable, cores=2)` 和旧 `harts=2` 写法都会同步两个字段。 +`replace()` 会自动补入原值，因此一边为原值、另一边为新值时按新值更新； +两边都是不同于原值的新值且互不相等时拒绝。建议一次只用一个名称更新核数。 -**生成接口与数值行为。** `generate(prompt, *, tokenizer, forward, max_new_tokens, -cancelled=None, timeout_seconds=None)` 返回不可变的 `GenerationResult`，包含 prompt -IDs、新生成 IDs、文本和 `stop_reason`。固定模型是 FP32 `Qwen/Qwen3-0.6B`， -batch=1、序列容量 256，无 KV cache。 +
-| 项目 | 行为 | +## 整体怎么实现 + + + +**从提示词到文字结果。** 本节按三条固定用例的实际路径说明：本机 Python 控制生成循环， +QEMU 内的 RISC-V 程序执行完整 Qwen3 模型，ONNX Runtime（ORT）在本机计算参考结果。 +`token` 是 tokenizer 划分和处理文字的单位，`token ID` 是它对应的编号； +`logits` 是模型为各候选 token 给出的分数。 +QEMU 返回的是这些分数，最终文字由 Python 转换得到。 + +**当前没有 KV cache（保存前面 token 的注意力中间结果的缓存）。** +每生成一个新 token，都把“提示词 + 已生成 token”作为输入，重新执行完整 28 层模型。 +每条固定用例因此需要 **8 次完整前向和 8 次 QEMU 运行**，尚未采用基于缓存、 +只计算新 token 的增量 decode，也不是只执行“一次 prefill + 一次 decode”。 +这里的 8 次指完整模型前向，不包括运行前的浮点自检小程序。 + +| 步骤 | 执行位置 | 输入、具体处理与输出 | 实现入口 | +|---|---|---|---| +| 运行前准备 | 本机 Python + 编译器；自检会调用 QEMU | 新构建时，将 ONNX 解析成中间表示，生成 C 程序、写权重包并编译链接出 ELF；复用时检查已有 ELF、权重包和配置。浮点自检与 ORT 参考程序初始化也在生成循环之前完成 | [CLI](../../../scripts/run_inference.py)的 `prepare_executable` / `main` | +| 提示词编码 | 本机 Python | 原始文字经 tokenizer 转成 token 编号。三条用例是原始文本补全，不套聊天模板，不额外插入起止 token | [生成循环](../../../scratchv/runtime/generation.py)的 `generate` | +| 准备本步输入 | 本机 Python | 将当前完整 token 历史右侧补齐到 256 个位置，并构造限制可关注位置的注意力遮罩，得到模型输入数组；有效长度另行保留，供选词使用 | [输入处理](../../../scratchv/runtime/llm_inputs.py)的 `prepare_inputs` | +| 本步参考计算 | 本机 Python 调用 ORT | 固定用例先独立核对本步实际输入与冻结历史一致，再用这些输入运行同一份 ONNX 模型，得到 ORT 参考 logits | [CLI](../../../scripts/run_inference.py)的 `forward`；[参考输入检查](../../../probes/w2_runtime_model/run.py)的 `checked_inputs` | +| 模型前向 | QEMU 模拟执行 RISC-V 程序 | QEMU 装载 ELF、输入二进制和权重；其中的 RISC-V 程序调用生成的模型函数，计算全部输出分数并写入输出内存，通过 UART 串口发送完成状态及输出字节数 | [运行器](../../../scratchv/runtime/riscv_external.py)的 `guest_harness` / `run_external` | +| 读回与逐步校验 | 本机 Python | 验证 UART 成功完成帧后，用 QMP 暂停并复核完成帧，将输出内存保存为 `output.bin`，退出并回收进程；检查输出、退出码和清理结果，再与 ORT 比较，失败立即停止 | [QEMU 适配器](../../../scratchv/runtime/qemu_generation.py)；[CLI](../../../scripts/run_inference.py)的 `compare_step` | +| 选择下一个 token | 本机 Python | 从 QEMU 输出中取最后有效位置的分数，用 greedy 选择最高分 token；同分取较小编号，无额外 Softmax。把实际选出的编号加入历史，未停止则回到“准备本步输入” | [输入与选词](../../../scratchv/runtime/llm_inputs.py)的 `last_valid_logits` / `greedy_token`；[生成循环](../../../scratchv/runtime/generation.py) | +| 结束与结果输出 | 本机 Python | 满足停止条件后，把新生成的编号序列解码为文字；固定用例再核对完整编号、文字、停止原因与步数，保存报告并打印文字 | [生成循环](../../../scratchv/runtime/generation.py)；[CLI](../../../scripts/run_inference.py)的 `main` | + +**QEMU 不直接接收提示词字符串，也不直接打印模型回答。** UART 完成消息不是文本生成结果； +模型输出是内存中的 logits，经 QMP 读回后，由 Python 选 token 和解码。 +表中的“本机 Python”表示 QEMU 外的控制程序；容器运行时，这套 Python 程序与 QEMU 一起在容器内运行，职责不变。 + +同一条生成循环复用同一 ELF 和权重包，每步只更新输入；**没有逐 token 重复编译**。 +但每步仍启动新的 QEMU 进程并装载权重，不能把复用构建文件理解为 QEMU 或 KV cache 常驻。 +前置编译与初始化、生成阶段、最终报告保存各自的计时范围见[结果表说明](#full-generation-results)。 + +
+展开模型输入输出的具体格式、遮罩和停止规则 + +`generate(prompt, *, tokenizer, forward, max_new_tokens, cancelled=None, timeout_seconds=None)` +返回 `GenerationResult`，包含提示词编号、新生成编号、最终文字和 `stop_reason`。 + +| 项目 | 实际行为 | |---|---| -| 原始文本模式 | 直接编码 prompt，不自动插入 BOS、EOS 或聊天模板；沿用 tokenizer 的 NFC 归一化 | -| 输入 | INT64 `input_ids[1,256]`；FP32 `attention_mask[1,1,256,256]`，包含 causal 与 padding mask | -| 输出 | FP32 `logits[1,256,151936]`；全部元素必须有限，包括补齐行 | -| 选词 | 取 `valid_length-1` 行的 greedy argmax，同值选较小 ID；不额外做 Softmax | -| 有效词表 | tokenizer 为 151669，模型输出宽度为 151936；选中不可解码 ID 时失败，不过滤后重选 | -| 停止 | 对新 token 按 EOS、容量、预算顺序判断；EOS IDs 为 `151645`、`151643` | -| 输入与预算 | prompt 非空且不超过 256，不隐式截断；零预算或初始上下文已满时不调用 backend | -| 异常与中断 | 不返回看似成功的部分生成；QEMU 适配器负责运行中的超时、取消和进程回收 | - -每步准备当前 token 输入和 mask、运行前向并取下一个 token。复用的是构建产物； -每 token 仍启动一个 QEMU 进程并装载权重。编译、FMA 预检和 ORT session 初始化 -不按 token 重做；完成后沿用 UART 完成帧及 QMP stop、dump、quit。已有构建元数据 -检查继续约束 ELF/bundle 身份与所选配置。 - -`--verify-onnx` 为相同输入运行 ORT 并比较完整输出，不以 ORT 替代 QEMU 生成。 -要求所有值有限、`max_abs < 1e-3`、`rtol=0`、greedy ID 相同，并记录 top-2 margin。 -`--reference-suite` 进一步比较冻结 prompt IDs、逐 token IDs、文本与停止原因； -它不会自动选择算术策略，也不接受另传 `--max-new-tokens`。完整三例显式配置为： +| 提示词 | 原始文本模式直接编码，`add_special_tokens=False`；沿用 tokenizer 的 NFC 归一化 | +| token 输入 | INT64 `input_ids[1,256]`；前面放有效编号，右侧用模型的 `pad_token_id` 补齐 | +| 注意力遮罩 | FP32 `attention_mask[1,1,256,256]`；位置 `q` 只允许关注 `k <= q` 且 `k < valid_length` 的位置。允许处为 0，其余为 FP32 最小有限值；不是把补齐的 query 行全部屏蔽 | +| 模型输出 | FP32 `logits[1,256,151936]`；包含全部位置的候选分数，所有元素都必须有限，包括补齐位置 | +| 用于选词的输出 | 取 `logits[0, valid_length-1, :]`，即最后有效位置的完整词表分数；不是固定取第 256 行 | +| 有效词表 | tokenizer 为 151669，模型输出宽度为 151936；若最高分落在不可解码 ID，报错，不过滤后重新选择 | +| 停止顺序 | 新 token 生成后依次检查 EOS、上下文容量 256、新 token 预算；EOS IDs 为 `151645`、`151643` | +| 空间和预算 | 提示词非空且不超过 256，不隐式截断；零预算或初始上下文已满时不调用模型前向 | +| 失败与中断 | 不把异常、超时或取消的部分生成当作成功；适配器负责 QEMU 进程回收 | + +形状、遮罩和停止顺序见[输入实现](../../../scratchv/runtime/llm_inputs.py)； +生成循环见[生成实现](../../../scratchv/runtime/generation.py)。 + +
+ +
+展开如何校验：相同输入、完整分数、逐 token 选择与最终文字 + +- **先确认模型程序成功执行。** UART 必须是完整合法帧，状态为 0、输出字节数正确； + 暂停 QEMU 后还会复核帧，并检查转储文件大小、输出有限性、退出码及清理结果。 + 收到一条消息，或仅看到进程退出，都不能代替后续数值校验。 +- **同输入的数值参考。** `--verify-onnx` 使用相同输入运行 ORT，比较整个输出数组， + 包括补齐位置；要求类型、形状正确、所有值有限、严格 `max_abs < 1e-3`、`rtol=0`， + 并且最后有效位置的 greedy ID 相同。top-2 margin 只作诊断记录，不是额外通过阈值。 + 误差如何计算及能说明什么见[误差解释](#max-absolute-error)。 +- **独立核对生成输入。** `--reference-suite` 从 + [冻结参考](../../../probes/w5_generation/references.json)维护独立的参考历史， + `checked_inputs` 逐元素检查实际 token 输入和遮罩，再把核对后的实际输入交给 ORT。 + QEMU 仍接收生成循环准备的实际输入。 +- **生成程序自己选择 token。** 每步要求 QEMU、ORT 和冻结参考选出的 ID 一致； + 只有通过后才推进独立参考历史。实际生成循环仍从 QEMU logits 中自行 greedy 选择， + 并把这个结果加入自己的历史，不能用冻结答案替代模型输出。 +- **最终用例检查。** 比较提示词 IDs、新 token IDs、解码文字、停止原因，以及前向次数和校验次数； + 同时检查要求出现的文字。只出现 `Paris` 等城市名，或只有最大误差达标，都不足以判整例通过。 +- **参考与实跑记录分开。** 冻结参考由[参考采集程序](../../../probes/w5_generation/run.py)运行 ORT 得到； + 其中 `baseline.qemu_e2e=NOT_RUN` 只表示采集参考时未运行 QEMU。之后的完整 QEMU 实跑 + 见[实际结果](#fixed-case-results)和[历史数据摘要](reports/validation-summary.json)，不能混用两种记录。 + +`--reference-suite` 必须使用 `raw` 模式并同时提供 `--verify-onnx` 参考模型， +使用冻结的 token 预算，不接受另传 `--max-new-tokens`；它也不会自动选择算术策略。 +完整三例历史实跑显式使用以下选项，不代表默认算术策略也完成了相同的完整模型验证： ```text ---harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 +--cores 4 --matmul-policy blocked_fma --matmul-tile 2x4 --matmul-rhs-layout column-panel --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference ``` +
+ +
+展开多轮对话、容器部署和失败处理 + **多轮文本。** `Conversation.from_directory(tokenizer_directory, system_prompt=None)` 加载同一快照的 `tokenizer_config.json` 官方 `chat_template`，通过 Jinja2 不可变沙箱 渲染，固定 `add_generation_prompt=True`、`enable_thinking=False`。生产路径使用 @@ -89,8 +192,369 @@ QEMU 适配器负责其子进程回收。CLI 最终保存报告失败时向 stde 生成成功而 `tee` 失败时返回日志退出码。下方固定 `8ad3474` 的历史复现命令保留原版 参数，仍须按其成功要求单独核对日志退出码，不向旧检查器传入新参数。 +
+ +## 实际复现结果 + +下文使用**“本机直接运行”**和**“容器内运行”**区分是否通过 Docker； +具体执行链路、运行环境和计时范围见单次前向表后的说明。 + + + +**历史完整模型运行配置。** 以下数字来自 **2026-10-23 的实跑**，代码功能基线为 +`8ad347477f7f5ba15b2da7c91178b3c6668456cc`。后续报告、取消、日志修复及 core 命名兼容 +工作只做相关验证；本次整理文档没有重跑模型，不能把旧数字当作当前版本的新测量。 + +| 项目 | 这次三条固定用例的配置 | +|---|---| +| 模型 | `Qwen/Qwen3-0.6B`，完整 28 层，FP32 | +| 输入与生成 | batch=1，固定序列长度 256；原始文本补全，不使用聊天模板；每条生成 8 个新 token | +| 生成方式 | greedy，每次取最高分 token；无 KV cache，每步执行完整前向 | +| 模拟器与核心数 | QEMU 模拟 RV64GC，显式使用 4 core；三条用例复用同一 ELF 和权重包 | +| 算术与布局 | `blocked_fma / 2x4 / column-panel`；Mean `two-four-lane-last-axis`；非线性 `fp32-reference`；工作区 padding `none` | +| 参考与环境 | 本机 WSL2 Ubuntu，4 个可用逻辑 CPU；ORT 通过 `CPUExecutionProvider` 计算参考输出 | + +
+展开模型快照和工具版本 + +- 模型快照：`c1899de289a04d12100db370d81485cdf75e47ca`，见 + [模型清单](../../../probes/w1_qwen3_export/manifest.json)及[冻结参考](../../../probes/w5_generation/references.json)。 +- QEMU 8.2.2（Ubuntu 包 `1:8.2.2+ds-0ubuntu1.18`），Zig 0.14.1； + 目标为 `riscv64 / RV64GC / LP64D / QEMU virt` 裸机，数学库为 Zig 随附 musl。 +- Python 3.12.3、NumPy 2.2.6、ONNX Runtime 1.22.1、tokenizers 0.21.4。 +- 以上为历史记录中的版本；完整字段见 + [数据摘要](reports/validation-summary.json)的 `configuration`。不据此推断当前机器或最新依赖版本。 + +
+ + + +**三条实际输出与停止原因。** 下表来自实际 QEMU 运行报告，并已与冻结参考逐项核对， +不是只展示预设的期望答案。每条均为 8 个新 token、8 次完整前向，`reference_passed=true`； +全部 24 次前向的数值和生成结果通过，QEMU 进程及进程组均已回收。 + +| 用例 | 原始输入提示词 | 实际新生成文字 | 实际停止原因 | +|---|---|---|---| +| France | `The capital of France is` | ` Paris. The capital of Italy is Rome` | `max_new_tokens`（达到 8 个 token 预算） | +| Germany | `The capital of Germany is` | ` Berlin. The capital of France is Paris` | `max_new_tokens`（达到 8 个 token 预算） | +| Japan | `The capital of Japan is` | ` Tokyo. The capital of the United States` | `max_new_tokens`（达到 8 个 token 预算） | + +三条输出开头均保留一个空格。**此次没有遇到 EOS，停止是因为预算用完，不代表自然回答完毕。** +例如 Japan 的输出停在 `the United States`，后续句子尚未补全； +这些是原始提示词的续写，不是聊天模板下的问答质量评测。 + +
+展开三条用例的实际输入及新生成 token 编号 + +| 用例 | 提示词 token IDs | 实际新生成 token IDs | +|---|---|---| +| France | `[785,6722,315,9625,374]` | `[12095,13,576,6722,315,15344,374,21718]` | +| Germany | `[785,6722,315,9856,374]` | `[19846,13,576,6722,315,9625,374,12095]` | +| Japan | `[785,6722,315,6323,374]` | `[26194,13,576,6722,315,279,3639,4180]` | + +
+ +对应的预期结果见[冻结参考](../../../probes/w5_generation/references.json)， +实际结果与 24 步误差见[历史数据摘要](reports/validation-summary.json)的 `three_case_generation`。 +[报告入口](#report-locations)说明哪些文件随仓库保存、哪些原始数据只保留在本机。 + + + +**三条固定用例的耗时与数值误差。** 对应上述实际文字结果，均为本机直接运行。 + +**每行对应同一条提示词生成 8 个新 token：串行执行 8 次前向，每次使用 4 core。** +本次按固定 token 预算结束，不代表让模型自由回答到自然结束。 +“QEMU 仿真总耗时”是这 8 次模拟器运行的实际经过时间相加。 +**本节各表的耗时均按实际经过时间展示，不把同时工作的多个线程的 CPU 用时相加。** +累计 CPU 用时仍保留在[原始数据摘要](reports/validation-summary.json)中，不作为运行速度指标列入表格。 + +| 用例 | 端到端生成与校验耗时（秒） | QEMU 仿真总耗时（秒） | 最大绝对误差 | +|---|---:|---:|---:| +| France | 3375.024 | 3199.695 | 0.000303983688 | +| Germany | 3668.852 | 3482.999 | 0.000262737274 | +| Japan | 3489.655 | 3315.397 | 0.000197887421 | + +
+表中名称说明： + +- **端到端生成与校验耗时**：从开始处理这条提示词，到全部 8 个新 token 生成、转换为最终文字并完成主要校验， + 实际经过的时间。包含输入处理、每步 QEMU 启动与权重装载、模型计算、结果读回和进程清理， + 以及参考计算、逐步数值校验和最终文字与冻结参考的核对。 + **最终文字已经生成并经过校验，这部分时间已包含在内；不含前置编译，也不含最后的报告整理、保存和终端显示。** + 计时顺序是：**处理提示词 → 逐个生成 8 个 token 并校验 → 转换为最终文字并核对 → 计时结束**； + 随后才整理、保存报告并打印文字。因此，它不是纯模型计算时间，也不是从部署到显示结果的全部链路耗时。 +- **QEMU 仿真总耗时**：同一条用例生成 8 个新 token 时，8 次 QEMU 仿真从启动到退出回收的 + **实际经过时间之和**。包含模拟器启动、装载权重、模型计算、输出转储和退出清理； + 不包含前置编译、QEMU 启动前的输入准备和文件检查，也不包含进程外的 ORT 参考计算和完整结果校验。 + **表中是 8 次仿真的累计耗时，不是单次前向耗时，也不是只执行模型算子的纯计算时间。** + 它已包含在“端到端生成与校验耗时”中，两者不能相加。 +- **最大绝对误差**：该用例中，模型输出分数与参考结果之间最大的绝对偏差，计算和判定方法见下文。 + +
+ +
+8 次 QEMU 运行如何生成一条结果？ + +三条提示词初始都为 5 个 token。第 1 步用这 5 个 token 选出第 1 个新 token； +第 2 步用提示词和已选结果，共 6 个 token，选出第 2 个；依次继续， +8 次前向的有效输入长度为 **5、6、7、8、9、10、11、12**，每次都右填充到 256。 +第 8 个新 token 产生后，历史长度变为 13，达到生成预算，不再执行第 9 次前向。 + +每步用 greedy 选出下一个 token，最终将 8 个新 token 转成同一条输出， +不是生成 8 个回答再择优。无 KV cache，复用的是同一 ELF 和权重文件； +Python/QEMU 的具体职责见[流程表](#generation-flow)。 + +
+ + + +
+最大绝对误差：和谁比、怎么算、能说明什么？ + +1. **和谁比较。** 同一份 Qwen3 模型、相同的 token 输入及遮罩，分别由 ScratchV 编译后在 QEMU + 中执行，以及由 ONNX Runtime（ORT）参考程序计算。对比的是模型输出的 **logits**， + 即选下一个 token 时使用的候选分数；这些原始分数不是概率，也不是最终文字。 + ORT 在这里提供数值基准，不是判断回答是否符合事实的裁判。 +2. **怎么算。** 每个对应位置的 QEMU 分数减去 ORT 分数，去掉正负号，再选出最大的差值： + `最大绝对误差 = max(|QEMU 分数 − ORT 分数|)`。 + 例如参考分数为 1.2500、QEMU 为 1.2503，这个位置的误差约为 0.0003。 + 每次前向比较全部 **38,895,616 个输出分数**（包括固定长度中补齐的位置）， + 表中再取该用例 **8 次前向中的最大值**，不是平均误差，也不只检查最终选中的 token。 +3. **为什么小于 0.001 才通过。** 这是[开发计划](../开发计划.md)为完整 QEMU 前向规定的数值容差， + W5 沿用 W4 的 `ATOL = 1e-3`。它给不同浮点计算实现之间的偏差设定上限； + FP32 舍入、运算顺序和数学函数实现差异可能使两个后端的数值不完全一样。 + 代码要求严格 **`max_abs < 0.001`**，等于 0.001 也不通过； + `rtol=0` 表示不再因分数较大而增加相对误差宽限。 + **0.001 是项目约定的分数偏差上限，不是“错误率低于 0.1%”，也不是普遍适用的正确性证明。** +4. **能否说明生成结果正确。** 单看误差小，只能说明这次计算的分数接近参考。 + 当两个候选 token 的分数很接近时，小偏差也可能改变排名、选出另一个 token。 + 因此，完整用例通过还要求：全部输出是有限数值，没有 NaN 或正负无穷； + 每一步实际选出的 token 与当步 ORT 结果及[冻结参考](../../../probes/w5_generation/references.json)一致； + 最终 token 序列、文字和停止原因也与冻结参考完全一致。不能仅凭误差小于 0.001 判整条用例通过。 + +上述计算和检查见[生成入口](../../../scripts/run_inference.py)的 `compare_step` 与 +`reference_suite` 路径。**这里的三条固定用例共 24 步同时通过了数值与生成结果检查**， +说明所列历史版本和显式配置在这些用例上与参考一致；不代表所有提示词都会得到相同结果， +也不保证模型生成的内容在事实层面永远正确。本次只解释已有结果，没有重新执行模型。 + +
+ +
+展开三例合计耗时、内存和采样限制 + +整次测量 10540.190 s（约 2 小时 55 分 40 秒），包括准备、ORT 和数值比较。 +24 次前向的**启动至完成通知耗时**合计 9990.355 s，平均 **416.265 s/token**； +**QEMU 仿真总耗时**合计 9998.091 s。这两个耗时指标的起止位置见[单次前向表说明](#container-results)。 +**采样到的内存峰值**：模拟器 5.406 GiB、主控程序 4.254 GiB、同时运行的进程合计 7.644 GiB。 +17/24 步各有一次内存采样读取失败，属于部分采样（`partial`），可能漏掉更高的瞬时占用。 +同时进程合计是在同一采样时刻求和后取最大值，不能直接相加各进程在不同时刻的峰值； +按 RSS 求和也可能重复计算共享内存页。 + +
+ + + +**独立单次完整前向与干净容器 Paris。** 两次都只生成 `[12095]`、文本 ` Paris`， +完整输出分数的最大绝对误差同为 `0.000172138214`；比较对象和通过条件与 +[上面的误差说明](#max-absolute-error)相同，这里每边只生成 1 个新 token。 +容器内重新构建一次，没有重复三例套件； +155,582,464 字节完整输出与本机单次前向逐字节相同。 + +**两列采用相同的程序内部计时方式，但构建方式不同；这是运行记录，不是容器性能开销对比。** + +| 测量项 | 本机直接运行（复用构建） | 容器内运行（重新编译） | +|---|---:|---:| +| 推理程序内部耗时（秒） | 444.416 | 547.454 | +| C 程序编译与链接耗时（秒） | 复用已有构建，本次未重新编译完整模型 | 93.868 | +| 启动至完成通知耗时（秒） | 418.954 | 415.058 | +| 模拟器进程总耗时（秒） | 419.383 | 415.478 | +| 采样到的模拟器内存峰值（GiB） | 5.398 | 5.398 | + +**本机直接运行**表示整条推理链路在 Docker 容器外执行，**容器内运行**表示同一链路在容器内执行。 +两边的模型计算都由 QEMU 完成。本次均在同一台电脑的 WSL2 Ubuntu 环境中， +不是 Windows 原生与另一台电脑的对比；也不表示每次都重新下载模型、安装依赖或编译。 + +**这张表的计时范围。** 以下时间项均为实际经过时间，包含关系不能相加。 +本机复用构建，容器的 **93.868 秒编译已经包含在 547.454 秒内部耗时中**； +构建工作不同，不能把两列总时间之差视为容器性能损耗。 + +| 名称 | 从哪里计到哪里 | 包含与排除 | +|---|---|---| +| 推理程序内部耗时 | 程序开始准备 → 按需构建、自检、参考程序初始化 → 生成与校验 → 程序内收尾 | 使用 `run/report.json` 的 `elapsed_seconds`；只计本次实际发生的工作，不计此前构建，也不含内部计时开始前的 Docker 镜像准备或容器启动 | +| C 程序编译与链接耗时 | 启动编译工具 → 生成并检查模型 ELF | `compile_seconds` 包含编译器启动、C 编译、汇编、链接、日志保存及 ELF 检查；不含此前的模型解析、C 源码生成、权重打包和之后的推理 | +| 端到端生成与校验耗时 | 处理提示词 → 得到最终文字并完成主要校验 | 是内部耗时的中间阶段，不含前置编译、浮点自检、ORT 初始化及最后报告保存、终端显示；含每步 ORT 参考计算。三例各生成 8 个 token，本表每边只生成 1 个 | +| 启动至完成通知耗时（原 UART 耗时） | QEMU 启动 → Python 确认合法的成功完成帧 | 包含启动、权重装载、模型计算和通知接收；不是串口传输时间，也不是纯算子时间。输出读回和数值校验尚未完成 | +| 模拟器进程总耗时 | 同一次 QEMU 启动 → 进程退出并回收 | 包含上项，以及暂停、输出转储和退出清理；不含进程外编译、ORT 参考计算和完整结果校验 | +| 采样到的模拟器内存峰值 | 定时读取 QEMU 驻留内存 RSS，取成功采样的最大值 | 包含模拟器、权重和运行缓冲；不是模型文件大小、配置内存上限或整机占用。可能漏掉瞬时峰值；1 GiB = 1024³ 字节 | + +UART 是串口通信接口；本项目用它发送完成状态，完整分数经 QMP 读回。 +关于 QEMU 的输入输出和通知顺序，见[流程表](#generation-flow)。 + +
+展开单次前向的历史计时细节与可比性限制 + +- 本机表中采用内部计时 **444.416 秒**。此前外层测量的 **448.263 秒**仍在原始记录中， + 不混入本表；其中端到端生成与校验为 **441.033 秒**。 + 容器没有独立记录同口径的生成与校验耗时，不能用其他计时项代替。 +- 本机复用产物中记录的 **85.680 秒**是历史模型编译时间，不计入本次。 + 小型浮点自检程序仍需编译，本机为 **0.110 秒**、容器为 **0.047 秒**， + 已包含在各自内部耗时中，与完整模型编译分开。自检用于检查工具链浮点乘加行为，不是重跑完整模型。 +- 两列内部耗时相差 **103.038 秒**，容器本次模型编译 **93.868 秒**是主要的已测量差别； + 其他准备、缓存和运行开销未完整分段计时，不能把差值全归于编译或 Docker。 + 模拟器进程耗时 **419.383 / 415.478 秒**接近，单次记录不能用于认定容器更快或更慢。 +- 本机从完成通知到退出回收约 **0.429 秒**，是 `419.383 - 418.954`； + 这是进程总耗时的一部分，不能再次累加。 +- 容器另有运行器计时 **430.481 秒**，包含权重传输、QEMU、后检查和清理， + 不含更早的输入打包、写入、ELF/bundle 检查及完整 ORT 参考计算，因此不是端到端生成与校验耗时。 +- 本机主控程序/同时进程采样内存峰值为 **4.119 / 7.210 GiB**。 + 容器 QEMU 有 **2071 个有效样本、1 次读取失败**，标为 `partial`； + 未清空本机 Linux 页缓存，不宣称冷缓存性能。 +- 容器单 prompt 的 `full_model_gate=NOT_RUN` 表示没有运行三例套件， + 不能据此说没有运行完整模型。 + +
+ + + +**小图与多轮行为。** 历史真实容器默认单核、手动 4 core、auto 解析为 4 的小图均 +生成 IDs `[2,3,4,6]`、文本 `2 3 4`，EOS 停止，逐步完整 logits 对 ORT 的最大误差为 0， +padding `none`。不同上下文和中断后的再次生成可复用同一配置 ELF/bundle； +历史 QEMU 超时/取消及 Docker 构建 SIGINT/SIGTERM 后未遗留目标进程。 +官方聊天模板的有/无 system 三轮渲染和 token 序列与 Transformers 一致；这些回复由 +模板验证预设，未表示完整模型三轮聊天实跑。 + +
+展开 2026-10-25 小图验证细节：报告保存、Docker 取消、清理与复用 + +**2026-10-25 局部修复验证。** 基于 `e93233143d2323de7de6888ff7d73bdbf50784c3` +加入报告保存与取消处理修复后，注入权限、磁盘满、I/O 和序列化失败，CLI 均保留原始生成错误 +或取消退出码；原本成功的运行在最终报告无法保存时返回失败，不误报成功。后续调用 +仍可正常保存。2026-10-25 的该项验证没有重新执行完整模型，上方完整数值与性能数据仍属于历史基线。 + +真实 Docker 小图在默认 1 core 和手动 4 core 下，正常生成均得到 `[2,3,4,6]`、 +文本 `2 3 4`，逐步 ORT 最大误差为 0。向 Docker 客户端发送 SIGINT/SIGTERM 后， +报告均为 `CANCELLED`，退出码分别为 130/143；容器外的客户端进程已回收、QEMU 进程组为空、 +`--rm` 容器消失，0.3 秒后复查仍无残留。中断检查以临时 `-S` 包装器控制真实 guest +的暂停时机，不修改生产 harness；正常生成和恢复使用实际小图前向。 + +两种信号之后均复用原 ELF/bundle 恢复生成，IDs、文本及全部逐步 logits 仍与基线一致； +4 core 的恢复还实测 `auto` 解析为 4，两组 padding 均为 `none`。这些结果说明小图 +取消、清理和复用正常，不扩大为完整模型重跑或完整多轮聊天结果。 + +
+ + + +**报告放在哪里。** + +| 要找的内容 | 固定入口 | 是否能随 PR 查看 | +|---|---|---| +| W5 功能、复现、数值、耗时、内存及未完成项 | 本 README；[完整结果](#实际复现结果) | 是 | +| W5 三例逐 token 数值和资源、本机/容器运行记录、小图与清理结论 | [validation-summary.json](reports/validation-summary.json) | 是，随仓库保存的历史数据摘要 | +| W4 软仿优化过程与单核/4 core 完整前向对照 | [W4 软仿报告](../W4/softsim-optimization-2026-10-22.md)及[汇总数据](../W4/reports/full-hart-comparison-2026-10-22.json) | 是，已有仓库文件 | +| CI 原始测试明细、benchmark 详细数据和诊断 | [固定历史 run 的附件](#ci-and-benchmark) | 在 Actions 中可查看/下载，有保留期限 | +| 本地交互页面 `comparison.html` 与大体积原始产物 | 忽略的 `output/`；[本地文件定位](#local-raw-artifacts) | 否；不能把本机文件链接当作 PR 报告链接 | + +公开 JSON 是现有记录的白名单摘录，保留测量单位、历史版本、逐步数值及采样缺失； +不含权重、ELF、大数组、机器绝对路径或进程 ID。它不是本次重新执行的结果。 +本地交互报告不再作为维护者查找 W5 结果的必经入口。 + + + +[公开数据摘要](reports/validation-summary.json)收录上述关键数值及逐 token 结果，可随仓库保存。 +下面只是本机诊断文件定位，不是 GitHub 下载链接。路径均相对于执行该次记录的 W5 +工作树根目录；原始产物在其 `output/` 中，不提交权重、ELF 和逐步 logits： + +
+展开本地原始文件清单（仅供本机定位，不是 GitHub 下载链接） + +- `output/w5-followup-20261023/qemu-reference-4hart/run/report.json`： + 完整三例；`run/` 的上一级目录内 `steps.json`、`numeric-steps.json`、`resources.json` 为计时与资源细节。 +- `output/w5-followup-20261023/qemu-final-single-forward-4hart/`：本机独立单次前向。 +- `output/w5-container-20261023/paris/run/report.json`：容器完整 Paris 单 token。 +- `output/w5-container-20261023/small-summary.json`、`small-runs/` 与 `build-cancel/`： + 容器小图、构建取消和进程清理记录。 +- `output/w5-hardening-20261025/docker-final/test_external_signals_reap_gue{0,1}/docker-cancellation-summary.json`： + 2026-10-25 的 1/4 core 小图外部取消、进程清理和恢复结果。 + +
+ + + +完整三例的独立容器运行以及其他执行人的独立复现记录仍未取得；不能把本机三例、 +容器单 token 或后续局部验证合计成这些结果。现有实现仍是无 KV cache、逐 token +重启 QEMU 与装载权重；常驻运行、多轮完整模型语义和其他算术策略的完整结果未在 +上述实跑范围内。 + +完整 W5 专项成功记录也尚未收录于上述历史快照；本次未检查远端是否已有后续运行。 +普通 CI 与 benchmark 的独立范围见下面折叠内容。 + + + +
+展开历史 CI、benchmark 与 W4 对照（不是本次完整模型运行结果） + +**版本与验证范围。** 以下固定保留 2026-10-27 整理的历史记录，不代表后续推送的 CI 状态。 +PR head `e8c94a7`、测试合并候选 `fa79ae4` 的普通 CI 为 +**14 个 job 成功、7 个按条件跳过**。本轮 core 命名兼容的相关验证范围与结果见 +[测试说明](../../../tests/llm-deploy/w5/README.md)。旧远端 CI 不能证明后续改动已通过验证， +局部验证也不表示当前版本重跑了完整模型。 + +**历史 CI 与 benchmark 快照（2026-10-27 核对）。** [普通 CI #40173064525](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525) +对应 PR head `e8c94a71becfae96c4171674e332ed1239335c4d`， +实际执行 `fa79ae4d0f697f968574d647347076a013429684`（与当时 main `735dd00` 的合并候选）。 +下表是该次运行的快照，不是每次打开 README 都会刷新的实时状态。 + +**CI 各项结果与附件下载（完整 W4/W5 本次未运行）** + +| 数据 | 该次结果与范围 | 直接查看 / 下载 | +|---|---|---| +| 通用测试 | **8321 passed、93 skipped**；额外模拟器检查 **7 passed** | [test job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121107820034)、[test-reports](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12591298436) | +| W4/W5 单测与真实小图 | **1769 passed、4 skipped**；不代表完整 28 层模型通过 | [小图 job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121107851526)、[w4-unit 附件](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12591112082)中的 `tests.xml` | +| 官方多轮模板 | **2 passed**；验证模板及 token 序列，不是完整模型三轮生成 | [w5-multiturn 附件](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12590439548) | +| 编译器 benchmark | job **SUCCESS**，但对照存在无效旧基线，不能表述为全项通过，详见下表 | [benchmark job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121107819853)、[benchmark-reports](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12590609469) | +| 课题 06 benchmark | job **SUCCESS**；全量报告实际为 **12 PASS / 11 FAIL**，CI 严格检查的范围更小 | [Topic 06 job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121107819972)、[topic06-benchmark-reports](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/artifacts/12591146751) | +| 完整 W4/W5 | **NOT RUN**：普通 PR 未显式启用完整前向和三例容器生成 | [范围汇总 job](https://github.com/ScratchV-Compiler/ScratchV/actions/runs/40173064525/job/121108398602)；本次没有完整模型性能附件 | + + +**已核对的 benchmark 数字与限制。** 这些是上述 CI 附件中已有的结果，2026-10-27 +只下载并整理，未重新执行。它们测试解释器/编译器等范围，不能替代 W5 完整生成计时。 + +**注意：Topic 06 为 12 PASS / 11 FAIL，单算子对照的旧基线无效；job 成功不代表所有基准通过。** + +**benchmark 详细数字、失败原因与统计范围** + +| 附件内的报告 | 实际数据 | 如何理解 | +|---|---|---| +| `benchmark-reports / dsl_bench` | **23/23**；总耗时 **0.088 s**，总指令数 **153** | 仅该 DSL 基准的结果 | +| `benchmark-reports / ir_interpreter` | **6/6**，最大误差 **4.6566e-10** | 仅解释器用例的数值对照 | +| `topic06-benchmark-reports / report.md` | **12/23 PASS（52.2%），11 FAIL** | 8 项 TinyFive 输入 ABI 不支持；3 项 `dynamic_add` 计数增加，即使总动态指令减少也被标为回退 | +| `benchmark-reports / single_op_bench.md` | 表中 **30 行 FAIL**；当前版本 **78/78** 组合有效，旧基线 **78/78** 无效 | 旧基线有标签解析错误，不能用旧版 0 指令数算加速比，也不能把表中 FAIL 都解释为当前代码失败 | + +Topic 06 的 CI 只对 activation/elementwise/loop 三组使用失败退出选项； +全量报告未设置该选项，所以“job 成功”和“全量 11 项 FAIL”可以同时出现。 +这些是当时核对发现的报告/基准限制，尚未定位其引入版本，不据此断言由 W5 改动造成。 +当前仅整理可见结果，没有修改 CI 或修复这些基准。 + + +GitHub 运行页的“Load 摘要”加载的是工作流写入 `GITHUB_STEP_SUMMARY` 的已保存 Markdown； +点击它不会重跑测试或模型。附件下载通常需登录 GitHub。上述小图/多轮附件预计 +2026-11-08 到期，通用测试/benchmark 附件预计 2026-11-24 到期；平台删除或权限变化也会 +使下载失效。本文保留核心结果，完整诊断日志仍以附件为准。 + +**历史 W4 单次前向对照。** 2026-10-22 的同配置记录中，1 core 为 **1384.65 s**， +4 core 为 **410.53 s**，约 **3.37×**；CPU 总时间 **+5.46%**，输出逐位相同。 +见[W4 软仿报告](../W4/softsim-optimization-2026-10-22.md)及 +[原始汇总 JSON](../W4/reports/full-hart-comparison-2026-10-22.json)。 +这只有一组有序对照，未使用旧计时异常记录，不能当作 W5 三例端到端生成的加速比。 + +
+ ## 如何复现 +
+展开完整复现说明、环境要求、命令与验收标准 + **本地已有环境。** 从仓库根执行，激活项目 Python 3.12 环境并按 [Linux 开发说明](../LINUX_REPRODUCTION.md)安装依赖。已有固定资产时，原始补全命令为： @@ -100,7 +564,7 @@ python scripts/run_inference.py \ --tokenizer-dir assets/qwen3-0.6b/tokenizer \ --prompt 'The capital of France is' --max-new-tokens 1 \ --verify-onnx assets/qwen3-0.6b/onnx/model.onnx \ - --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ + --cores 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ --matmul-rhs-layout column-panel \ --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ --timeout 1800 --total-timeout 3600 --output-dir output/w5-paris-new @@ -108,8 +572,8 @@ python scripts/run_inference.py \ 这会执行一次完整 28 层模型。只构建可加 `--build-only`，其 `BUILD_ONLY` 不表示 完成生成。复用时以 `--reuse-build output/w5-paris-new/executable.json` 替换 -`--model ...`，仍传相同 tokenizer 和所有原有算术/hart 配置，并使用新的输出目录。 -元数据可能含绝对路径，不直接把宿主构建搬进容器复用；跨环境从模型重新构建。 +`--model ...`，仍传相同 tokenizer 和所有原有算术/core 配置，并使用新的输出目录。 +元数据可能含绝对路径，不直接把本机构建的产物搬进容器复用；跨环境从模型重新构建。 聊天模式将上例 `--prompt ...` 换为下列参数，其他模型、参考策略及输出目录参数保留： @@ -126,8 +590,10 @@ python scripts/run_inference.py \ Ubuntu 24.04 x86_64 环境，不复制作者的虚拟环境、`output/`、ELF、bundle 或报告。 三例共用一次新构建、执行 24 次前向，不先另跑 Paris，也不重新录制 ORT 参考。 固定提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 是已完成完整数值实跑的功能基线； -该提交用于复现既有完整数值结果，不含本轮报告保存及 SIGTERM 分类修复；这些行为 -使用当前分支的相关小图验证，不将旧完整结果标为本轮重跑。 +该提交用于复现既有完整数值结果，不含后续报告保存、SIGTERM 分类及 core 名称兼容修复； +这些行为另用相关小图或 Python 测试验证，不将旧完整结果标为修改后的重跑。 +该固定提交也尚不支持 `--cores`，所以下面的历史复现命令保留 `--harts`； +运行当前分支时优先使用 `--cores`。这次名称兼容修改没有重跑完整模型。 | 条件 | 固定内容或要求 | |---|---| @@ -199,7 +665,7 @@ python3 -c 'import sys; assert sys.version_info[:2] == (3, 12)' printf 'Run output: %s\n' "$SCRATCHV_W5_OUTPUT" ``` -检查 `runner-resources.txt`：宿主应为 Ubuntu 24.04，Docker daemon 也是本机 Linux； +检查 `runner-resources.txt`：容器外的 Linux 环境应为 Ubuntu 24.04，Docker daemon 也运行在该 Linux 环境中； 若使用远端 daemon、容器资源配额不足或没有引擎，先停止并回报，不用修改 WSL 来凑条件。 挂载目录不能包含逗号。不要把私人凭据写入报告。 @@ -233,7 +699,7 @@ cat "$SCRATCHV_W5_OUTPUT/image-tools.txt" **3. 独立获取固定资产。** 下载只调用现有 `acquire_model`，沿用 W1 manifest 中已有的下载/分片完整性检查。 -它只用 Python 标准库，不执行模型、不重新导出、不安装宿主推理依赖。 +它只用 Python 标准库，不执行模型、不重新导出、不在容器外的 Linux 环境安装推理依赖。 Tokenizer 从同一提交的 Git LFS 获取。 ```bash @@ -356,15 +822,8 @@ printf 'Preserved output: %s\n' "$SCRATCHV_W5_OUTPUT" 主机掉电、daemon 失联或强制终止可能阻止清理；如清理日志有非零码或最后仍列出容器， 标为未完成并回报，不视为通过。 -**三条用例与结果判定。** 全部为 raw completion、greedy、8 个新 token， -停止原因均为 `max_new_tokens`。文本含前导空格；只出现城市名不足以判定一致， -需同时核对[冻结参考](../../../probes/w5_generation/references.json)的 IDs 和文本： - -| 原始 prompt | 新生成 IDs | 新生成文本 | -|---|---|---| -| `The capital of France is` | `[12095,13,576,6722,315,15344,374,21718]` | ` Paris. The capital of Italy is Rome` | -| `The capital of Germany is` | `[19846,13,576,6722,315,9625,374,12095]` | ` Berlin. The capital of France is Paris` | -| `The capital of Japan is` | `[26194,13,576,6722,315,279,3639,4180]` | ` Tokyo. The capital of the United States` | +**三条用例与结果判定。** 输入、应得到的文字、token 编号及停止原因统一见 +[三条实际输出与冻结参考对照](#fixed-case-results)。复现必须逐项匹配，不能只检查是否出现城市名。 成功要求：三例、24 次前向全部完成，prompt IDs/新 IDs/文本/停止原因匹配；每步全部 `256×151936=38,895,616` 个 logits 有限且严格 `max_abs < 1e-3`；28 层完整图检查、 @@ -409,84 +868,4 @@ PY RSS 使用 `run/steps/*/run.json`，partial 采样明确注明；未采集的 CPU 写“未独立采集”， 不填作者历史数据，也不据不同机器的记录计算加速比。 -## 实际复现结果 - -以下完整模型数字来自 **2026-10-23 的历史实跑**，其功能基线为上述 `8ad3474`。 -本轮只补异常处理与文档相关验证，没有重新执行完整模型。完整模型均为 -**28 层 Qwen3-0.6B**，使用显式 `4 hart / blocked_fma / 2x4 / column-panel / -two-four-lane-last-axis / fp32-reference / padding none`。 - -**宿主 QEMU 三条固定生成。** 三例各 8 步，24 步的文本、IDs 与停止原因全部符合上表， -每步完整 logits 均满足严格误差要求，全部 QEMU 进程和进程组已回收。 - -| 用例 | 生成调用耗时（s） | QEMU CPU（s） | Host CPU（s） | 最大绝对误差 | -|---|---:|---:|---:|---:| -| France | 3375.024 | 11517.789 | 167.657 | 0.000303983688 | -| Germany | 3668.852 | 12574.174 | 190.673 | 0.000262737274 | -| Japan | 3489.655 | 11958.881 | 178.759 | 0.000197887421 | - -整次测量 10540.190 s（约 2 小时 55 分 40 秒），包括准备、ORT 和数值比较。 -24 步 UART 耗时合计 9990.355 s，平均 **416.265 s/token**；QEMU 完整进程合计 -9998.091 s，CPU 合计 36050.844 s，平均约 360.58%（100% 表示一个逻辑 CPU）。 -Host 总 CPU 540.717 s；表中生成阶段合计 537.089 s。 -RSS 观测峰值 QEMU 5.406 GiB、Host 4.254 GiB、同时进程合计 7.644 GiB。 -17/24 步的采样各有一次读取失败，属于 partial，不能当作无缺失峰值。 - -**独立单次完整前向与干净容器 Paris。** 两次都只生成 `[12095]`、文本 ` Paris`， -完整 logits 最大误差同为 `0.000172138214`。容器内重新构建一次，没有重复三例套件； -155,582,464 字节完整输出与宿主单次前向逐字节相同。 - -| 测量项 | 宿主单次前向 | 容器单次前向 | -|---|---:|---:| -| 全过程耗时（s） | 448.263 | 547.454 | -| UART 耗时（s） | 418.954 | 415.058 | -| QEMU 完整进程耗时（s） | 419.383 | 415.478 | -| QEMU CPU（s） | 1498.851 | 未独立采集 | -| QEMU RSS 观测峰值（GiB） | 5.398 | 5.398 | - -宿主生成调用耗时 441.033 s，Host 总 CPU 25.332 s（生成期间 21.604 s）， -Host/同时进程 RSS 观测峰值分别 4.119/7.210 GiB。容器总耗时含新构建、FMA 和 ORT； -运行器记录 430.481 s 包括权重传输、QEMU、后检查及清理，不含更早的输入打包、写入和 -ELF/bundle 检查。容器 RSS 有 2071 个有效样本、1 次读取失败，同样为 partial。 - -UART 耗时包含 QEMU 启动和权重装载，不是纯算子时间；完整进程另含 stop/dump/quit。 -宿主单次前向启动了新 QEMU，但未清空宿主页缓存，不宣称冷缓存性能。 -容器单 prompt 的 `full_model_gate=NOT_RUN` 表示未运行三例，不表示未运行完整模型。 - -**小图与多轮行为。** 历史真实容器默认单 hart、手动 4 hart、auto 解析为 4 的小图均 -生成 IDs `[2,3,4,6]`、文本 `2 3 4`，EOS 停止，逐步完整 logits 对 ORT 的最大误差为 0， -padding `none`。不同上下文和中断后的再次生成可复用同一配置 ELF/bundle； -历史 QEMU 超时/取消及 Docker 构建 SIGINT/SIGTERM 后未遗留目标进程。 -官方聊天模板的有/无 system 三轮渲染和 token 序列与 Transformers 一致；这些回复由 -模板验证预设，未表示完整模型三轮聊天实跑。 - -**2026-10-25 局部修复验证。** 基于 `e93233143d2323de7de6888ff7d73bdbf50784c3` -加入本轮报告保存与取消处理修复后，注入权限、磁盘满、I/O 和序列化失败，CLI 均保留原始生成错误 -或取消退出码；原本成功的运行在最终报告无法保存时返回失败，不误报成功。后续调用 -仍可正常保存。本轮没有重新执行完整模型，上方完整数值与性能数据仍属于历史基线。 - -真实 Docker 小图在默认 1 hart 和手动 4 hart 下，正常生成均得到 `[2,3,4,6]`、 -文本 `2 3 4`，逐步 ORT 最大误差为 0。向 Docker 客户端发送 SIGINT/SIGTERM 后， -报告均为 `CANCELLED`，退出码分别为 130/143；宿主 PID 被回收、QEMU 进程组为空、 -`--rm` 容器消失，0.3 秒后复查仍无残留。中断检查以临时 `-S` 包装器控制真实 guest -的暂停时机，不修改生产 harness；正常生成和恢复使用实际小图前向。 - -两种信号之后均复用原 ELF/bundle 恢复生成，IDs、文本及全部逐步 logits 仍与基线一致； -4 hart 的恢复还实测 `auto` 解析为 4，两组 padding 均为 `none`。这些结果说明小图 -取消、清理和复用正常，不扩大为完整模型重跑或完整多轮聊天结果。 - -原始产物为本地 `output/` 内容，不提交权重、ELF 和逐步 logits： - -- `output/w5-followup-20261023/qemu-reference-4hart/run/report.json`： - 完整三例；同目录 `steps.json`、`numeric-steps.json`、`resources.json` 为计时与资源细节。 -- `output/w5-followup-20261023/qemu-final-single-forward-4hart/`：宿主独立单次前向。 -- `output/w5-container-20261023/paris/run/report.json`：容器完整 Paris 单 token。 -- `output/w5-container-20261023/small-summary.json`、`small-runs/` 与 `build-cancel/`： - 容器小图、构建取消和进程清理记录。 -- `output/w5-hardening-20261025/docker-final/test_external_signals_reap_gue{0,1}/docker-cancellation-summary.json`： - 本轮 1/4 hart 小图外部取消、进程清理和恢复结果。 - -完整三例的独立容器运行以及其他执行人的独立复现记录仍未取得；不能把宿主三例、 -容器单 token 或本轮局部验证合计成这些结果。现有实现仍是无 KV cache、逐 token -重启 QEMU 与装载权重；常驻运行、多轮完整模型语义和其他算术策略的完整结果未在 -上述实跑范围内。 +
diff --git a/docs/llm-deploy-v1.0/W5/reports/validation-summary.json b/docs/llm-deploy-v1.0/W5/reports/validation-summary.json new file mode 100644 index 00000000..f33c13b8 --- /dev/null +++ b/docs/llm-deploy-v1.0/W5/reports/validation-summary.json @@ -0,0 +1,826 @@ +{ + "schema_version": 1, + "title": "W5 历史实跑数据摘要", + "scope": { + "historical_result": true, + "new_model_execution": false, + "current_commit_ci_result": false, + "full_model_measurement_date": "2026-10-23", + "full_model_baseline_commit": "8ad347477f7f5ba15b2da7c91178b3c6668456cc", + "baseline_source": "baseline_and_scope", + "baseline_note": "完整模型数据沿用 README 记录的历史功能基线，不代表后续命名、兼容修复或最新 PR 提交已经重跑完整模型。", + "sources_base": "所有来源路径相对于当时运行所在的 W5 仓库工作树根目录；output 原始文件保留本地，本摘要公开可读数据。", + "raw_report_schema_version": 1, + "field_mapping": "本摘要 cores/cores_requested 对应历史原始 schema 1 的 harts/harts_requested；没有重写原始记录。", + "excluded": "不复制机器绝对路径、PID、编译命令、制品哈希、源码、权重、ELF 或张量数组。token ID 是冻结生成用例结果。" + }, + "units": { + "time": "带 _s 的字段单位为秒；wall 是经过时间，CPU 是线程累计处理器时间，不能互相相加。", + "memory": "带 _bytes 的字段单位为 byte；RSS 是采样观测值，不是内存硬上限；GiB 换算为 bytes / 1073741824。", + "cpu_percent": "QEMU CPU / QEMU process wall * 100；100% 表示一个逻辑 CPU。", + "numerical": "max_abs 为所比较 logits 元素最大绝对误差；*_frozen_selected_abs 为选中 token logit 相对冻结参考的绝对误差。", + "step": "numeric_steps 的 step 是各用例内部前向序号，从 0 到 7；case 与 step 共同定位一条记录。" + }, + "measurement_definitions": { + "generation_wall_s": "单条用例生成调用经过时间，包含逐 token QEMU 调用与 Host 参考数值检查，不是编译耗时。", + "uart_completion_wall_s": "QEMU launch through host validation of the UART completion frame", + "qemu_process_wall_s": "QEMU 子进程寿命；包括 UART 完成后的停止、输出转储与退出清理。", + "runner_elapsed_s": "运行器调用总耗时，包含权重传输、QEMU、完成后检查和清理；不等同于纯算子执行。", + "measurement_total_wall_s": "外层测量全过程，包含准备、ORT 参考和数值检查。", + "qemu_cpu_s": "每步 RUSAGE_CHILDREN CPU 差值由测量工具归入 QEMU，三例汇总取 cases[*].qemu_cpu；不使用包含其他子进程的 total_child_cpu。", + "host_cpu_s": "Host 进程 CPU，含 ORT 及验证开销；case 与整个测量的范围分别记录。", + "rss": "Host+live descendants simultaneous RSS sum；共享页可能重复计数。各进程独立峰值不能相加当作同一时刻峰值。", + "cache": "复用相同 ELF/bundle；每次前向启动新 QEMU，未清空宿主页缓存，不能作为冷缓存结果。", + "performance_comparison": "这些数据没有单核完整生成对照，不据此计算 W5 1/4 核生成加速比。", + "inference_program_elapsed_s": "推理程序开始准备至程序内部计时结束，取对应 run/report.json 的 elapsed_seconds；包含当次实际发生的构建、自检、参考程序初始化、生成与校验及程序内收尾，不包含此前构建、Docker 镜像准备或容器启动。", + "compile_wall_s": "本次 C 程序编译与链接经过时间，取新构建的 build.compile_seconds；包含编译器启动、C 编译、汇编、链接、日志保存和 ELF 检查，不含此前模型解析、C 源码生成及权重打包。复用构建时为 null，不能计入旧构建记录中的编译时间。", + "deployment_elapsed_s": "保留的历史字段名；容器记录中与 inference_program_elapsed_s 同取 run/report.json.elapsed_seconds，并非 Docker 镜像准备到退出的外层全过程。" + }, + "configuration": { + "model_id": "Qwen/Qwen3-0.6B", + "layers": 28, + "generation": { + "batch_size": 1, + "sequence_length": 256, + "dtype": "float32", + "use_cache": false, + "prompt_format": "raw_completion", + "add_special_tokens": false, + "sampling": "greedy_argmax_lowest_id" + }, + "cores_requested": 4, + "cores": 4, + "matmul_policy": "blocked_fma", + "matmul_tile": "2x4", + "matmul_rhs_layout": "column-panel", + "mean_policy": "two-four-lane-last-axis", + "nonlinear_policy": "fp32-reference", + "qemu_workspace_layout": "none", + "execution": "one ELF/bundle; a fresh QEMU process per forward; no KV cache", + "versions": { + "compiler": "0.14.1", + "qemu": "QEMU emulator version 8.2.2 (Debian 1:8.2.2+ds-0ubuntu1.18)", + "target": "riscv64 / RV64GC / LP64D / QEMU virt / bare metal", + "math_library": "Zig-bundled musl" + }, + "host_environment": { + "platform": "Linux-6.18.40.1-microsoft-standard-WSL2-x86_64-with-glibc2.39", + "python": "3.12.3", + "packages": { + "numpy": "2.2.6", + "onnxruntime": "1.22.1", + "tokenizers": "0.21.4" + }, + "available_logical_cpus": 4, + "clocksource": "tsc", + "host_tree_sample_interval_s": 0.5, + "reference_provider": "CPUExecutionProvider" + }, + "scope_note": "显式优化策略的历史完整模型结果；不是默认算术策略完整验收。默认单核与手动/auto 的覆盖另见小图记录。" + }, + "three_case_generation": { + "sources": [ + "three_case_report", + "three_case_resources", + "three_case_metadata" + ], + "status": "PASS", + "passed": true, + "full_model_gate": "PASS", + "container_reproduction": "NOT_ASSERTED", + "forward_count": 24, + "cases": [ + { + "name": "france", + "prompt": "The capital of France is", + "max_new_tokens": 8, + "generated_token_ids": [ + 12095, + 13, + 576, + 6722, + 315, + 15344, + 374, + 21718 + ], + "generated_text": " Paris. The capital of Italy is Rome", + "stop_reason": "max_new_tokens", + "forward_count": 8, + "reference_passed": true, + "generation_wall_s": 3375.0240403530006, + "qemu_cpu_s": 11517.789080999999, + "host_cpu_s": 167.656669, + "sampled_host_peak_rss_bytes": 4568129536, + "sampled_simultaneous_tree_peak_rss_bytes": 8123035648, + "max_abs": 0.0003039836883544922, + "numeric_steps_passed": true + }, + { + "name": "germany", + "prompt": "The capital of Germany is", + "max_new_tokens": 8, + "generated_token_ids": [ + 19846, + 13, + 576, + 6722, + 315, + 9625, + 374, + 12095 + ], + "generated_text": " Berlin. The capital of France is Paris", + "stop_reason": "max_new_tokens", + "forward_count": 8, + "reference_passed": true, + "generation_wall_s": 3668.8521105910004, + "qemu_cpu_s": 12574.174352, + "host_cpu_s": 190.67315399999998, + "sampled_host_peak_rss_bytes": 4404678656, + "sampled_simultaneous_tree_peak_rss_bytes": 8207597568, + "max_abs": 0.0002627372741699219, + "numeric_steps_passed": true + }, + { + "name": "japan", + "prompt": "The capital of Japan is", + "max_new_tokens": 8, + "generated_token_ids": [ + 26194, + 13, + 576, + 6722, + 315, + 279, + 3639, + 4180 + ], + "generated_text": " Tokyo. The capital of the United States", + "stop_reason": "max_new_tokens", + "forward_count": 8, + "reference_passed": true, + "generation_wall_s": 3489.654915645, + "qemu_cpu_s": 11958.880560000001, + "host_cpu_s": 178.75917299999992, + "sampled_host_peak_rss_bytes": 4400611328, + "sampled_simultaneous_tree_peak_rss_bytes": 7957729280, + "max_abs": 0.00019788742065429688, + "numeric_steps_passed": true + } + ], + "totals": { + "measurement_total_wall_s": 10540.189622884001, + "generation_wall_s": 10533.531066589001, + "uart_completion_wall_s": 9990.355179675997, + "qemu_process_wall_s": 9998.090941066994, + "qemu_cpu_s": 36050.843993, + "qemu_average_cpu_percent": 360.58, + "host_measurement_cpu_s": 540.716702, + "host_generation_cpu_s": 537.088996, + "sampled_qemu_peak_rss_bytes": 5804662784, + "sampled_host_peak_rss_bytes": 4568129536, + "sampled_simultaneous_tree_peak_rss_bytes": 8207597568, + "numeric_max_abs": 0.0003039836883544922, + "all_qemu_reaped": true + }, + "sampling": { + "qemu_rss_partial_steps": [ + 0, + 1, + 5, + 6, + 7, + 9, + 11, + 12, + 14, + 15, + 16, + 17, + 18, + 20, + 21, + 22, + 23 + ], + "qemu_rss_sample_errors": 17, + "host_tree_sample_count": 21032, + "host_tree_sample_errors": 0, + "max_abs_perf_minus_raw_s": 0.0007727560023340629, + "max_abs_utc_minus_perf_s": 0.0005040991982241394, + "note": "17/24 步 QEMU RSS 各失读一次，峰值属于 partial；Host/tree 零采样错误不能替代 QEMU 采样完整性。" + }, + "numeric_comparison": { + "atol": 0.001, + "rtol": 0.0, + "elements_compared_per_step": 38895616, + "reference": "冻结 reference token 与本次 ORT/QEMU greedy token 逐步一致，并检查 logits 数值误差；不含原始张量数组。", + "all_24_steps": { + "selected_row_passed": true, + "frozen_step_passed": true, + "reference_token_passed": true + } + }, + "numeric_steps": [ + { + "case": "france", + "step": 0, + "valid_length": 5, + "qemu_token_id": 12095, + "ort_token_id": 12095, + "frozen_token_id": 12095, + "max_abs": 0.00017213821411132812, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "france", + "step": 1, + "valid_length": 6, + "qemu_token_id": 13, + "ort_token_id": 13, + "frozen_token_id": 13, + "max_abs": 9.34600830078125e-05, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "france", + "step": 2, + "valid_length": 7, + "qemu_token_id": 576, + "ort_token_id": 576, + "frozen_token_id": 576, + "max_abs": 0.00011712312698364258, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "france", + "step": 3, + "valid_length": 8, + "qemu_token_id": 6722, + "ort_token_id": 6722, + "frozen_token_id": 6722, + "max_abs": 0.0003039836883544922, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "france", + "step": 4, + "valid_length": 9, + "qemu_token_id": 315, + "ort_token_id": 315, + "frozen_token_id": 315, + "max_abs": 0.00013625621795654297, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "france", + "step": 5, + "valid_length": 10, + "qemu_token_id": 15344, + "ort_token_id": 15344, + "frozen_token_id": 15344, + "max_abs": 6.151199340820312e-05, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "france", + "step": 6, + "valid_length": 11, + "qemu_token_id": 374, + "ort_token_id": 374, + "frozen_token_id": 374, + "max_abs": 0.00016880035400390625, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 1.9073486328125e-06, + "passed": true + }, + { + "case": "france", + "step": 7, + "valid_length": 12, + "qemu_token_id": 21718, + "ort_token_id": 21718, + "frozen_token_id": 21718, + "max_abs": 0.0001971721649169922, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 1.9073486328125e-06, + "passed": true + }, + { + "case": "germany", + "step": 0, + "valid_length": 5, + "qemu_token_id": 19846, + "ort_token_id": 19846, + "frozen_token_id": 19846, + "max_abs": 0.00018739700317382812, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "germany", + "step": 1, + "valid_length": 6, + "qemu_token_id": 13, + "ort_token_id": 13, + "frozen_token_id": 13, + "max_abs": 5.5789947509765625e-05, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "germany", + "step": 2, + "valid_length": 7, + "qemu_token_id": 576, + "ort_token_id": 576, + "frozen_token_id": 576, + "max_abs": 0.0001690387725830078, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "germany", + "step": 3, + "valid_length": 8, + "qemu_token_id": 6722, + "ort_token_id": 6722, + "frozen_token_id": 6722, + "max_abs": 0.0002627372741699219, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 1.9073486328125e-06, + "passed": true + }, + { + "case": "germany", + "step": 4, + "valid_length": 9, + "qemu_token_id": 315, + "ort_token_id": 315, + "frozen_token_id": 315, + "max_abs": 0.0002231597900390625, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 5.7220458984375e-06, + "passed": true + }, + { + "case": "germany", + "step": 5, + "valid_length": 10, + "qemu_token_id": 9625, + "ort_token_id": 9625, + "frozen_token_id": 9625, + "max_abs": 0.00014162063598632812, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "germany", + "step": 6, + "valid_length": 11, + "qemu_token_id": 374, + "ort_token_id": 374, + "frozen_token_id": 374, + "max_abs": 0.00011169910430908203, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 1.9073486328125e-06, + "passed": true + }, + { + "case": "germany", + "step": 7, + "valid_length": 12, + "qemu_token_id": 12095, + "ort_token_id": 12095, + "frozen_token_id": 12095, + "max_abs": 0.00017213821411132812, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 1.9073486328125e-06, + "passed": true + }, + { + "case": "japan", + "step": 0, + "valid_length": 5, + "qemu_token_id": 26194, + "ort_token_id": 26194, + "frozen_token_id": 26194, + "max_abs": 3.3289194107055664e-05, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "japan", + "step": 1, + "valid_length": 6, + "qemu_token_id": 13, + "ort_token_id": 13, + "frozen_token_id": 13, + "max_abs": 0.00019788742065429688, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "japan", + "step": 2, + "valid_length": 7, + "qemu_token_id": 576, + "ort_token_id": 576, + "frozen_token_id": 576, + "max_abs": 0.0001494884490966797, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "japan", + "step": 3, + "valid_length": 8, + "qemu_token_id": 6722, + "ort_token_id": 6722, + "frozen_token_id": 6722, + "max_abs": 8.016824722290039e-05, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "japan", + "step": 4, + "valid_length": 9, + "qemu_token_id": 315, + "ort_token_id": 315, + "frozen_token_id": 315, + "max_abs": 0.00010776519775390625, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "japan", + "step": 5, + "valid_length": 10, + "qemu_token_id": 279, + "ort_token_id": 279, + "frozen_token_id": 279, + "max_abs": 9.918212890625e-05, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 0.0, + "passed": true + }, + { + "case": "japan", + "step": 6, + "valid_length": 11, + "qemu_token_id": 3639, + "ort_token_id": 3639, + "frozen_token_id": 3639, + "max_abs": 5.27501106262207e-05, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 5.7220458984375e-06, + "passed": true + }, + { + "case": "japan", + "step": 7, + "valid_length": 12, + "qemu_token_id": 4180, + "ort_token_id": 4180, + "frozen_token_id": 4180, + "max_abs": 5.817413330078125e-05, + "ort_frozen_selected_abs": 0.0, + "qemu_frozen_selected_abs": 1.9073486328125e-06, + "passed": true + } + ] + }, + "single_full_forward_comparison": { + "date": "2026-10-23", + "scope": "相同完整 28 层模型及显式策略，各生成第一个 Paris token；本机复用构建，容器重新编译，不能用总时间差判断容器额外开销；不是三例完整生成的容器对照。", + "host": { + "sources": [ + "host_single_report", + "host_single_resources", + "repeat_and_cleanup" + ], + "passed": true, + "forward_count": 1, + "full_reference_suite_evaluated": false, + "case": { + "prompt": "The capital of France is", + "max_new_tokens": 1, + "generated_token_ids": [ + 12095 + ], + "generated_text": " Paris", + "stop_reason": "max_new_tokens" + }, + "measurement_total_wall_s": 448.2632728360004, + "generation_wall_s": 441.0328804979981, + "uart_completion_wall_s": 418.95395780800027, + "qemu_process_wall_s": 419.38253989200166, + "qemu_cpu_s": 1498.851419, + "host_measurement_cpu_s": 25.33218, + "host_generation_cpu_s": 21.604175, + "sampled_host_peak_rss_bytes": 4423188480, + "sampled_simultaneous_tree_peak_rss_bytes": 7741919232, + "qemu_rss": { + "sampled_peak_bytes": 5796397056, + "availability": "available", + "successful_samples": 2091, + "sample_errors": 0, + "sample_interval_s": 0.2 + }, + "max_abs": 0.00017213821411132812, + "qemu_reaped": true, + "inference_program_elapsed_s": 444.4163706699983, + "build_reused": true, + "compile_wall_s": null, + "compile_note": "本次复用已有 ELF 和权重包，没有重新编译完整模型；原始报告 build 中保留的旧 compile_seconds 不计入此次运行。" + }, + "container": { + "sources": [ + "container_paris_report", + "container_paris_summary" + ], + "passed": true, + "forward_count": 1, + "case": { + "prompt": "The capital of France is", + "max_new_tokens": 1, + "generated_token_ids": [ + 12095 + ], + "generated_text": " Paris", + "stop_reason": "max_new_tokens" + }, + "full_model_gate": "NOT_RUN", + "container_reproduction": "NOT_ASSERTED", + "acceptance": "PASS: clean-container single Paris forward; existing host 3x8 gate is separate", + "deployment_elapsed_s": 547.4544949770025, + "runner_elapsed_s": 430.48115924100057, + "uart_completion_wall_s": 415.05810176999876, + "qemu_process_wall_s": 415.47787108400007, + "qemu_cpu_s": null, + "cpu_note": "No independent container CPU-total measurement in this deployment acceptance; Docker client GNU time is not container CPU time.", + "qemu_rss": { + "sampled_peak_bytes": 5795934208, + "availability": "partial", + "successful_samples": 2071, + "sample_errors": 1, + "sample_interval_s": 0.2 + }, + "max_abs": 0.00017213821411132812, + "qemu_reaped": true, + "process_exit_code": 0, + "matches_host_full_logits_byte_for_byte": true, + "compared_bytes": 155582464, + "scope_note": "full_model_gate=NOT_RUN 表示未执行容器三例 24 步整套验收；本次确实执行了完整模型一个前向。容器 CPU 未独立采集，不能使用 Docker 客户端 CPU 代替。", + "inference_program_elapsed_s": 547.4544949770025, + "build_reused": false, + "compile_wall_s": 93.86813328300195, + "compile_note": "本次在容器内重新编译完整模型；compile_wall_s 已包含在 inference_program_elapsed_s 中，不能再次相加。" + } + }, + "container_small_graph": { + "date": "2026-10-23", + "source": "container_small_summary", + "passed": true, + "scope": "独立的小型确定性图；不代表完整模型生成性能。", + "cases": [ + { + "mode": "default", + "cores_requested": 1, + "cores": 1, + "passed": true, + "tokens": [ + 2, + 3, + 4, + 6 + ], + "max_abs": 0, + "padding": "none" + }, + { + "mode": "manual", + "cores_requested": 4, + "cores": 4, + "passed": true, + "tokens": [ + 2, + 3, + 4, + 6 + ], + "max_abs": 0, + "padding": "none" + }, + { + "mode": "auto_reuse", + "cores_requested": "auto", + "cores": 4, + "passed": true, + "tokens": [ + 2, + 3, + 4, + 6 + ], + "max_abs": 0, + "padding": "none" + } + ] + }, + "process_cleanup": { + "full_model_history": { + "date": "2026-10-23", + "source": "repeat_and_cleanup", + "formal_suite_passed": true, + "suite_forwards": 24, + "single_forward_passed": true, + "same_existing_elf_and_bundle_identity": true, + "repeat_full_output_byte_identical": true, + "compared_bytes": 155582464, + "qemu_processes_and_groups_absent": 25, + "all_recorded_processes_absent": true + }, + "docker_external_cancellation": { + "date": "2026-10-25", + "base_commit": "e93233143d2323de7de6888ff7d73bdbf50784c3", + "version_note": "README 记录为该基线上加入报告保存与取消修复后的局部验证，不是 2026-10-23 完整模型重跑。", + "scope": "真实 Docker 小图；取消时用临时 -S 包装器控制 guest 暂停时机。0.3 秒后复查清理；恢复沿用相同 ELF/bundle。", + "cases": [ + { + "source": "docker_cancellation_1_core", + "cores": 1, + "passed": true, + "available_auto_cores": 4, + "baseline": { + "status": "PASS", + "passed": true, + "cores_requested": 1, + "cores": 1, + "padding": "none", + "generated_token_ids": [ + 2, + 3, + 4, + 6 + ], + "generated_text": "2 3 4", + "max_abs": 0.0, + "all_numeric_steps_passed": true + }, + "signals": [ + { + "signal": "SIGINT", + "exit_code": 130, + "container_removed": true, + "host_processes_reaped": true, + "qemu_group_empty": true, + "delayed_recheck_seconds": 0.3, + "report_status": "CANCELLED", + "report_passed": false, + "recovery": { + "status": "PASS", + "passed": true, + "cores_requested": 1, + "cores": 1, + "max_abs": 0.0, + "all_numeric_steps_passed": true, + "output_matches_baseline": true, + "padding": "none" + } + }, + { + "signal": "SIGTERM", + "exit_code": 143, + "container_removed": true, + "host_processes_reaped": true, + "qemu_group_empty": true, + "delayed_recheck_seconds": 0.3, + "report_status": "CANCELLED", + "report_passed": false, + "recovery": { + "status": "PASS", + "passed": true, + "cores_requested": 1, + "cores": 1, + "max_abs": 0.0, + "all_numeric_steps_passed": true, + "output_matches_baseline": true, + "padding": "none" + } + } + ] + }, + { + "source": "docker_cancellation_4_core", + "cores": 4, + "passed": true, + "available_auto_cores": 4, + "baseline": { + "status": "PASS", + "passed": true, + "cores_requested": 4, + "cores": 4, + "padding": "none", + "generated_token_ids": [ + 2, + 3, + 4, + 6 + ], + "generated_text": "2 3 4", + "max_abs": 0.0, + "all_numeric_steps_passed": true + }, + "signals": [ + { + "signal": "SIGINT", + "exit_code": 130, + "container_removed": true, + "host_processes_reaped": true, + "qemu_group_empty": true, + "delayed_recheck_seconds": 0.3, + "report_status": "CANCELLED", + "report_passed": false, + "recovery": { + "status": "PASS", + "passed": true, + "cores_requested": "auto", + "cores": 4, + "max_abs": 0.0, + "all_numeric_steps_passed": true, + "output_matches_baseline": true, + "padding": "none" + } + }, + { + "signal": "SIGTERM", + "exit_code": 143, + "container_removed": true, + "host_processes_reaped": true, + "qemu_group_empty": true, + "delayed_recheck_seconds": 0.3, + "report_status": "CANCELLED", + "report_passed": false, + "recovery": { + "status": "PASS", + "passed": true, + "cores_requested": "auto", + "cores": 4, + "max_abs": 0.0, + "all_numeric_steps_passed": true, + "output_matches_baseline": true, + "padding": "none" + } + } + ] + } + ] + } + }, + "remaining_scope": { + "current_head_full_model": "未因本次摘要整理重跑；历史完整模型结果不得描述为当前代码的新执行结果。", + "container_three_cases_24_steps": "上述容器数据只覆盖 Paris 单步及小图，未提供容器三例 24 步完整运行结果。", + "second_person_reproduction": "这些作者本机记录不构成第二人独立复现。", + "ci_and_benchmark": "本文件不包含最新 Actions CI 或 benchmark 状态；它们与历史 W5 模型测量分开查看。", + "chat_and_cache": "raw completion 与无 KV cache；三例文本不是完整多轮聊天质量验收。" + }, + "sources": { + "three_case_report": "output/w5-followup-20261023/qemu-reference-4hart/run/report.json", + "three_case_resources": "output/w5-followup-20261023/qemu-reference-4hart/resources.json", + "three_case_metadata": "output/w5-followup-20261023/qemu-reference-4hart/metadata.json", + "host_single_report": "output/w5-followup-20261023/qemu-final-single-forward-4hart/run/report.json", + "host_single_resources": "output/w5-followup-20261023/qemu-final-single-forward-4hart/resources.json", + "repeat_and_cleanup": "output/w5-followup-20261023/qemu-final-single-forward-4hart/repeat-and-cleanup-check.json", + "container_paris_report": "output/w5-container-20261023/paris/run/report.json", + "container_paris_summary": "output/w5-container-20261023/paris-summary.json", + "container_small_summary": "output/w5-container-20261023/small-summary.json", + "docker_cancellation_1_core": "output/w5-hardening-20261025/docker-final/test_external_signals_reap_gue0/docker-cancellation-summary.json", + "docker_cancellation_4_core": "output/w5-hardening-20261025/docker-final/test_external_signals_reap_gue1/docker-cancellation-summary.json", + "frozen_reference": "probes/w5_generation/references.json", + "baseline_and_scope": "docs/llm-deploy-v1.0/W5/README.md" + } +} diff --git a/probes/w4_qwen3_full/diagnostic.py b/probes/w4_qwen3_full/diagnostic.py index 1f007ad0..71c5ad31 100644 --- a/probes/w4_qwen3_full/diagnostic.py +++ b/probes/w4_qwen3_full/diagnostic.py @@ -28,10 +28,10 @@ from probes.w3_qwen3_full.cases import CASE_NAMES, input_cases from probes.w3_qwen3_full.comparison import compare_positions from probes.w3_qwen3_full.worker import checkpoint_schema, checked_tensor, save_array -from probes.w4_qwen3_full.run import ATOL, save_executable, spec_json +from probes.w4_qwen3_full.run import ATOL, CoreNamespace, add_core_argument, save_executable, spec_json from scratchv.backend.tensor_c_codegen import TensorCCodegen from scratchv.ir.types import BasicBlock, DataType, Function, Instruction, OpCode, Program -from scratchv.runtime.riscv_harts import resolve_harts +from scratchv.runtime.riscv_harts import resolve_cores from scratchv.runtime.riscv_external import build_external, run_external from scratchv.runtime.riscv_tensor import discover_toolchain from scratchv.runtime.weight_bundle import write_weight_bundle @@ -196,9 +196,7 @@ def main(argv=None): default="none", help="QEMU-only experimental two-buffer workspace layout (default: none); " "padding adds 4 KiB and may reduce performance; unsupported graphs fail") - cli.add_argument("--harts", choices=("1", "2", "4", "auto"), default="1", - help="RV64 harts (default: 1); auto selects 1/2/4 from available CPUs, " - "up to 4; parallel MatMul requires blocked_fma") + add_core_argument(cli) cli.add_argument("--matmul-policy", choices=("sequential", "blocked_fma"), default="sequential") cli.add_argument("--matmul-tile", choices=("1x4", "2x4", "1x8"), default="1x4", help="MatMul output tile (default: 1x4); 2x4/1x8 require blocked_fma") @@ -209,15 +207,15 @@ def main(argv=None): cli.add_argument("--matmul-rhs-layout", choices=("original", "column-panel"), default="original", help="Experimental immutable MatMul RHS packing (default: original); " "column-panel requires blocked_fma") - args = cli.parse_args(argv) + args = cli.parse_args(argv, namespace=CoreNamespace()) if args.matmul_tile != "1x4" and args.matmul_policy != "blocked_fma": cli.error("--matmul-tile 2x4/1x8 requires --matmul-policy blocked_fma") if args.matmul_rhs_layout != "original" and args.matmul_policy != "blocked_fma": cli.error("--matmul-rhs-layout column-panel requires --matmul-policy blocked_fma") - requested_harts = args.harts if args.harts == "auto" else int(args.harts) - args.harts = resolve_harts(requested_harts) - if requested_harts == "auto": - print(f"[W4] auto selected {args.harts} hart(s)", flush=True) + requested_cores = args.cores if args.cores == "auto" else int(args.cores) + args.cores = resolve_cores(requested_cores) + if requested_cores == "auto": + print(f"[W4] auto selected {args.cores} core(s)", flush=True) if any(not math.isfinite(limit) or limit <= 0 for limit in (args.timeout, args.build_timeout)): cli.error("Timeouts must be finite and positive") out = new_output_dir(args.output_dir) @@ -226,7 +224,8 @@ def main(argv=None): "passed": False, "status": "FAIL", "stage": "initialization", "stages_seconds": {}, "checkpoint": args.checkpoint, "case": args.case, "matmul_policy": args.matmul_policy, "kernel_calls": args.kernel_calls, - "harts_requested": requested_harts, "harts": args.harts, + "cores_requested": requested_cores, "cores": args.cores, + "harts_requested": requested_cores, "harts": args.cores, "matmul_tile": args.matmul_tile, "matmul_rhs_layout": args.matmul_rhs_layout, "qemu_workspace_layout_requested": args.qemu_workspace_layout, @@ -290,9 +289,9 @@ def stage(name, operation): artifact.external_initializers, out / "weights")) tools = discover_toolchain(args.cc, args.qemu) exe = stage("build_riscv_checkpoint", lambda: build_external(artifact, out / "weights", - out / "build", tools, timeout=args.build_timeout, harts=args.harts)) + out / "build", tools, timeout=args.build_timeout, cores=args.cores)) save_executable(exe, out) - report.update(build=exe.evidence, memory_layout=exe.layout, harts=args.harts) + report.update(build=exe.evidence, memory_layout=exe.layout, cores=args.cores) report["derived_onnx"] = stage("slice_onnx", lambda: slice_onnx(model, row, model_dir, out / "checkpoint.onnx")) del artifact, bindings, sliced, program, parser, target, model gc.collect() diff --git a/probes/w4_qwen3_full/run.py b/probes/w4_qwen3_full/run.py index ad5fa9c0..0d21f9e4 100644 --- a/probes/w4_qwen3_full/run.py +++ b/probes/w4_qwen3_full/run.py @@ -26,7 +26,7 @@ from probes.w4_qwen3_full.fma_conformance import run_probe as run_fma_probe from scratchv.backend.tensor_c_codegen import TensorCCodegen, TensorSpec, _shape from scratchv.ir.types import DataType -from scratchv.runtime.riscv_harts import resolve_harts +from scratchv.runtime.riscv_harts import read_cores, resolve_cores from scratchv.runtime.riscv_external import build_external, run_external, ExternalExecutable, validate_external_elf from scratchv.runtime.riscv_tensor import discover_toolchain, RiscVToolchain, input_layout, toolchain_versions from scratchv.runtime.weight_bundle import write_weight_bundle, validate_weight_bundle, plan_guest_memory @@ -103,11 +103,16 @@ def spec_json(spec): def save_executable(exe, out): + cores = read_cores(exe.evidence) + if cores != exe.cores: + raise ValueError("Executable core count does not match build evidence") + # Keep equal legacy fields so existing readers can consume newly saved builds. + evidence = {**exe.evidence, "cores": cores, "harts": cores} record = {"elf": str(exe.elf_path), "bundle_dir": str(exe.bundle_dir), "manifest": exe.manifest, "layout": exe.layout, "inputs": [spec_json(s) for s in exe.inputs], "weights": [spec_json(s) for s in exe.weights], "output": spec_json(exe.output), - "cc": list(exe.toolchain.cc), "qemu": exe.toolchain.qemu, "evidence": exe.evidence, - "harts": exe.harts} + "cc": list(exe.toolchain.cc), "qemu": exe.toolchain.qemu, "evidence": evidence, + "cores": cores, "harts": cores} atomic_text(out / "executable.json", json.dumps(record, indent=2) + "\n") return sha256_file(out / "executable.json") @@ -138,11 +143,9 @@ def spec(value): exe = ExternalExecutable(Path(record["elf"]), Path(record["bundle_dir"]), record["manifest"], record["layout"], tuple(map(spec, record["inputs"])), spec(record["output"]), tuple(map(spec, record["weights"])), RiscVToolchain(tuple(record["cc"]), record["qemu"]), - record["evidence"], harts=record.get("harts", 1)) - if (type(exe.harts) is not int or exe.harts not in (1, 2, 4) - or type(exe.evidence.get("harts", 1)) is not int - or exe.evidence.get("harts", 1) != exe.harts): - raise ValueError("Reused hart count does not match build evidence") + record["evidence"], cores=read_cores(record)) + if read_cores(exe.evidence) != exe.cores: + raise ValueError("Reused core count does not match build evidence") mean_policy = exe.evidence.get("mean_policy", "sequential") if not isinstance(mean_policy, str) or mean_policy not in ("sequential", "two-four-lane-last-axis"): raise ValueError("Unsupported reused Mean policy") @@ -155,7 +158,7 @@ def spec(value): _, input_bytes = input_layout(exe.inputs) layout = plan_guest_memory(workspace_bytes=exe.layout["regions"]["workspace"]["nbytes"], weight_bytes=manifest["total_bytes"], input_bytes=input_bytes, output_bytes=exe.output.nbytes, - harts=exe.harts) + cores=exe.cores) if layout != exe.layout: raise ValueError("Reused memory layout is not the canonical non-overlapping plan") evidence = validate_external_elf(exe.elf_path, layout) @@ -166,7 +169,7 @@ def spec(value): for name, digest in exe.evidence["sources"].items(): if Path(name).name != name or sha256_file(exe.elf_path.parent / name) != digest: raise ValueError("Reused generated source changed after build") - return exe + return replace(exe, evidence={**exe.evidence, "cores": exe.cores, "harts": exe.cores}) def ort_reference(model_dir, feed, output): @@ -185,6 +188,39 @@ def ort_reference(model_dir, feed, output): gc.collect() +class CoreNamespace(argparse.Namespace): + """Keep the legacy attribute as a live alias, including caller updates.""" + + @property + def harts(self): + return self.cores + + @harts.setter + def harts(self, value): + self.cores = value + + +class _CoreCountAction(argparse.Action): + """One count for both spellings; never silently let one override the other.""" + + def __call__(self, parser, namespace, values, option_string=None): + provided = getattr(namespace, "_core_count_arguments", {}) + if any(name != option_string and value != values for name, value in provided.items()): + parser.error("--cores and --harts must specify the same value") + # Preserve argparse's last-value behavior when a wrapper repeats one spelling. + provided[option_string] = values + namespace._core_count_arguments = provided + setattr(namespace, self.dest, values) + + +def add_core_argument(parser): + parser.add_argument("--cores", "--harts", dest="cores", + choices=("1", "2", "4", "auto"), default="1", + action=_CoreCountAction, + help="QEMU CPU cores (default: 1); auto selects 1/2/4 from available CPUs, " + "up to 4; --harts is a compatibility alias; parallel MatMul requires blocked_fma") + + def main(argv=None): cli = argparse.ArgumentParser(description=__doc__) cli.add_argument("--model-dir", type=Path, required=True) @@ -196,9 +232,7 @@ def main(argv=None): default="none", help="QEMU-only experimental two-buffer workspace layout (default: none); " "padding adds 4 KiB and may reduce performance; unsupported graphs fail") - cli.add_argument("--harts", choices=("1", "2", "4", "auto"), default="1", - help="RV64 harts (default: 1); auto selects 1/2/4 from available CPUs, " - "up to 4; parallel MatMul requires blocked_fma") + add_core_argument(cli) cli.add_argument("--timeout", type=float, default=10800) cli.add_argument("--build-timeout", type=float, default=900) cli.add_argument("--build-only", action="store_true") @@ -213,21 +247,22 @@ def main(argv=None): cli.add_argument("--matmul-rhs-layout", choices=("original", "column-panel"), default="original", help="Experimental immutable MatMul RHS packing (default: original); " "column-panel requires blocked_fma") - args = cli.parse_args(argv) + args = cli.parse_args(argv, namespace=CoreNamespace()) if args.matmul_tile != "1x4" and args.matmul_policy != "blocked_fma": cli.error("--matmul-tile 2x4/1x8 requires --matmul-policy blocked_fma") if args.matmul_rhs_layout != "original" and args.matmul_policy != "blocked_fma": cli.error("--matmul-rhs-layout column-panel requires --matmul-policy blocked_fma") - requested_harts = args.harts if args.harts == "auto" else int(args.harts) - args.harts = resolve_harts(requested_harts) - if requested_harts == "auto": - print(f"[W4] auto selected {args.harts} hart(s)", flush=True) + requested_cores = args.cores if args.cores == "auto" else int(args.cores) + args.cores = resolve_cores(requested_cores) + if requested_cores == "auto": + print(f"[W4] auto selected {args.cores} core(s)", flush=True) out = args.output_dir.resolve() out.mkdir(parents=True, exist_ok=False) start = time.perf_counter() report = {"schema_version": 1, "gate": "numeric:qemu-full-qwen3", "passed": False, "status": "FAIL", "stage": "initialization", "stages_seconds": {}, "cases": [], - "harts_requested": requested_harts, "harts": args.harts, + "cores_requested": requested_cores, "cores": args.cores, + "harts_requested": requested_cores, "harts": args.cores, "matmul_tile": args.matmul_tile, "mean_policy": args.mean_policy, "nonlinear_policy": args.nonlinear_policy, "matmul_rhs_layout": args.matmul_rhs_layout, @@ -258,7 +293,6 @@ def stage(name, operation): model_dir = args.model_dir.resolve() report["model_dir"] = str(model_dir) report["matmul_policy"] = args.matmul_policy - report["harts"] = args.harts report["model_files"] = stage("asset_hashes", lambda: assets.verify_files(model_dir)) import onnx model = stage("metadata_load", lambda: onnx.load(str(model_dir / "model.onnx"), load_external_data=False)) @@ -270,8 +304,11 @@ def stage(name, operation): if sha256_file(args.reuse_build) != previous.get("executable_sha256"): raise ValueError("Reused executable metadata hash is missing or changed") exe = stage("load_existing_build", lambda: load_executable(args.reuse_build)) - if exe.harts != args.harts: - raise ValueError("Reused build hart count differs from requested --harts") + if (("cores" in previous or "harts" in previous) + and read_cores(previous) != exe.cores): + raise ValueError("Reused report core count differs from executable") + if exe.cores != args.cores: + raise ValueError("Reused build core count differs from requested --cores") if (exe.evidence.get("mean_policy", "sequential") != args.mean_policy or previous.get("mean_policy", "sequential") != args.mean_policy): raise ValueError("Reused build Mean policy differs from requested --mean-policy; " @@ -343,7 +380,7 @@ def stage(name, operation): artifact.external_initializers, out / "weights")) toolchain = discover_toolchain(args.cc, args.qemu) exe = stage("build_riscv_full", lambda: build_external(artifact, out / "weights", - out / "build", toolchain, timeout=args.build_timeout, harts=args.harts)) + out / "build", toolchain, timeout=args.build_timeout, cores=args.cores)) del artifact, program, parser gc.collect() report["executable_sha256"] = save_executable(exe, out) diff --git a/probes/w5_generation/README.md b/probes/w5_generation/README.md index 9b6b0d1c..20cf039f 100644 --- a/probes/w5_generation/README.md +++ b/probes/w5_generation/README.md @@ -38,4 +38,5 @@ python probes/w5_generation/run.py \ 每步同时记录 ORT forward wall/CPU 时间；每条用例另记包括 Host 输入准备、检查、采样及解码的 wall/CPU 时间。session 加载单独计时。CPU 时间含进程内所有线程；Linux `ru_maxrss` 是进程截至当时的累计峰值，不能作为每个用例的独立内存增量。这些是 ORT/Host 数据，不能与既有 QEMU 计时混用或推算 QEMU 加速比。 -本探针不读写 hart、padding、CI 或 WSL 设置。QEMU 的默认 1 hart、手动 1/2/4、启动时 auto 和默认关闭的 padding 仍由既有后端负责。 +本探针不读写 core 数、padding、CI 或 WSL 设置。QEMU 的默认 1 core、手动 1/2/4、启动时 auto 和默认关闭的 padding 仍由既有后端负责。 +QEMU 生成入口使用 `--cores`，仍接受旧 `--harts`；它们不属于本 Host 参考探针的参数，详见 [W5 开发说明](../../docs/llm-deploy-v1.0/W5/README.md)。 diff --git a/scratchv/backend/tensor_c_codegen.py b/scratchv/backend/tensor_c_codegen.py index a4edec3d..00335304 100644 --- a/scratchv/backend/tensor_c_codegen.py +++ b/scratchv/backend/tensor_c_codegen.py @@ -189,10 +189,23 @@ def _literal(value, dtype): ''' _EXTERNAL_PRELUDE = r''' +#if defined(SCRATCHV_CORES) && defined(SCRATCHV_HARTS) && SCRATCHV_CORES != SCRATCHV_HARTS +#error "SCRATCHV_CORES and legacy SCRATCHV_HARTS disagree" +#endif +#ifndef SCRATCHV_CORES +#ifdef SCRATCHV_HARTS +#define SCRATCHV_CORES SCRATCHV_HARTS +#else +#define SCRATCHV_CORES 1 +#endif +#endif +#ifndef SCRATCHV_HARTS +#define SCRATCHV_HARTS SCRATCHV_CORES +#endif /* External storage targets 64-bit hosts/guests; lengths are caller-validated. */ _Static_assert(sizeof(size_t) == 8 && sizeof(uintptr_t) == 8, "external tensor ABI requires a 64-bit target"); -#if defined(SCRATCHV_HARTS) && SCRATCHV_HARTS > 1 +#if defined(SCRATCHV_CORES) && SCRATCHV_CORES > 1 /* Synchronous: all callbacks finish before workspace reuse or validation. */ extern void sv_parallel_for(size_t count, void (*fn)(void *, size_t, size_t), void *context); @@ -960,7 +973,7 @@ def _emit(self, instruction, output, details): # Each output retains increasing K order, fused FP32 products, # and a separate FP32 addition after each 128-element K block. # Tiles only share input loads across independent outputs; they - # never split a reduction between harts or change its grouping. + # never split a reduction between cores or change its grouping. # __builtin_fmaf retains explicit FMA with -ffp-contract=off. tile_rows, tile_columns = (int(part) for part in self.matmul_tile.split("x")) lanes = tile_rows * tile_columns @@ -1039,8 +1052,8 @@ def _emit(self, instruction, output, details): # Each task owns one (batch, row tile, column tile); its K loop # and 128-element grouping are shared verbatim with serial. if tasks >= 2 and math.prod(batch) * m * n * k >= 32768: - condition = ("defined(SCRATCHV_HARTS) && SCRATCHV_HARTS > 1 " - f"&& SCRATCHV_HARTS <= {tasks}") + condition = ("defined(SCRATCHV_CORES) && SCRATCHV_CORES > 1 " + f"&& SCRATCHV_CORES <= {tasks}") context = f"sv_matmul_{dst}" aliases = [] seen = set() @@ -1104,7 +1117,7 @@ def _emit(self, instruction, output, details): axis = details["axis"] outer, dimension, inner = math.prod(x.shape[:axis]), x.shape[axis], math.prod(x.shape[axis+1:]) off = f"o*{dimension*inner}ULL+j*{inner}ULL+t" - # Only independent (outer, inner) rows may run on different harts. + # Only independent (outer, inner) rows may run on different cores. # The explicitly selected arithmetic is identical in both paths. initialization, exponential = "sum=0.0f", "expf" accumulate, finish, normalize = "sum+=v;", "", "/=sum" @@ -1129,8 +1142,8 @@ def _emit(self, instruction, output, details): # Keep small/empty graphs and inline storage on the original path. # This floor avoids a barrier for tiny graphs; it is not autotuning. if self.constant_storage == "external" and rows >= 2 and size >= 32768: - condition = ("defined(SCRATCHV_HARTS) && SCRATCHV_HARTS > 1 " - f"&& SCRATCHV_HARTS <= {rows}") + condition = ("defined(SCRATCHV_CORES) && SCRATCHV_CORES > 1 " + f"&& SCRATCHV_CORES <= {rows}") # Reuse code only: each operation keeps its own input/output # pointers and status, initialized below before the joined call. key = (x.dtype, x.shape, axis) diff --git a/scratchv/runtime/riscv_external.py b/scratchv/runtime/riscv_external.py index 594d82f3..436f69f3 100644 --- a/scratchv/runtime/riscv_external.py +++ b/scratchv/runtime/riscv_external.py @@ -6,7 +6,7 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import InitVar, dataclass import ctypes import errno import hashlib @@ -32,7 +32,10 @@ _terminate_invocation, _windows_job, input_layout, pack_inputs, toolchain_versions, ) from .weight_bundle import MIB, plan_guest_memory, validate_weight_bundle -from .riscv_harts import STARTUP_ASSEMBLY as MULTIHART_STARTUP, WORKER_SOURCE, resolve_harts, validate_harts +from .riscv_harts import ( + STARTUP_ASSEMBLY as MULTICORE_STARTUP, CORE_DEFINITIONS, WORKER_SOURCE, UNSET, + read_cores, resolve_cores, select_cores, validate_cores, +) from .weight_transport import ( stage_transport, validate_transport, remove_transport, save_transport_report, refresh_transport_residuals, @@ -111,7 +114,7 @@ def identity(info): "sha256": hashlib.sha256(data).hexdigest()} -@dataclass(frozen=True) +@dataclass(frozen=True, init=False) class ExternalExecutable: elf_path: Path bundle_dir: Path @@ -122,7 +125,36 @@ class ExternalExecutable: weights: tuple toolchain: RiscVToolchain evidence: dict - harts: int = 1 + cores: int + harts: int + _replace_cores: InitVar[int] + + def __init__(self, elf_path, bundle_dir, manifest, layout, inputs, output, + weights, toolchain, evidence, cores=UNSET, *, harts=UNSET, _replace_cores=UNSET): + if _replace_cores is not UNSET: + # replace() supplies the original values for omitted fields. Its + # InitVar carries the baseline without adding a key to asdict(). + # One changed alias updates both; two different new values fail. + baseline = validate_cores(_replace_cores) + if cores is not UNSET: + validate_cores(cores) + if harts is not UNSET: + validate_cores(harts) + if cores is not UNSET and harts is not UNSET and cores != harts: + if cores == baseline: + cores = harts + elif harts == baseline: + harts = cores + count = select_cores(cores, harts=harts) + for name, value in (("elf_path", elf_path), ("bundle_dir", bundle_dir), + ("manifest", manifest), ("layout", layout), ("inputs", inputs), + ("output", output), ("weights", weights), ("toolchain", toolchain), + ("evidence", evidence), ("cores", count), ("harts", count)): + object.__setattr__(self, name, value) + + @property + def _replace_cores(self): + return self.cores def guest_harness(artifact, manifest, layout): @@ -133,7 +165,7 @@ def guest_harness(artifact, manifest, layout): inputs = ",".join(f"(const void *)0x{layout['input_base'] + n:x}UL" for n in offsets) or "0" weights = ",".join(f"(const void *)0x{layout['weights_base'] + t['offset']:x}UL" for t in manifest["tensors"]) or "0" - return r''' + return CORE_DEFINITIONS + r''' typedef unsigned long usize; typedef unsigned char u8; typedef unsigned int u32; @@ -155,7 +187,7 @@ def guest_harness(artifact, manifest, layout): for (int i=0;i<4;++i) uart_byte((u8)(status>>(8*i))); for (int i=0;i<8;++i) uart_byte((u8)(size>>(8*i))); __asm__ volatile("fence rw,rw" ::: "memory"); -#if SCRATCHV_HARTS > 1 +#if SCRATCHV_CORES > 1 __asm__ volatile("csrw mie, zero" ::: "memory"); #endif for (;;) __asm__ volatile("wfi"); @@ -173,8 +205,8 @@ def guest_harness(artifact, manifest, layout): ''' -def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, harts=1): - harts = resolve_harts(harts) +def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, cores=UNSET, harts=UNSET): + cores = resolve_cores(select_cores(cores, harts=harts, allow_auto=True)) if artifact.constant_storage != "external": raise ValueError("External runtime requires external code generation") mean_policy = getattr(artifact, "mean_policy", "sequential") @@ -187,7 +219,7 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, _, input_bytes = input_layout(artifact.inputs) layout = plan_guest_memory(workspace_bytes=artifact.workspace_bytes, weight_bytes=manifest["total_bytes"], input_bytes=input_bytes, - output_bytes=artifact.output.nbytes, harts=harts) + output_bytes=artifact.output.nbytes, cores=cores) build_dir = Path(build_dir).resolve() build_dir.mkdir(parents=True, exist_ok=False) linker = LINKER_SCRIPT.replace("0x1c000000", hex(layout["code_capacity"])) @@ -200,11 +232,11 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, " .text.expf : ALIGN(4096) { *(.text.expf) } > RAM\n" " .text : { *(.text .text.*) } > RAM", ) - if harts != 1: - linker = linker.replace("0x100000", hex(harts * MIB)).replace( - "1 MiB guest stack", f"{harts} MiB guest stacks") + if cores != 1: + linker = linker.replace("0x100000", hex(cores * MIB)).replace( + "1 MiB guest stack", f"{cores} MiB guest stacks") sources = {"model.c": artifact.source, "guest.c": guest_harness(artifact, manifest, layout), - "start.S": STARTUP_ASSEMBLY if harts == 1 else MULTIHART_STARTUP, "link.ld": linker} + "start.S": STARTUP_ASSEMBLY if cores == 1 else MULTICORE_STARTUP, "link.ld": linker} for name, source in sources.items(): (build_dir / name).write_text(source, encoding="utf-8") target = (["-target", "riscv64-linux-musl", "-static"] if toolchain.is_zig else @@ -215,7 +247,7 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, "-fno-stack-protector", "-fno-pie", "-fPIC" if toolchain.is_zig else "-fno-pic", "-fno-fast-math", "-ffp-contract=off", "-fno-strict-aliasing", "-ffunction-sections", "-fdata-sections", *artifact.compile_flags, - f"-DSCRATCHV_HARTS={harts}", + f"-DSCRATCHV_CORES={cores}", f"-DSCRATCHV_HARTS={cores}", "model.c", "guest.c", "start.S", "-Wl,-T,link.ld", "-Wl,-e,sv_boot", "-Wl,--gc-sections", "-Wl,--build-id=none", *(["-lm"] if toolchain.is_zig else []), "-o", "model.elf"] @@ -239,7 +271,7 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, if result.returncode: raise RuntimeError(result.stderr.decode(errors="replace")[-12000:]) evidence = validate_external_elf(build_dir / "model.elf", layout) - evidence.update(compile_seconds=time.perf_counter() - start, command=command, harts=harts, + evidence.update(compile_seconds=time.perf_counter() - start, command=command, cores=cores, harts=cores, mean_policy=mean_policy, nonlinear_policy=nonlinear_policy, matmul_tile=artifact.matmul_tile, matmul_rhs_layout=artifact.matmul_rhs_layout, @@ -253,7 +285,7 @@ def build_external(artifact, bundle_dir, build_dir, toolchain, *, timeout=600.0, (build_dir / "build.json").write_text(json.dumps(evidence, indent=2) + "\n", encoding="utf-8") return ExternalExecutable(build_dir / "model.elf", Path(bundle_dir).resolve(), manifest, layout, tuple(artifact.inputs), artifact.output, tuple(artifact.external_weights), - toolchain, evidence, harts) + toolchain, evidence, cores) def _check_cancelled(cancelled): @@ -612,16 +644,17 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0, cancelled=None) _check_cancelled(cancelled) if not np.isfinite(timeout) or timeout <= 0: raise ValueError("timeout must be positive and finite") - harts = validate_harts(getattr(executable, "harts", 1)) - built_harts = validate_harts(executable.evidence.get("harts", 1)) + cores = select_cores(getattr(executable, "cores", UNSET), + harts=getattr(executable, "harts", UNSET)) + built_cores = read_cores(executable.evidence) mean_policy = executable.evidence.get("mean_policy", "sequential") if not isinstance(mean_policy, str) or mean_policy not in ("sequential", "two-four-lane-last-axis"): raise ValueError("Unsupported executable Mean policy") nonlinear_policy = executable.evidence.get("nonlinear_policy", "libm") if not isinstance(nonlinear_policy, str) or nonlinear_policy not in ("libm", "fp32-reference"): raise ValueError("Unsupported executable nonlinear policy") - if harts != built_harts or executable.layout["regions"]["stack"]["nbytes"] != harts * MIB: - raise ValueError("Executable hart count disagrees with its build or stack layout") + if cores != built_cores or executable.layout["regions"]["stack"]["nbytes"] != cores * MIB: + raise ValueError("Executable core count (legacy harts) disagrees with its build or stack layout") if sha256_file(executable.elf_path) != executable.evidence["sha256"]: raise ValueError("ELF changed after build") manifest = validate_weight_bundle(executable.bundle_dir, executable.weights) @@ -636,7 +669,7 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0, cancelled=None) port = reservation.getsockname()[1] layout = executable.layout command = [executable.toolchain.qemu, "-machine", "virt", "-cpu", "rv64", "-accel", - "tcg" if harts == 1 else "tcg,thread=multi", "-smp", str(harts), "-m", str(layout["memory_mib"]), "-bios", "none", "-display", "none", + "tcg" if cores == 1 else "tcg,thread=multi", "-smp", str(cores), "-m", str(layout["memory_mib"]), "-bios", "none", "-display", "none", "-monitor", "none", "-serial", "file:uart.bin", "-no-reboot", "-qmp", f"tcp:127.0.0.1:{port},server=on,wait=off,nodelay=on", "-kernel", str(executable.elf_path), "-device", @@ -646,7 +679,7 @@ def run_external(executable, inputs, run_dir, *, timeout=3600.0, cancelled=None) options["start_new_session"] = True started = time.perf_counter() report = {"passed": False, "status": "FAIL", "command": command, "timeout_s": timeout, - "harts": harts, "mean_policy": mean_policy, "nonlinear_policy": nonlinear_policy, + "cores": cores, "harts": cores, "mean_policy": mean_policy, "nonlinear_policy": nonlinear_policy, "qemu_workspace_layout": executable.evidence.get("qemu_workspace_layout", "none"), "transport": "QMP physical memory dump after guest completion", "guest_completed": False} transport_dir = run_dir / "weight-transport" diff --git a/scratchv/runtime/riscv_harts.py b/scratchv/runtime/riscv_harts.py index 6f0b7e07..d6192bfe 100644 --- a/scratchv/runtime/riscv_harts.py +++ b/scratchv/runtime/riscv_harts.py @@ -2,20 +2,46 @@ import os -def validate_harts(harts): - if type(harts) is not int or harts not in (1, 2, 4): - raise ValueError("harts must be an integer in {1, 2, 4}") - return harts +UNSET = object() -def resolve_harts(harts): +def validate_cores(cores): + if type(cores) is not int or cores not in (1, 2, 4): + raise ValueError("cores (legacy harts) must be an integer in {1, 2, 4}") + return cores + + +def select_cores(cores=UNSET, *, harts=UNSET, allow_auto=False): + """Validate either spelling, rejecting conflicting explicit requests.""" + def check(value): + return value if allow_auto and type(value) is str and value == "auto" else validate_cores(value) + if cores is not UNSET: + cores = check(cores) + if harts is not UNSET: + harts = check(harts) + if cores is not UNSET and harts is not UNSET and cores != harts: + raise ValueError("cores and legacy harts disagree") + return cores if cores is not UNSET else harts if harts is not UNSET else 1 + + +def read_cores(record, *, default=1): + """Read current/legacy metadata; UNSET requires an explicit count.""" + cores, harts = record.get("cores", UNSET), record.get("harts", UNSET) + if cores is UNSET and harts is UNSET: + if default is UNSET: + raise ValueError("Missing cores (or legacy harts) field") + return validate_cores(default) + return select_cores(cores, harts=harts) + + +def resolve_cores(cores): """Resolve an explicit auto request once, using process CPU availability. Prefer Linux affinity over machine-wide counts. This is a startup choice, not a load monitor or cgroup bandwidth-quota policy. """ - if harts != "auto": - return validate_harts(harts) + if type(cores) is not str or cores != "auto": + return validate_cores(cores) try: available = len(os.sched_getaffinity(0)) except (AttributeError, OSError, NotImplementedError): @@ -28,7 +54,35 @@ def resolve_harts(harts): return 4 if available >= 4 else 2 if available >= 2 else 1 -STARTUP_ASSEMBLY = r""" +def validate_harts(harts): + """Compatibility alias for validate_cores.""" + return validate_cores(harts) + + +def resolve_harts(harts): + """Compatibility alias for resolve_cores.""" + return resolve_cores(harts) + + + +# C and preprocessed assembly both accept the old compiler definition. +CORE_DEFINITIONS = r""" +#if defined(SCRATCHV_CORES) && defined(SCRATCHV_HARTS) && SCRATCHV_CORES != SCRATCHV_HARTS +#error "SCRATCHV_CORES and legacy SCRATCHV_HARTS disagree" +#endif +#ifndef SCRATCHV_CORES +#ifdef SCRATCHV_HARTS +#define SCRATCHV_CORES SCRATCHV_HARTS +#else +#define SCRATCHV_CORES 1 +#endif +#endif +#ifndef SCRATCHV_HARTS +#define SCRATCHV_HARTS SCRATCHV_CORES +#endif +""" + +STARTUP_ASSEMBLY = CORE_DEFINITIONS + r""" .section .data,"aw" .balign 4 sv_boot_ready: @@ -37,7 +91,7 @@ def resolve_harts(harts): .global sv_boot sv_boot: csrr a0, mhartid - li t0, SCRATCHV_HARTS + li t0, SCRATCHV_CORES bgeu a0, t0, .Lpark .option push .option norelax @@ -92,24 +146,24 @@ def resolve_harts(harts): # Included after the external runtime's types and its one-owner UART frame(). -WORKER_SOURCE = r''' +WORKER_SOURCE = CORE_DEFINITIONS + r''' typedef void (*sv_task_fn)(void *, usize, usize); -#if SCRATCHV_HARTS > 1 -static usize sv_generation, sv_done[SCRATCHV_HARTS]; +#if SCRATCHV_CORES > 1 +static usize sv_generation, sv_done[SCRATCHV_CORES]; static unsigned sv_workers_ready, sv_failure; static usize sv_count; static sv_task_fn sv_function; static void *sv_context; -/* QEMU virt CLINT/ACLINT MSWI: one 32-bit MSIP register per hart. +/* QEMU virt CLINT/ACLINT MSWI: one 32-bit MSIP register per core. * Publish RAM state before the MMIO doorbell. Notifications can coalesce: * generation/ready/done/failure, rather than the interrupt, own the state. */ -static void sv_notify(usize hart) { +static void sv_notify(usize core) { __asm__ volatile("fence rw,ow" ::: "memory"); - ((volatile u32 *)0x02000000UL)[hart]=1; + ((volatile u32 *)0x02000000UL)[core]=1; } -static void sv_clear_doorbell(usize hart) { - ((volatile u32 *)0x02000000UL)[hart]=0; +static void sv_clear_doorbell(usize core) { + ((volatile u32 *)0x02000000UL)[core]=0; __asm__ volatile("fence iorw,iorw" ::: "memory"); } static void sv_park_worker(void) { @@ -121,28 +175,28 @@ def resolve_harts(harts): unsigned failure = __atomic_load_n(&sv_failure, __ATOMIC_ACQUIRE); if (failure) frame(failure, 0); } -static void sv_execute(usize hart, usize count, sv_task_fn fn, void *context) { - usize width=count/SCRATCHV_HARTS, remainder=count%SCRATCHV_HARTS; - usize begin=hart*width+(hart +展开本轮实际执行的相关验证命令 + +在仓库根目录、已满足下方环境要求时执行。使用新的输出目录保留本轮 JUnit。 + +```bash +python -m pytest tests/test_w4_full_probe_contract.py tests/test_w5_inference_cli.py tests/test_w5_ci.py tests/test_w4_diagnostic.py tests/test_w5_deployment.py -q --junitxml=output/w5-cores-publish-20261030/interfaces.xml +``` + +```bash +python -m pytest tests/test_riscv_external_harts.py -q -k 'core or auto or manual or each_hart or invalid_hart or macro' --junitxml=output/w5-cores-publish-20261030/cores.xml +``` + +```bash +python -m pytest tests/test_qemu_generation.py -q --junitxml=output/w5-cores-publish-20261030/qemu-generation.xml +``` + +```bash +python -B -m pytest tests/test_inst_counter.py -q -p no:cacheprovider --junitxml=output/w5-cores-publish-20261030/main-integration.xml +``` + + + ## 环境与约定 +本文的 core 指 QEMU 模拟的 CPU 核。当前 CLI 使用 `--cores`，旧 `--harts` 继续兼容。 + 在仓库根目录执行命令，使用已安装 `.[llm-deploy]` 的 Ubuntu 24.04 / Python 3.12 虚拟环境。`conftest.py` 复用既有 CLI fixtures；Docker 测试复用 `tests/test_qemu_generation.py` 的小图生成器和原 `scripts/deploy_w5.sh`。 真实容器测试要求本机 Linux Docker Engine 正常运行、当前用户有访问权限，且能够构建 `deployment/w5/Dockerfile`；模型和输出通过临时目录挂载。宿主不需要额外安装 QEMU，镜像提供它及编译工具链。可通过 `SCRATCHV_W5_BASE_IMAGE` 指定可访问的 Ubuntu 24.04 基础镜像，默认使用 `public.ecr.aws/ubuntu/ubuntu:24.04`。首次构建需要下载依赖，后续复用 Docker 缓存。 -测试不会安装或启动 Docker，也不修改 WSL 配置。每次使用独立镜像标签，仅清理自己启动的客户端、容器及标签。中断诊断使用临时 QEMU 启动包装器的 `-S` 暂停真实 guest，不修改生产 harness。默认单核、手动 4 hart 和 padding 关闭均保持；4 hart 的恢复阶段在容器可用 CPU 自动选择为 4 时使用 `--harts auto`。 +测试不会安装或启动 Docker，也不修改 WSL 配置。每次使用独立镜像标签，仅清理自己启动的客户端、容器及标签。中断诊断使用临时 QEMU 启动包装器的 `-S` 暂停真实 guest，不修改生产 harness。默认单核、手动 4 core 和 padding 关闭均保持；4 core 的恢复阶段在容器可用 CPU 自动选择为 4 时仍使用旧别名 `--harts auto`，同时覆盖兼容调用。 ## 运行全部本目录测试 @@ -25,7 +72,7 @@ SCRATCHV_W5_DOCKER_TESTS=1 python -B -m pytest tests/llm-deploy/w5 -q -p no:cach | 文件 | 覆盖内容 | 通过标准 | | --- | --- | --- | | `test_report_persistence.py` | 权限、磁盘满、I/O/序列化失败；生成、超时、取消、数值/参考文本错误；中间保存恢复与下一次调用 | 原错误和退出码保留；保存失败不误报 PASS；信号处理恢复、清理执行；后续可正常保存 | -| `test_docker_cancellation.py` | 默认 1 / 手动 4 hart 的真实多 token、Docker 客户端 SIGINT/SIGTERM、同 ELF/bundle 恢复 | 正常输出 `[2,3,4,6]` / `2 3 4`，四步 ORT 误差为 0；SIGINT/SIGTERM 分别返回 130/143、报告 CANCELLED；容器、宿主 PID 和 QEMU 进程组消失；恢复结果一致 | +| `test_docker_cancellation.py` | 默认 1 / 手动 4 core 的真实多 token、Docker 客户端 SIGINT/SIGTERM、同 ELF/bundle 恢复 | 正常输出 `[2,3,4,6]` / `2 3 4`，四步 ORT 误差为 0；SIGINT/SIGTERM 分别返回 130/143、报告 CANCELLED；容器、宿主 PID 和 QEMU 进程组消失；恢复结果一致 | 仅报告保存边界： @@ -41,7 +88,9 @@ SCRATCHV_W5_DOCKER_TESTS=1 python -B -m pytest tests/llm-deploy/w5/test_docker_c Docker 测试在每项临时目录写入 `baseline.log`、`sigint.log`、`sigterm.log`、相应恢复日志和 `docker-cancellation-summary.json`；`deployment/<运行名>/report.json` 保留逐步数值及 QEMU 清理状态。清理断言来自独立的宿主进程和容器检查，失败后的兜底清理不能使测试变为通过。 -既有 CLI 与部署编排回归仍位于原位置，可按影响范围单独执行： +CLI 测试还检查 `--cores` 的默认、手动和自动选择，旧 `--harts` 别名，以及新旧参数冲突； +新旧核数字段的读取与构建复用保持一致；还检查 `parse_args().harts` 的读取、双向赋值及旧包装脚本继续执行。 +CLI 与部署编排测试仍位于原位置，可按影响范围单独执行： ```bash python -B -m pytest tests/test_w5_inference_cli.py -q -p no:cacheprovider @@ -53,6 +102,14 @@ python -B -m pytest tests/test_w5_deployment.py -q -p no:cacheprovider 后者覆盖引擎不可用、构建失败、参数透传，以及 info/build 阶段的取消清理；使用假引擎，不替代本目录的真实 Docker run 阶段检查。 +执行对象的构造、`asdict()` 和 `replace()` 兼容检查沿用已有运行时测试：确认新旧核数字段等值、 +没有私有字段进入结果、复制相互独立，替换核数时同步且拒绝无效值。下面只运行内存中的对象检查，不启动 QEMU： + +```bash +python -B -m pytest tests/test_riscv_external_harts.py -k 'executable_constructor or executable_asdict or executable_replace' -q -p no:cacheprovider +``` + + W5 专项的日志退出状态沿用现有工作流测试，直接执行 YAML 中的 Bash，并以短小假生成器 替代模型。需要 Bash、GNU tee、`/dev/full` 和 PyYAML，不需要 Docker 或模型资产： 生成或日志任一失败必须失败，分别保留退出码；状态文件本身写失败也不能成功。 @@ -62,7 +119,7 @@ python -B -m pytest tests/test_w4_workflows.py -k 'w5_deployment_pipeline or w5_ ``` 报告检查器的无模型测试覆盖有效、非零、缺失、不可读和无效的日志状态；即使生成报告 -为 PASS，也不能覆盖日志失败。旧版未传日志检查参数的调用仍可用： +为 PASS，也不能覆盖日志失败。还检查报告、构建记录和每次前向中的 `cores` / `harts` 字段：新格式、旧格式及混合格式均可读取，缺失、无效或冲突值必须失败。旧版未传日志检查参数的调用仍可用： ```bash python -B -m pytest tests/test_w5_ci.py -q -p no:cacheprovider diff --git a/tests/llm-deploy/w5/test_docker_cancellation.py b/tests/llm-deploy/w5/test_docker_cancellation.py index d0897e26..1724e051 100644 --- a/tests/llm-deploy/w5/test_docker_cancellation.py +++ b/tests/llm-deploy/w5/test_docker_cancellation.py @@ -185,7 +185,7 @@ def test_external_signals_reap_guest_and_reuse_build(tmp_path, docker_assets, ha "--max-new-tokens", "8", "--verify-onnx", "/assets/model.onnx", "--qemu", "/assets/qemu-control", "--matmul-policy", "blocked_fma", "--timeout", "90", "--total-timeout", "180", "--build-timeout", "180"] - # The single-hart baseline really exercises the CLI default. + # The single-core baseline really exercises the CLI default. hart_args = [] if harts == 1 else ["--harts", "4"] active = None evidence = {"harts": harts, "image": image, "cancellations": []} diff --git a/tests/test_qemu_generation.py b/tests/test_qemu_generation.py index cd2b214b..c327145c 100644 --- a/tests/test_qemu_generation.py +++ b/tests/test_qemu_generation.py @@ -1,4 +1,4 @@ -"""Generation ABI, live guest cancellation, and reusable 1/4-hart small graphs.""" +"""Generation ABI, live guest cancellation, and reusable 1/4-core small graphs.""" import json import os from pathlib import Path diff --git a/tests/test_riscv_external_harts.py b/tests/test_riscv_external_harts.py index a8ea458d..59cbe58a 100644 --- a/tests/test_riscv_external_harts.py +++ b/tests/test_riscv_external_harts.py @@ -1,5 +1,5 @@ """Real small-guest checks for external-runtime hart ownership and barriers.""" -from dataclasses import replace +from dataclasses import FrozenInstanceError, asdict, fields, replace import os import shutil import subprocess @@ -10,8 +10,8 @@ from scratchv.backend.tensor_c_codegen import TensorCArtifact, TensorCCodegen, TensorSpec from scratchv.ir.builder import IRBuilder from scratchv.ir.types import DataType, Value -from scratchv.runtime.riscv_external import build_external, run_external -from scratchv.runtime.riscv_harts import resolve_harts +from scratchv.runtime.riscv_external import ExternalExecutable, build_external, run_external +from scratchv.runtime.riscv_harts import UNSET, read_cores, resolve_cores, resolve_harts, select_cores from scratchv.runtime.riscv_tensor import RiscVToolchain, discover_toolchain from scratchv.runtime.weight_bundle import MIB, plan_guest_memory, write_weight_bundle @@ -367,3 +367,183 @@ def test_doorbells_before_wait_coalesce_and_spurious_wakes_do_not_repeat_tasks(t np.testing.assert_array_equal(output[:65], expected) assert report["passed"] and report["harts"] == harts np.testing.assert_array_equal(output[73:73+harts], 0) + + +@pytest.mark.parametrize("cores", [1, 2, 4]) +def test_cores_and_legacy_harts_select_the_same_layout(cores): + assert select_cores(cores) == select_cores(harts=cores) == select_cores(cores, harts=cores) + assert resolve_cores(cores) == resolve_harts(cores) + assert plan_guest_memory(cores=cores) == plan_guest_memory(harts=cores) + assert select_cores() == 1 + + +@pytest.mark.parametrize("bad", [None, True, False, 0, 3, 1.0, "2"]) +def test_invalid_cores_and_alias_conflicts_fail_before_build(tmp_path, bad): + for kwargs in ({"cores": bad}, {"cores": 4, "harts": bad}): + with pytest.raises(ValueError, match="cores"): + plan_guest_memory(**kwargs) + with pytest.raises(ValueError, match="cores"): + build_external(artifact(), tmp_path / "missing", tmp_path / "build", + RiscVToolchain(("unused",), "unused"), **kwargs) + assert not (tmp_path / "build").exists() + with pytest.raises(ValueError, match="disagree"): + select_cores(4, harts=1) + with pytest.raises(ValueError, match="disagree"): + select_cores("auto", harts=4, allow_auto=True) + + +@pytest.mark.parametrize("record,expected", [ + ({}, 1), ({"cores": 4}, 4), ({"harts": 4}, 4), + ({"cores": 2, "harts": 2}, 2), +]) +def test_core_metadata_reads_both_generations(record, expected): + assert read_cores(record) == expected + + +@pytest.mark.parametrize("record", [ + {"cores": 4, "harts": 1}, {"cores": 1, "harts": True}, + {"cores": False, "harts": 1}, {"cores": None}, + {"cores": "4"}, {"harts": 1.0}, {"cores": "auto"}, +]) +def test_core_metadata_rejects_invalid_or_conflicting_aliases(record): + with pytest.raises(ValueError, match="cores"): + read_cores(record) + + +def test_required_core_metadata_does_not_invent_a_single_core(): + with pytest.raises(ValueError, match="Missing cores"): + read_cores({}, default=UNSET) + + +def test_executable_constructor_and_dataclass_replace_accept_both_spellings(tmp_path): + positional = (tmp_path / "model.elf", tmp_path, {}, {}, (), None, (), + RiscVToolchain(("unused",), "unused"), {}) + original = ExternalExecutable(*positional, cores=4) + assert original.cores == original.harts == 4 + assert ExternalExecutable(*positional, harts=4) == original + assert ExternalExecutable(*positional, cores=4, harts=4) == original + assert ExternalExecutable(*positional).cores == 1 + assert replace(original, evidence={"changed": True}).cores == 4 + assert replace(original, cores=2).cores == replace(original, harts=2).cores == 2 + with pytest.raises(ValueError, match="disagree"): + ExternalExecutable(*positional, cores=4, harts=1) + with pytest.raises(ValueError, match="disagree"): + replace(original, cores=2, harts=1) + + +@pytest.mark.skipif(not os.environ.get("SCRATCHV_QEMU"), reason="explicit RV64 tools required") +@pytest.mark.parametrize("cores", [1, 4]) +def test_new_core_api_runs_small_graph_and_reuses_legacy_build_evidence(tmp_path, cores): + model = artifact() + write_weight_bundle((), (), tmp_path / "weights") + executable = build_external(model, tmp_path / "weights", tmp_path / "build", + discover_toolchain(), cores=cores) + assert executable.cores == executable.harts == cores + assert executable.evidence["cores"] == executable.evidence["harts"] == cores + first, report = run_external(executable, {}, tmp_path / "current", timeout=15) + assert report["passed"] and report["cores"] == report["harts"] == cores + evidence = dict(executable.evidence) + del evidence["cores"] + legacy, old_report = run_external(replace(executable, evidence=evidence), {}, + tmp_path / "legacy", timeout=15) + assert old_report["passed"] and old_report["cores"] == cores + np.testing.assert_array_equal(first, legacy) + expected = np.full(65, sum(range(1, 33)), dtype=np.int64) + expected[0] += 1000 + expected[:3] += 2000 + np.testing.assert_array_equal(first[:65], expected) + bad = replace(executable, evidence={**evidence, "cores": 2 if cores == 1 else 1}) + with pytest.raises(ValueError, match="disagree"): + run_external(bad, {}, tmp_path / "conflicting", timeout=1) + assert not (tmp_path / "conflicting").exists() + + +@pytest.mark.parametrize("definitions", [ + [], ["-DSCRATCHV_CORES=4"], ["-DSCRATCHV_HARTS=4"], + ["-DSCRATCHV_CORES=4", "-DSCRATCHV_HARTS=4"], +]) +def test_runtime_macro_aliases_support_existing_c_and_assembly(definitions): + from scratchv.runtime.riscv_harts import STARTUP_ASSEMBLY, WORKER_SOURCE + from scratchv.backend.tensor_c_codegen import _EXTERNAL_PRELUDE + compiler = shutil.which("cc") + if not compiler: + pytest.skip("host C preprocessor required") + expected = 4 if definitions else 1 + for source, language in ((STARTUP_ASSEMBLY, "assembler-with-cpp"), + (WORKER_SOURCE, "c"), (_EXTERNAL_PRELUDE, "c")): + completed = subprocess.run([compiler, "-E", "-P", "-x", language, *definitions, "-"], + input=source + "\nSCRATCHV_CORES SCRATCHV_HARTS\n", + text=True, capture_output=True, check=True) + assert completed.stdout.rstrip().endswith(f"{expected} {expected}") + + +def test_runtime_macro_alias_conflicts_are_rejected(): + from scratchv.runtime.riscv_harts import STARTUP_ASSEMBLY, WORKER_SOURCE + from scratchv.backend.tensor_c_codegen import _EXTERNAL_PRELUDE + compiler = shutil.which("cc") + if not compiler: + pytest.skip("host C preprocessor required") + for source, language in ((STARTUP_ASSEMBLY, "assembler-with-cpp"), + (WORKER_SOURCE, "c"), (_EXTERNAL_PRELUDE, "c")): + completed = subprocess.run([compiler, "-E", "-P", "-x", language, + "-DSCRATCHV_CORES=4", "-DSCRATCHV_HARTS=1", "-"], + input=source, text=True, capture_output=True) + assert completed.returncode != 0 and "disagree" in completed.stderr + + +@pytest.mark.parametrize("kwargs,expected", [ + ({}, 1), ({"cores": 1}, 1), ({"cores": 2}, 2), ({"cores": 4}, 4), + ({"harts": 4}, 4), ({"cores": 4, "harts": 4}, 4), +]) +def test_executable_asdict_preserves_public_core_and_legacy_fields(tmp_path, kwargs, expected): + original = ExternalExecutable( + tmp_path / "model.elf", tmp_path, {}, {}, (), None, (), + RiscVToolchain(("unused",), "unused"), {"labels": ["original"]}, **kwargs) + record = asdict(original) + assert record["cores"] == record["harts"] == expected + assert type(record["cores"]) is type(record["harts"]) is int + assert set(record) == { + "elf_path", "bundle_dir", "manifest", "layout", "inputs", + "output", "weights", "toolchain", "evidence", "cores", "harts", + } + record["evidence"]["labels"].append("copy") + assert original.evidence == {"labels": ["original"]} + shallow = {field.name: getattr(original, field.name) for field in fields(original)} + assert ExternalExecutable(**shallow) == original + shallow.pop("cores") + assert ExternalExecutable(**shallow) == original + with pytest.raises(FrozenInstanceError): + original.cores = 2 + with pytest.raises(FrozenInstanceError): + original.harts = 2 + + +@pytest.mark.parametrize("initial", [1, 2, 4]) +@pytest.mark.parametrize("updated", [1, 2, 4]) +def test_executable_replace_keeps_serialized_core_aliases_equal(tmp_path, initial, updated): + original = ExternalExecutable( + tmp_path / "model.elf", tmp_path, {}, {}, (), None, (), + RiscVToolchain(("unused",), "unused"), {}, cores=initial) + for change in ({"cores": updated}, {"harts": updated}, + {"cores": updated, "harts": updated}, + {"cores": initial, "harts": updated}, + {"cores": updated, "harts": initial}): + changed = replace(original, **change) + assert changed.cores == changed.harts == updated + copied = replace(changed, evidence={"modified": True}) + record = asdict(copied) + assert record["cores"] == record["harts"] == updated + assert original.cores == original.harts == initial + others = [value for value in (1, 2, 4) if value != initial] + with pytest.raises(ValueError, match="disagree"): + replace(original, cores=others[0], harts=others[1]) + + +@pytest.mark.parametrize("field", ["cores", "harts"]) +@pytest.mark.parametrize("bad", [None, True, False, 1.0, "4", 0, 3, "auto"]) +def test_executable_replace_rejects_invalid_count_before_alias_sync(tmp_path, field, bad): + original = ExternalExecutable( + tmp_path / "model.elf", tmp_path, {}, {}, (), None, (), + RiscVToolchain(("unused",), "unused"), {}, cores=1) + with pytest.raises(ValueError, match="cores"): + replace(original, **{field: bad}) diff --git a/tests/test_tensor_c_external.py b/tests/test_tensor_c_external.py index 3559ed04..1d6e2225 100644 --- a/tests/test_tensor_c_external.py +++ b/tests/test_tensor_c_external.py @@ -1365,7 +1365,7 @@ def test_column_panel_rv64_errors_recover_in_same_guest_workspace(tmp_path): from scratchv.runtime.riscv_tensor import discover_toolchain from scratchv.runtime.weight_bundle import write_weight_bundle - # Only the final row is poisoned, owned by a worker in the 4-hart 2x4 partition. + # Only the final row is poisoned, owned by a worker in the 4-core 2x4 partition. program, bindings, data = _panel_graph(k=129, n=17, m=17, transpose=True) bindings["panel_weight"].fill(2.0) data.fill(0.25) diff --git a/tests/test_w4_diagnostic.py b/tests/test_w4_diagnostic.py index 5a2d5c04..60521364 100644 --- a/tests/test_w4_diagnostic.py +++ b/tests/test_w4_diagnostic.py @@ -191,19 +191,19 @@ def test_diagnostic_cli_rejects_unbounded_timeout_before_creating_output(tmp_pat @pytest.mark.parametrize("policy,kernel_calls,flags", [ ("sequential", True, []), ("sequential", False, ["--no-kernel-calls"]), - ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--kernel-calls", "--harts", "4"]), + ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--kernel-calls", "--cores", "4"]), ("blocked_fma", False, ["--matmul-policy", "blocked_fma", "--no-kernel-calls"]), - ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--harts", "auto"]), + ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--cores", "auto"]), ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--matmul-tile", "1x4"]), - ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--matmul-tile", "2x4", "--harts", "4"]), + ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--matmul-tile", "2x4", "--cores", "4"]), ("blocked_fma", False, ["--matmul-policy", "blocked_fma", "--matmul-tile", "1x8", - "--no-kernel-calls", "--harts", "auto"]), + "--no-kernel-calls", "--cores", "auto"]), ("blocked_fma", True, ["--matmul-policy", "blocked_fma", "--matmul-rhs-layout", "column-panel", - "--matmul-tile", "2x4", "--harts", "4"]), + "--matmul-tile", "2x4", "--cores", "4"]), ("blocked_fma", False, ["--matmul-policy", "blocked_fma", "--matmul-rhs-layout", "column-panel", - "--no-kernel-calls", "--harts", "auto"]), + "--no-kernel-calls", "--cores", "auto"]), ("sequential", True, ["--qemu-workspace-layout", "two-buffer-pad-4k"]), - ("sequential", True, ["--qemu-workspace-layout", "two-buffer-pad-4k", "--harts", "4"]), + ("sequential", True, ["--qemu-workspace-layout", "two-buffer-pad-4k", "--cores", "4"]), ]) def test_diagnostic_codegen_policy_reaches_build_and_failed_build_never_accepts_gate( tmp_path, monkeypatch, policy, kernel_calls, flags): @@ -240,7 +240,7 @@ def test_diagnostic_codegen_policy_reaches_build_and_failed_build_never_accepts_ monkeypatch.setattr(onnx_parser, "ONNXParser", lambda: parser) monkeypatch.setattr(diagnostic, "discover_toolchain", lambda *a: object()) generated = [] - requested_harts = flags[flags.index("--harts") + 1] if "--harts" in flags else "1" + requested_harts = flags[flags.index("--cores") + 1] if "--cores" in flags else "1" requested_harts = "auto" if requested_harts == "auto" else int(requested_harts) expected_harts = 2 if requested_harts == "auto" else requested_harts probes = [] @@ -251,7 +251,7 @@ def available_cpus(pid): monkeypatch.setattr(os, "sched_getaffinity", available_cpus, raising=False) def stopped_build(artifact, *a, **kw): - assert type(kw["harts"]) is int and kw["harts"] == expected_harts + assert type(kw["cores"]) is int and kw["cores"] == expected_harts generated.append(artifact) raise RuntimeError("intentional unit-test build stop") @@ -276,7 +276,7 @@ def stopped_build(artifact, *a, **kw): assert report["matmul_rhs_layout"] == report["codegen"]["matmul_rhs_layout"] == requested_rhs_layout assert report["codegen"]["matmul_rhs_panels"] == list(artifact.matmul_rhs_panels) assert report["harts_requested"] == requested_harts - assert report["harts"] == expected_harts + assert report["cores"] == expected_harts assert report["qemu_workspace_layout_requested"] == requested_layout assert report["codegen"]["qemu_workspace_layout"] == requested_layout assert probes == ([0] if requested_harts == "auto" else []) diff --git a/tests/test_w4_full_probe_contract.py b/tests/test_w4_full_probe_contract.py index a1852076..deff5236 100644 --- a/tests/test_w4_full_probe_contract.py +++ b/tests/test_w4_full_probe_contract.py @@ -370,7 +370,7 @@ def test_reused_executable_rejects_invalid_harts(tmp_path, monkeypatch, harts): path, record = _saved_executable(tmp_path, monkeypatch) record["harts"] = record["evidence"]["harts"] = harts path.write_text(json.dumps(record), encoding="utf-8") - with pytest.raises(ValueError, match="hart"): + with pytest.raises(ValueError, match="core|hart"): run.load_executable(path) @@ -378,7 +378,7 @@ def test_reused_executable_rejects_hart_count_changed_without_rebuild(tmp_path, path, record = _saved_executable(tmp_path, monkeypatch) record["harts"] = 4 path.write_text(json.dumps(record), encoding="utf-8") - with pytest.raises(ValueError, match="hart"): + with pytest.raises(ValueError, match="core|hart"): run.load_executable(path) @@ -416,15 +416,15 @@ def available_cpus(pid): expected = 2 if requested == "auto" else requested or 1 built = [] def stopped_build(artifact, *a, **kw): - built.append(kw["harts"]) - assert type(kw["harts"]) is int and kw["harts"] == expected + built.append(kw["cores"]) + assert type(kw["cores"]) is int and kw["cores"] == expected assert artifact.qemu_workspace_layout == layout assert artifact.mean_policy == mean_policy assert artifact.nonlinear_policy == nonlinear_policy raise RuntimeError("intentional tiny-graph build stop") monkeypatch.setattr(run, "build_external", stopped_build) if requested is not None: - arguments += ["--harts", str(requested)] + arguments += ["--cores" if layout == "none" else "--harts", str(requested)] if layout != "none": arguments += ["--qemu-workspace-layout", layout] arguments += ["--mean-policy", mean_policy, "--nonlinear-policy", nonlinear_policy] @@ -433,7 +433,8 @@ def stopped_build(artifact, *a, **kw): assert built == [expected] assert probes == ([0] if requested == "auto" else []) assert saved["harts_requested"] == (requested if requested is not None else 1) - assert saved["harts"] == expected + assert saved["cores"] == saved["harts"] == expected + assert saved["cores_requested"] == saved["harts_requested"] assert saved["qemu_workspace_layout_requested"] == saved["codegen"]["qemu_workspace_layout"] == layout assert saved["mean_policy"] == saved["codegen"]["mean_policy"] == mean_policy assert saved["nonlinear_policy"] == saved["codegen"]["nonlinear_policy"] == nonlinear_policy @@ -457,7 +458,7 @@ def test_full_cli_auto_reuse_requires_resolved_harts_to_match_build(tmp_path, mo if built_harts == 4: assert result == 0 and saved["status"] == "BUILD_ONLY" else: - assert result == 1 and "hart count" in saved["error"] + assert result == 1 and "core count" in saved["error"] assert saved["stage"] == "load_existing_build" @@ -510,7 +511,7 @@ def test_full_cli_matmul_tile_reaches_tiny_graph_build(tmp_path, monkeypatch, ti def stopped_build(artifact, *a, **kw): generated.append(artifact) - assert kw["harts"] == 1 and artifact.qemu_workspace_layout == "none" + assert kw["cores"] == 1 and artifact.qemu_workspace_layout == "none" raise RuntimeError("intentional tile build stop") monkeypatch.setattr(run, "build_external", stopped_build) @@ -606,7 +607,7 @@ def test_full_cli_rhs_layout_reaches_tiny_graph_build(tmp_path, monkeypatch, rhs def stopped_build(artifact, *a, **kw): generated.append(artifact) - assert kw["harts"] == 1 and artifact.qemu_workspace_layout == "none" + assert kw["cores"] == 1 and artifact.qemu_workspace_layout == "none" raise RuntimeError("intentional RHS layout build stop") monkeypatch.setattr(run, "build_external", stopped_build) @@ -827,3 +828,66 @@ def test_w4_unknown_nonlinear_policy_fails_before_output(tmp_path, policy): run.main(["--model-dir", str(tmp_path), "--output-dir", str(target), "--nonlinear-policy", policy]) assert exc.value.code == 2 and not target.exists() + + +@pytest.mark.parametrize("cores", [1, 4]) +@pytest.mark.parametrize("top_names,evidence_names", [ + (("cores",), ("cores",)), (("harts",), ("harts",)), + (("cores",), ("harts",)), (("harts",), ("cores",)), + (("cores", "harts"), ("cores", "harts")), +]) +def test_core_metadata_aliases_roundtrip_without_rewriting_source( + tmp_path, monkeypatch, cores, top_names, evidence_names): + path, record = _saved_executable(tmp_path, monkeypatch) + record.update({name: cores for name in top_names}) + record["evidence"].update({name: cores for name in evidence_names}) + record["layout"] = plan_guest_memory(input_bytes=2048, output_bytes=256 * 151936 * 4, + workspace_bytes=64, cores=cores) + original = json.dumps(record) + path.write_text(original) + executable = run.load_executable(path) + assert executable.cores == executable.harts == cores + assert path.read_text() == original + out = tmp_path / "resaved" + out.mkdir() + run.save_executable(executable, out) + saved = json.loads((out / "executable.json").read_text()) + assert saved["cores"] == saved["harts"] == cores + assert saved["evidence"]["cores"] == saved["evidence"]["harts"] == cores + reloaded = run.load_executable(out / "executable.json") + assert reloaded.cores == cores and reloaded.elf_path == executable.elf_path + + +@pytest.mark.parametrize("container", ["top", "evidence"]) +@pytest.mark.parametrize("fields", [ + {"cores": 4, "harts": 1}, {"cores": 1, "harts": True}, + {"cores": None}, {"cores": True}, {"cores": "1"}, {"cores": "auto"}, {"cores": 3}, +]) +def test_core_metadata_rejects_conflicts_and_invalid_values(tmp_path, monkeypatch, container, fields): + path, record = _saved_executable(tmp_path, monkeypatch) + (record if container == "top" else record["evidence"]).update(fields) + path.write_text(json.dumps(record)) + with pytest.raises(ValueError, match="core|hart"): + run.load_executable(path) + + +def test_core_metadata_without_either_field_stays_single_core(tmp_path, monkeypatch): + path, _ = _saved_executable(tmp_path, monkeypatch) + executable = run.load_executable(path) + assert executable.cores == executable.harts == 1 + assert executable.evidence["cores"] == executable.evidence["harts"] == 1 + + +@pytest.mark.parametrize("entry", ["full", "diagnostic"]) +@pytest.mark.parametrize("flags", [ + ["--cores", "1", "--harts", "4"], ["--harts", "4", "--cores", "1"], + ["--cores", "auto", "--harts", "4"], ["--cores", "3"], +]) +def test_w4_core_alias_conflict_stops_before_output(tmp_path, entry, flags): + from probes.w4_qwen3_full import diagnostic + module = run if entry == "full" else diagnostic + output = tmp_path / "output" + with pytest.raises(SystemExit) as exc: + module.main(["--model-dir", str(tmp_path / "model"), "--output-dir", str(output), *flags]) + assert exc.value.code == 2 + assert not output.exists() diff --git a/tests/test_w5_ci.py b/tests/test_w5_ci.py index c6adf267..ff27baa3 100644 --- a/tests/test_w5_ci.py +++ b/tests/test_w5_ci.py @@ -236,3 +236,38 @@ def test_missing_run_report_produces_failure_artifact(gate, tmp_path): "--exit-code-file", str(exit_path), "--output", str(output)]) == 1 result = json.loads(output.read_text()) assert result["status"] == "FAIL" and result["passed"] is False + + +@pytest.mark.parametrize("form", ["cores", "harts", "both", "mixed"]) +def test_core_field_compatibility_at_report_build_and_each_forward(gate, report, reference, form): + records = [report, report["build"], *report["runs"]] + for index, record in enumerate(records): + count = record.pop("harts") + selected = ("cores" if index % 2 else "harts") if form == "mixed" else form + if selected in ("cores", "both"): + record["cores"] = count + if selected in ("harts", "both"): + record["harts"] = count + assert gate.validate_report(report, reference) is None + + +@pytest.mark.parametrize("location", ["top", "build", "run"]) +@pytest.mark.parametrize("fields", [ + {}, + {"cores": 1}, + {"cores": True}, + {"cores": 4.0}, + {"cores": "4"}, + {"cores": None}, + {"cores": 4, "harts": 1}, + {"cores": 1, "harts": 4}, + {"cores": 4, "harts": 4.0}, + {"cores": 4, "harts": None}, +]) +def test_missing_invalid_or_conflicting_core_fields_cannot_pass( + gate, report, reference, location, fields): + target = report if location == "top" else report["build"] if location == "build" else report["runs"][7] + target.pop("harts") + target.update(fields) + with pytest.raises(ValueError): + gate.validate_report(report, reference) diff --git a/tests/test_w5_deployment.py b/tests/test_w5_deployment.py index b9f038b9..09c4464f 100644 --- a/tests/test_w5_deployment.py +++ b/tests/test_w5_deployment.py @@ -93,10 +93,11 @@ def test_build_failure_never_starts_generation(setup): assert [call[0] for call in recorded(calls)] == ["info", "build"] -def test_arguments_mounts_and_invoking_user_are_preserved(setup): +@pytest.mark.parametrize("option", ["--cores", "--harts"]) +def test_arguments_mounts_and_invoking_user_are_preserved(setup, option): env, calls, output = setup args = ["--model", "/assets/model.onnx", "--tokenizer-dir", "/assets/tokenizer", - "--prompt", "quotes '$HOME'; * stay literal", "--harts", "auto", + "--prompt", "quotes '$HOME'; * stay literal", option, "auto", "--output-dir", "/output/second run"] result = run(env, *args) assert result.returncode == 0, result.stderr diff --git a/tests/test_w5_inference_cli.py b/tests/test_w5_inference_cli.py index 49fa90b6..a0d238c4 100644 --- a/tests/test_w5_inference_cli.py +++ b/tests/test_w5_inference_cli.py @@ -55,13 +55,13 @@ def runtime(cli, monkeypatch): tokenizer = Tokenizer() executable = SimpleNamespace( evidence={"matmul_policy": "sequential", "qemu_workspace_layout": "none"}, - toolchain=SimpleNamespace(cc="unused-cc", qemu="unused-qemu"), harts=1, + toolchain=SimpleNamespace(cc="unused-cc", qemu="unused-qemu"), cores=1, ) state = SimpleNamespace(tokenizer=tokenizer, exe=executable, prepared=[], adapters=[], - saved=[], fma=[], failure=None, oracle=None, harts=[]) + saved=[], fma=[], failure=None, oracle=None, cores=[]) - def prepare(args, out, harts): - state.prepared.append((args, out, harts)) + def prepare(args, out, cores): + state.prepared.append((args, out, cores)) return executable class Adapter: @@ -89,7 +89,7 @@ def fma(*args, **kwargs): monkeypatch.setattr(cli, "QemuForward", Adapter) monkeypatch.setattr(cli, "run_fma_probe", fma) monkeypatch.setattr(cli, "oracle_session", lambda path: SimpleNamespace(run=lambda names, feed: [state.oracle])) - monkeypatch.setattr(cli, "resolve_harts", lambda requested: state.harts.append(requested) or (4 if requested == "auto" else requested)) + monkeypatch.setattr(cli, "resolve_cores", lambda requested: state.cores.append(requested) or (4 if requested == "auto" else requested)) return state @@ -101,9 +101,10 @@ def report(tmp_path): return json.loads((tmp_path / "run/report.json").read_text(encoding="utf-8")) -@pytest.mark.parametrize("requested,resolved", [(None, 1), ("2", 2), ("4", 4), ("auto", 4)]) -def test_raw_multitoken_one_build_one_adapter_and_harts_resolved_once(cli, runtime, tmp_path, requested, resolved): - options = [] if requested is None else ["--harts", requested] +@pytest.mark.parametrize("flag", ["--cores", "--harts"]) +@pytest.mark.parametrize("requested,resolved", [(None, 1), ("1", 1), ("2", 2), ("4", 4), ("auto", 4)]) +def test_raw_multitoken_one_build_one_adapter_and_cores_resolved_once(cli, runtime, tmp_path, flag, requested, resolved): + options = [] if requested is None else [flag, requested] assert cli.main(argv(tmp_path, "--prompt", " raw text ", "--max-new-tokens", "3", *options)) == 0 value = report(tmp_path) assert value["status"] == "PASS" and value["passed"] @@ -111,8 +112,9 @@ def test_raw_multitoken_one_build_one_adapter_and_harts_resolved_once(cli, runti assert value["qemu_workspace_layout"] == "none" and value["mean_policy"] == "sequential" assert value["nonlinear_policy"] == "libm" assert value["executable_sha256"] == "fixture-metadata-identity" - assert value["harts"] == resolved - assert runtime.harts == ["auto" if requested == "auto" else int(requested or 1)] + assert value["cores"] == value["harts"] == resolved + assert value["cores_requested"] == value["harts_requested"] + assert runtime.cores == ["auto" if requested == "auto" else int(requested or 1)] assert len(runtime.prepared) == len(runtime.saved) == len(runtime.adapters) == 1 assert len(runtime.fma) == 1 assert runtime.adapters[0].exe is runtime.exe @@ -199,24 +201,24 @@ def matching_reuse_metadata(cli, monkeypatch, path): monkeypatch.setattr(cli, "sha256_file", lambda metadata: "matching") -@pytest.mark.parametrize("changes,harts", [ - ({"harts": 4}, 1), +@pytest.mark.parametrize("changes,cores", [ + ({"cores": 4}, 1), ({"matmul_policy": "blocked_fma"}, 1), ({"matmul_tile": "2x4"}, 1), ({"matmul_rhs_layout": "column-panel"}, 1), ({"qemu_workspace_layout": "two-buffer-pad-4k"}, 1), ]) -def test_reuse_rejects_changed_configuration_or_padding(cli, monkeypatch, tmp_path, changes, harts): +def test_reuse_rejects_changed_configuration_or_padding(cli, monkeypatch, tmp_path, changes, cores): args = cli.parse_args(["--reuse-build", str(tmp_path / "old/executable.json"), "--tokenizer-dir", "tokens", "--output-dir", str(tmp_path / "new"), "--prompt", "text"]) matching_reuse_metadata(cli, monkeypatch, args.reuse_build) evidence = {"matmul_policy": "sequential", "matmul_tile": "1x4", "matmul_rhs_layout": "original", "qemu_workspace_layout": "none"} - evidence.update({k: v for k, v in changes.items() if k != "harts"}) - exe = SimpleNamespace(harts=changes.get("harts", 1), evidence=evidence) + evidence.update({k: v for k, v in changes.items() if k != "cores"}) + exe = SimpleNamespace(cores=changes.get("cores", 1), evidence=evidence) monkeypatch.setattr(cli, "load_executable", lambda path: exe) with pytest.raises(ValueError, match="differs"): - cli.prepare_executable(args, tmp_path, harts) + cli.prepare_executable(args, tmp_path, cores) def test_matching_reuse_returns_same_executable_without_compile(cli, monkeypatch, tmp_path): @@ -224,7 +226,7 @@ def test_matching_reuse_returns_same_executable_without_compile(cli, monkeypatch "--tokenizer-dir", "tokens", "--output-dir", str(tmp_path / "new"), "--prompt", "text"]) matching_reuse_metadata(cli, monkeypatch, args.reuse_build) - exe = SimpleNamespace(harts=1, evidence={"matmul_policy": "sequential"}) + exe = SimpleNamespace(cores=1, evidence={"matmul_policy": "sequential"}) monkeypatch.setattr(cli, "load_executable", lambda path: exe) monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("must not rebuild")) assert cli.prepare_executable(args, tmp_path, 1) is exe @@ -327,7 +329,7 @@ def test_verified_legacy_report_cannot_overwrite_native_failed_audit(cli, monkey @dataclass class Executable: evidence: dict - harts: int = 1 + cores: int = 1 metadata = tmp_path / "old/executable.json" matching_reuse_metadata(cli, monkeypatch, metadata) @@ -588,15 +590,15 @@ def guarded(prepared): (None, 8, 1), ("1", 8, 1), ("2", 8, 2), ("4", 1, 4), ("auto", 1, 1), ("auto", 3, 2), ("auto", 12, 4), ]) -def test_cli_uses_real_hart_resolver_without_changing_default(cli, runtime, tmp_path, monkeypatch, requested, cpu_count, resolved): +def test_cli_uses_real_core_resolver_without_changing_default(cli, runtime, tmp_path, monkeypatch, requested, cpu_count, resolved): from scratchv.runtime import riscv_harts affinity_reads = [] monkeypatch.setattr(riscv_harts.os, "sched_getaffinity", lambda pid: affinity_reads.append(pid) or set(range(cpu_count))) - monkeypatch.setattr(cli, "resolve_harts", riscv_harts.resolve_harts) - options = [] if requested is None else ["--harts", requested] + monkeypatch.setattr(cli, "resolve_cores", riscv_harts.resolve_cores) + options = [] if requested is None else ["--cores", requested] assert cli.main(argv(tmp_path, "--build-only", *options)) == 0 value = report(tmp_path) - assert value["harts"] == resolved and runtime.prepared[0][2] == resolved + assert value["cores"] == resolved and runtime.prepared[0][2] == resolved assert value["qemu_workspace_layout"] == "none" and value["qemu_forwards"] == 0 assert affinity_reads == ([0] if requested == "auto" else []) @@ -613,7 +615,7 @@ def test_w5_reuse_mean_policy_requires_explicit_matching_build(cli, monkeypatch, evidence = {"matmul_policy": "sequential"} if built is not None: evidence["mean_policy"] = built - exe = SimpleNamespace(harts=1, evidence=evidence) + exe = SimpleNamespace(cores=1, evidence=evidence) monkeypatch.setattr(cli, "load_executable", lambda path: exe) monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("reuse must not compile")) args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", @@ -628,7 +630,7 @@ def test_w5_reuse_mean_policy_requires_explicit_matching_build(cli, monkeypatch, def test_w5_reuse_mean_evidence_and_sibling_report_must_agree(cli, monkeypatch, tmp_path): metadata = tmp_path / "old/executable.json" matching_reuse_metadata(cli, monkeypatch, metadata) - exe = SimpleNamespace(harts=1, evidence={"matmul_policy": "sequential", "mean_policy": "two-four-lane-last-axis"}) + exe = SimpleNamespace(cores=1, evidence={"matmul_policy": "sequential", "mean_policy": "two-four-lane-last-axis"}) monkeypatch.setattr(cli, "load_executable", lambda path: exe) args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--mean-policy", "two-four-lane-last-axis"]) @@ -660,7 +662,7 @@ def build(artifact, bundle, out, tools, **kwargs): args = cli.parse_args(argv(tmp_path, "--build-only", "--mean-policy", policy)) exe = cli.prepare_executable(args, tmp_path, 1) assert len(built) == 1 and built[0].mean_policy == policy - assert exe.evidence["mean_policy"] == policy and exe.harts == 1 + assert exe.evidence["mean_policy"] == policy and exe.cores == 1 assert built[0].matmul_policy == "sequential" and built[0].qemu_workspace_layout == "none" @@ -682,7 +684,7 @@ def test_w5_reuse_nonlinear_policy_requires_explicit_matching_build(cli, monkeyp evidence = {"matmul_policy": "sequential"} if built is not None: evidence["nonlinear_policy"] = built - exe = SimpleNamespace(harts=1, evidence=evidence) + exe = SimpleNamespace(cores=1, evidence=evidence) monkeypatch.setattr(cli, "load_executable", lambda path: exe) monkeypatch.setattr(cli, "build_external", lambda *a, **kw: pytest.fail("reuse must not compile")) args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", @@ -697,7 +699,7 @@ def test_w5_reuse_nonlinear_policy_requires_explicit_matching_build(cli, monkeyp def test_w5_reuse_nonlinear_evidence_and_sibling_report_must_agree(cli, monkeypatch, tmp_path): metadata = tmp_path / "old/executable.json" matching_reuse_metadata(cli, monkeypatch, metadata) - exe = SimpleNamespace(harts=1, evidence={"matmul_policy": "sequential", "nonlinear_policy": "fp32-reference"}) + exe = SimpleNamespace(cores=1, evidence={"matmul_policy": "sequential", "nonlinear_policy": "fp32-reference"}) monkeypatch.setattr(cli, "load_executable", lambda path: exe) args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", "--output-dir", str(tmp_path / "out"), "--prompt", "text", "--nonlinear-policy", "fp32-reference"]) @@ -729,7 +731,7 @@ def build(artifact, bundle, out, tools, **kwargs): args = cli.parse_args(argv(tmp_path, "--build-only", "--nonlinear-policy", policy)) exe = cli.prepare_executable(args, tmp_path, 1) assert len(built) == 1 and built[0].nonlinear_policy == policy - assert exe.evidence["nonlinear_policy"] == policy and exe.harts == 1 + assert exe.evidence["nonlinear_policy"] == policy and exe.cores == 1 assert built[0].matmul_policy == "sequential" and built[0].qemu_workspace_layout == "none" @@ -755,3 +757,93 @@ def test_finite_extreme_logits_have_json_safe_diagnostic_margin(cli): assert comparison["qemu_token_id"] == comparison["ort_token_id"] == 3 assert comparison["ort_top2_margin"] == 2.0 * float(np.finfo(np.float32).max) json.dumps(comparison, allow_nan=False) + + +@pytest.mark.parametrize("flags", [ + ["--cores", "4", "--harts", "4"], ["--harts", "auto", "--cores", "auto"], + ["--harts", "1", "--harts", "4", "--cores", "4"], + ["--cores", "1", "--cores", "4", "--harts", "4"], +]) +def test_equal_cli_aliases_resolve_once(cli, runtime, tmp_path, flags): + assert cli.main(argv(tmp_path, "--build-only", *flags)) == 0 + saved = report(tmp_path) + assert saved["cores"] == saved["harts"] == 4 + assert len(runtime.cores) == len(runtime.prepared) == 1 + + +@pytest.mark.parametrize("flags", [ + ["--cores", "4", "--harts", "1"], ["--harts", "4", "--cores", "1"], + ["--cores", "auto", "--harts", "4"], ["--harts", "auto", "--cores", "1"], + ["--harts", "1", "--harts", "4", "--cores", "1"], + ["--cores", "1", "--cores", "4", "--harts", "1"], + ["--cores", "3"], ["--harts", "0"], +]) +def test_core_alias_conflict_or_invalid_count_has_no_side_effects(cli, runtime, tmp_path, flags): + with pytest.raises(SystemExit) as exc: + cli.main(argv(tmp_path, "--build-only", *flags)) + assert exc.value.code == 2 + assert not (tmp_path / "run").exists() + assert runtime.prepared == runtime.cores == [] + + +@pytest.mark.parametrize("fields", [{"cores": 4}, {"harts": 4}, {"cores": 1, "harts": 4}]) +def test_reused_report_rejects_inconsistent_core_aliases(cli, monkeypatch, tmp_path, fields): + metadata = tmp_path / "old/executable.json" + matching_reuse_metadata(cli, monkeypatch, metadata) + previous = metadata.with_name("report.json") + record = json.loads(previous.read_text()) + record.update(fields) + previous.write_text(json.dumps(record)) + monkeypatch.setattr(cli, "load_executable", lambda path: SimpleNamespace( + cores=1, evidence={"matmul_policy": "sequential"})) + args = cli.parse_args(["--reuse-build", str(metadata), "--tokenizer-dir", "tokens", + "--output-dir", str(tmp_path / "new"), "--prompt", "text"]) + with pytest.raises(ValueError, match="core|hart"): + cli.prepare_executable(args, tmp_path, cores=1) + + +@pytest.mark.parametrize("flag", ["--cores", "--harts"]) +@pytest.mark.parametrize("first,last,expected", [("1", "4", 4), ("4", "1", 1), ("1", "auto", 4)]) +def test_repeated_core_option_preserves_argparse_last_value(cli, runtime, tmp_path, flag, first, last, expected): + assert cli.main(argv(tmp_path, "--build-only", flag, first, flag, last)) == 0 + saved = report(tmp_path) + requested = "auto" if last == "auto" else int(last) + assert saved["cores"] == saved["harts"] == expected + assert saved["cores_requested"] == saved["harts_requested"] == requested + assert runtime.cores == [requested] + assert len(runtime.prepared) == 1 + + +@pytest.mark.parametrize("flag", ["--cores", "--harts"]) +@pytest.mark.parametrize("requested", [None, "1", "2", "4", "auto"]) +def test_parse_args_retains_legacy_harts_attribute_and_string_type(cli, tmp_path, flag, requested): + flags = [] if requested is None else [flag, requested] + args = cli.parse_args(argv(tmp_path, "--build-only", *flags)) + assert args.cores == args.harts == (requested or "1") + assert type(args.cores) is type(args.harts) is str + + +def test_parsed_core_attributes_stay_synchronized_when_callers_update_either_name(cli, tmp_path): + args = cli.parse_args(argv(tmp_path, "--build-only")) + args.harts = "4" + assert args.cores == args.harts == "4" + args.cores = "auto" + assert args.harts == args.cores == "auto" + # W4 resolves its selected count on the namespace after parsing. + args.cores = 4 + assert args.harts == args.cores == 4 + + +@pytest.mark.parametrize("value,expected", [("4", 4), ("auto", 4)]) +def test_old_wrapper_harts_assignment_reaches_core_selection_once( + cli, runtime, monkeypatch, tmp_path, value, expected): + args = cli.parse_args(argv(tmp_path, "--build-only")) + args.harts = value + monkeypatch.setattr(cli, "parse_args", lambda _argv: args) + assert cli.main([]) == 0 + saved = report(tmp_path) + assert saved["cores"] == saved["harts"] == expected + assert saved["cores_requested"] == saved["harts_requested"] == ( + value if value == "auto" else int(value)) + assert runtime.cores == [value if value == "auto" else int(value)] + assert len(runtime.prepared) == 1 From e72cd2dde55e7a84e653c651ba3012e0d24428ca Mon Sep 17 00:00:00 2001 From: yuki-328 Date: Fri, 30 Oct 2026 22:04:26 +0800 Subject: [PATCH 7/7] docs(w5): simplify results and centralize environment setup Organize the W5 report around changes, measured results and reproduction. Keep timing and accuracy explanations below the results table, and move Linux/container preparation into the shared reproduction guide. --- docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md | 358 +++++++++++++- docs/llm-deploy-v1.0/W5/README.md | 546 +++++++-------------- 2 files changed, 526 insertions(+), 378 deletions(-) diff --git a/docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md b/docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md index 91948318..f251d5a7 100644 --- a/docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md +++ b/docs/llm-deploy-v1.0/LINUX_REPRODUCTION.md @@ -1,14 +1,16 @@ # Linux 交付与独立复现约定 -W1/W2 的正式交付、第二人复现和 CI 使用 **Ubuntu 24.04 x86_64 / Python 3.12**。 -Linux 是编译器、ORT、IR 解释器和 QEMU 的宿主环境；目标程序仍为 RV64GC/LP64D 裸机程序。 +本文集中说明 LLM-Deploy 的 Linux 环境、固定模型资产和复现准备。W1/W2 的既有流程见第 1～4 节，W5 的本机运行与容器准备见第 5～6 节。 + +使用 **Ubuntu 24.04 x86_64 / Python 3.12**。环境检查通过只代表准备完成，不等于模型数值或生成结果已经通过。 +Linux 是编译器、ORT、IR 解释器和 QEMU 的运行环境；目标程序仍为 RV64GC/LP64D 裸机程序。 ## 1. 安装与运行前提 在仓库根目录、已选定的 Python 3.12 环境中安装一次： ```bash -pip install -v -e ".[llm-deploy]" +python -m pip install -v -e ".[llm-deploy]" ``` 所有 Python 依赖统一声明在 [pyproject.toml](../../pyproject.toml) 的 `llm-deploy` 中， @@ -81,3 +83,353 @@ W2 入口只读取上述本地资产并执行测试，不提供资产下载功 ```bash python scripts/check_linux_repro_docs.py --check-bash ``` + + + +## 5. W5 本机运行环境与模型准备 + +本节供 W5 的三条固定生成用例和单条提示词运行使用。已有可用的 W4/W5 环境与同版本模型时，可直接回到 [W5 运行命令](W5/README.md#如何复现)，不重复创建环境、下载资产或运行模型。 + +| 准备项 | 要求与入口 | +|---|---| +| Python 与依赖 | 使用 Python 3.12 虚拟环境；从仓库根目录按第 1 节安装 `.[llm-deploy]`，始终使用该环境的 `python` | +| 系统工具 | Git、Git LFS、`qemu-system-riscv64`；Zig 由项目 Python 依赖提供，运行入口自动发现 | +| 固定模型 | `Qwen/Qwen3-0.6B` 的 FP32 ONNX、外置权重和同版本 tokenizer；本机准备方式见第 3 节，独立容器准备见第 6 节 | +| 资产位置 | 本机使用 `assets/qwen3-0.6b/onnx/model.onnx`、同目录权重文件及 `assets/qwen3-0.6b/tokenizer/` | +| 资源 | 复现历史 4 core 配置需至少 4 个可用逻辑 CPU；准备后至少 10 GiB 可用内存、10 GiB 剩余磁盘，初始建议预留 25 GiB 磁盘 | +| 输出 | 使用新的 `output/` 子目录；已有 ELF/bundle 可在原环境复用，不能把包含本机绝对路径的构建描述直接搬进容器 | + +只检查工具是否可用时，从已激活的虚拟环境执行： + +```bash +python --version +python -m pip --version +python -m ziglang version +qemu-system-riscv64 --version +git lfs version +``` + +这些命令不执行模型。第 3 节的 ORT 验证命令会运行模型，仅在需要该项验证时执行，不能把它当成每次生成前都要重复的环境准备。 + + + +
+历史实跑的模型快照与工具版本 + +- 模型快照：`c1899de289a04d12100db370d81485cdf75e47ca`，见 + [模型清单](../../probes/w1_qwen3_export/manifest.json)及[冻结参考](../../probes/w5_generation/references.json)。 +- QEMU 8.2.2（Ubuntu 包 `1:8.2.2+ds-0ubuntu1.18`），Zig 0.14.1； + 目标为 `riscv64 / RV64GC / LP64D / QEMU virt` 裸机，数学库为 Zig 随附 musl。 +- Python 3.12.3、NumPy 2.2.6、ONNX Runtime 1.22.1、tokenizers 0.21.4。 +- 以上为历史记录中的版本；完整字段见 + [数据摘要](W5/reports/validation-summary.json)的 `configuration`。不据此推断当前机器或最新依赖版本。 + + +
+ +环境准备完成后，按 [W5 报告中的生成命令与结果判定](W5/README.md#如何复现)执行；本节不把环境检查或镜像构建当作生成成功。 + + + +## 6. W5 容器准备与独立复现 + +入口为 [部署脚本](../../scripts/deploy_w5.sh)和 [Dockerfile](../../deployment/w5/Dockerfile)。需要已安装并可访问的 Docker Engine/Buildx；部署脚本会检查引擎和资产、构建镜像，再启动生成，不负责安装引擎或下载模型，也不更改 WSL 配置。 + +**镜像、挂载与路径。** 容器只打包运行所需源码和依赖，资产只读挂载到 `/assets`，构建及报告 +写入 `/output`，以调用者 UID/GID 运行。默认镜像 `scratchv-w5:local`、基础镜像 +`ubuntu:24.04`；可显式设置 `SCRATCHV_CONTAINER_ENGINE`、`SCRATCHV_W5_IMAGE`、 +`SCRATCHV_W5_BASE_IMAGE`、`SCRATCHV_W5_ASSETS` 和 `SCRATCHV_W5_OUTPUT`。 +路径可含空格，挂载路径不能含逗号。脚本检查引擎和资产后构建、运行，不安装引擎， +不下载模型，不更改 WSL 或 Windows。依赖来自 +[pyproject.toml](../../pyproject.toml) 的 `llm-deploy`，不另维护一份版本锁。 + +容器中的 Python 与 QEMU 仍承担相同职责；容器不会替代 QEMU 执行模型。生成结果和计时见 [W5 实验结果](W5/README.md#实验结果)。 + +
+独立容器复现：固定版本、环境准备、三例生成与报告回传 + +以下是一套需要在同一 Bash 会话中顺序执行的完整流程。准备阶段设置的目录、镜像与输出变量会被后续运行和清理步骤继续使用。 + +**固定版本的独立容器复现。** 以下步骤在同一 Bash 会话依次执行，使用独立的 +Ubuntu 24.04 x86_64 环境，不复制作者的虚拟环境、`output/`、ELF、bundle 或报告。 +三例共用一次新构建、执行 24 次前向，不先另跑 Paris，也不重新录制 ORT 参考。 +固定提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 是已完成完整数值实跑的功能基线； +该提交用于复现既有完整数值结果，不含后续报告保存、SIGTERM 分类及 core 名称兼容修复； +这些行为另用相关小图或 Python 测试验证，不将旧完整结果标为修改后的重跑。 +该固定提交也尚不支持 `--cores`，所以下面的历史复现命令保留 `--harts`； +运行当前分支时优先使用 `--cores`。这次名称兼容修改没有重跑完整模型。 + +| 条件 | 固定内容或要求 | +|---|---| +| 主机工具 | Bash、Git/Git LFS、Python 3.12、Docker Engine/Buildx、GNU time、timeout、setsid、tar | +| 模型 | `Qwen/Qwen3-0.6B`，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 ONNX 及同版本 tokenizer | +| 基础镜像 | `public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca` | +| 原实跑工具 | Python 3.12.3、QEMU 8.2.2（Ubuntu `1:8.2.2+ds-0ubuntu1.18`）、Zig 0.14.1、ORT 1.22.1、NumPy 2.2.6、Torch 2.7.1+cpu | +| 原容器引擎 | Docker Engine 29.9.2、Buildx 0.38.1 | +| 准备后的资源 | 至少 4 个可用逻辑 CPU、10 GiB 可用内存、10 GiB 剩余磁盘；初始建议预留 25 GiB 磁盘 | +| 时间预算 | 单步 1800 s，总预算 16500 s，外层 280 分钟，强制清理再留 30 s | + +镜像 digest 只固定基础镜像；apt 与未逐项锁定的间接依赖仍可能变化。记录实际镜像、 +工具和依赖版本，先核对差异是否影响结果比较，不默认为相同环境。这里沿用原资产 +身份检查，不增加源码、输出或报告哈希。保留部署前后镜像 ID 作为运行记录， +不要求两者必须相等：Buildx 的缓存构建仍可能更新 provenance/attestation 元数据， +使 manifest ID 改变。ID 变化本身不足以说明运行内容改变，也不代表版本差异可忽略； +继续记录固定源码、模型和实际工具/依赖版本，复用既有 ELF/bundle 检查。 + +**1. 新 clone 并固定提交。** + +以下命令在同一个 Bash 会话按顺序执行，从有足够空间的父目录开始。 +目录名 `scratchv-w5-repro` 必须不存在；重试时改用新的目录名，保留失败记录。 + +```bash +set -euo pipefail +export W5_COMMIT=8ad347477f7f5ba15b2da7c91178b3c6668456cc +test ! -e scratchv-w5-repro +GIT_LFS_SKIP_SMUDGE=1 git clone --no-checkout --filter=blob:none \ + https://github.com/ScratchV-Compiler/ScratchV.git scratchv-w5-repro +cd scratchv-w5-repro +git fetch --no-tags origin refs/pull/115/head +GIT_LFS_SKIP_SMUDGE=1 git checkout --detach "$W5_COMMIT" +test "$(git rev-parse HEAD)" = "$W5_COMMIT" +test -z "$(git status --porcelain)" + +export W5_RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" +export SCRATCHV_CONTAINER_ENGINE=docker +export SCRATCHV_W5_IMAGE="scratchv-w5:independent-$W5_RUN_ID" +export SCRATCHV_W5_ASSETS="$PWD/output/w5-independent-assets" +export SCRATCHV_W5_OUTPUT="$PWD/output/w5-independent-$W5_RUN_ID" +export SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca +test ! -e "$SCRATCHV_W5_ASSETS" +test ! -e "$SCRATCHV_W5_OUTPUT" +mkdir -p "$SCRATCHV_W5_OUTPUT" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/start-utc.txt" +{ + git rev-parse HEAD + git status --porcelain + git remote -v +} > "$SCRATCHV_W5_OUTPUT/source.txt" +{ + cat /etc/os-release + uname -a + lscpu + free -b + df -B1 . + python3 --version + git --version + git lfs version + docker version + docker buildx version + docker info --format 'OS={{.OSType}} Arch={{.Architecture}} CPUs={{.NCPU}} Memory={{.MemTotal}}' + command -v setsid timeout tar + /usr/bin/time --version +} > "$SCRATCHV_W5_OUTPUT/runner-resources.txt" 2>&1 +docker info > /dev/null +test "$(uname -m)" = x86_64 +python3 -c 'import sys; assert sys.version_info[:2] == (3, 12)' +printf 'Run output: %s\n' "$SCRATCHV_W5_OUTPUT" +``` + +检查 `runner-resources.txt`：容器外的 Linux 环境应为 Ubuntu 24.04，Docker daemon 也运行在该 Linux 环境中； +若使用远端 daemon、容器资源配额不足或没有引擎，先停止并回报，不用修改 WSL 来凑条件。 +挂载目录不能包含逗号。不要把私人凭据写入报告。 + +**2. 构建镜像并记录实际环境。** + +复用现有 Dockerfile；随后部署脚本会再检查构建并复用 Docker 缓存。 +基础镜像和构建参数不变，不使用作者构建好的镜像。 + +```bash +docker build --file deployment/w5/Dockerfile \ + --build-arg "BASE_IMAGE=$SCRATCHV_W5_BASE_IMAGE" \ + --tag "$SCRATCHV_W5_IMAGE" . \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/image-build.log" +docker image inspect "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/container-image.json" +docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/image-id-before.txt" +docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" -m pip freeze \ + > "$SCRATCHV_W5_OUTPUT/dependencies.txt" +docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" \ + -c 'import json; from probes.w1_qwen3_export.run import environment; print(json.dumps(environment(export=True), indent=2))' \ + > "$SCRATCHV_W5_OUTPUT/image-environment.json" +docker run --rm --entrypoint sh "$SCRATCHV_W5_IMAGE" -c \ + 'python --version; python -m ziglang version; qemu-system-riscv64 --version; dpkg-query -W python3.12 qemu-system-misc' \ + > "$SCRATCHV_W5_OUTPUT/image-tools.txt" +cat "$SCRATCHV_W5_OUTPUT/image-tools.txt" +``` + +核对 `image-tools.txt` 与上表；环境差异应保留在报告中，部署脚本本身不锁 apt 补丁。 + +**3. 独立获取固定资产。** + +下载只调用现有 `acquire_model`，沿用 W1 manifest 中已有的下载/分片完整性检查。 +它只用 Python 标准库，不执行模型、不重新导出、不在容器外的 Linux 环境安装推理依赖。 +Tokenizer 从同一提交的 Git LFS 获取。 + +```bash +git lfs install --local +git lfs pull --include="assets/qwen3-0.6b/tokenizer/*" --exclude="" \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/tokenizer-acquisition.log" +python3 - <<'PY' 2>&1 | tee "$SCRATCHV_W5_OUTPUT/model-acquisition.log" +import json, os, shutil +from pathlib import Path +from probes.w1_qwen3_export.run import acquire_model +assets = Path(os.environ["SCRATCHV_W5_ASSETS"]) +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +result = acquire_model(assets / "onnx") +shutil.copytree("assets/qwen3-0.6b/tokenizer", assets / "tokenizer") +(out / "acquisition.json").write_text(json.dumps(result, indent=2) + "\n") +PY +test -z "$(git status --porcelain)" +``` + +不要另跑参考采集脚本或添加 `--record`：suite 内部已经逐步对照当轮 ORT 与冻结参考， +无需再运行一遍完整 ORT 生成作为“预热”。 + +**4. 资源检查。** 镜像和资产准备后核对可用资源，Docker 配额不能低于要求。 + +```bash +python3 - <<'PY' +import json, os, shutil +from pathlib import Path +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +mem = {line.split(":")[0]: int(line.split()[1]) * 1024 + for line in Path("/proc/meminfo").read_text().splitlines() + if line.startswith(("MemTotal:", "MemAvailable:"))} +evidence = {"stage": "after_image_and_assets", + "available_cpus": len(os.sched_getaffinity(0)), "memory": mem, + "disk_free_bytes": shutil.disk_usage(out).free, + "minimum_available_cpus": 4, + "minimum_available_memory_bytes": 10 * 1024**3, + "minimum_disk_free_bytes": 10 * 1024**3} +evidence["passed"] = (evidence["available_cpus"] >= 4 + and mem.get("MemAvailable", 0) >= 10 * 1024**3 + and evidence["disk_free_bytes"] >= 10 * 1024**3) +(out / "resource-preflight.json").write_text(json.dumps(evidence, indent=2) + "\n") +if not evidence["passed"]: + raise SystemExit("Insufficient W5 CPU/RAM/disk; generation NOT RUN.") +PY +``` + +资源不足时保留记录，不运行完整生成。 + +**5. 一次运行三条完整用例并保存报告。** + +整块复制执行。`--reference-suite` 自带三条用例及每例 8-token 预算， +不要同时传 `--max-new-tokens`，不要传 `--reuse-build`。 +代码从固定模型重新构建一次，跨用例自动复用同一 ELF/bundle。 + +```bash +test "$(git rev-parse HEAD)" = "$W5_COMMIT" +test -z "$(git status --porcelain)" +test ! -e "$SCRATCHV_W5_OUTPUT/run" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-start-utc.txt" + +set +e +timeout --signal=INT --kill-after=30s 280m \ + /usr/bin/time -v -o "$SCRATCHV_W5_OUTPUT/process-time.txt" \ + bash scripts/deploy_w5.sh \ + --model /assets/onnx/model.onnx --tokenizer-dir /assets/tokenizer \ + --reference-suite --verify-onnx /assets/onnx/model.onnx \ + --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ + --matmul-rhs-layout column-panel \ + --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ + --timeout 1800 --total-timeout 16500 \ + 2>&1 | tee "$SCRATCHV_W5_OUTPUT/runner.log" +pipeline_status=("${PIPESTATUS[@]}") +printf '%s\n' "${pipeline_status[0]}" > "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" +printf '%s\n' "${pipeline_status[1]}" > "$SCRATCHV_W5_OUTPUT/log-exit-code.txt" +date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-end-utc.txt" + +# 此唯一 tag 只供本次使用；不停止其他人的容器。 +{ + containers=$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE") + list_status=$? + printf 'list_exit_code=%s\n' "$list_status" + for container in $containers; do + docker stop --time 30 "$container" + printf 'stop_exit_code=%s\n' "$?" + done + docker ps --filter "ancestor=$SCRATCHV_W5_IMAGE" \ + --format '{{.ID}} {{.Status}}' + printf 'final_list_exit_code=%s\n' "$?" +} > "$SCRATCHV_W5_OUTPUT/container-cleanup.log" 2>&1 + +docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ + > "$SCRATCHV_W5_OUTPUT/image-id-after.txt" +python3 -B -X utf8 scripts/check_w5_ci.py \ + --report "$SCRATCHV_W5_OUTPUT/run/report.json" \ + --reference probes/w5_generation/references.json \ + --exit-code-file "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" \ + --output "$SCRATCHV_W5_OUTPUT/result-check.json" \ + > "$SCRATCHV_W5_OUTPUT/result-check.log" 2>&1 +printf '%s\n' "$?" > "$SCRATCHV_W5_OUTPUT/check-exit-code.txt" +cat "$SCRATCHV_W5_OUTPUT/result-check.log" +{ + date -u +%FT%TZ + git rev-parse HEAD + git status --porcelain + free -b + df -B1 . +} > "$SCRATCHV_W5_OUTPUT/final-resources.txt" +set -e +printf 'Preserved output: %s\n' "$SCRATCHV_W5_OUTPUT" +``` + +这里为失败留证临时关闭 `errexit`，**脚本走到最后不等于 PASS**。 +即使 suite 非零退出，仍运行容器清理和已有报告检查器；报告缺失时 checker 应失败， +不能补造成功字段。`result-check.json` 在本地的 `commit` 可为 null；实际 SHA 由 +`source.txt` 和最终 Git 记录确认。 + +若手动 Ctrl-C 使整段 shell 中断，重新进入同一 clone，恢复上面输出目录和镜像变量， +执行本节从“此唯一 tag”注释之后的收尾段，再打包已有报告。不要直接重新跑模型。 +主机掉电、daemon 失联或强制终止可能阻止清理；如清理日志有非零码或最后仍列出容器， +标为未完成并回报，不视为通过。 + +**三条用例与结果判定。** 输入、应得到的文字、token 编号及停止原因统一见 +[三条实际输出与冻结参考对照](W5/README.md#fixed-case-results)。复现必须逐项匹配，不能只检查是否出现城市名。 + +成功要求：三例、24 次前向全部完成，prompt IDs/新 IDs/文本/停止原因匹配；每步全部 +`256×151936=38,895,616` 个 logits 有限且严格 `max_abs < 1e-3`；28 层完整图检查、 +FMA 12 行 binary32 位模式与 FCSR 检查通过；同一 ELF/bundle 与所选策略匹配、 +padding `none`；每个 QEMU 正常退出并回收，没有清理错误。套件、日志管道、报告 +检查器退出码均为 0，最终无本次容器残留，源码没有改动。 +`container_reproduction=NOT_ASSERTED` 是内部 CLI 不会自行宣称容器复现的字段， +不能手改；容器身份由外层构建、运行和退出记录说明。 + +**回传报告。** 附 `handoff.txt` 说明执行人、UTC 时间、完整 SHA、独立 clone/构建、 +环境差异、三例 IDs/文本/误差、计时与缺失采样、清理结果以及失败阶段，保留： + +- 来源、主机资源、镜像/依赖/工具、下载记录、所有退出码与构建/运行/清理日志。 +- `run/report.json`、`result-check.json`、`run/executable.json`、`run/build/` 的 JSON。 +- `run/steps/step-0000…0023/run.json`、UART 完成帧 `uart.bin`、错误日志及 + `run/fma-conformance/` 的报告。 + +不传权重、ELF、原始 logits 或 Zig 缓存；失败时也回传已有报告并标明缺失项。 +原始产物保留至复现结果确认；重试使用新目录。沿用原小文件筛选方式打包： + +```bash +python3 - <<'PY' +import os, tarfile +from pathlib import Path +out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) +allowed = {".json", ".xml", ".txt", ".log", ".stderr", ".stdout"} +archive = out.parent / (out.name + "-reports.tar.gz") +if archive.exists(): + raise SystemExit(f"Keep earlier evidence; archive already exists: {archive}") +with tarfile.open(archive, "w:gz") as bundle: + for path in sorted(out.rglob("*")): + relative = path.relative_to(out) + if (path.is_file() and not path.is_symlink() + and ".zig-cache" not in relative.parts + and (path.suffix in allowed or path.name == "uart.bin")): + bundle.add(path, arcname=str(Path(out.name) / relative), recursive=False) +print(archive) +PY +``` + +`process-time.txt` 的 CPU/RSS 属于 Docker 客户端，不是容器或 QEMU 总量。逐步耗时、 +RSS 使用 `run/steps/*/run.json`，partial 采样明确注明；未采集的 CPU 写“未独立采集”， +不填作者历史数据，也不据不同机器的记录计算加速比。 + +
diff --git a/docs/llm-deploy-v1.0/W5/README.md b/docs/llm-deploy-v1.0/W5/README.md index e1f4050b..4ab56d23 100644 --- a/docs/llm-deploy-v1.0/W5/README.md +++ b/docs/llm-deploy-v1.0/W5/README.md @@ -1,6 +1,160 @@ # W5：端到端文本生成 -## 本分支增加了什么 +## 改动与功能 + +W5 把 W4 的“单次模型前向”接成了**逐 token 文本生成**：本机 Python 处理提示词、选择下一个 token 并校验；QEMU 中的 RISC-V 程序执行模型，返回用于选词的分数（logits）。 + +| 改动 | 实现的功能 | +|---|---| +| 生成循环与 QEMU 适配 | 提示词转为 token 编号 → 模型计算 → greedy 选最高分 token → 加入历史并继续；同一轮生成复用同一 ELF 和权重文件，不逐 token 编译 | +| 命令行与部署 | 提供单条提示词、三条固定参考用例、官方模板多轮文本和容器部署入口 | +| 校验与失败处理 | 逐步对比完整模型输出、选出的 token 和最终文字；支持超时、取消、进程清理及失败后复用；报告或日志写入失败也会报错 | +| 本轮命名与兼容修复 | 统一使用 `--cores` / `cores`；兼容旧 `--harts` / `harts`、`parse_args().harts`、dataclass 导出和更新 | +| 本轮文档与数据整理 | 集中说明实跑结果、计时范围和复现方法，补齐可随仓库查看的[数据摘要](reports/validation-summary.json) | + +**默认行为不变：**默认单核，支持手动 `--cores 1/2/4` 和启动时 `--cores auto`，上限 4；padding 关闭。默认算术策略不变，下面完整模型结果使用显式参考配置。 + +**目前的限制：**没有 KV cache，每生成一个新 token 都重新计算完整模型、启动一次新的 QEMU 并装载权重。完整模型多轮聊天、三例独立容器运行和第二人复现，尚无本文可用的完整实跑记录。 + +## 实验结果 + +**完整模型数据来自 2026-10-23 的历史实跑，基线为 `8ad3474`；不是本轮命名、兼容或文档修改后重新测量。** + +- **模型与输入：**Qwen3-0.6B，完整 28 层、FP32，batch=1、固定长度 256；原始文本补全，不套聊天模板。 +- **运行配置：**QEMU / RV64GC，4 core；`blocked_fma / 2x4 / column-panel`，Mean 为 `two-four-lane-last-axis`，非线性为 `fp32-reference`，padding 为 `none`。 +- **生成规模：**每条生成 8 个新 token、执行 8 次完整前向；三条共 24 次，复用同一 ELF 和权重文件。 + +| 输入提示词 | 实际新生成文字 | 端到端生成与校验耗时（秒） | QEMU 仿真总耗时（秒） | 最大绝对误差 | +|---|---|---:|---:|---:| +| `The capital of France is` | ` Paris. The capital of Italy is Rome` | 3375.024 | 3199.695 | 0.000303983688 | +| `The capital of Germany is` | ` Berlin. The capital of France is Paris` | 3668.852 | 3482.999 | 0.000262737274 | +| `The capital of Japan is` | ` Tokyo. The capital of the United States` | 3489.655 | 3315.397 | 0.000197887421 | + + + +
+表中名称说明： + +- **端到端生成与校验耗时**：从开始处理这条提示词，到全部 8 个新 token 生成、转换为最终文字并完成主要校验， + 实际经过的时间。包含输入处理、每步 QEMU 启动与权重装载、模型计算、结果读回和进程清理， + 以及参考计算、逐步数值校验和最终文字与冻结参考的核对。 + **最终文字已经生成并经过校验，这部分时间已包含在内；不含前置编译，也不含最后的报告整理、保存和终端显示。** + 计时顺序是：**处理提示词 → 逐个生成 8 个 token 并校验 → 转换为最终文字并核对 → 计时结束**； + 随后才整理、保存报告并打印文字。因此，它不是纯模型计算时间，也不是从部署到显示结果的全部链路耗时。 +- **QEMU 仿真总耗时**：同一条用例生成 8 个新 token 时，8 次 QEMU 仿真从启动到退出回收的 + **实际经过时间之和**。包含模拟器启动、装载权重、模型计算、输出转储和退出清理； + 不包含前置编译、QEMU 启动前的输入准备和文件检查，也不包含进程外的 ORT 参考计算和完整结果校验。 + **表中是 8 次仿真的累计耗时，不是单次前向耗时，也不是只执行模型算子的纯计算时间。** + 它已包含在“端到端生成与校验耗时”中，两者不能相加。 +- **最大绝对误差**：该用例中，模型输出分数与参考结果之间最大的绝对偏差，计算和判定方法见下文。 + +
+ +
+8 次 QEMU 运行如何生成一条结果？ + +三条提示词初始都为 5 个 token。第 1 步用这 5 个 token 选出第 1 个新 token； +第 2 步用提示词和已选结果，共 6 个 token，选出第 2 个；依次继续， +8 次前向的有效输入长度为 **5、6、7、8、9、10、11、12**，每次都右填充到 256。 +第 8 个新 token 产生后，历史长度变为 13，达到生成预算，不再执行第 9 次前向。 + +每步用 greedy 选出下一个 token，最终将 8 个新 token 转成同一条输出， +不是生成 8 个回答再择优。无 KV cache，复用的是同一 ELF 和权重文件； +Python/QEMU 的具体职责见[流程表](#generation-flow)。 + +
+ + + +
+最大绝对误差：和谁比、怎么算、能说明什么？ + +1. **和谁比较。** 同一份 Qwen3 模型、相同的 token 输入及遮罩，分别由 ScratchV 编译后在 QEMU + 中执行，以及由 ONNX Runtime（ORT）参考程序计算。对比的是模型输出的 **logits**， + 即选下一个 token 时使用的候选分数；这些原始分数不是概率，也不是最终文字。 + ORT 在这里提供数值基准，不是判断回答是否符合事实的裁判。 +2. **怎么算。** 每个对应位置的 QEMU 分数减去 ORT 分数，去掉正负号，再选出最大的差值： + `最大绝对误差 = max(|QEMU 分数 − ORT 分数|)`。 + 例如参考分数为 1.2500、QEMU 为 1.2503，这个位置的误差约为 0.0003。 + 每次前向比较全部 **38,895,616 个输出分数**（包括固定长度中补齐的位置）， + 表中再取该用例 **8 次前向中的最大值**，不是平均误差，也不只检查最终选中的 token。 +3. **为什么小于 0.001 才通过。** 这是[开发计划](../开发计划.md)为完整 QEMU 前向规定的数值容差， + W5 沿用 W4 的 `ATOL = 1e-3`。它给不同浮点计算实现之间的偏差设定上限； + FP32 舍入、运算顺序和数学函数实现差异可能使两个后端的数值不完全一样。 + 代码要求严格 **`max_abs < 0.001`**，等于 0.001 也不通过； + `rtol=0` 表示不再因分数较大而增加相对误差宽限。 + **0.001 是项目约定的分数偏差上限，不是“错误率低于 0.1%”，也不是普遍适用的正确性证明。** +4. **能否说明生成结果正确。** 单看误差小，只能说明这次计算的分数接近参考。 + 当两个候选 token 的分数很接近时，小偏差也可能改变排名、选出另一个 token。 + 因此，完整用例通过还要求：全部输出是有限数值，没有 NaN 或正负无穷； + 每一步实际选出的 token 与当步 ORT 结果及[冻结参考](../../../probes/w5_generation/references.json)一致； + 最终 token 序列、文字和停止原因也与冻结参考完全一致。不能仅凭误差小于 0.001 判整条用例通过。 + +上述计算和检查见[生成入口](../../../scripts/run_inference.py)的 `compare_step` 与 +`reference_suite` 路径。**这里的三条固定用例共 24 步同时通过了数值与生成结果检查**， +说明所列历史版本和显式配置在这些用例上与参考一致；不代表所有提示词都会得到相同结果， +也不保证模型生成的内容在事实层面永远正确。本次只解释已有结果，没有重新执行模型。 + +
+ +
+展开三例合计耗时、内存和采样限制 + +整次测量 10540.190 s（约 2 小时 55 分 40 秒），包括准备、ORT 和数值比较。 +24 次前向的**启动至完成通知耗时**合计 9990.355 s，平均 **416.265 s/token**； +**QEMU 仿真总耗时**合计 9998.091 s。这两个耗时指标的起止位置见[单次前向表说明](#container-results)。 +**采样到的内存峰值**：模拟器 5.406 GiB、主控程序 4.254 GiB、同时运行的进程合计 7.644 GiB。 +17/24 步各有一次内存采样读取失败，属于部分采样（`partial`），可能漏掉更高的瞬时占用。 +同时进程合计是在同一采样时刻求和后取最大值，不能直接相加各进程在不同时刻的峰值； +按 RSS 求和也可能重复计算共享内存页。 + +
+ +**结果：3/3 用例、24/24 步通过。** 每步完整 logits 与同模型、同输入的 ONNX Runtime（ORT）结果比较，均满足 `max(|QEMU − ORT|) < 0.001`，且全部数值有限；每步 token ID、最终文字和停止原因也与冻结参考一致。这说明所列用例与参考一致，不代表所有提示词的回答质量。 + +**停止原因：**三条都是 `max_new_tokens`，即达到 8-token 预算，未遇到 EOS；Japan 的续写尚未成句，不能理解为模型已自然回答完毕。表中生成文字开头均有一个空格。 + +**计时范围：**两列都是实际经过时间。端到端时间包括最终文字生成、ORT 参考计算和校验，不含前置编译、最终报告保存及终端显示；QEMU 时间是 8 次仿真从启动到退出回收的总和，已包含在端到端时间内，不能相加。 + +**干净容器单次前向：**本机与容器都生成 ` Paris`，最大绝对误差均为 `0.000172138214`，两边完整输出逐字节相同。本机内部计时 **444.416 秒**且复用构建；容器 **547.454 秒**，其中本次 C 编译与链接 **93.868 秒**。构建条件不同，这不是容器性能对比，也不代表容器三例全部完成。 + +本轮命名兼容修复另有真实 1/4 core 小图对照，逐步输出与 ORT 完全一致、最大误差为 **0**，取消和超时后的清理与复用通过；完整模型数据仍沿用上面的历史记录。 + +数据入口：[逐 token 数值与资源摘要](reports/validation-summary.json) · [冻结参考](../../../probes/w5_generation/references.json) · [本轮相关验证记录](../../../tests/llm-deploy/w5/README.md)。计时、误差、逐 token 生成和内存说明见表格下方的折叠项；输入输出细节与历史 CI 记录见[其他](#其他)。 + +## 如何复现 + +**下面是在当前代码上运行三条固定用例的最短路径，不是本次已重新执行的记录。** 从仓库根目录执行；使用新输出目录，以免覆盖已有结果。 + +1. **准备环境。** 按 [Linux 复现指南：W5 环境与固定资产](../LINUX_REPRODUCTION.md#w5-local-environment)完成准备；需要容器或第二人独立复现时，使用同一指南的[容器流程](../LINUX_REPRODUCTION.md#w5-container-reproduction)。已有环境可直接继续。 +2. **一次运行三例。** `--reference-suite` 已包含三条提示词及每条 8-token 预算，不再传 `--max-new-tokens`。只构建一次，跨用例复用同一 ELF 和权重文件。历史三例整次测量约 2 小时 56 分钟，实际时间取决于机器与环境。 + +```bash +python scripts/run_inference.py \ + --model assets/qwen3-0.6b/onnx/model.onnx \ + --tokenizer-dir assets/qwen3-0.6b/tokenizer \ + --reference-suite --verify-onnx assets/qwen3-0.6b/onnx/model.onnx \ + --cores 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ + --matmul-rhs-layout column-panel \ + --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ + --timeout 1800 --total-timeout 16500 \ + --output-dir output/w5-readme-repro-new +``` + +3. **核对结果。** 查看 `output/w5-readme-repro-new/report.json`：应有三条用例、24 次模型前向，文字与上表一致，逐步完整 logits 有限且最大绝对误差严格小于 0.001；token ID 与参考一致，停止原因均为 `max_new_tokens`，无进程清理错误。只出现城市名不能代替全部检查。构建描述在 `executable.json`，每步记录在 `steps/`。 + +已有构建可用 `--reuse-build ` 替换 `--model ...`，保留原核数和全部算术配置，仍使用新的输出目录。上述完整三例命令不是小图测试，只有确实需要复现完整结果时才执行。 + +若要**复现表中的历史版本**，使用[独立容器复现流程](../LINUX_REPRODUCTION.md#w5-container-reproduction)中的固定提交 `8ad3474` 和旧 `--harts` 参数；不要把旧版本复现结果当作本轮兼容修复的验证。 + +## 其他 + +
+实现细节、接口兼容与补充数据 + + + +### 接口与默认配置 W5 将 W4 的单次 QEMU 前向接成文本生成：本机 Python 准备输入、选择 token、控制循环并校验， QEMU 中的 RISC-V 程序执行模型。支持原始文本补全、三条固定参考用例及 @@ -63,7 +217,9 @@ W5 没有 `--qemu-workspace-layout` 参数，padding 始终关闭。
-## 整体怎么实现 + + +### 输入输出与生成流程 @@ -157,7 +313,7 @@ QEMU 返回的是这些分数，最终文字由 Python 转换得到。
-展开多轮对话、容器部署和失败处理 +多轮对话与失败处理 **多轮文本。** `Conversation.from_directory(tokenizer_directory, system_prompt=None)` 加载同一快照的 `tokenizer_config.json` 官方 `chat_template`，通过 Jinja2 不可变沙箱 @@ -171,13 +327,7 @@ system 只在构造时提供；每轮重新渲染完整历史并编码，不截 不承诺下一轮 token IDs 直接拼接；超过 256-token 容量会在前向前失败。单个会话拒绝 并发或重入 turn/reset。原始三例不使用聊天模板。 -**部署与报告。** 容器只打包运行所需源码和依赖，资产只读挂载到 `/assets`，构建及报告 -写入 `/output`，以调用者 UID/GID 运行。默认镜像 `scratchv-w5:local`、基础镜像 -`ubuntu:24.04`；可显式设置 `SCRATCHV_CONTAINER_ENGINE`、`SCRATCHV_W5_IMAGE`、 -`SCRATCHV_W5_BASE_IMAGE`、`SCRATCHV_W5_ASSETS` 和 `SCRATCHV_W5_OUTPUT`。 -路径可含空格，挂载路径不能含逗号。脚本检查引擎和资产后构建、运行，不安装引擎， -不下载模型，不更改 WSL 或 Windows。依赖来自 -[pyproject.toml](../../../pyproject.toml) 的 `llm-deploy`，不另维护一份版本锁。 +**容器运行。** Python 与 QEMU 在容器内仍承担相同职责；镜像、资产挂载及运行准备见 [Linux 复现指南](../LINUX_REPRODUCTION.md#w5-container-reproduction)。 部署预检和构建客户端各有独立进程组，SIGINT/SIGTERM 时转发终止并等待清理，必要时 5 秒后强制结束，返回 130/143。容器运行阶段保留 `exec ... run --init` 信号转发， @@ -189,12 +339,14 @@ QEMU 适配器负责其子进程回收。CLI 最终保存报告失败时向 stde 本分支的报告检查器支持 `--log-exit-code-file`：指定后要求生成与日志退出码均为 0， 日志状态缺失、不可读或内容无效也返回 FAIL。外层 Bash 分别保存 `runner-exit-code.txt` 和 `log-exit-code.txt`；生成失败优先保留原退出码， -生成成功而 `tee` 失败时返回日志退出码。下方固定 `8ad3474` 的历史复现命令保留原版 +生成成功而 `tee` 失败时返回日志退出码。环境指南中固定 `8ad3474` 的复现命令保留原版 参数，仍须按其成功要求单独核对日志退出码，不向旧检查器传入新参数。
-## 实际复现结果 + + +### 详细测量与验证记录 下文使用**“本机直接运行”**和**“容器内运行”**区分是否通过 Docker； 具体执行链路、运行环境和计时范围见单次前向表后的说明。 @@ -214,18 +366,7 @@ QEMU 适配器负责其子进程回收。CLI 最终保存报告失败时向 stde | 算术与布局 | `blocked_fma / 2x4 / column-panel`；Mean `two-four-lane-last-axis`；非线性 `fp32-reference`；工作区 padding `none` | | 参考与环境 | 本机 WSL2 Ubuntu，4 个可用逻辑 CPU；ORT 通过 `CPUExecutionProvider` 计算参考输出 | -
-展开模型快照和工具版本 - -- 模型快照：`c1899de289a04d12100db370d81485cdf75e47ca`，见 - [模型清单](../../../probes/w1_qwen3_export/manifest.json)及[冻结参考](../../../probes/w5_generation/references.json)。 -- QEMU 8.2.2（Ubuntu 包 `1:8.2.2+ds-0ubuntu1.18`），Zig 0.14.1； - 目标为 `riscv64 / RV64GC / LP64D / QEMU virt` 裸机，数学库为 Zig 随附 musl。 -- Python 3.12.3、NumPy 2.2.6、ONNX Runtime 1.22.1、tokenizers 0.21.4。 -- 以上为历史记录中的版本；完整字段见 - [数据摘要](reports/validation-summary.json)的 `configuration`。不据此推断当前机器或最新依赖版本。 - -
+模型快照与工具版本见 [Linux 复现指南](../LINUX_REPRODUCTION.md#w5-measured-environment)。 @@ -274,83 +415,15 @@ QEMU 适配器负责其子进程回收。CLI 最终保存报告失败时向 stde | Germany | 3668.852 | 3482.999 | 0.000262737274 | | Japan | 3489.655 | 3315.397 | 0.000197887421 | -
-表中名称说明： -- **端到端生成与校验耗时**：从开始处理这条提示词，到全部 8 个新 token 生成、转换为最终文字并完成主要校验， - 实际经过的时间。包含输入处理、每步 QEMU 启动与权重装载、模型计算、结果读回和进程清理， - 以及参考计算、逐步数值校验和最终文字与冻结参考的核对。 - **最终文字已经生成并经过校验，这部分时间已包含在内；不含前置编译，也不含最后的报告整理、保存和终端显示。** - 计时顺序是：**处理提示词 → 逐个生成 8 个 token 并校验 → 转换为最终文字并核对 → 计时结束**； - 随后才整理、保存报告并打印文字。因此，它不是纯模型计算时间，也不是从部署到显示结果的全部链路耗时。 -- **QEMU 仿真总耗时**：同一条用例生成 8 个新 token 时，8 次 QEMU 仿真从启动到退出回收的 - **实际经过时间之和**。包含模拟器启动、装载权重、模型计算、输出转储和退出清理； - 不包含前置编译、QEMU 启动前的输入准备和文件检查，也不包含进程外的 ORT 参考计算和完整结果校验。 - **表中是 8 次仿真的累计耗时，不是单次前向耗时，也不是只执行模型算子的纯计算时间。** - 它已包含在“端到端生成与校验耗时”中，两者不能相加。 -- **最大绝对误差**：该用例中，模型输出分数与参考结果之间最大的绝对偏差，计算和判定方法见下文。 - -
-
-8 次 QEMU 运行如何生成一条结果？ -三条提示词初始都为 5 个 token。第 1 步用这 5 个 token 选出第 1 个新 token； -第 2 步用提示词和已选结果，共 6 个 token，选出第 2 个；依次继续， -8 次前向的有效输入长度为 **5、6、7、8、9、10、11、12**，每次都右填充到 256。 -第 8 个新 token 产生后，历史长度变为 13，达到生成预算，不再执行第 9 次前向。 -每步用 greedy 选出下一个 token，最终将 8 个新 token 转成同一条输出， -不是生成 8 个回答再择优。无 KV cache，复用的是同一 ELF 和权重文件； -Python/QEMU 的具体职责见[流程表](#generation-flow)。 - -
- - - -
-最大绝对误差：和谁比、怎么算、能说明什么？ - -1. **和谁比较。** 同一份 Qwen3 模型、相同的 token 输入及遮罩，分别由 ScratchV 编译后在 QEMU - 中执行，以及由 ONNX Runtime（ORT）参考程序计算。对比的是模型输出的 **logits**， - 即选下一个 token 时使用的候选分数；这些原始分数不是概率，也不是最终文字。 - ORT 在这里提供数值基准，不是判断回答是否符合事实的裁判。 -2. **怎么算。** 每个对应位置的 QEMU 分数减去 ORT 分数，去掉正负号，再选出最大的差值： - `最大绝对误差 = max(|QEMU 分数 − ORT 分数|)`。 - 例如参考分数为 1.2500、QEMU 为 1.2503，这个位置的误差约为 0.0003。 - 每次前向比较全部 **38,895,616 个输出分数**（包括固定长度中补齐的位置）， - 表中再取该用例 **8 次前向中的最大值**，不是平均误差，也不只检查最终选中的 token。 -3. **为什么小于 0.001 才通过。** 这是[开发计划](../开发计划.md)为完整 QEMU 前向规定的数值容差， - W5 沿用 W4 的 `ATOL = 1e-3`。它给不同浮点计算实现之间的偏差设定上限； - FP32 舍入、运算顺序和数学函数实现差异可能使两个后端的数值不完全一样。 - 代码要求严格 **`max_abs < 0.001`**，等于 0.001 也不通过； - `rtol=0` 表示不再因分数较大而增加相对误差宽限。 - **0.001 是项目约定的分数偏差上限，不是“错误率低于 0.1%”，也不是普遍适用的正确性证明。** -4. **能否说明生成结果正确。** 单看误差小，只能说明这次计算的分数接近参考。 - 当两个候选 token 的分数很接近时，小偏差也可能改变排名、选出另一个 token。 - 因此，完整用例通过还要求：全部输出是有限数值，没有 NaN 或正负无穷； - 每一步实际选出的 token 与当步 ORT 结果及[冻结参考](../../../probes/w5_generation/references.json)一致； - 最终 token 序列、文字和停止原因也与冻结参考完全一致。不能仅凭误差小于 0.001 判整条用例通过。 -上述计算和检查见[生成入口](../../../scripts/run_inference.py)的 `compare_step` 与 -`reference_suite` 路径。**这里的三条固定用例共 24 步同时通过了数值与生成结果检查**， -说明所列历史版本和显式配置在这些用例上与参考一致；不代表所有提示词都会得到相同结果， -也不保证模型生成的内容在事实层面永远正确。本次只解释已有结果，没有重新执行模型。 -
-
-展开三例合计耗时、内存和采样限制 -整次测量 10540.190 s（约 2 小时 55 分 40 秒），包括准备、ORT 和数值比较。 -24 次前向的**启动至完成通知耗时**合计 9990.355 s，平均 **416.265 s/token**； -**QEMU 仿真总耗时**合计 9998.091 s。这两个耗时指标的起止位置见[单次前向表说明](#container-results)。 -**采样到的内存峰值**：模拟器 5.406 GiB、主控程序 4.254 GiB、同时运行的进程合计 7.644 GiB。 -17/24 步各有一次内存采样读取失败，属于部分采样（`partial`），可能漏掉更高的瞬时占用。 -同时进程合计是在同一采样时刻求和后取最大值，不能直接相加各进程在不同时刻的峰值； -按 RSS 求和也可能重复计算共享内存页。 -
@@ -550,13 +623,12 @@ GitHub 运行页的“Load 摘要”加载的是工作流写入 `GITHUB_STEP_SUM -## 如何复现 +### 运行示例
-展开完整复现说明、环境要求、命令与验收标准 +单条提示词、构建复用与多轮运行示例 -**本地已有环境。** 从仓库根执行，激活项目 Python 3.12 环境并按 -[Linux 开发说明](../LINUX_REPRODUCTION.md)安装依赖。已有固定资产时，原始补全命令为： +**单条提示词。** 完成 [W5 环境与固定资产准备](../LINUX_REPRODUCTION.md#w5-local-environment)后，从仓库根目录执行： ```bash python scripts/run_inference.py \ @@ -586,286 +658,10 @@ python scripts/run_inference.py \ 完整模型三轮生成已实跑。相关测试命令集中在 [W5 测试说明](../../../tests/llm-deploy/w5/README.md)。 -**固定版本的独立容器复现。** 以下步骤在同一 Bash 会话依次执行，使用独立的 -Ubuntu 24.04 x86_64 环境，不复制作者的虚拟环境、`output/`、ELF、bundle 或报告。 -三例共用一次新构建、执行 24 次前向，不先另跑 Paris，也不重新录制 ORT 参考。 -固定提交 `8ad347477f7f5ba15b2da7c91178b3c6668456cc` 是已完成完整数值实跑的功能基线； -该提交用于复现既有完整数值结果，不含后续报告保存、SIGTERM 分类及 core 名称兼容修复； -这些行为另用相关小图或 Python 测试验证，不将旧完整结果标为修改后的重跑。 -该固定提交也尚不支持 `--cores`，所以下面的历史复现命令保留 `--harts`； -运行当前分支时优先使用 `--cores`。这次名称兼容修改没有重跑完整模型。 - -| 条件 | 固定内容或要求 | -|---|---| -| 主机工具 | Bash、Git/Git LFS、Python 3.12、Docker Engine/Buildx、GNU time、timeout、setsid、tar | -| 模型 | `Qwen/Qwen3-0.6B`，revision `c1899de289a04d12100db370d81485cdf75e47ca`，FP32 ONNX 及同版本 tokenizer | -| 基础镜像 | `public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca` | -| 原实跑工具 | Python 3.12.3、QEMU 8.2.2（Ubuntu `1:8.2.2+ds-0ubuntu1.18`）、Zig 0.14.1、ORT 1.22.1、NumPy 2.2.6、Torch 2.7.1+cpu | -| 原容器引擎 | Docker Engine 29.9.2、Buildx 0.38.1 | -| 准备后的资源 | 至少 4 个可用逻辑 CPU、10 GiB 可用内存、10 GiB 剩余磁盘；初始建议预留 25 GiB 磁盘 | -| 时间预算 | 单步 1800 s，总预算 16500 s，外层 280 分钟，强制清理再留 30 s | - -镜像 digest 只固定基础镜像；apt 与未逐项锁定的间接依赖仍可能变化。记录实际镜像、 -工具和依赖版本，先核对差异是否影响结果比较，不默认为相同环境。这里沿用原资产 -身份检查，不增加源码、输出或报告哈希。保留部署前后镜像 ID 作为运行记录， -不要求两者必须相等：Buildx 的缓存构建仍可能更新 provenance/attestation 元数据， -使 manifest ID 改变。ID 变化本身不足以说明运行内容改变，也不代表版本差异可忽略； -继续记录固定源码、模型和实际工具/依赖版本，复用既有 ELF/bundle 检查。 - -**1. 新 clone 并固定提交。** - -以下命令在同一个 Bash 会话按顺序执行，从有足够空间的父目录开始。 -目录名 `scratchv-w5-repro` 必须不存在；重试时改用新的目录名，保留失败记录。 - -```bash -set -euo pipefail -export W5_COMMIT=8ad347477f7f5ba15b2da7c91178b3c6668456cc -test ! -e scratchv-w5-repro -GIT_LFS_SKIP_SMUDGE=1 git clone --no-checkout --filter=blob:none \ - https://github.com/ScratchV-Compiler/ScratchV.git scratchv-w5-repro -cd scratchv-w5-repro -git fetch --no-tags origin refs/pull/115/head -GIT_LFS_SKIP_SMUDGE=1 git checkout --detach "$W5_COMMIT" -test "$(git rev-parse HEAD)" = "$W5_COMMIT" -test -z "$(git status --porcelain)" - -export W5_RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" -export SCRATCHV_CONTAINER_ENGINE=docker -export SCRATCHV_W5_IMAGE="scratchv-w5:independent-$W5_RUN_ID" -export SCRATCHV_W5_ASSETS="$PWD/output/w5-independent-assets" -export SCRATCHV_W5_OUTPUT="$PWD/output/w5-independent-$W5_RUN_ID" -export SCRATCHV_W5_BASE_IMAGE=public.ecr.aws/ubuntu/ubuntu@sha256:5ce7043d3beb84e91bfeecacb5fafc80602bf247a1dd6016bc0fe7d69d707eca -test ! -e "$SCRATCHV_W5_ASSETS" -test ! -e "$SCRATCHV_W5_OUTPUT" -mkdir -p "$SCRATCHV_W5_OUTPUT" -date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/start-utc.txt" -{ - git rev-parse HEAD - git status --porcelain - git remote -v -} > "$SCRATCHV_W5_OUTPUT/source.txt" -{ - cat /etc/os-release - uname -a - lscpu - free -b - df -B1 . - python3 --version - git --version - git lfs version - docker version - docker buildx version - docker info --format 'OS={{.OSType}} Arch={{.Architecture}} CPUs={{.NCPU}} Memory={{.MemTotal}}' - command -v setsid timeout tar - /usr/bin/time --version -} > "$SCRATCHV_W5_OUTPUT/runner-resources.txt" 2>&1 -docker info > /dev/null -test "$(uname -m)" = x86_64 -python3 -c 'import sys; assert sys.version_info[:2] == (3, 12)' -printf 'Run output: %s\n' "$SCRATCHV_W5_OUTPUT" -``` - -检查 `runner-resources.txt`：容器外的 Linux 环境应为 Ubuntu 24.04，Docker daemon 也运行在该 Linux 环境中； -若使用远端 daemon、容器资源配额不足或没有引擎，先停止并回报，不用修改 WSL 来凑条件。 -挂载目录不能包含逗号。不要把私人凭据写入报告。 - -**2. 构建镜像并记录实际环境。** + -复用现有 Dockerfile；随后部署脚本会再检查构建并复用 Docker 缓存。 -基础镜像和构建参数不变，不使用作者构建好的镜像。 - -```bash -docker build --file deployment/w5/Dockerfile \ - --build-arg "BASE_IMAGE=$SCRATCHV_W5_BASE_IMAGE" \ - --tag "$SCRATCHV_W5_IMAGE" . \ - 2>&1 | tee "$SCRATCHV_W5_OUTPUT/image-build.log" -docker image inspect "$SCRATCHV_W5_IMAGE" \ - > "$SCRATCHV_W5_OUTPUT/container-image.json" -docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ - > "$SCRATCHV_W5_OUTPUT/image-id-before.txt" -docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" -m pip freeze \ - > "$SCRATCHV_W5_OUTPUT/dependencies.txt" -docker run --rm --entrypoint python "$SCRATCHV_W5_IMAGE" \ - -c 'import json; from probes.w1_qwen3_export.run import environment; print(json.dumps(environment(export=True), indent=2))' \ - > "$SCRATCHV_W5_OUTPUT/image-environment.json" -docker run --rm --entrypoint sh "$SCRATCHV_W5_IMAGE" -c \ - 'python --version; python -m ziglang version; qemu-system-riscv64 --version; dpkg-query -W python3.12 qemu-system-misc' \ - > "$SCRATCHV_W5_OUTPUT/image-tools.txt" -cat "$SCRATCHV_W5_OUTPUT/image-tools.txt" -``` +**容器运行与独立复现。** 固定提交、环境准备、资产获取、三例生成、清理及报告回传步骤集中在 [Linux 复现指南](../LINUX_REPRODUCTION.md#w5-container-reproduction)，按同一 Bash 会话中的顺序执行。 -核对 `image-tools.txt` 与上表；环境差异应保留在报告中，部署脚本本身不锁 apt 补丁。 - -**3. 独立获取固定资产。** - -下载只调用现有 `acquire_model`，沿用 W1 manifest 中已有的下载/分片完整性检查。 -它只用 Python 标准库，不执行模型、不重新导出、不在容器外的 Linux 环境安装推理依赖。 -Tokenizer 从同一提交的 Git LFS 获取。 - -```bash -git lfs install --local -git lfs pull --include="assets/qwen3-0.6b/tokenizer/*" --exclude="" \ - 2>&1 | tee "$SCRATCHV_W5_OUTPUT/tokenizer-acquisition.log" -python3 - <<'PY' 2>&1 | tee "$SCRATCHV_W5_OUTPUT/model-acquisition.log" -import json, os, shutil -from pathlib import Path -from probes.w1_qwen3_export.run import acquire_model -assets = Path(os.environ["SCRATCHV_W5_ASSETS"]) -out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) -result = acquire_model(assets / "onnx") -shutil.copytree("assets/qwen3-0.6b/tokenizer", assets / "tokenizer") -(out / "acquisition.json").write_text(json.dumps(result, indent=2) + "\n") -PY -test -z "$(git status --porcelain)" -``` - -不要另跑参考采集脚本或添加 `--record`：suite 内部已经逐步对照当轮 ORT 与冻结参考， -无需再运行一遍完整 ORT 生成作为“预热”。 - -**4. 资源检查。** 镜像和资产准备后核对可用资源，Docker 配额不能低于要求。 - -```bash -python3 - <<'PY' -import json, os, shutil -from pathlib import Path -out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) -mem = {line.split(":")[0]: int(line.split()[1]) * 1024 - for line in Path("/proc/meminfo").read_text().splitlines() - if line.startswith(("MemTotal:", "MemAvailable:"))} -evidence = {"stage": "after_image_and_assets", - "available_cpus": len(os.sched_getaffinity(0)), "memory": mem, - "disk_free_bytes": shutil.disk_usage(out).free, - "minimum_available_cpus": 4, - "minimum_available_memory_bytes": 10 * 1024**3, - "minimum_disk_free_bytes": 10 * 1024**3} -evidence["passed"] = (evidence["available_cpus"] >= 4 - and mem.get("MemAvailable", 0) >= 10 * 1024**3 - and evidence["disk_free_bytes"] >= 10 * 1024**3) -(out / "resource-preflight.json").write_text(json.dumps(evidence, indent=2) + "\n") -if not evidence["passed"]: - raise SystemExit("Insufficient W5 CPU/RAM/disk; generation NOT RUN.") -PY -``` - -资源不足时保留记录，不运行完整生成。 - -**5. 一次运行三条完整用例并保存报告。** - -整块复制执行。`--reference-suite` 自带三条用例及每例 8-token 预算， -不要同时传 `--max-new-tokens`，不要传 `--reuse-build`。 -代码从固定模型重新构建一次，跨用例自动复用同一 ELF/bundle。 - -```bash -test "$(git rev-parse HEAD)" = "$W5_COMMIT" -test -z "$(git status --porcelain)" -test ! -e "$SCRATCHV_W5_OUTPUT/run" -date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-start-utc.txt" - -set +e -timeout --signal=INT --kill-after=30s 280m \ - /usr/bin/time -v -o "$SCRATCHV_W5_OUTPUT/process-time.txt" \ - bash scripts/deploy_w5.sh \ - --model /assets/onnx/model.onnx --tokenizer-dir /assets/tokenizer \ - --reference-suite --verify-onnx /assets/onnx/model.onnx \ - --harts 4 --matmul-policy blocked_fma --matmul-tile 2x4 \ - --matmul-rhs-layout column-panel \ - --mean-policy two-four-lane-last-axis --nonlinear-policy fp32-reference \ - --timeout 1800 --total-timeout 16500 \ - 2>&1 | tee "$SCRATCHV_W5_OUTPUT/runner.log" -pipeline_status=("${PIPESTATUS[@]}") -printf '%s\n' "${pipeline_status[0]}" > "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" -printf '%s\n' "${pipeline_status[1]}" > "$SCRATCHV_W5_OUTPUT/log-exit-code.txt" -date -u +%FT%TZ > "$SCRATCHV_W5_OUTPUT/generation-end-utc.txt" - -# 此唯一 tag 只供本次使用；不停止其他人的容器。 -{ - containers=$(docker ps --quiet --filter "ancestor=$SCRATCHV_W5_IMAGE") - list_status=$? - printf 'list_exit_code=%s\n' "$list_status" - for container in $containers; do - docker stop --time 30 "$container" - printf 'stop_exit_code=%s\n' "$?" - done - docker ps --filter "ancestor=$SCRATCHV_W5_IMAGE" \ - --format '{{.ID}} {{.Status}}' - printf 'final_list_exit_code=%s\n' "$?" -} > "$SCRATCHV_W5_OUTPUT/container-cleanup.log" 2>&1 - -docker image inspect --format '{{.Id}}' "$SCRATCHV_W5_IMAGE" \ - > "$SCRATCHV_W5_OUTPUT/image-id-after.txt" -python3 -B -X utf8 scripts/check_w5_ci.py \ - --report "$SCRATCHV_W5_OUTPUT/run/report.json" \ - --reference probes/w5_generation/references.json \ - --exit-code-file "$SCRATCHV_W5_OUTPUT/runner-exit-code.txt" \ - --output "$SCRATCHV_W5_OUTPUT/result-check.json" \ - > "$SCRATCHV_W5_OUTPUT/result-check.log" 2>&1 -printf '%s\n' "$?" > "$SCRATCHV_W5_OUTPUT/check-exit-code.txt" -cat "$SCRATCHV_W5_OUTPUT/result-check.log" -{ - date -u +%FT%TZ - git rev-parse HEAD - git status --porcelain - free -b - df -B1 . -} > "$SCRATCHV_W5_OUTPUT/final-resources.txt" -set -e -printf 'Preserved output: %s\n' "$SCRATCHV_W5_OUTPUT" -``` - -这里为失败留证临时关闭 `errexit`，**脚本走到最后不等于 PASS**。 -即使 suite 非零退出，仍运行容器清理和已有报告检查器；报告缺失时 checker 应失败， -不能补造成功字段。`result-check.json` 在本地的 `commit` 可为 null；实际 SHA 由 -`source.txt` 和最终 Git 记录确认。 - -若手动 Ctrl-C 使整段 shell 中断，重新进入同一 clone，恢复上面输出目录和镜像变量， -执行本节从“此唯一 tag”注释之后的收尾段，再打包已有报告。不要直接重新跑模型。 -主机掉电、daemon 失联或强制终止可能阻止清理；如清理日志有非零码或最后仍列出容器， -标为未完成并回报，不视为通过。 - -**三条用例与结果判定。** 输入、应得到的文字、token 编号及停止原因统一见 -[三条实际输出与冻结参考对照](#fixed-case-results)。复现必须逐项匹配，不能只检查是否出现城市名。 - -成功要求：三例、24 次前向全部完成，prompt IDs/新 IDs/文本/停止原因匹配；每步全部 -`256×151936=38,895,616` 个 logits 有限且严格 `max_abs < 1e-3`；28 层完整图检查、 -FMA 12 行 binary32 位模式与 FCSR 检查通过；同一 ELF/bundle 与所选策略匹配、 -padding `none`；每个 QEMU 正常退出并回收，没有清理错误。套件、日志管道、报告 -检查器退出码均为 0，最终无本次容器残留，源码没有改动。 -`container_reproduction=NOT_ASSERTED` 是内部 CLI 不会自行宣称容器复现的字段， -不能手改；容器身份由外层构建、运行和退出记录说明。 - -**回传报告。** 附 `handoff.txt` 说明执行人、UTC 时间、完整 SHA、独立 clone/构建、 -环境差异、三例 IDs/文本/误差、计时与缺失采样、清理结果以及失败阶段，保留： - -- 来源、主机资源、镜像/依赖/工具、下载记录、所有退出码与构建/运行/清理日志。 -- `run/report.json`、`result-check.json`、`run/executable.json`、`run/build/` 的 JSON。 -- `run/steps/step-0000…0023/run.json`、UART 完成帧 `uart.bin`、错误日志及 - `run/fma-conformance/` 的报告。 - -不传权重、ELF、原始 logits 或 Zig 缓存；失败时也回传已有报告并标明缺失项。 -原始产物保留至复现结果确认；重试使用新目录。沿用原小文件筛选方式打包： - -```bash -python3 - <<'PY' -import os, tarfile -from pathlib import Path -out = Path(os.environ["SCRATCHV_W5_OUTPUT"]) -allowed = {".json", ".xml", ".txt", ".log", ".stderr", ".stdout"} -archive = out.parent / (out.name + "-reports.tar.gz") -if archive.exists(): - raise SystemExit(f"Keep earlier evidence; archive already exists: {archive}") -with tarfile.open(archive, "w:gz") as bundle: - for path in sorted(out.rglob("*")): - relative = path.relative_to(out) - if (path.is_file() and not path.is_symlink() - and ".zig-cache" not in relative.parts - and (path.suffix in allowed or path.name == "uart.bin")): - bundle.add(path, arcname=str(Path(out.name) / relative), recursive=False) -print(archive) -PY -``` - -`process-time.txt` 的 CPU/RSS 属于 Docker 客户端，不是容器或 QEMU 总量。逐步耗时、 -RSS 使用 `run/steps/*/run.json`，partial 采样明确注明；未采集的 CPU 写“未独立采集”， -不填作者历史数据，也不据不同机器的记录计算加速比。 +