diff --git a/.github/workflows/publish-worker-container.yml b/.github/workflows/publish-worker-container.yml index 81b620a..46d9ba7 100644 --- a/.github/workflows/publish-worker-container.yml +++ b/.github/workflows/publish-worker-container.yml @@ -8,7 +8,7 @@ on: required: true type: choice options: - - server-v0.3.5 + - server-v0.3.6 confirm: description: I have reviewed the release evidence and worker capability notes required: true @@ -17,7 +17,7 @@ on: permissions: {} concurrency: - group: publish-server-container-v0.3.5 + group: publish-server-container-v0.3.6 cancel-in-progress: false jobs: @@ -45,7 +45,7 @@ jobs: set -euo pipefail test "$GITHUB_REF" = "refs/heads/main" test "$CONFIRM" = "true" - test "$RELEASE_TAG" = "server-v0.3.5" + test "$RELEASE_TAG" = "server-v0.3.6" git fetch --force --tags origin test "$(git cat-file -t "refs/tags/${RELEASE_TAG}")" = "tag" source_sha="$(git rev-parse "${RELEASE_TAG}^{commit}")" @@ -102,12 +102,12 @@ jobs: sbom: true provenance: mode=max build-args: | - VYRAL_IMAGE_VERSION=0.3.5 + VYRAL_IMAGE_VERSION=0.3.6 VYRAL_IMAGE_REVISION=${{ needs.validate.outputs.source_sha }} - tags: ghcr.io/univeracity/vyral-server:0.3.5 + tags: ghcr.io/univeracity/vyral-server:0.3.6 labels: | org.opencontainers.image.source=https://github.com/Univeracity/vyral - org.opencontainers.image.version=0.3.5 + org.opencontainers.image.version=0.3.6 org.opencontainers.image.revision=${{ needs.validate.outputs.source_sha }} - name: Qualify the exact published worker image @@ -146,7 +146,7 @@ jobs: - name: Retain published-image evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: server-container-server-v0.3.5 + name: server-container-server-v0.3.6 path: artifacts/publish if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/python-runtime-qualification.yml b/.github/workflows/python-runtime-qualification.yml index f7fcb03..14f8ea3 100644 --- a/.github/workflows/python-runtime-qualification.yml +++ b/.github/workflows/python-runtime-qualification.yml @@ -36,6 +36,8 @@ jobs: steps: - name: Check out repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 @@ -61,6 +63,12 @@ jobs: python -m coverage report --fail-under=77.5 --precision=1 python -m mypy --config-file runtimes/python/pyproject.toml runtimes/python/src/vyral_runtime runtimes/python/tests/typecheck_consumer.py runtimes/python/tests/test_record_store.py + - name: Qualify high-dimensional retrieval against the frozen source baseline + run: >- + python scripts/benchmark-python-vector-search.py + --records 100 --dimensions 384 --repeats 1 + --output artifacts/qualification/vector-parity.json + - name: Build and consume clean artifacts run: | python -m build --sdist --wheel --outdir artifacts/python-runtime runtimes/python diff --git a/.github/workflows/release-integrity.yml b/.github/workflows/release-integrity.yml index 006dcca..b420b7b 100644 --- a/.github/workflows/release-integrity.yml +++ b/.github/workflows/release-integrity.yml @@ -148,7 +148,7 @@ jobs: docker buildx build \ --file Dockerfile \ --tag "vyral-server-mcp-qualification:${{ github.sha }}" \ - --build-arg VYRAL_IMAGE_VERSION=0.3.5 \ + --build-arg VYRAL_IMAGE_VERSION=0.3.6 \ --build-arg VYRAL_IMAGE_REVISION="${{ github.sha }}" \ --sbom=true \ --provenance=mode=max \ diff --git a/.gitignore b/.gitignore index 4f033fa..9f17778 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,8 @@ docs/reference/* !docs/reference/execution-runtime-limitations.md !docs/reference/python-retrieval-qualification.md !docs/reference/stability.md +!docs/reference/local-runtime-providers.md +!docs/reference/cloudflare-r2-qualification.md .agent-state/ .agent-artifacts/ .vyral/ diff --git a/Dockerfile b/Dockerfile index c6fe866..f37bbc9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN dotnet restore src/Vyral.Server/Vyral.Server.csproj --locked-mode --disable- FROM mcr.microsoft.com/dotnet/aspnet:10.0-noble-chiseled-extra@sha256:00e0ad6a7ef8c0c1391b87f05c7ac757a15740455688f2bfcd146a3f4b987efd -ARG VYRAL_IMAGE_VERSION=0.3.5 +ARG VYRAL_IMAGE_VERSION=0.3.6 ARG VYRAL_IMAGE_REVISION=local LABEL org.opencontainers.image.title="Vyral Server" \ org.opencontainers.image.description="Provider-portable records, retrieval, durable execution, and MCP server" \ diff --git a/PUBLIC-EXPORT-MANIFEST.json b/PUBLIC-EXPORT-MANIFEST.json index a6ee83f..c930d4c 100644 --- a/PUBLIC-EXPORT-MANIFEST.json +++ b/PUBLIC-EXPORT-MANIFEST.json @@ -1,5 +1,5 @@ { - "fileCount": 1028, + "fileCount": 1041, "files": [ { "mode": "644", @@ -79,12 +79,12 @@ { "mode": "644", "path": ".github/workflows/codeql.yml", - "sha256": "e21fb90edbf82811a12a70056067e7857b4169f4cf660eabd8107d8672286d2d" + "sha256": "18bb07386efb58748b8c2cce319c6081601db53ca758fd63b8326a70bbf91255" }, { "mode": "644", "path": ".github/workflows/container-security.yml", - "sha256": "e40f87eec74e7e24b7d96566d5d5475ac9bf07e138e830d16e3799cca54fdea9" + "sha256": "f3bfbc271c14c0cecf2fc4b9076747e5fdd5c098df9eb3256e9c11d3ba0154c8" }, { "mode": "644", @@ -119,17 +119,17 @@ { "mode": "644", "path": ".github/workflows/publish-worker-container.yml", - "sha256": "acb215ca6f2b2e0c3e873e76c30c5273999c363873e1d9408e94fbd214363d66" + "sha256": "e570b027b62d07a833a8ac4cbeb45bafc3f2a3f4fd06708715d5b29690d780d6" }, { "mode": "644", "path": ".github/workflows/python-runtime-qualification.yml", - "sha256": "d0d5a1ab92b7986486ec5142d30f23ba9696ebc2f8e88366e17ab3a956aa4f91" + "sha256": "5cbad16456871bb1c7fd0b92c5b9dae052335db39dd924b2a83bff85547e3579" }, { "mode": "644", "path": ".github/workflows/release-integrity.yml", - "sha256": "dba728941d38e0b91c717a1160d0edcee8d51d006ec18dd1e9892a695df53103" + "sha256": "68f8899a5578311ff536968718283d58e2867fb31e21787e6b1ed09ea2dc654b" }, { "mode": "644", @@ -144,7 +144,7 @@ { "mode": "644", "path": ".gitignore", - "sha256": "541e8bfa0775968129c22414fded5c2356a6d0ec401535e98725bdc61a821406" + "sha256": "2b091f1a8237a2edea8b7300e8c3292ed243fd1e5e363917d3e9ac268990c304" }, { "mode": "644", @@ -164,7 +164,7 @@ { "mode": "644", "path": "Dockerfile", - "sha256": "77449e886e24693a0cb4b9e3d7845a449503900ca147958a272452d6a1a0d0a5" + "sha256": "48f56932e28db91b1b3a309bc5b52173363c997937c1347c2864767ae22a4bb5" }, { "mode": "644", @@ -174,7 +174,7 @@ { "mode": "644", "path": "README.md", - "sha256": "13c1be266759a482396a97430d5800b89d018a66dbf4e3d4a9ea4462e211a0fb" + "sha256": "4f7a95934641fd69276bc73d6e70c8850be0c3e52c9793d9e3ddaf91a1f6298b" }, { "mode": "644", @@ -254,7 +254,7 @@ { "mode": "644", "path": "clients/javascript/README.md", - "sha256": "2b36cd4afa5958ab398641055c1de7692f4a58bc9da12fd63708c601e7e2ce2d" + "sha256": "f288836cda6fce9f4be077527a31bdbf96100d514489d35106d5d5ff2e9971e3" }, { "mode": "644", @@ -269,17 +269,17 @@ { "mode": "644", "path": "clients/javascript/src/index.d.ts", - "sha256": "5542a25684f2be4a617f25e52f2e6624651a214351ef40a4ffadbd4e36e59b92" + "sha256": "43b145184f1f199c4e260c1ff2e14b5d05b7de5098a4a843901dea1ec0a316ee" }, { "mode": "644", "path": "clients/javascript/src/index.js", - "sha256": "cc6b5d8754cc0bbdd4c5846785dfc72eaae99e19f59b426a3c757849ec6cab7c" + "sha256": "dd0d6740ee5802008ac62dc9d454f10e7f9ce1ded185a0c39e5252da194f981c" }, { "mode": "644", "path": "clients/javascript/test/index.test.js", - "sha256": "68897f28099baea3e03166c28f0f175e28cd65e609500c98aed24ff8930ad889" + "sha256": "911455d75fd67ae58d849e427597ba288377143a96d58550ce1dae78008d8f5a" }, { "mode": "644", @@ -329,17 +329,17 @@ { "mode": "644", "path": "clients/python/src/vyral_client/client.py", - "sha256": "95a9f6b5deccaad3c3e9a54736d0e01f193e630a9e180f48a3c4c26cf4bb8521" + "sha256": "dc29236ac79dd2b76606a197c184caa573e4eff9cd0a16e7457ba13b32d1dc18" }, { "mode": "644", "path": "clients/python/src/vyral_client/client.pyi", - "sha256": "bc70472418ad3a27cba73e0c96ee3b4a543851189809130ffc39e6cf70e2daa1" + "sha256": "a3d4c02f37f4d7bf15c9ce59f71e585f839e5ac81e26f1441b3076bc175b5e17" }, { "mode": "644", "path": "clients/python/src/vyral_client/contracts.py", - "sha256": "172205dec3b61ae2e9351c15b8f15efc4e45821bdc0361a7d7d2569366ce3861" + "sha256": "de328b2c9571d0a848cfb15be37e2c21051c847f5fdc636a4007c4c9d817af2c" }, { "mode": "644", @@ -354,7 +354,7 @@ { "mode": "644", "path": "clients/python/tests/test_client.py", - "sha256": "2b4f57c045174f94db8e6b2798e44c33be68a060f82c80e03f9d9944069c5980" + "sha256": "a8f8d1544ebb0b0dc65e3e2cc5047eb569da8df0de77de197b992df8f1745437" }, { "mode": "644", @@ -479,7 +479,7 @@ { "mode": "644", "path": "contracts/schemas/vyral-public.schema.json", - "sha256": "4d5c41bd26abc3f3b9acca066f8a4564152ea2156f223af105db6a53b556f8ab" + "sha256": "37c9584b5bc08e67cfb89685f1e7846c83869af3b58b0c7959645b300244fd24" }, { "mode": "644", @@ -601,6 +601,11 @@ "path": "design/aws-opensearch-record-projection.md", "sha256": "7276fc3852ab4b6e0a29d8d0db4dcb883591521bd65120c299ebd19baea77618" }, + { + "mode": "644", + "path": "design/cloudflare-queues-bridge.md", + "sha256": "adcf68eb6502e5e1e3a88caf102c14647126c92feb2f59a06546c23149d0b1a4" + }, { "mode": "644", "path": "design/developer-adoption-and-evidence-growth.md", @@ -624,7 +629,7 @@ { "mode": "644", "path": "design/local-logprob-judge.md", - "sha256": "a3fb8bb6583f204406c6b65b8e5ec31469cec31ceb997a833d09e457ad55449d" + "sha256": "459257549318bf0a9ee76b55fb04b11c29ed930338aaa180ef5ecf2b1c655df0" }, { "mode": "644", @@ -694,7 +699,7 @@ { "mode": "644", "path": "docs/guides/consumer-handoff.md", - "sha256": "b70770e5be45b6fd87d4f6c4cdb50753d0e40ec7eceea72d2e77c1ac70e62225" + "sha256": "fc6f532260a1603bdb173acb6f9272fc82d13c105590ac0654630d3e38733f45" }, { "mode": "644", @@ -724,17 +729,27 @@ { "mode": "644", "path": "docs/maintainers/releasing.md", - "sha256": "c8f7ac3bf7f07973464f2e071f82ae02d0b14f96ee76c5f00e6885b637135950" + "sha256": "d2477735688d28e812116ee51e3a43240bd74b1f24b0f8121297ae6545485cbf" + }, + { + "mode": "644", + "path": "docs/reference/cloudflare-r2-qualification.md", + "sha256": "acba44e4d9d7da0a341b985dd8bdb479e0b9f7b6a0e50e5870acf78d8b5ba985" }, { "mode": "644", "path": "docs/reference/execution-runtime-limitations.md", "sha256": "106da29f1fbab9f8dc8a8b4a40e83e0c402a29dd959ef194ec8862188583577d" }, + { + "mode": "644", + "path": "docs/reference/local-runtime-providers.md", + "sha256": "9bf76d07cca8f5a41465439342edf392e6f7ecc542200072556dfc88b9c8d5c4" + }, { "mode": "644", "path": "docs/reference/python-retrieval-qualification.md", - "sha256": "ae929e75e31ca6460c0489aed49af77229b566f5822bf58ade86bae39c56bb2d" + "sha256": "a36eadef09c3644c07836a579b6b2a76cb76639027e5d48cd9116a50aa828c66" }, { "mode": "644", @@ -824,7 +839,7 @@ { "mode": "644", "path": "packaging/worker-container-release.json", - "sha256": "84cbca5e6d7ee73f605427deb4f9fa52cf460ec7518a31e64431d12c9cb0e902" + "sha256": "76d3d91ee8f669499f3ab611ecfa6ab999db4d02797751d7edf32b49d6dd4d95" }, { "mode": "644", @@ -841,6 +856,16 @@ "path": "qualification/adapter-qualification.schema.json", "sha256": "71b90464e7bc237e7d871f95788f2c75ea93398a6234c8e5ecc974a9e9f38dfc" }, + { + "mode": "644", + "path": "qualification/local-runtime-mechanics-2026-10-03.json", + "sha256": "9e33008fcad8a3bb6e9d140e94284f9c0f57cdef7151a63eb29cc8cd8e6a7bef" + }, + { + "mode": "644", + "path": "qualification/python-vector-mechanics-2026-10-03.json", + "sha256": "3abddd46637129ceb549a4667ed1a2a8e9a3ac4ff572bccce97a58ba586a1973" + }, { "mode": "644", "path": "qualification/retrieval-projection-qualification.json", @@ -879,7 +904,7 @@ { "mode": "644", "path": "runtimes/python/pyproject.toml", - "sha256": "d77685188efdc0c38a69b78566884336ae920a96362f6dee0fb861f58213cf9f" + "sha256": "013471874021a8c1ed09d90ae94baff2645c1682732fd1b76b09f47dd80cabb7" }, { "mode": "644", @@ -1014,12 +1039,12 @@ { "mode": "644", "path": "runtimes/python/src/vyral_runtime/_contracts/vyral-public.schema.json", - "sha256": "4d5c41bd26abc3f3b9acca066f8a4564152ea2156f223af105db6a53b556f8ab" + "sha256": "37c9584b5bc08e67cfb89685f1e7846c83869af3b58b0c7959645b300244fd24" }, { "mode": "644", "path": "runtimes/python/src/vyral_runtime/_contracts/vyral.openapi.json", - "sha256": "49c4159426b9f8cf527df3ca827b925acc784851e2a017167e69504892e75c63" + "sha256": "5c9c9daad9ee28308ae11bebfc3db9e60e9932264bad2c77720d2aac2118ddd9" }, { "mode": "644", @@ -1299,7 +1324,7 @@ { "mode": "644", "path": "runtimes/python/src/vyral_runtime/local/record_store.py", - "sha256": "b04d156e9df36297487896dc108b8bfb74b9fd46dc53063134c2b962346b3658" + "sha256": "dd882719f63ccfaf8ce392b7d7080677dc35bea56248af89aa0b41d9d48b082b" }, { "mode": "644", @@ -1524,7 +1549,7 @@ { "mode": "644", "path": "runtimes/python/tests/test_runtime.py", - "sha256": "6dca53fc7b9960ddf2fe7321bbfebb39b06963491d4abb924708a390d41b8472" + "sha256": "e552e18a33887af571b1e0465d72b6f08ba36743c838a6e3132638e3958e676a" }, { "mode": "644", @@ -1706,6 +1731,11 @@ "path": "scripts/benchmark-python-runtime.py", "sha256": "1cb7a1080ba809a52eedc7bfb3882a5e3e935946094587cf9a46fc7d59336a98" }, + { + "mode": "644", + "path": "scripts/benchmark-python-vector-search.py", + "sha256": "96e110a2be60c5ecdf9ef2dfaba3dd34faf24fc1a3679b4d56dc7194546b4bcf" + }, { "mode": "755", "path": "scripts/benchmark-ripgrep-retrieval.py", @@ -1734,7 +1764,7 @@ { "mode": "755", "path": "scripts/export-public-tree.py", - "sha256": "ba25d438dc2f9d4de816346b4c8a56a735c6a8a1195d66bea39ba203e6fd87c0" + "sha256": "eb159bca1b2268f3cb44a8d037f05f5f27dafff1d10e0b7fa6d6cf7caa5f7769" }, { "mode": "755", @@ -2139,7 +2169,7 @@ { "mode": "644", "path": "scripts/verify-publication-policy.py", - "sha256": "a2bce3d19754b184250ba8f95db2d0a6f071b879c59ad62410c681e8c0611023" + "sha256": "739242cb23627e0069bad49635a570a6451c34dbe507f9d49e5a696e33d7d030" }, { "mode": "755", @@ -2174,7 +2204,7 @@ { "mode": "755", "path": "scripts/verify-python-runtime-install.py", - "sha256": "427e456cff1e1d94bf85bf2a464dc6445c2c6fa15a474518e9b12189ad16cf8b" + "sha256": "d885596f3ec90788c12aa70c6a6ecf02a83123df7fd60d919fbcd28cdf048101" }, { "mode": "755", @@ -2199,7 +2229,7 @@ { "mode": "755", "path": "scripts/verify-python-runtime.sh", - "sha256": "61f8a2e9fb166a6ea2c8c5bb1c88ce9b574f7c6086fd9fbd4cbeaa70ced79910" + "sha256": "6c5f7f8ac30cefc5bcddd222032168fd1a888a101922af8b3edb52778c2dde45" }, { "mode": "644", @@ -2234,12 +2264,12 @@ { "mode": "755", "path": "scripts/verify-version-policy.py", - "sha256": "fcab9abc0d3bb1574a3977746f7d667f51033e67943daa69e6a96af0ce67b0d5" + "sha256": "237562e88401c2dbb2337f856b8b86993f2aca72cc73044ef35e2a9720913e73" }, { "mode": "755", "path": "scripts/verify-worker-container-release.py", - "sha256": "2775945a5829d468b1230ff0efc79284b81020e16ecf3a41b882fd5887115695" + "sha256": "3d6dbe19cdc55ed6dbfcb003b3d4fcc07c1eb0d4bb8bbea5e87fa32d7b8b9ffd" }, { "mode": "755", @@ -2589,7 +2619,7 @@ { "mode": "644", "path": "src/Vyral.Aws/S3ObjectStore.cs", - "sha256": "47e1369883bef5f623652b5ba5dab851c9f0f918aaf3b2af3336dc11dfc1f462" + "sha256": "767cc33cc2e490173edd1f51fbe1e443cd2635443f5fe09c5ac0076ca6731af2" }, { "mode": "644", @@ -2689,12 +2719,12 @@ { "mode": "644", "path": "src/Vyral.Cloudflare/CloudflareR2Options.cs", - "sha256": "653c1810dbe20e2bb0af52108aa5def8fa3d47b432f4f82091251a4fbbe15991" + "sha256": "80892c313c4802b4e14e96ffe074db2f7daac63c8e7d7a72f8ae9d5fa3a43954" }, { "mode": "644", "path": "src/Vyral.Cloudflare/R2ObjectStore.cs", - "sha256": "28e7a3d2363f566fccfb024983e63bcf324062e54588c92d36bb65c8d4ef175d" + "sha256": "5ef61f94fa25ece2923818a8dfc6dc7309a58261d33a73b843f92d2957f3a18e" }, { "mode": "644", @@ -2709,12 +2739,12 @@ { "mode": "644", "path": "src/Vyral.Cloudflare/WorkerR2GenerationProjection/package-lock.json", - "sha256": "ee0f7405b91d512a093ae5ee26674b6ca0aa7f3e5ed0ff62eb537c3574df1d21" + "sha256": "0014346e66d56938df63b04325b1bb409cb01487c273da9565d34f3e93e694c0" }, { "mode": "644", "path": "src/Vyral.Cloudflare/WorkerR2GenerationProjection/package.json", - "sha256": "76367a6d69c996e003b76a24cb4b50744b2f16b170bea5bb7f32b60f08d94d86" + "sha256": "661d1412b394d163221fb9b6e99c68be22a52ce2a061532dfc0c42f40a09f166" }, { "mode": "644", @@ -3589,12 +3619,12 @@ { "mode": "644", "path": "src/Vyral.Providers.Abstractions/AiCapabilityModels.cs", - "sha256": "7431bd568979b7e59182e8fc053a6c02f42375ced497d2995324e1db0748fec4" + "sha256": "3578c5a287c8cbceb5056415f093777729d7c876b68f2562efebf33c9455d7ca" }, { "mode": "644", "path": "src/Vyral.Providers.Abstractions/AiJudgeModels.cs", - "sha256": "f1baeeec3a71795cc7242f24dfa5c23f99170e0cc7849ba7716beaf65054b811" + "sha256": "899ee8bad60e03f35e1477d1ff9488154f829fac10b16170f3b3b3ee1dea357e" }, { "mode": "644", @@ -3764,7 +3794,7 @@ { "mode": "644", "path": "src/Vyral.Providers.Abstractions/ProviderRunRequests.cs", - "sha256": "519e0ce2a71fcb30a71c2c3b774e73909961b2a4e9ec83f42b68f1e943761fb5" + "sha256": "a4435a32ea1f52de90ed6391123500e7a22f8df6c54145777a005cd08769a71d" }, { "mode": "644", @@ -3774,7 +3804,7 @@ { "mode": "644", "path": "src/Vyral.Providers.Abstractions/ProviderRunResults.cs", - "sha256": "2239ab76ef472bd7985fe14f61464c0f85fcd2648d4d9ae39c0f6ecfc16775b7" + "sha256": "6a4b75fb38e7d3c5c28b2e7e270faafce14db73924877b553f5844d567c601c2" }, { "mode": "644", @@ -3829,7 +3859,7 @@ { "mode": "644", "path": "src/Vyral.Providers.Cli/CliProviderTarget.cs", - "sha256": "9c383da6e8b391de1fe93510bc69c5fc91f0a07de48a43c2001fd382383e10fc" + "sha256": "a8bd5959c56f761a294b6a3ba2e1f9a4d88ba009b4c21a85126436e789a2c887" }, { "mode": "644", @@ -3844,7 +3874,7 @@ { "mode": "644", "path": "src/Vyral.Providers.Cli/CliWorkspaceCodingAgentRunner.cs", - "sha256": "92fb8e2d924c9600bd2368bf55bc5fc35719e18a065ccca088fb576b51ece3f4" + "sha256": "622e10f461ab6f918f4c3f2e1e1b01078452dd8a714f1e80def7fbd17efd7a07" }, { "mode": "644", @@ -3859,7 +3889,7 @@ { "mode": "644", "path": "src/Vyral.Providers.Cli/ProviderProcessRunner.cs", - "sha256": "88a3791275756d59ca234d892fd3fb6193ba03709b15cd76f418d4d4de38717f" + "sha256": "94468deccfa7acf85c8b2db73f075621dec7e52622d7a773fe8eda4365eba445" }, { "mode": "644", @@ -3934,7 +3964,27 @@ { "mode": "644", "path": "src/Vyral.Providers.Local/DeterministicAiProviderTarget.cs", - "sha256": "e1e70e56c836b8dd42ae3280975401371cfc0b348965c0a9984882f31d86b85e" + "sha256": "f1c52659bd0c3edf87eb85c55d7c2ec856c414baf9060ac82084eea4779b6d47" + }, + { + "mode": "644", + "path": "src/Vyral.Providers.Local/LlamaCppRuntimeSessionFactory.cs", + "sha256": "adbfafe0595364533a17a3ed45f5ca36eb737aa80ecc10d17b5da82179b73c0a" + }, + { + "mode": "644", + "path": "src/Vyral.Providers.Local/LocalLlamaCppProviderTarget.cs", + "sha256": "f30f8e11331ae126db9b2322136899a9661c215532f5d2f3b7c7eb55061e610c" + }, + { + "mode": "644", + "path": "src/Vyral.Providers.Local/LocalLogprobJudgeProviderTarget.cs", + "sha256": "c7c003673e38e1cc8973bd3344639185514c73ef11b44493cda4b6968ad10303" + }, + { + "mode": "644", + "path": "src/Vyral.Providers.Local/LocalLogprobSession.cs", + "sha256": "13561224bf68b81ca3a822fb34cc8562fdc82e182b7db95e4c98fb3ca6f5345f" }, { "mode": "644", @@ -4089,7 +4139,7 @@ { "mode": "644", "path": "src/Vyral.Server/Program.cs", - "sha256": "45dce9a4961206b7fae9c3f0697c393351d23d6400d770cf18e72a997a7c1302" + "sha256": "247e8157f87ec1074a0f17de5a633d9c2852e6ee1d15f6ba04be43e632d93a73" }, { "mode": "644", @@ -4139,12 +4189,12 @@ { "mode": "644", "path": "src/Vyral.Server/ServerStorageFactory.cs", - "sha256": "3bfd564aa8994d7837946a61fbb5aecc03339a2eed57d983dac9a7a3a8d617df" + "sha256": "e4646d136a60d090f51a10e6c1c3887f7d33a7554a108e6bc5b0919ec08ed92a" }, { "mode": "644", "path": "src/Vyral.Server/ServerStorageOptions.cs", - "sha256": "cb9981c68ae136b16df502d07dcedfd1f1ec49baa0173d96a0c7d564dc5af793" + "sha256": "d186fe138290769b3a223ba61f0fd49a7f6129b2dab86705653f7d7dd57b42a7" }, { "mode": "644", @@ -4199,7 +4249,7 @@ { "mode": "644", "path": "src/Vyral.Server/contracts/vyral.openapi.json", - "sha256": "49c4159426b9f8cf527df3ca827b925acc784851e2a017167e69504892e75c63" + "sha256": "5c9c9daad9ee28308ae11bebfc3db9e60e9932264bad2c77720d2aac2118ddd9" }, { "mode": "644", @@ -4209,7 +4259,7 @@ { "mode": "644", "path": "tests/Vyral.Tests.Aws/AwsConformanceTests.cs", - "sha256": "1f672b8f856c27d61512728cfc30c60b6dd5ed75b495fba055fc2bf9669bd065" + "sha256": "fc3020709d0e52e8711abafab90ea790971d945f407b984e9cd07201f4377736" }, { "mode": "644", @@ -4354,18 +4404,23 @@ { "mode": "644", "path": "tests/Vyral.Tests.Cloudflare/CloudflareR2ConformanceTests.cs", - "sha256": "8fb52b4ef9bd926e7ab324553abb1e2c4d96eb55a66bde4a588b61ff5cf821b5" + "sha256": "4e9a0e2c797a2cb85185c75b9199a2e4b8a53e9fea893926590ca1908c40a288" }, { "mode": "644", "path": "tests/Vyral.Tests.Cloudflare/CloudflareR2LiveSettings.cs", - "sha256": "52437faef8fe241cf7da36e064e37fc4ad037c4df7f0e9968b05726bcaec10b8" + "sha256": "0d1f4e6e8a877156881fa306696c64d97dca23fcd5a732267f76af431683b73e" }, { "mode": "644", "path": "tests/Vyral.Tests.Cloudflare/CloudflareR2OptionsTests.cs", "sha256": "3f08e93732d5a2aacf1336314ecdebeb01d41b25a08d0b182b21b6f25adcc4e6" }, + { + "mode": "644", + "path": "tests/Vyral.Tests.Cloudflare/R2TransportTests.cs", + "sha256": "c2500435b37dc05997997ede73f30e0eec3f6388ba6f6b3e116d81b8e0712f39" + }, { "mode": "644", "path": "tests/Vyral.Tests.Cloudflare/Vyral.Tests.Cloudflare.csproj", @@ -4794,7 +4849,7 @@ { "mode": "644", "path": "tests/Vyral.Tests.Local/ServerWorkflowTests.cs", - "sha256": "9fe32bfcd2b032f53d52ee4ecdf0fedb64e211d17baf9e60b687b1b75dd43086" + "sha256": "017452c083dc8d4a7c533c57d5efad9eea168a2aad95416c036d163450c204b8" }, { "mode": "644", @@ -4894,7 +4949,7 @@ { "mode": "644", "path": "tests/Vyral.Tests.Providers/CliWorkspaceCodingAgentRunnerTests.cs", - "sha256": "ad6bc209bc7af95c5be3293c43ec5e74de383c124ba0ec85053a7e665886b3c0" + "sha256": "73c0d46958af9694b10734bd0515191a60a1912067cd558a6859ce600f8d0c8a" }, { "mode": "644", @@ -4926,6 +4981,16 @@ "path": "tests/Vyral.Tests.Providers/JulesProviderTargetTests.cs", "sha256": "2433fec4a130b5353e33cbe157f27185f771f6fc8d4a3b3f1d92e4c782462989" }, + { + "mode": "644", + "path": "tests/Vyral.Tests.Providers/LocalLogprobJudgeProviderTests.cs", + "sha256": "2c0b9e5f02926000066ee3adb7c6d5e30c70eccc7c8a30c9853325f495028027" + }, + { + "mode": "644", + "path": "tests/Vyral.Tests.Providers/LocalNativeRuntimeTests.cs", + "sha256": "a815b018fd5093c072449bf3feda2bff479e9c2f57d83a868950606e1b926bb6" + }, { "mode": "644", "path": "tests/Vyral.Tests.Providers/LocalTokenOverlapRerankerProviderTargetTests.cs", @@ -4954,7 +5019,7 @@ { "mode": "644", "path": "tests/Vyral.Tests.Providers/ProviderProcessRunnerTests.cs", - "sha256": "537fa69248edc4343983a68839a8f13d4d70d3806d026f82f1a3c645834194fc" + "sha256": "a585f9750e854b1f2a9a8407ed40a13403a6d00cf4ab886bf886817003fc7749" }, { "mode": "644", @@ -5144,5 +5209,5 @@ ], "schemaVersion": 1, "sourceDirty": false, - "treeSha256": "c890e131a900cde9b7d8d5b989c31b897e7537cc68965e14289cf4992c0e493d" + "treeSha256": "32eda2d67e62a4a9f37764f7c63a6c5e2a3b374b4ed527f4a7951855d957cd52" } diff --git a/README.md b/README.md index ed503e2..b141711 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ also need TLS, rate limits, authenticated ingress, and deployment-specific identity policy. See the [CanonicalStore guide](docs/concepts/canonical-store.md) and the [deployment guide](deploy). -The server image version described by this source is `ghcr.io/univeracity/vyral-server:0.3.5`; +The server image version described by this source is `ghcr.io/univeracity/vyral-server:0.3.6`; verify registry availability before deploying, pin its published digest, and use the same API-key and runtime restrictions shown above. The default entrypoint remains the public API server. A separate, least-privilege deployment can run the preview generic @@ -424,10 +424,13 @@ Every answer carries a `calibrated` flag so callers cannot mistake an uncalibrated or vendor-claimed-but-unverified score for a validated likelihood. -The [local log-probability judge proposal](design/local-logprob-judge.md) -describes a possible Choice-only extension with verified label coverage and -an optional runtime-session adapter. It is a design proposal, not a fourth -implemented or qualified provider. +The [local runtime provider guide](docs/reference/local-runtime-providers.md) +describes the opt-in native chat and Choice-only log-probability judge. +The judge uses verified label bytes, cyclic rotations, explicit raw label mass +and rotation agreement. It reports calibrated=false. Local runtime sessions +are injectable; no particular host service is required. Both targets remain +prototype, with a bounded native smoke profile rather than production worker +or trusted-issuer qualification. ## Execution runtime @@ -473,7 +476,7 @@ Start with the [execution design](design/execution-runtime.md), | AWS storage and execution | Live qualified for isolated S3, DynamoDB, and SQS; managed OpenSearch remains subject to end-user validation | | Google execution | Live qualified; other Google adapters retain their separate qualification posture | | Temporal coordinator and projection | Prototype; disposable and operator-provisioned qualification gates | -| Cloudflare R2 | Preview object-store adapter | +| Cloudflare R2 | Preview; supported object subset live tested, conditional deletion refused | | Generation-bound retrieval projections | Local reference plus OpenSearch and Worker/R2 shapes; maturity is topology- and evidence-scoped | An adapter is `live_qualified` only when the versioned diff --git a/clients/javascript/src/index.d.ts b/clients/javascript/src/index.d.ts index 7b3ea65..d3cec83 100644 --- a/clients/javascript/src/index.d.ts +++ b/clients/javascript/src/index.d.ts @@ -2509,7 +2509,7 @@ export interface ProviderRunRequest { operation?: string; mode?: string; modelId?: string | null; - payload: AiChatPayload | AiExtractPayload | AiRerankPayload | AiReviewPayload | AiScaffoldPayload | AiToolPlanPayload | JsonObject; + payload: AiChatPayload | AiExtractPayload | AiRerankPayload | AiReviewPayload | AiScaffoldPayload | AiToolPlanPayload | AiJudgePayload | JsonObject; correlationId?: string; contextRefs?: Array; timeoutSeconds?: number | null; @@ -2524,7 +2524,7 @@ export interface ProviderRunResult { capability: string; operation: string; mode: string; - output: AiChatResult | AiExtractResult | AiRerankResult | AiReviewResult | AiScaffoldResult | AiToolPlanResult | JsonObject; + output: AiChatResult | AiExtractResult | AiRerankResult | AiReviewResult | AiScaffoldResult | AiToolPlanResult | AiJudgeResult | JsonObject; failureClass?: string | null; providerStatus?: string | null; error?: string | null; @@ -2732,6 +2732,7 @@ export interface AiChatPayload { messages?: Array; system?: string | null; maxOutputChars?: number | null; + maxOutputTokens?: number | null; } export interface AiChatResult { @@ -3089,6 +3090,45 @@ export interface RagContextGraphEvaluationFailureModes { budgetTruncated?: boolean; } +export interface AiJudgeOption { + id: string; + label: string; +} + +export interface AiJudgeQuestion { + id: string; + type: "choice" | "noul" | "score"; + prompt: string; + options?: Array; +} + +export interface AiJudgePayload { + context?: string | null; + questions: Array; + references?: Array; +} + +export interface AiJudgeAnswer { + questionId: string; + choice?: string | null; + calibrated: boolean; + score?: number | null; + legend?: JsonObject | null; + probability?: number | null; + confidence?: number | null; + labelMassCoverage?: number | null; + rotationAgreement?: number | null; + probabilities?: JsonObject | null; + rawLabelProbabilities?: JsonObject | null; + rawClassProbabilities?: JsonObject | null; +} + +export interface AiJudgeResult { + answers: Array; + validationStatus: string; + modelId?: string | null; +} + export interface VyralRequestOptions { headers?: HeadersInit; signal?: AbortSignal; diff --git a/clients/javascript/src/index.js b/clients/javascript/src/index.js index aa0c75e..897b4f4 100644 --- a/clients/javascript/src/index.js +++ b/clients/javascript/src/index.js @@ -813,7 +813,8 @@ export function buildProviderChatRequest(messages, options = {}) { }; assignDefined(payload, { system: options.system, - maxOutputChars: options.maxOutputChars + maxOutputChars: options.maxOutputChars, + maxOutputTokens: options.maxOutputTokens }); return buildProviderRunRequest("ai.chat", payload, options); } diff --git a/clients/javascript/test/index.test.js b/clients/javascript/test/index.test.js index 8195c4b..b587bce 100644 --- a/clients/javascript/test/index.test.js +++ b/clients/javascript/test/index.test.js @@ -1045,8 +1045,10 @@ test("provider request builders expose typed AI payloads", () => { { role: "user", content: "Summarize." } ], { modelId: "gpt-5.3-codex-spark", - timeoutSeconds: 30 + timeoutSeconds: 30, + maxOutputTokens: 12 }); + assert.equal(chat.payload.maxOutputTokens, 12); assert.equal(chat.capability, "ai.chat"); assert.equal(chat.modelId, "gpt-5.3-codex-spark"); assert.equal(chat.payload.messages[0].content, "Summarize."); diff --git a/clients/python/src/vyral_client/client.py b/clients/python/src/vyral_client/client.py index 64c8070..912272f 100644 --- a/clients/python/src/vyral_client/client.py +++ b/clients/python/src/vyral_client/client.py @@ -1326,6 +1326,7 @@ def build_provider_chat_request( *, system: str | None = None, max_output_chars: int | None = None, + max_output_tokens: int | None = None, **run_options: Any, ) -> dict[str, Any]: if not messages: @@ -1335,6 +1336,8 @@ def build_provider_chat_request( payload["system"] = system if max_output_chars is not None: payload["maxOutputChars"] = max_output_chars + if max_output_tokens is not None: + payload["maxOutputTokens"] = max_output_tokens return build_provider_run_request("ai.chat", payload, **run_options) diff --git a/clients/python/src/vyral_client/client.pyi b/clients/python/src/vyral_client/client.pyi index f11fb2f..0342222 100644 --- a/clients/python/src/vyral_client/client.pyi +++ b/clients/python/src/vyral_client/client.pyi @@ -158,7 +158,7 @@ def build_rerank_options(*, provider: str | None=None, enabled: bool=True, mode: def build_provider_run_request(capability: str, payload: dict[str, Any], *, provider: str | None=None, operation: str='run', mode: str='advisory', model_id: str | None=None, correlation_id: str | None=None, context_refs: Sequence[str] | None=None, timeout_seconds: int | None=None, max_output_bytes: int | None=None, artifact_directory: str | None=None) -> dict[str, Any]: ... -def build_provider_chat_request(messages: Sequence[dict[str, Any]], *, system: str | None=None, max_output_chars: int | None=None, **run_options: Any) -> dict[str, Any]: +def build_provider_chat_request(messages: Sequence[dict[str, Any]], *, system: str | None=None, max_output_chars: int | None=None, max_output_tokens: int | None=None, **run_options: Any) -> dict[str, Any]: ... def build_provider_extract_request(text: str, *, schema: dict[str, Any] | None=None, instructions: str | None=None, **run_options: Any) -> dict[str, Any]: diff --git a/clients/python/src/vyral_client/contracts.py b/clients/python/src/vyral_client/contracts.py index 80b54c5..e8528c0 100644 --- a/clients/python/src/vyral_client/contracts.py +++ b/clients/python/src/vyral_client/contracts.py @@ -2640,7 +2640,7 @@ class ProviderTargetDescriptor(_ProviderTargetDescriptorRequired, total=False): class _ProviderRunRequestRequired(TypedDict): capability: str - payload: AiChatPayload | AiExtractPayload | AiRerankPayload | AiReviewPayload | AiScaffoldPayload | AiToolPlanPayload | dict[str, JSONValue] + payload: AiChatPayload | AiExtractPayload | AiRerankPayload | AiReviewPayload | AiScaffoldPayload | AiToolPlanPayload | AiJudgePayload | dict[str, JSONValue] class ProviderRunRequest(_ProviderRunRequestRequired, total=False): provider: str | None @@ -2660,7 +2660,7 @@ class _ProviderRunResultRequired(TypedDict): capability: str operation: str mode: str - output: AiChatResult | AiExtractResult | AiRerankResult | AiReviewResult | AiScaffoldResult | AiToolPlanResult | dict[str, JSONValue] + output: AiChatResult | AiExtractResult | AiRerankResult | AiReviewResult | AiScaffoldResult | AiToolPlanResult | AiJudgeResult | dict[str, JSONValue] class ProviderRunResult(_ProviderRunResultRequired, total=False): failureClass: str | None @@ -2886,6 +2886,7 @@ class AiChatPayload(TypedDict, total=False): messages: list[AiMessage] system: str | None maxOutputChars: int | None + maxOutputTokens: int | None class _AiChatResultRequired(TypedDict): message: AiMessage @@ -3257,3 +3258,48 @@ class RagContextGraphEvaluationFailureModes(TypedDict, total=False): graphContextTextMissing: bool contextTextTruncated: bool budgetTruncated: bool + +class _AiJudgeOptionRequired(TypedDict): + id: str + label: str + +class AiJudgeOption(_AiJudgeOptionRequired, total=False): + pass + +class _AiJudgeQuestionRequired(TypedDict): + id: str + type: Literal['choice', 'noul', 'score'] + prompt: str + +class AiJudgeQuestion(_AiJudgeQuestionRequired, total=False): + options: list[AiJudgeOption] + +class _AiJudgePayloadRequired(TypedDict): + questions: list[AiJudgeQuestion] + +class AiJudgePayload(_AiJudgePayloadRequired, total=False): + context: str | None + references: list[AiReference] + +class _AiJudgeAnswerRequired(TypedDict): + questionId: str + calibrated: bool + +class AiJudgeAnswer(_AiJudgeAnswerRequired, total=False): + choice: str | None + score: float | None + legend: dict[str, str] | None + probability: float | None + confidence: float | None + labelMassCoverage: float | None + rotationAgreement: float | None + probabilities: dict[str, float] | None + rawLabelProbabilities: dict[str, float] | None + rawClassProbabilities: dict[str, float] | None + +class _AiJudgeResultRequired(TypedDict): + answers: list[AiJudgeAnswer] + validationStatus: str + +class AiJudgeResult(_AiJudgeResultRequired, total=False): + modelId: str | None diff --git a/clients/python/tests/test_client.py b/clients/python/tests/test_client.py index 6e4a61f..e5024a0 100644 --- a/clients/python/tests/test_client.py +++ b/clients/python/tests/test_client.py @@ -1277,7 +1277,9 @@ def test_provider_request_builders_expose_typed_ai_payloads(self) -> None: [{"role": "user", "content": "Summarize."}], model_id="gpt-5.3-codex-spark", timeout_seconds=30, + max_output_tokens=12, ) + self.assertEqual(12, chat["payload"]["maxOutputTokens"]) self.assertEqual("ai.chat", chat["capability"]) self.assertEqual("gpt-5.3-codex-spark", chat["modelId"]) self.assertEqual("Summarize.", chat["payload"]["messages"][0]["content"]) diff --git a/contracts/schemas/vyral-public.schema.json b/contracts/schemas/vyral-public.schema.json index 9afaeaf..3d60660 100644 --- a/contracts/schemas/vyral-public.schema.json +++ b/contracts/schemas/vyral-public.schema.json @@ -11286,6 +11286,9 @@ { "$ref": "#/$defs/AiToolPlanPayload" }, + { + "$ref": "#/$defs/AiJudgePayload" + }, { "type": "object", "additionalProperties": true, @@ -11298,7 +11301,8 @@ "ai.rerank": "#/components/schemas/AiRerankPayload", "ai.review": "#/components/schemas/AiReviewPayload", "ai.scaffold": "#/components/schemas/AiScaffoldPayload", - "ai.toolPlan": "#/components/schemas/AiToolPlanPayload" + "ai.toolPlan": "#/components/schemas/AiToolPlanPayload", + "ai.judge": "#/components/schemas/AiJudgePayload" } }, "correlationId": { @@ -11402,6 +11406,9 @@ { "$ref": "#/$defs/AiToolPlanResult" }, + { + "$ref": "#/$defs/AiJudgeResult" + }, { "type": "object", "additionalProperties": true, @@ -11414,7 +11421,8 @@ "ai.rerank": "#/components/schemas/AiRerankResult", "ai.review": "#/components/schemas/AiReviewResult", "ai.scaffold": "#/components/schemas/AiScaffoldResult", - "ai.toolPlan": "#/components/schemas/AiToolPlanResult" + "ai.toolPlan": "#/components/schemas/AiToolPlanResult", + "ai.judge": "#/components/schemas/AiJudgeResult" } }, "failureClass": { @@ -12626,6 +12634,15 @@ "null" ], "format": "int32" + }, + "maxOutputTokens": { + "type": [ + "integer", + "null" + ], + "format": "int32", + "minimum": 1, + "description": "Positive ceiling on all generated tokens, including reasoning. Adapters that cannot enforce it reject before inference; character and byte limits remain separate." } } }, @@ -14157,6 +14174,206 @@ "type": "boolean" } } + }, + "AiJudgeOption": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "AiJudgeQuestion": { + "type": "object", + "required": [ + "id", + "type", + "prompt" + ], + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "choice", + "noul", + "score" + ] + }, + "prompt": { + "type": "string" + }, + "options": { + "type": "array", + "items": { + "$ref": "#/$defs/AiJudgeOption" + } + } + } + }, + "AiJudgePayload": { + "type": "object", + "required": [ + "questions" + ], + "properties": { + "context": { + "type": [ + "string", + "null" + ] + }, + "questions": { + "type": "array", + "items": { + "$ref": "#/$defs/AiJudgeQuestion" + } + }, + "references": { + "type": "array", + "items": { + "$ref": "#/$defs/AiReference" + } + } + } + }, + "AiJudgeAnswer": { + "type": "object", + "required": [ + "questionId", + "calibrated" + ], + "properties": { + "questionId": { + "type": "string" + }, + "choice": { + "type": [ + "string", + "null" + ] + }, + "calibrated": { + "type": "boolean" + }, + "score": { + "type": [ + "number", + "null" + ], + "description": "Expected zero-based ordered level index." + }, + "legend": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "probability": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "confidence": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "labelMassCoverage": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Minimum unconditioned decision-label mass over rotations; null means unavailable." + }, + "rotationAgreement": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Share of rotations whose argmax agrees with the final winner; null means unavailable." + }, + "probabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "rawLabelProbabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "description": "Mean unconditioned mass per original option; not a normalized probability distribution." + }, + "rawClassProbabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + } + }, + "AiJudgeResult": { + "type": "object", + "required": [ + "answers", + "validationStatus" + ], + "properties": { + "answers": { + "type": "array", + "items": { + "$ref": "#/$defs/AiJudgeAnswer" + } + }, + "validationStatus": { + "type": "string" + }, + "modelId": { + "type": [ + "string", + "null" + ] + } + } } } } diff --git a/design/cloudflare-queues-bridge.md b/design/cloudflare-queues-bridge.md new file mode 100644 index 0000000..d1fdb91 --- /dev/null +++ b/design/cloudflare-queues-bridge.md @@ -0,0 +1,67 @@ +# Bounded Cloudflare Queues bridge + +Status: design decision; no bridge is deployed or registered. Add a prototype +only for a selected deployment that needs to submit projection jobs from a +hosted Vyral service to a Worker-owned Queue. Reuse Vyral's durable execution +identity, hosted-handler catalog, projection generation checks and object-store +references. Queues is a delivery transport; it grants no execution authority. + +## Submission boundary + +The Vyral sender builds a versioned envelope with an opaque job ID, audience, +issuer/key ID, issued/expiry times, collection and projection generation, an +allowlisted handler ID, an immutable payload digest and object reference, and +a bounded attempt identifier. It signs canonical bytes with a rotated, +deployment-specific signing key held outside source. The Worker verifies the +signature, audience, lifetime and exact schema before dispatch. The Worker +owns its Queue binding and rejects arbitrary queue names, handler IDs, URLs, +inline credentials, executable commands and oversized envelopes. + +Use at most 64 KiB of UTF-8 envelope bytes, including signature and metadata. +Large content stays in object storage behind scoped access; the consumer +verifies its digest. Cloudflare currently limits messages to 128 KB, including +metadata, and batches to 100 messages. The tighter envelope bound leaves room +for deployment metadata. Keep concurrency, batch size, max retries and dispatch +rate explicit in the selected profile. + +## Delivery and acknowledgment + +Queues delivers at least once, so duplicate and reordered messages are expected. +The Worker consumer validates again and claims the issuer, job ID, generation +and payload digest tuple in durable deduplication state before submitting the +approved Vyral operation. The same identity with a different digest is a conflict. +An expired or obsolete generation is a permanent refusal, never an implicit +request to rewrite a newer projection. + +Queue acknowledgment follows a durable acceptance/terminal record, depending +on the selected operation's documented completion boundary. A network timeout +after submission is unresolved: look up the same durable Vyral job before +retrying and never assign a fresh job ID to hide uncertainty. An in-memory map +is insufficient deduplication across restarts. Completion must not depend on +an atomic transaction spanning Queue delivery and an HTTP effect. + +Retry only recognized transient errors with bounded delay and a fixed maximum. +Permanent authentication, schema, generation, digest and authorization failures +go to a configured dead-letter queue with a safe reason and opaque identity. +If no DLQ is configured, exhausted messages can be deleted by the service; +the profile must reject that configuration when terminal accounting is required. +DLQ redrive repeats validation and preserves original identity and digest. + +## Ownership and acceptance + +The Worker deployment owns Queue bindings, secrets, deduplication state, +retry/DLQ settings and observability. Vyral owns its request validation, +handler allowlist, job status, authorization and projection freshness. +Neither successful submission nor Queue acknowledgment establishes successful +projection, model execution or a production qualification. + +A future prototype must demonstrate tampered/expired/wrong-audience envelopes, +duplicate/conflicting IDs, stale generations, bounded bytes and concurrency, +lost submission responses, restart/lookup, retry exhaustion, DLQ redrive and +retained terminal inventory. Run fixtures first, then one isolated live profile. +There is no current deployment requirement for a prototype. + +Sources: [delivery guarantees](https://developers.cloudflare.com/queues/reference/delivery-guarantees/), +[limits](https://developers.cloudflare.com/queues/platform/limits/), +[retries](https://developers.cloudflare.com/queues/configuration/batching-retries/), +[dead-letter queues](https://developers.cloudflare.com/queues/configuration/dead-letter-queues/). diff --git a/design/local-logprob-judge.md b/design/local-logprob-judge.md index 3a421c0..5f33fbd 100644 --- a/design/local-logprob-judge.md +++ b/design/local-logprob-judge.md @@ -1,17 +1,18 @@ # Local log-probability judgment -Status: proposal. No new provider is registered by this document, and no -backend has been qualified. The existing in-process ONNX judge remains the -available local implementation. This proposal addresses fixed-option Choice -scoring with a generative model without coupling Vyral to a particular host, -service, model package, deployment, or consumer task. +Status: implemented as an opt-in prototype. The portable Choice-scoring target, +injected session contract and bounded native loopback adapter are available. +See [the runtime guide](../docs/reference/local-runtime-providers.md) for supported +contracts, controls and reproduction. Fake-session, transport and pinned real-model +mechanics checks pass; this is not an accuracy, calibration or production-issuer +qualification. The existing in-process ONNX classifier remains available. ## Boundary Keep `ai.judge` and its shared-context batch request as the caller-facing -contract. A proposed `LocalLogprobJudgeProviderTarget` would own prompt +contract. The `LocalLogprobJudgeProviderTarget` owns prompt construction, option rotations, probability accounting, validation, and -mapping results back to caller option IDs. It would initially advertise only +mapping results back to caller option IDs. It advertises only Choice questions; Noul and Score must return `unsupported` until implemented and tested explicitly. @@ -39,7 +40,10 @@ bounded cleanup token even when the caller token is cancelled. Renewal failure terminates the batch. Credentials and lease secrets stay in memory and are excluded from traces, config hashes, errors, and raw output. -## HTTP compatibility +## Optional host transport compatibility + +The following streaming transport is an extension point; it is not implemented +by the native adapter shipped here. An optional HTTP session can use an OpenAI-compatible chat-completions response shape. That shape alone is insufficient evidence of scoring support. @@ -125,7 +129,8 @@ backend error code. Preserve the original failure if cleanup also fails. Before registering a target, require offline tests for rotation invariance, complete and missing label sets, byte/ID mismatch, full-vocabulary mass, low coverage, unsupported question types, exact context boundaries, -cancellation, split/truncated SSE frames, and disposal on every exit path. +cancellation and disposal on every exit path. A future streaming adapter must +also qualify split/truncated SSE frames before registration. Exercise the batch with a counting session to prove one acquisition/load and one release, including failures partway through a batch. Test serialization of the additive answer evidence and keep existing judge conformance passing. diff --git a/docs/guides/consumer-handoff.md b/docs/guides/consumer-handoff.md index 392b390..5398e5a 100644 --- a/docs/guides/consumer-handoff.md +++ b/docs/guides/consumer-handoff.md @@ -32,6 +32,7 @@ Useful startup variables: | `VYRAL_CLOUDFLARE_ACCOUNT_ID` | Cloudflare account id for R2 when `VYRAL_OBJECT_STORE=cloudflare-r2`. | | `VYRAL_R2_ACCESS_KEY_ID` | Cloudflare R2 S3 API access key id. | | `VYRAL_R2_SECRET_ACCESS_KEY` | Cloudflare R2 S3 API secret access key. | +| `VYRAL_R2_SESSION_TOKEN` | Optional security token for short-lived, bucket-scoped S3 credentials. | | `VYRAL_R2_BUCKET` | R2 bucket used by readiness probes and artifact ingestion when R2 is selected. | | `VYRAL_PROVIDER_ARTIFACT_DIR` | Provider run artifact directory. | | `VYRAL_API_KEY` | Enables API-key protection when set. Send as bearer token or `X-Vyral-Api-Key`. | @@ -94,8 +95,8 @@ and `auto` signing region and defaults to disabling payload signing and automati put checksum validation for compatibility. These settings are not an integrity qualification: verify byte-for-byte round trips, conditional no-replace writes, metadata, pagination, cancellation, and deletion against an isolated bucket -before deployment. The R2 adapter remains preview; no live qualification is -claimed here. +before deployment. The R2 adapter remains preview. The [bounded live qualification](../reference/cloudflare-r2-qualification.md) +covers its supported object subset; conditional deletion is explicitly refused. Any consumer can submit a record and its artifact to: diff --git a/docs/maintainers/releasing.md b/docs/maintainers/releasing.md index d7d0ab1..d9a79f9 100644 --- a/docs/maintainers/releasing.md +++ b/docs/maintainers/releasing.md @@ -113,11 +113,11 @@ GitHub-verified signed `server-v0.3.1` tag at current `main` and a successful Release Integrity run for that commit. It does not republish the unaffected NuGet, PyPI, or npm artifacts. -The server's `0.3.5` source authorization is likewise container-only: +The server's `0.3.6` source authorization is likewise container-only: [`packaging/worker-container-release.json`](../../packaging/worker-container-release.json) and the manual [`Publish server container`](../../.github/workflows/publish-worker-container.yml) -workflow authorize only `ghcr.io/univeracity/vyral-server:0.3.5`. It requires a -GitHub-verified signed `server-v0.3.5` tag at current `main`, successful canonical +workflow authorize only `ghcr.io/univeracity/vyral-server:0.3.6`. It requires a +GitHub-verified signed `server-v0.3.6` tag at current `main`, successful canonical Release Integrity evidence containing the hosted-worker receipt, and a second MCP and hosted-worker qualification plus pinned Trivy scan against the exact published digest. The hosted-worker entrypoint is preview and initially hosts only diff --git a/docs/reference/cloudflare-r2-qualification.md b/docs/reference/cloudflare-r2-qualification.md new file mode 100644 index 0000000..97fa4c2 --- /dev/null +++ b/docs/reference/cloudflare-r2-qualification.md @@ -0,0 +1,38 @@ +# Cloudflare R2 object qualification + +The R2 object adapter remains preview. On 2026-10-03 a bounded live run passed +8 live checks against a newly created isolated bucket, using short-lived credentials +scoped to that bucket. No existing objects or buckets were used. Retained raw +test receipts and account details are private operator evidence. + +The supported subset covers content/metadata/ETag round trips, no-replace and +matching/nonmatching replacement preconditions, portable names and metadata, +pagination (including empty listings), idempotent unconditional deletion, and a +pre-cancelled write that leaves no object. Nine local transport checks additionally +cover a 503 failure without automatic retry and cancellation before dispatch. + +The live run found that R2 accepted a delete with a mismatched If-Match value and +deleted the object. Vyral now refuses every conditional R2 delete before sending +it, including a matching value; a HEAD followed by DELETE would introduce a race. +The live check confirms both refusals retain the object, followed by successful +unconditional deletion. This does not qualify R2 for the complete portable +IObjectStore contract, which includes conditional deletion. + +The AWS SDK v4 can return a null S3Objects collection for an empty listing. +The shared S3 implementation now treats it as empty and disposes read responses. +R2 sets MaxErrorRetry=0 by default (explicitly configurable from 0 through 3), +and accepts optional session credentials. A caller must reconcile effects after +a dispatched request fails or is cancelled; pre-cancellation is the only live +cancellation case qualified here. + +Reproduction requires VYRAL_R2_ACCOUNT_ID, VYRAL_R2_ACCESS_KEY_ID, +VYRAL_R2_SECRET_ACCESS_KEY, optional VYRAL_R2_SESSION_TOKEN and VYRAL_R2_BUCKET, +then dotnet test tests/Vyral.Tests.Cloudflare/Vyral.Tests.Cloudflare.csproj. +Use an isolated bucket and temporary credentials. The tests use random prefixes +and delete their own objects. Account identifiers, bucket names and secrets must +stay outside public receipts. + +Cloudflare documents supported conditions for object reads/writes, without +conditional DeleteObject support, in its [S3 API matrix](https://developers.cloudflare.com/r2/api/s3/api/). +Its [temporary credentials guide](https://developers.cloudflare.com/r2/api/s3/temporary-credentials/) +describes the bucket/object permissions used by this qualification. diff --git a/docs/reference/local-runtime-providers.md b/docs/reference/local-runtime-providers.md new file mode 100644 index 0000000..77881b5 --- /dev/null +++ b/docs/reference/local-runtime-providers.md @@ -0,0 +1,96 @@ +# Local runtime providers + +The native chat target and Choice-only log-probability judge are opt-in prototypes. +Ordinary CI uses fake sessions and HTTP fixtures; a real model is never downloaded +or launched implicitly. The previous proposal's scoring mechanics are implemented +in Vyral. Host-specific acquisition, renewal and release belong behind the injected +ILocalLogprobSessionFactory, allowing an in-process engine or explicitly configured +local service to implement the same shape. + +## Contracts and scoring + +AiChatRequest.maxOutputTokens is a positive ceiling on every generated token, +including hidden reasoning. An adapter that cannot enforce it must reject before +inference. The deterministic and CLI adapters reject it; they never approximate it +with characters or bytes. The native target requires an explicit ceiling and sends +that exact value as n_predict. maxOutputChars and MaxOutputBytes remain separate +output controls. Unknown payload fields, tools, attachments and source writes are +unsupported. + +LocalLogprobJudgeProviderTarget answers only Choice questions. It acquires one +session per batch, resolves decision labels to verified single token IDs/bytes, +and counts every complete model-templated prompt plus one reserved output token +before any inference. It refuses overflow without truncation. The default scores +three cyclic rotations, capped by option count; options bound question count, +option count and total scoring calls. A host adapter owns bounded cleanup even +after cancellation. + +Each answer exposes normalized Probabilities, mean unconditioned +rawLabelProbabilities, minimum labelMassCoverage across rotations, and +rotationAgreement (the share of rotation winners agreeing with the final winner). +Ties use original option order. Missing labels, mismatched bytes, duplicate IDs, +multiple scored positions, nonfinite values or impossible mass reject the batch. +Low positive coverage remains visible. Callers choose their own abstention policy. +These fields are null for providers without this evidence. RawClassProbabilities +keeps its existing native Noul-classifier meaning. Every log-probability answer +has calibrated=false; no relevance or accuracy claim follows from normalization. + +## Native transport profile + +LlamaCppRuntimeSessionFactory connects only to a literal loopback HTTP origin, +with redirects and proxies disabled. It uses bounded response streaming and one +explicit send per route, with no application retry. It checks configured model +file bytes, the peer's absolute model path, runtime build, original template hash, +one-slot context window and effective template probe against an operator-owned +profile. These are consistency checks inside a trusted local process boundary, +not cryptographic proof against an arbitrary peer impersonating the runtime. + +Its closed adapter route inventory is GET /props and POST /apply-template, +/tokenize, /detokenize (judge label verification) and /completion. It calls no +tools, helper processes or remote providers. The profile operator must pin the +actual binary/library closure and model bytes, disable context shifting, +speculative decoding, tools and remote routes, and qualify that launch separately. +The adapter does not manage the process or establish complete host egress denial. + +Native scoring requests one output position, negative temperature (greedy choice +with original softmax probabilities), no sampler chain, pre-sampling probabilities, +and a bounded top-token distribution. Every requested decision label must appear; +missing from top-k means unknown and rejects the answer. The two explicitly +supported native response fields are completion_probabilities and probs; simultaneous +fields are rejected. Backend support for that distribution must be qualified at +the pinned build. SSE transports can implement the injected session contract, +but this native adapter uses bounded nonstreaming completion. + +## Explicit registration and reproduction + +Set Providers:LocalRuntime:Enabled=true, separately from remote CLI registration. +Configure Endpoint, ModelId, absolute ModelPath, ModelSha256, BuildInfo, +TemplateSha256, TemplateProbeSha256, ContextTokens and MaxOutputTokens under +Providers:LocalRuntime. Pin only an operator-controlled runtime. The server +registers local-llamacpp and local-logprob-judge; the latter uses mechanics mode. +WorkspaceAgent opt-in is also independent of remote CLI enablement and retains +API-key authentication and its existing source-writing guards. + +For the optional backend gate, set VYRAL_LOCAL_RUNTIME_PROFILE to an ignored JSON +file containing the equivalent camelCase runtime options, then run the provider +tests filtered by LocalNativeRuntimeTests. The fixtures exercise token/byte +ceilings, unknown and negative usage, model/template/window mismatch, preflight +refusals and cancellation. The real smoke verifies chat and Choice response +mechanics for one pinned model/runtime; it establishes no model-quality claim. + +Provider-native input/output counters remain reported observations. A total +derived by summing them is explicitly consumer_inference/estimated, and absent +counters stay unknown. Receipts remain partial; a local result or signature +cannot establish complete helper-route accounting or independent production +issuance. Reuse the existing durable provider jobs, public lookup, constrained +workspace runner, signing hooks and independent review contract. Cancellation +after a completion send leaves the native effect unresolved; it does not prove +model execution stopped or consumed zero work. A deployed worker, private IPC, +restart/owner-death containment and protected independent issuer require their +own joined qualification before source-writing or trusted issuance is enabled. + +Primary protocol reference: [llama.cpp server](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md). + +The [dated mechanics receipt](../../qualification/local-runtime-mechanics-2026-10-03.json) +pins the tested source, binary/library closure, model and effective template. +It records two real-model smoke cases alongside deterministic boundary checks. diff --git a/docs/reference/python-retrieval-qualification.md b/docs/reference/python-retrieval-qualification.md index 63cc2a5..4480403 100644 --- a/docs/reference/python-retrieval-qualification.md +++ b/docs/reference/python-retrieval-qualification.md @@ -59,12 +59,14 @@ embedded `VyralRuntime` drains its executor and clears its record cache. ## Flat vector search -The dependency-free Python baseline retains individual float32 rounding steps +The dependency-free Python baseline uses the standard-library C float scalar +conversion and retains individual float32 rounding steps and their explicit overflow/nonfinite errors. Cosine query-norm preparation is reused within one search, and diagnostics are constructed for retained top results. Every eligible record and vector row is still hydrated and validated, including nonselected records: malformed JSON, dimensions or vector byte lengths -fail the search. Namespace/partition filtering, thresholds, stable ties and +fail the search. A bounded top-result heap replaces retaining and sorting every +matching record; it still exhausts and validates all eligible rows. Namespace/partition filtering, thresholds, stable ties and continuation retain their existing behavior. This remains exhaustive flat search. High-dimensional Python arithmetic and @@ -94,3 +96,45 @@ For a frozen evidence comparison, preserve the cache manifest, source revisions, eligibility filters, fields, prefix/phrase rules and candidate budgets. Use one final output renderer and the consuming tokenizer for equal output budgets; RAG `maxChars` bounds excerpt text before citation formatting and other output. + +## High-dimensional qualification and grouping decision + +From a Git source checkout, run: + + python3 scripts/benchmark-python-vector-search.py --output /tmp/vector-search.json + +The bounded default uses 1,000 deterministic synthetic 384-dimensional records, +three metrics and three repeats. It compares complete records, scores, diagnostics +and continuation against the pinned previous source, plus 2,075 scalar/vector bit +and error controls. It binds source/input hashes and platform versions. Use the +existing Python platform workflow to qualify supported Python 3.10–3.12 on Linux, +macOS and Windows; a single-host timing receipt does not replace that matrix. +No compiled extension, NumPy dependency, relaxed score tolerance, fast-math or +model download is introduced. + +The current fail-closed policy deliberately prevents ranking before complete record +validation: a malformed nonselected record must still fail the query. Splitting +storage validation from hydration would require a separately versioned contract +and corruption controls. Keep full validation in this profile. + +Budget passages and distinct investigation items as separate experiments. Preserve +one larger candidate pool, group by an explicit caller-owned item identifier after +retrieval, and retain the first ranked passage per item (or an explicitly bounded +number). Compare final rendered output under the same tokenizer budget. Report +pool size, eligible items, selected passages and selected items; grouping after a +small top pool can hide additional items and cannot establish recall equivalence. +The probe reports passage/distinct-item counts without changing Vyral scores or +introducing domain-specific identifiers into the engine. This closes the mechanics +investigation, not application relevance qualification or full runtime promotion. + +The [dated synthetic receipt](../../qualification/python-vector-mechanics-2026-10-03.json) +retains all 2,075 numeric/error controls and nine complete responses from a +1,000-record, 384-dimension fixture. Median complete retrieval/serialization was +1,020.7 ms before and 933.4 ms after on this host. Treat this as bounded mechanics +evidence. The supported-platform workflow repeats the same parity controls on +Python 3.10–3.12 across Linux, macOS and Windows with a smaller record count. + +Nine additional complete baseline/candidate responses agree at candidate pools +of 10, 30 and 100. Caller-side grouping caps distinct output at ten items, with +returned rows, consumed passages and distinct outputs recorded separately. +A candidate pool is not the number of rows the caller has actually received. diff --git a/packaging/worker-container-release.json b/packaging/worker-container-release.json index 00a96e7..5ccec2b 100644 --- a/packaging/worker-container-release.json +++ b/packaging/worker-container-release.json @@ -3,7 +3,7 @@ "publicationAuthorized": true, "image": { "name": "ghcr.io/univeracity/vyral-server", - "version": "0.3.5", + "version": "0.3.6", "source": "Dockerfile", "environment": "publish-container", "maturity": "mixed", @@ -17,7 +17,7 @@ }, "authorization": { "mode": "manual-protected-environment", - "releaseTag": "server-v0.3.5", + "releaseTag": "server-v0.3.6", "workflow": ".github/workflows/publish-worker-container.yml", "requirements": [ "a GitHub-verified signed annotated server tag that resolves to current main", diff --git a/qualification/local-runtime-mechanics-2026-10-03.json b/qualification/local-runtime-mechanics-2026-10-03.json new file mode 100644 index 0000000..18e274e --- /dev/null +++ b/qualification/local-runtime-mechanics-2026-10-03.json @@ -0,0 +1,63 @@ +{ + "schemaVersion": "vyral.local-runtime-mechanics.v1", + "testedAtUtc": "2026-10-03", + "level": "prototype", + "buildInfo": "b1-4c5957c", + "model": "Qwen3-1.7B-Q8_0.gguf", + "modelSha256": "061b54daade076b5d3362dac252678d17da8c68f07560be70818cace6590cb1a", + "runtimeClosureSha256": { + "libggml-base.so": "0015c6661c5763ab9d5f22d66c226ce69f638a0558499e94701478ccadb5f25e", + "libggml-base.so.0": "0015c6661c5763ab9d5f22d66c226ce69f638a0558499e94701478ccadb5f25e", + "libggml-base.so.0.25.1": "0015c6661c5763ab9d5f22d66c226ce69f638a0558499e94701478ccadb5f25e", + "libggml-cpu.so": "2039be98c6ee0d1cee1d3793a95f2deb061706e8ef39c9dd33a1cc3941227d92", + "libggml-cpu.so.0": "2039be98c6ee0d1cee1d3793a95f2deb061706e8ef39c9dd33a1cc3941227d92", + "libggml-cpu.so.0.25.1": "2039be98c6ee0d1cee1d3793a95f2deb061706e8ef39c9dd33a1cc3941227d92", + "libggml.so": "dea80d51796e35a29860f7ed0727a9461e6f0c9b1647be09eb877a2cb1aa856a", + "libggml.so.0": "dea80d51796e35a29860f7ed0727a9461e6f0c9b1647be09eb877a2cb1aa856a", + "libggml.so.0.25.1": "dea80d51796e35a29860f7ed0727a9461e6f0c9b1647be09eb877a2cb1aa856a", + "libllama-cli-impl.so": "ec277578883ed08cd981178a7f3a6c317a3cab8bb52f58b6f7270128423203ca", + "libllama-common.so": "a492f075d9555e332901409a25df359b5a4981ebfc3172f552080d57039621d1", + "libllama-common.so.0": "a492f075d9555e332901409a25df359b5a4981ebfc3172f552080d57039621d1", + "libllama-common.so.0.5.0": "a492f075d9555e332901409a25df359b5a4981ebfc3172f552080d57039621d1", + "libllama-server-impl.so": "26bc3756d41f06fabb25c39ab9441d5905878ada1798f8691fd058df5313e35d", + "libllama.so": "01c93d2cb0f3716eddca46ba438c5135a252211236f079bc7161862863e94fc5", + "libllama.so.0": "01c93d2cb0f3716eddca46ba438c5135a252211236f079bc7161862863e94fc5", + "libllama.so.0.5.0": "01c93d2cb0f3716eddca46ba438c5135a252211236f079bc7161862863e94fc5", + "libmtmd.so": "9ad8161b860a5deaac9d0dc3d10ed5f3fb3162981d9455c8862fd96ee8f5f71c", + "libmtmd.so.0": "9ad8161b860a5deaac9d0dc3d10ed5f3fb3162981d9455c8862fd96ee8f5f71c", + "libmtmd.so.0.5.0": "9ad8161b860a5deaac9d0dc3d10ed5f3fb3162981d9455c8862fd96ee8f5f71c", + "llama-server": "5ed18fe1236449aa2001f68f4e5fc56828a13d42c76b6f2335848e533f095d33" + }, + "templateSha256": "57f1fd00f0013a2be96aa79b857391f27e23df5b5f847072b524c897e24d0361", + "effectiveTemplateProbeSha256": "323214ed1434eb18d9d20cd120d363b3364184c1790b3116228e9a7c55dc23e9", + "contextTokens": 2048, + "slots": 1, + "launchControls": [ + "literal loopback", + "CPU only", + "no context shift", + "no web UI", + "thinking disabled through template arguments" + ], + "sourceSha256": { + "src/Vyral.Providers.Local/LocalLlamaCppProviderTarget.cs": "f30f8e11331ae126db9b2322136899a9661c215532f5d2f3b7c7eb55061e610c", + "src/Vyral.Providers.Local/LlamaCppRuntimeSessionFactory.cs": "adbfafe0595364533a17a3ed45f5ca36eb737aa80ecc10d17b5da82179b73c0a", + "src/Vyral.Providers.Local/LocalLogprobJudgeProviderTarget.cs": "c7c003673e38e1cc8973bd3344639185514c73ef11b44493cda4b6968ad10303", + "src/Vyral.Providers.Local/LocalLogprobSession.cs": "13561224bf68b81ca3a822fb34cc8562fdc82e182b7db95e4c98fb3ca6f5345f", + "tests/Vyral.Tests.Providers/LocalNativeRuntimeTests.cs": "a815b018fd5093c072449bf3feda2bff479e9c2f57d83a868950606e1b926bb6", + "tests/Vyral.Tests.Providers/LocalLogprobJudgeProviderTests.cs": "2c0b9e5f02926000066ee3adb7c6d5e30c70eccc7c8a30c9853325f495028027" + }, + "tests": { + "focusedPassed": 33, + "realModelSmokeCases": 2, + "scoringMode": "Choice; first output position; two rotations", + "receiptSha256": "cd99e0f101ec5258fb2449a31bf7ed8517202ebed568b28b0cf6d81ee5f1f965" + }, + "limits": [ + "mechanics only; no accuracy or calibration qualification", + "operator consistency binding; no proof against a malicious HTTP peer", + "no implicit model download or runtime launch", + "no complete host egress, descendant or owner-death qualification", + "no complete attempt-accounting or protected issuance claim" + ] +} diff --git a/qualification/python-vector-mechanics-2026-10-03.json b/qualification/python-vector-mechanics-2026-10-03.json new file mode 100644 index 0000000..fcc26e1 --- /dev/null +++ b/qualification/python-vector-mechanics-2026-10-03.json @@ -0,0 +1,105 @@ +{ + "schemaVersion": "vyral.python-vector-qualification.v1", + "sourceSha256": "dd882719f63ccfaf8ce392b7d7080677dc35bea56248af89aa0b41d9d48b082b", + "referenceSha256": "b04d156e9df36297487896dc108b8bfb74b9fd46dc53063134c2b962346b3658", + "inputSha256": "0c5b7833ac045a568f34c1e19153b5c5befd8f7ceb03c5eb660d4cd5dfdf3e94", + "python": "3.12.3", + "platform": "Linux-6.8.0-142-lowlatency-x86_64-with-glibc2.39", + "sqlite": "3.45.1", + "records": 1000, + "dimensions": 384, + "numericErrorControls": 2075, + "completeResponses": 9, + "completeBudgetResponses": 9, + "completeParity": true, + "timingScope": "retrieval and serialization; no encoder/model", + "timings": { + "baseline": { + "p50Ms": 1020.7279659953201, + "samples": 9 + }, + "candidate": { + "p50Ms": 933.4267340018414, + "samples": 9 + } + }, + "grouping": [ + { + "metric": "cosine", + "candidatePool": 10, + "returnedBeforeGrouping": 10, + "passagesConsumed": 10, + "outputItemBudget": 10, + "distinctItemsReturned": 10 + }, + { + "metric": "cosine", + "candidatePool": 30, + "returnedBeforeGrouping": 30, + "passagesConsumed": 10, + "outputItemBudget": 10, + "distinctItemsReturned": 10 + }, + { + "metric": "cosine", + "candidatePool": 100, + "returnedBeforeGrouping": 100, + "passagesConsumed": 10, + "outputItemBudget": 10, + "distinctItemsReturned": 10 + }, + { + "metric": "dotproduct", + "candidatePool": 10, + "returnedBeforeGrouping": 10, + "passagesConsumed": 10, + "outputItemBudget": 10, + "distinctItemsReturned": 10 + }, + { + "metric": "dotproduct", + "candidatePool": 30, + "returnedBeforeGrouping": 30, + "passagesConsumed": 10, + "outputItemBudget": 10, + "distinctItemsReturned": 10 + }, + { + "metric": "dotproduct", + "candidatePool": 100, + "returnedBeforeGrouping": 100, + "passagesConsumed": 10, + "outputItemBudget": 10, + "distinctItemsReturned": 10 + }, + { + "metric": "euclidean", + "candidatePool": 10, + "returnedBeforeGrouping": 10, + "passagesConsumed": 10, + "outputItemBudget": 10, + "distinctItemsReturned": 9 + }, + { + "metric": "euclidean", + "candidatePool": 30, + "returnedBeforeGrouping": 30, + "passagesConsumed": 11, + "outputItemBudget": 10, + "distinctItemsReturned": 10 + }, + { + "metric": "euclidean", + "candidatePool": 100, + "returnedBeforeGrouping": 100, + "passagesConsumed": 11, + "outputItemBudget": 10, + "distinctItemsReturned": 10 + } + ], + "limits": [ + "one synthetic host fixture; no SLA or relevance claim", + "full eligible record validation retained; no rank-before-hydration", + "distinct-item grouping is caller-owned and evaluated separately" + ] +} diff --git a/runtimes/python/src/vyral_runtime/_contracts/vyral-public.schema.json b/runtimes/python/src/vyral_runtime/_contracts/vyral-public.schema.json index 9afaeaf..3d60660 100644 --- a/runtimes/python/src/vyral_runtime/_contracts/vyral-public.schema.json +++ b/runtimes/python/src/vyral_runtime/_contracts/vyral-public.schema.json @@ -11286,6 +11286,9 @@ { "$ref": "#/$defs/AiToolPlanPayload" }, + { + "$ref": "#/$defs/AiJudgePayload" + }, { "type": "object", "additionalProperties": true, @@ -11298,7 +11301,8 @@ "ai.rerank": "#/components/schemas/AiRerankPayload", "ai.review": "#/components/schemas/AiReviewPayload", "ai.scaffold": "#/components/schemas/AiScaffoldPayload", - "ai.toolPlan": "#/components/schemas/AiToolPlanPayload" + "ai.toolPlan": "#/components/schemas/AiToolPlanPayload", + "ai.judge": "#/components/schemas/AiJudgePayload" } }, "correlationId": { @@ -11402,6 +11406,9 @@ { "$ref": "#/$defs/AiToolPlanResult" }, + { + "$ref": "#/$defs/AiJudgeResult" + }, { "type": "object", "additionalProperties": true, @@ -11414,7 +11421,8 @@ "ai.rerank": "#/components/schemas/AiRerankResult", "ai.review": "#/components/schemas/AiReviewResult", "ai.scaffold": "#/components/schemas/AiScaffoldResult", - "ai.toolPlan": "#/components/schemas/AiToolPlanResult" + "ai.toolPlan": "#/components/schemas/AiToolPlanResult", + "ai.judge": "#/components/schemas/AiJudgeResult" } }, "failureClass": { @@ -12626,6 +12634,15 @@ "null" ], "format": "int32" + }, + "maxOutputTokens": { + "type": [ + "integer", + "null" + ], + "format": "int32", + "minimum": 1, + "description": "Positive ceiling on all generated tokens, including reasoning. Adapters that cannot enforce it reject before inference; character and byte limits remain separate." } } }, @@ -14157,6 +14174,206 @@ "type": "boolean" } } + }, + "AiJudgeOption": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + "AiJudgeQuestion": { + "type": "object", + "required": [ + "id", + "type", + "prompt" + ], + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "choice", + "noul", + "score" + ] + }, + "prompt": { + "type": "string" + }, + "options": { + "type": "array", + "items": { + "$ref": "#/$defs/AiJudgeOption" + } + } + } + }, + "AiJudgePayload": { + "type": "object", + "required": [ + "questions" + ], + "properties": { + "context": { + "type": [ + "string", + "null" + ] + }, + "questions": { + "type": "array", + "items": { + "$ref": "#/$defs/AiJudgeQuestion" + } + }, + "references": { + "type": "array", + "items": { + "$ref": "#/$defs/AiReference" + } + } + } + }, + "AiJudgeAnswer": { + "type": "object", + "required": [ + "questionId", + "calibrated" + ], + "properties": { + "questionId": { + "type": "string" + }, + "choice": { + "type": [ + "string", + "null" + ] + }, + "calibrated": { + "type": "boolean" + }, + "score": { + "type": [ + "number", + "null" + ], + "description": "Expected zero-based ordered level index." + }, + "legend": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "probability": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "confidence": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "labelMassCoverage": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Minimum unconditioned decision-label mass over rotations; null means unavailable." + }, + "rotationAgreement": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Share of rotations whose argmax agrees with the final winner; null means unavailable." + }, + "probabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "rawLabelProbabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "description": "Mean unconditioned mass per original option; not a normalized probability distribution." + }, + "rawClassProbabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + } + }, + "AiJudgeResult": { + "type": "object", + "required": [ + "answers", + "validationStatus" + ], + "properties": { + "answers": { + "type": "array", + "items": { + "$ref": "#/$defs/AiJudgeAnswer" + } + }, + "validationStatus": { + "type": "string" + }, + "modelId": { + "type": [ + "string", + "null" + ] + } + } } } } diff --git a/runtimes/python/src/vyral_runtime/_contracts/vyral.openapi.json b/runtimes/python/src/vyral_runtime/_contracts/vyral.openapi.json index 8c3a7ae..388152d 100644 --- a/runtimes/python/src/vyral_runtime/_contracts/vyral.openapi.json +++ b/runtimes/python/src/vyral_runtime/_contracts/vyral.openapi.json @@ -16397,6 +16397,9 @@ { "$ref": "#/components/schemas/AiToolPlanPayload" }, + { + "$ref": "#/components/schemas/AiJudgePayload" + }, { "type": "object", "additionalProperties": true, @@ -16409,7 +16412,8 @@ "ai.rerank": "#/components/schemas/AiRerankPayload", "ai.review": "#/components/schemas/AiReviewPayload", "ai.scaffold": "#/components/schemas/AiScaffoldPayload", - "ai.toolPlan": "#/components/schemas/AiToolPlanPayload" + "ai.toolPlan": "#/components/schemas/AiToolPlanPayload", + "ai.judge": "#/components/schemas/AiJudgePayload" } }, "correlationId": { @@ -16514,6 +16518,9 @@ { "$ref": "#/components/schemas/AiToolPlanResult" }, + { + "$ref": "#/components/schemas/AiJudgeResult" + }, { "type": "object", "additionalProperties": true, @@ -16526,7 +16533,8 @@ "ai.rerank": "#/components/schemas/AiRerankResult", "ai.review": "#/components/schemas/AiReviewResult", "ai.scaffold": "#/components/schemas/AiScaffoldResult", - "ai.toolPlan": "#/components/schemas/AiToolPlanResult" + "ai.toolPlan": "#/components/schemas/AiToolPlanResult", + "ai.judge": "#/components/schemas/AiJudgeResult" } }, "failureClass": { @@ -17755,6 +17763,15 @@ "null" ], "format": "int32" + }, + "maxOutputTokens": { + "type": [ + "integer", + "null" + ], + "format": "int32", + "minimum": 1, + "description": "Positive ceiling on all generated tokens, including reasoning. Adapters that cannot enforce it reject before inference; character and byte limits remain separate." } }, "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiChatPayload" @@ -19317,6 +19334,211 @@ } }, "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/RagContextGraphEvaluationFailureModes" + }, + "AiJudgeOption": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgeOption" + }, + "AiJudgeQuestion": { + "type": "object", + "required": [ + "id", + "type", + "prompt" + ], + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "choice", + "noul", + "score" + ] + }, + "prompt": { + "type": "string" + }, + "options": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AiJudgeOption" + } + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgeQuestion" + }, + "AiJudgePayload": { + "type": "object", + "required": [ + "questions" + ], + "properties": { + "context": { + "type": [ + "string", + "null" + ] + }, + "questions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AiJudgeQuestion" + } + }, + "references": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AiReference" + } + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgePayload" + }, + "AiJudgeAnswer": { + "type": "object", + "required": [ + "questionId", + "calibrated" + ], + "properties": { + "questionId": { + "type": "string" + }, + "choice": { + "type": [ + "string", + "null" + ] + }, + "calibrated": { + "type": "boolean" + }, + "score": { + "type": [ + "number", + "null" + ], + "description": "Expected zero-based ordered level index." + }, + "legend": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "probability": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "confidence": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "labelMassCoverage": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Minimum unconditioned decision-label mass over rotations; null means unavailable." + }, + "rotationAgreement": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Share of rotations whose argmax agrees with the final winner; null means unavailable." + }, + "probabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "rawLabelProbabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "description": "Mean unconditioned mass per original option; not a normalized probability distribution." + }, + "rawClassProbabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgeAnswer" + }, + "AiJudgeResult": { + "type": "object", + "required": [ + "answers", + "validationStatus" + ], + "properties": { + "answers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AiJudgeAnswer" + } + }, + "validationStatus": { + "type": "string" + }, + "modelId": { + "type": [ + "string", + "null" + ] + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgeResult" } } }, diff --git a/runtimes/python/src/vyral_runtime/local/record_store.py b/runtimes/python/src/vyral_runtime/local/record_store.py index 4c939de..24f018f 100644 --- a/runtimes/python/src/vyral_runtime/local/record_store.py +++ b/runtimes/python/src/vyral_runtime/local/record_store.py @@ -1,9 +1,11 @@ from __future__ import annotations from contextlib import contextmanager +from ctypes import c_float, sizeof from dataclasses import dataclass from datetime import datetime, timezone import json +import heapq import math from pathlib import Path import re @@ -60,6 +62,8 @@ _SUPPORTED_DISTANCES = frozenset({"cosine", "dotproduct", "euclidean"}) _SUPPORTED_INDEX_TYPES = frozenset({"flat", "quantizedflat", "diskann"}) _VALUE_BOUNDARY_TOKEN = "x" * 257 +if sizeof(c_float) != 4: + raise RuntimeError("The local vector profile requires a four-byte C float.") class RecordStoreError(RuntimeError): @@ -903,41 +907,42 @@ def _search_vector_records_page( policy.indexed_metadata, vector_search.field, ) - matches: list[tuple[float, VyralRecord]] = [] + candidate_count = 0 similarity = _PreparedSimilarity(field_policy.distance_function, query_vector) - for row in connection.execute(plan.sql, plan.parameters): - record = VyralRecord.from_value(json.loads(row["record_json"])) - stored_dimensions = int(row["dimensions"]) - if stored_dimensions != field_policy.dimensions: - raise RecordStoreError( - f"Stored vector for record {record.id!r} has dimensions " - f"{stored_dimensions}, but policy expects {field_policy.dimensions}." - ) - vector_data = bytes(row["vector_data"]) - if len(vector_data) != stored_dimensions * 4: - raise RecordStoreError( - f"Stored vector for record {record.id!r} has an invalid byte length." + + def candidates() -> Iterator[tuple[float, VyralRecord]]: + nonlocal candidate_count + for row in connection.execute(plan.sql, plan.parameters): + record = VyralRecord.from_value(json.loads(row["record_json"])) + stored_dimensions = int(row["dimensions"]) + if stored_dimensions != field_policy.dimensions: + raise RecordStoreError( + f"Stored vector for record {record.id!r} has dimensions " + f"{stored_dimensions}, but policy expects {field_policy.dimensions}." + ) + vector_data = bytes(row["vector_data"]) + if len(vector_data) != stored_dimensions * 4: + raise RecordStoreError( + f"Stored vector for record {record.id!r} has an invalid byte length." + ) + stored_vector = struct.unpack( + f"<{stored_dimensions}f", + vector_data, ) - stored_vector = struct.unpack( - f"<{stored_dimensions}f", - vector_data, - ) - score = similarity.score(stored_vector) - if ( - vector_search.min_score is not None - and score < vector_search.min_score - ): - continue - matches.append((score, record)) - matches.sort( - key=lambda match: ( - -match[0], - match[1].partition_key, - match[1].id, + score = similarity.score(stored_vector) + if ( + vector_search.min_score is not None + and score < vector_search.min_score + ): + continue + candidate_count += 1 + yield score, record + + # Exhaust and validate every eligible row, retaining only the bounded top pool. + selected = heapq.nsmallest( + vector_search.top, candidates(), + key=lambda match: (-match[0], match[1].partition_key, match[1].id), ) - ) - selected = matches[: vector_search.top] - candidate_count = len(matches) ranked = [ VyralRecordMatch( record=record, @@ -1291,10 +1296,8 @@ def _validate_vectors( def _float32(value: float, label: str) -> float: if not math.isfinite(value): raise RecordValidationError(f"{label} contains a non-finite value.") - try: - normalized = float(struct.unpack(" None: self.assertEqual("3.1.0", bundle.summary.openapi_version) self.assertEqual(129, bundle.summary.operation_count) self.assertEqual(133, bundle.summary.rest_operation_count) - self.assertEqual(276, bundle.summary.schema_count) + self.assertEqual(281, bundle.summary.schema_count) self.assertEqual( "sha256:" + sha256((ROOT / "contracts/public-sdk-surface.json").read_bytes()).hexdigest(), bundle.summary.catalog_sha256, diff --git a/scripts/benchmark-python-vector-search.py b/scripts/benchmark-python-vector-search.py new file mode 100644 index 0000000..9b64e8b --- /dev/null +++ b/scripts/benchmark-python-vector-search.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Qualify high-dimensional flat-vector arithmetic and complete synthetic retrieval.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import importlib.util +import json +import platform +from pathlib import Path +import random +import sqlite3 +import statistics +import struct +import subprocess +import sys +from tempfile import TemporaryDirectory +from time import perf_counter + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "runtimes/python/src")) +from vyral_runtime.local import record_store # noqa: E402 + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline-ref", default="bc2739023d1cf1f0b39aa1c20910b7c4de9738ba") + parser.add_argument("--records", type=int, default=1000) + parser.add_argument("--dimensions", type=int, default=384) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if not 100 <= args.records <= 4000 or not 8 <= args.dimensions <= 4096 or not 1 <= args.repeats <= 10: + parser.error("Bound records to 100..4000, dimensions to 8..4096 and repeats to 1..10") + source_path = "runtimes/python/src/vyral_runtime/local/record_store.py" + baseline_bytes = subprocess.check_output(["git", "show", f"{args.baseline_ref}:{source_path}"], cwd=ROOT) + rng = random.Random(381) + + def vector() -> list[float]: + return [struct.unpack(" int: for requirement in ( "workflow_dispatch:", - "server-v0.3.5", + "server-v0.3.6", 'test "$GITHUB_REF" = "refs/heads/main"', 'git cat-file -t "refs/tags/${RELEASE_TAG}"', ".verification.verified == true", "release-integrity.yml/runs?head_sha=", "name: publish-container", "docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc", - "VYRAL_IMAGE_VERSION=0.3.5", - "ghcr.io/univeracity/vyral-server:0.3.5", + "VYRAL_IMAGE_VERSION=0.3.6", + "ghcr.io/univeracity/vyral-server:0.3.6", "scripts/verify-hosted-worker-container.sh", "scripts/verify-mcp-container.sh", "aquasec/trivy:0.73.0@sha256:", - "server-container-server-v0.3.5", + "server-container-server-v0.3.6", ): if requirement not in worker_container_publisher: errors.append( diff --git a/scripts/verify-python-runtime-install.py b/scripts/verify-python-runtime-install.py index 53cb2b4..68315c5 100755 --- a/scripts/verify-python-runtime-install.py +++ b/scripts/verify-python-runtime-install.py @@ -36,7 +36,7 @@ assert readiness.contract is not None assert readiness.contract.operation_count == 129 assert readiness.contract.rest_operation_count == 133 -assert readiness.contract.schema_count == 276 +assert readiness.contract.schema_count == 281 assert len(run_bundled_goldens()) == 13 assert len(run_bundled_projection_generation_scenario()) == 15 assert len(run_bundled_record_store_scenario()) == 17 diff --git a/scripts/verify-python-runtime.sh b/scripts/verify-python-runtime.sh index bd3c28a..040a3f5 100755 --- a/scripts/verify-python-runtime.sh +++ b/scripts/verify-python-runtime.sh @@ -172,7 +172,7 @@ assert readiness.full_local_ready is False assert readiness.contract is not None assert readiness.contract.operation_count == 129 assert readiness.contract.rest_operation_count == 133 -assert readiness.contract.schema_count == 276 +assert readiness.contract.schema_count == 281 assert len(run_bundled_goldens()) == 13 assert len(run_bundled_projection_generation_scenario()) == 15 assert len(run_bundled_record_store_scenario()) == 17 diff --git a/scripts/verify-version-policy.py b/scripts/verify-version-policy.py index 8a87283..509577a 100755 --- a/scripts/verify-version-policy.py +++ b/scripts/verify-version-policy.py @@ -14,7 +14,7 @@ ROOT = Path(__file__).resolve().parent.parent PRODUCT_VERSION = "0.3.0" CORE_PACKAGE_VERSION = "0.3.2" -SERVER_IMAGE_VERSION = "0.3.5" +SERVER_IMAGE_VERSION = "0.3.6" EXECUTION_VERSION = "0.2.0" PYTHON_RUNTIME_VERSION = "0.1.3" PYTHON_RUNTIME_FIXTURE_MINIMUM = "0.1.0" diff --git a/scripts/verify-worker-container-release.py b/scripts/verify-worker-container-release.py index d1940bc..fcb5a52 100755 --- a/scripts/verify-worker-container-release.py +++ b/scripts/verify-worker-container-release.py @@ -16,7 +16,7 @@ "publicationAuthorized": True, "image": { "name": "ghcr.io/univeracity/vyral-server", - "version": "0.3.5", + "version": "0.3.6", "source": "Dockerfile", "environment": "publish-container", "maturity": "mixed", @@ -28,7 +28,7 @@ }, "authorization": { "mode": "manual-protected-environment", - "releaseTag": "server-v0.3.5", + "releaseTag": "server-v0.3.6", "workflow": ".github/workflows/publish-worker-container.yml", "requirements": [ "a GitHub-verified signed annotated server tag that resolves to current main", @@ -55,7 +55,7 @@ def main() -> int: workflow = WORKFLOW.read_text(encoding="utf-8") for requirement in ( "workflow_dispatch:", - "server-v0.3.5", + "server-v0.3.6", 'test "$GITHUB_REF" = "refs/heads/main"', 'git cat-file -t "refs/tags/${RELEASE_TAG}"', ".verification.verified == true", @@ -71,12 +71,12 @@ def main() -> int: "docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc", "sbom: true", "provenance: mode=max", - "VYRAL_IMAGE_VERSION=0.3.5", - "ghcr.io/univeracity/vyral-server:0.3.5", + "VYRAL_IMAGE_VERSION=0.3.6", + "ghcr.io/univeracity/vyral-server:0.3.6", "scripts/verify-hosted-worker-container.sh", "scripts/verify-mcp-container.sh", "aquasec/trivy:0.73.0@sha256:", - "server-container-server-v0.3.5", + "server-container-server-v0.3.6", ): if requirement not in workflow: raise SystemExit(f"Worker container publisher is missing {requirement!r}.") @@ -86,7 +86,7 @@ def main() -> int: if "\n push:" in workflow: raise SystemExit("Worker container publisher must not have an automatic push trigger.") - print("worker-container-release=ok tag=server-v0.3.5 image=0.3.5") + print("worker-container-release=ok tag=server-v0.3.6 image=0.3.6") return 0 diff --git a/src/Vyral.Aws/S3ObjectStore.cs b/src/Vyral.Aws/S3ObjectStore.cs index e9489aa..64b23f0 100644 --- a/src/Vyral.Aws/S3ObjectStore.cs +++ b/src/Vyral.Aws/S3ObjectStore.cs @@ -106,7 +106,7 @@ public async Task PutObjectAsync(ObjectWriteRequest request, Cancell try { - var response = await _client.GetObjectAsync(new GetObjectRequest + using var response = await _client.GetObjectAsync(new GetObjectRequest { BucketName = request.Container, Key = key @@ -185,7 +185,8 @@ public async Task ListObjectsAsync(ObjectListRequest request, var response = await _client.ListObjectsV2Async(listRequest, ct); var items = new List(); - foreach (var obj in response.S3Objects) + // AWS SDK v4 leaves absent response collections null, including an empty bucket. + foreach (var obj in response.S3Objects ?? new List()) { // S3 ListObjectsV2 does not return metadata per object — would need individual HEAD // We return empty metadata and the S3 ETag; content hash requires a separate HEAD. diff --git a/src/Vyral.Cloudflare/CloudflareR2Options.cs b/src/Vyral.Cloudflare/CloudflareR2Options.cs index aa83c83..bdfcfac 100644 --- a/src/Vyral.Cloudflare/CloudflareR2Options.cs +++ b/src/Vyral.Cloudflare/CloudflareR2Options.cs @@ -7,6 +7,9 @@ public sealed class CloudflareR2Options public string? AccountId { get; init; } public string? AccessKeyId { get; init; } public string? SecretAccessKey { get; init; } + public string? SessionToken { get; init; } + /// Explicit SDK retries; zero preserves unresolved write outcomes for caller lookup. + public int MaxErrorRetry { get; init; } = 0; public string? ServiceUrl { get; init; } public string AuthenticationRegion { get; init; } = "auto"; public bool ForcePathStyle { get; init; } = true; @@ -37,6 +40,8 @@ public string ResolveServiceUrl() public void ValidateCredentials() { + if (MaxErrorRetry is < 0 or > 3) + throw new InvalidOperationException("Cloudflare R2 MaxErrorRetry must be between 0 and 3."); if (string.IsNullOrWhiteSpace(AccessKeyId)) { throw new InvalidOperationException("Cloudflare R2 object store requires Cloudflare:R2:AccessKeyId or VYRAL_R2_ACCESS_KEY_ID."); diff --git a/src/Vyral.Cloudflare/R2ObjectStore.cs b/src/Vyral.Cloudflare/R2ObjectStore.cs index 496275c..db46b38 100644 --- a/src/Vyral.Cloudflare/R2ObjectStore.cs +++ b/src/Vyral.Cloudflare/R2ObjectStore.cs @@ -33,14 +33,17 @@ public static R2ObjectStore Create(CloudflareR2Options options) { options.ValidateCredentials(); - var credentials = new BasicAWSCredentials(options.AccessKeyId!, options.SecretAccessKey!); + AWSCredentials credentials = string.IsNullOrWhiteSpace(options.SessionToken) + ? new BasicAWSCredentials(options.AccessKeyId!, options.SecretAccessKey!) + : new SessionAWSCredentials(options.AccessKeyId!, options.SecretAccessKey!, options.SessionToken); var client = new AmazonS3Client(credentials, new AmazonS3Config { ServiceURL = options.ResolveServiceUrl(), AuthenticationRegion = string.IsNullOrWhiteSpace(options.AuthenticationRegion) ? "auto" : options.AuthenticationRegion.Trim(), - ForcePathStyle = options.ForcePathStyle + ForcePathStyle = options.ForcePathStyle, + MaxErrorRetry = options.MaxErrorRetry }); return new R2ObjectStore(client, new S3ObjectStoreOptions @@ -56,8 +59,14 @@ public Task PutObjectAsync(ObjectWriteRequest request, CancellationT public Task GetObjectAsync(ObjectReadRequest request, CancellationToken ct = default) => _inner.GetObjectAsync(request, ct); - public Task DeleteObjectAsync(ObjectDeleteRequest request, CancellationToken ct = default) => - _inner.DeleteObjectAsync(request, ct); + public Task DeleteObjectAsync(ObjectDeleteRequest request, CancellationToken ct = default) + { + // The qualified R2 S3 endpoint ignored DeleteObject If-Match. A HEAD followed + // by DELETE would race, so never substitute that for an atomic precondition. + if (!string.IsNullOrWhiteSpace(request.IfMatch)) + throw new InvalidOperationException("Conditional object deletion is unsupported by the R2 profile."); + return _inner.DeleteObjectAsync(request, ct); + } public Task ListObjectsAsync(ObjectListRequest request, CancellationToken ct = default) => _inner.ListObjectsAsync(request, ct); diff --git a/src/Vyral.Providers.Abstractions/AiCapabilityModels.cs b/src/Vyral.Providers.Abstractions/AiCapabilityModels.cs index 82db28c..6487904 100644 --- a/src/Vyral.Providers.Abstractions/AiCapabilityModels.cs +++ b/src/Vyral.Providers.Abstractions/AiCapabilityModels.cs @@ -30,6 +30,11 @@ public sealed class AiChatRequest [JsonPropertyName("maxOutputChars")] public int? MaxOutputChars { get; set; } + + /// Optional positive ceiling on generated tokens, including hidden reasoning. + /// An adapter that cannot enforce it must reject the request before inference. + [JsonPropertyName("maxOutputTokens")] + public int? MaxOutputTokens { get; set; } } public sealed class AiChatResult diff --git a/src/Vyral.Providers.Abstractions/AiJudgeModels.cs b/src/Vyral.Providers.Abstractions/AiJudgeModels.cs index 86ad3d8..b0177bb 100644 --- a/src/Vyral.Providers.Abstractions/AiJudgeModels.cs +++ b/src/Vyral.Providers.Abstractions/AiJudgeModels.cs @@ -131,6 +131,18 @@ public sealed class AiJudgeAnswer [JsonPropertyName("rawClassProbabilities")] public Dictionary? RawClassProbabilities { get; set; } + /// Minimum full-vocabulary decision-label mass over scored rotations. Null means unavailable. + [JsonPropertyName("labelMassCoverage")] + public double? LabelMassCoverage { get; set; } + + /// Mean unconditioned probability per original option; distinct from normalized Probabilities. + [JsonPropertyName("rawLabelProbabilities")] + public Dictionary? RawLabelProbabilities { get; set; } + + /// Share of rotations whose argmax equals the final winning option. Null means unavailable. + [JsonPropertyName("rotationAgreement")] + public double? RotationAgreement { get; set; } + /// /// True only when the underlying probability model has been fit against labeled outcomes /// (e.g. a temperature-scaling calibration). False means the probability is a heuristic proxy diff --git a/src/Vyral.Providers.Abstractions/ProviderRunRequests.cs b/src/Vyral.Providers.Abstractions/ProviderRunRequests.cs index 259c199..d2cbd34 100644 --- a/src/Vyral.Providers.Abstractions/ProviderRunRequests.cs +++ b/src/Vyral.Providers.Abstractions/ProviderRunRequests.cs @@ -25,6 +25,14 @@ public static ProviderRunRequest ForChat( int? timeoutSeconds = null) => Build(ProviderCapabilityIds.AiChat, chat, provider, modelId, mode, timeoutSeconds); + public static ProviderRunRequest ForJudge( + AiJudgeRequest judge, + string? provider = null, + string? modelId = null, + string mode = ProviderModes.Mechanics, + int? timeoutSeconds = null) + => Build(ProviderCapabilityIds.AiJudge, judge, provider, modelId, mode, timeoutSeconds); + public static ProviderRunRequest ForExtract( AiExtractRequest extract, string? provider = null, diff --git a/src/Vyral.Providers.Abstractions/ProviderRunResults.cs b/src/Vyral.Providers.Abstractions/ProviderRunResults.cs index df08f17..6bb8950 100644 --- a/src/Vyral.Providers.Abstractions/ProviderRunResults.cs +++ b/src/Vyral.Providers.Abstractions/ProviderRunResults.cs @@ -13,6 +13,9 @@ public static class ProviderRunResults public static AiChatResult GetChat(ProviderRunResult result) => Deserialize(result); + public static AiJudgeResult GetJudge(ProviderRunResult result) + => Deserialize(result); + public static AiExtractResult GetExtract(ProviderRunResult result) => Deserialize(result); diff --git a/src/Vyral.Providers.Cli/CliProviderTarget.cs b/src/Vyral.Providers.Cli/CliProviderTarget.cs index 9796371..5c8e93a 100644 --- a/src/Vyral.Providers.Cli/CliProviderTarget.cs +++ b/src/Vyral.Providers.Cli/CliProviderTarget.cs @@ -394,6 +394,12 @@ public async Task RunAsync(ProviderRunRequest request, Cancel string guardedPrompt; var policy = ProviderModePolicies.Resolve(_policies, request.Mode); + if (request.Capability == ProviderCapabilityIds.AiChat && request.Payload["maxOutputTokens"] is not null) + { + return CreateResult(request, ProviderRunStatus.Unsupported, trace, stopwatch.Elapsed, + "This CLI adapter cannot enforce a generated-token ceiling.", ProviderFailureClasses.Unsupported, "token_ceiling_unsupported"); + } + if (!Capabilities.Any(c => string.Equals(c.Id, request.Capability, StringComparison.OrdinalIgnoreCase))) { return CreateResult(request, ProviderRunStatus.Unsupported, trace, stopwatch.Elapsed, null, ProviderFailureClasses.Unsupported, "unsupported_capability"); diff --git a/src/Vyral.Providers.Local/DeterministicAiProviderTarget.cs b/src/Vyral.Providers.Local/DeterministicAiProviderTarget.cs index 808a7a0..b684de1 100644 --- a/src/Vyral.Providers.Local/DeterministicAiProviderTarget.cs +++ b/src/Vyral.Providers.Local/DeterministicAiProviderTarget.cs @@ -158,6 +158,10 @@ public Task RunAsync(ProviderRunRequest request, Cancellation return Task.FromResult(CreateResult(request, trace, stopwatch, ProviderRunStatus.Rejected, "Unknown provider mode.", ProviderFailureClasses.Policy, "unknown_mode")); } + if (request.Capability == ProviderCapabilityIds.AiChat && request.Payload["maxOutputTokens"] is not null) + return Task.FromResult(CreateResult(request, trace, stopwatch, ProviderRunStatus.Unsupported, + "This provider cannot enforce a generated-token ceiling.", ProviderFailureClasses.Unsupported, "token_ceiling_unsupported")); + var payloadBytes = Encoding.UTF8.GetByteCount(request.Payload.ToJsonString(ProviderJson.Options)); if (payloadBytes > policy.MaxInputBytes) { diff --git a/src/Vyral.Providers.Local/LlamaCppRuntimeSessionFactory.cs b/src/Vyral.Providers.Local/LlamaCppRuntimeSessionFactory.cs new file mode 100644 index 0000000..a8f0925 --- /dev/null +++ b/src/Vyral.Providers.Local/LlamaCppRuntimeSessionFactory.cs @@ -0,0 +1,188 @@ +using System.Net; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; +using Vyral.Providers.Abstractions; + +namespace Vyral.Providers.Local; + +/// Explicit binding to an operator-managed, single-slot local llama.cpp process. +public sealed class LlamaCppRuntimeOptions +{ + public Uri Endpoint { get; init; } = new("http://127.0.0.1:8080/"); + public string ModelId { get; init; } = ""; + public string ModelPath { get; init; } = ""; + public string ModelSha256 { get; init; } = ""; + public string BuildInfo { get; init; } = ""; + public string TemplateSha256 { get; init; } = ""; + public string TemplateProbeSha256 { get; init; } = ""; + public int ContextTokens { get; init; } = 2048; + public int MaxOutputTokens { get; init; } = 512; + public int MaxResponseBytes { get; init; } = 1048576; + + internal void Validate() + { + if (Endpoint.Scheme != "http" || !IPAddress.TryParse(Endpoint.Host, out var ip) || !IPAddress.IsLoopback(ip) + || Endpoint.AbsolutePath != "/" || Endpoint.Query != "" || Endpoint.UserInfo != "" || Endpoint.Fragment != "") + throw new ArgumentException("Runtime endpoint must be a literal loopback HTTP origin."); + if (string.IsNullOrWhiteSpace(ModelId) || !Path.IsPathFullyQualified(ModelPath) || string.IsNullOrWhiteSpace(BuildInfo) + || !Regex.IsMatch(ModelSha256, "^[a-f0-9]{64}$") || !Regex.IsMatch(TemplateSha256, "^[a-f0-9]{64}$") + || !Regex.IsMatch(TemplateProbeSha256, "^[a-f0-9]{64}$") + || ContextTokens < 2 || MaxOutputTokens < 1 || MaxOutputTokens >= ContextTokens || MaxResponseBytes is < 256 or > 4194304) + throw new ArgumentException("Pin model bytes, runtime build, template, context window and bounded output."); + } +} + +public sealed class LlamaCppRuntimeSessionFactory : ILocalLogprobSessionFactory +{ + private readonly LlamaCppRuntimeOptions _options; + public LlamaCppRuntimeSessionFactory(LlamaCppRuntimeOptions options) + { + options.Validate(); _options = options; + BindingId = ProviderHash.Sha256($"llamacpp-native-v1|{options.Endpoint}|{options.ModelId}|{options.ModelSha256}|{options.BuildInfo}|{options.TemplateSha256}|{options.TemplateProbeSha256}|{options.ContextTokens}|{options.MaxOutputTokens}|{options.MaxResponseBytes}"); + } + public string BindingId { get; } + public string ModelId => _options.ModelId; + public string Auth => "none"; + public bool RequiresNetwork => true; // Loopback HTTP is still network transport. + public async ValueTask OpenAsync(CancellationToken ct) + { + var session = new LlamaCppRuntimeSession(_options); + try { await session.ValidateBindingAsync(ct); return session; } + catch { await session.DisposeAsync(); throw; } + } +} + +internal sealed class LlamaCppRuntimeSession : ILocalLogprobSession +{ + private readonly LlamaCppRuntimeOptions _options; + private readonly HttpClient _client; + public LlamaCppRuntimeSession(LlamaCppRuntimeOptions options) + { + _options = options; + _client = new(new HttpClientHandler { AllowAutoRedirect = false, UseProxy = false }) + { BaseAddress = options.Endpoint, Timeout = Timeout.InfiniteTimeSpan }; + } + public int ContextTokens => _options.ContextTokens; + public ValueTask DisposeAsync() { _client.Dispose(); return ValueTask.CompletedTask; } + + internal async Task ValidateBindingAsync(CancellationToken ct) + { + // These are consistency checks within the trusted operator-owned local process boundary, + // not cryptographic proof that an arbitrary HTTP peer loaded these bytes. + await using var file = File.OpenRead(_options.ModelPath); + var digest = Convert.ToHexStringLower(await SHA256.HashDataAsync(file, ct)); + if (digest != _options.ModelSha256) throw new LocalScoringException("model_digest_mismatch", ProviderFailureClasses.Configuration); + var props = await SendAsync(HttpMethod.Get, "props", null, ct); + if (props["model_path"] is not JsonValue path || !path.TryGetValue(out var reportedPath) + || !Path.IsPathFullyQualified(reportedPath) || Path.GetFullPath(reportedPath) != Path.GetFullPath(_options.ModelPath) + || props["build_info"]?.GetValue() != _options.BuildInfo + || props["chat_template"] is not JsonValue template || !template.TryGetValue(out var text) + || Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text))) != _options.TemplateSha256 + || props["default_generation_settings"]?["n_ctx"]?.GetValue() != ContextTokens + || props["total_slots"]?.GetValue() != 1) + throw new LocalScoringException("runtime_binding_mismatch", ProviderFailureClasses.Configuration); + var probe = await SendAsync(HttpMethod.Post, "apply-template", new() { + ["messages"] = new JsonArray(new JsonObject { ["role"] = "user", ["content"] = "vyral-template-binding-v1" }) }, ct); + if (probe["prompt"] is not JsonValue rendered || !rendered.TryGetValue(out var probeText) + || Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(probeText))) != _options.TemplateProbeSha256) + throw new LocalScoringException("template_probe_mismatch", ProviderFailureClasses.Configuration); + } + + internal async Task SendAsync(HttpMethod method, string path, JsonObject? payload, CancellationToken ct) + { + using var request = new HttpRequestMessage(method, path); + if (payload is not null) request.Content = JsonContent.Create(payload, options: ProviderJson.Options); + using var response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ct); + if (!response.IsSuccessStatusCode) + throw new LocalScoringException(response.StatusCode switch { + HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden => "runtime_auth_failed", + HttpStatusCode.TooManyRequests => "runtime_rate_limited", + _ => "runtime_unavailable" }, response.StatusCode switch { + HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden => ProviderFailureClasses.Auth, + HttpStatusCode.TooManyRequests => ProviderFailureClasses.RateLimit, + _ => ProviderFailureClasses.ProviderUnavailable }); + await using var stream = await response.Content.ReadAsStreamAsync(ct); + using var output = new MemoryStream(); var buffer = new byte[8192]; + int read; + while ((read = await stream.ReadAsync(buffer, ct)) != 0) + { + if (output.Length + read > _options.MaxResponseBytes) + throw new LocalScoringException("runtime_response_limit", ProviderFailureClasses.Policy); + output.Write(buffer, 0, read); + } + return JsonNode.Parse(output.ToArray()) as JsonObject + ?? throw new LocalScoringException("invalid_runtime_response", ProviderFailureClasses.Schema); + } + + private async Task TokenizeAsync(string content, bool special, CancellationToken ct) + { + var response = await SendAsync(HttpMethod.Post, "tokenize", + new() { ["content"] = content, ["add_special"] = special, ["parse_special"] = special }, ct); + if (response["tokens"] is not JsonArray tokens || tokens.Count == 0 + || tokens.Any(t => t is not JsonValue value || !value.TryGetValue(out var id) || id < 0)) + throw new LocalScoringException("invalid_runtime_tokens", ProviderFailureClasses.Schema); + return tokens.Select(t => t!.GetValue()).ToArray(); + } + + public async ValueTask ResolveLabelAsync(string label, CancellationToken ct) + { + var tokens = await TokenizeAsync(label, false, ct); + if (tokens.Length != 1) throw new LocalScoringException("label_not_single_token", ProviderFailureClasses.Unsupported); + var decoded = await SendAsync(HttpMethod.Post, "detokenize", new() { ["tokens"] = new JsonArray(JsonValue.Create(tokens[0])) }, ct); + if (decoded["content"]?.GetValue() != label) + throw new LocalScoringException("label_bytes_mismatch", ProviderFailureClasses.Schema); + return new(tokens[0], Encoding.UTF8.GetBytes(label)); + } + + public async ValueTask PrepareAsync(string prompt, CancellationToken ct) + => await PrepareChatAsync(new() { new() { Role = AiRoles.User, Content = prompt } }, ct); + + internal async Task PrepareChatAsync(List messages, CancellationToken ct) + { + var templated = await SendAsync(HttpMethod.Post, "apply-template", + new() { ["messages"] = ProviderJson.ToJsonObject(new AiChatRequest { Messages = messages })["messages"]!.DeepClone() }, ct); + if (templated["prompt"] is not JsonValue value || !value.TryGetValue(out var prompt) || string.IsNullOrEmpty(prompt)) + throw new LocalScoringException("invalid_template_response", ProviderFailureClasses.Schema); + return new(await TokenizeAsync(prompt, true, ct)); + } + + internal Task CompleteAsync(LocalScoringPrompt prompt, int maxTokens, bool score, CancellationToken ct) + { + if (maxTokens < 1 || maxTokens > _options.MaxOutputTokens || prompt.TokenIds.Length > ContextTokens - maxTokens) + throw new LocalScoringException("context_limit_exceeded", ProviderFailureClasses.Policy); + return SendAsync(HttpMethod.Post, "completion", new() { + ["prompt"] = new JsonArray(prompt.TokenIds.Select(id => JsonValue.Create(id)).ToArray()), + ["n_predict"] = maxTokens, ["cache_prompt"] = false, ["stream"] = false, + ["temperature"] = -1, ["seed"] = 0, + ["n_probs"] = score ? 256 : 0, ["post_sampling_probs"] = false, + ["samplers"] = new JsonArray(), ["return_tokens"] = true }, ct); + } + + public async ValueTask ScoreAsync(LocalScoringPrompt prompt, + IReadOnlyList labels, CancellationToken ct) + { + var response = await CompleteAsync(prompt, 1, true, ct); + if (response["probs"] is not null && response["completion_probabilities"] is not null) + throw new LocalScoringException("ambiguous_scoring_response", ProviderFailureClasses.Schema); + var scoredPositions = response["probs"] ?? response["completion_probabilities"]; + if (response["truncated"]?.GetValue() != false || response["tokens_predicted"]?.GetValue() != 1 + || scoredPositions is not JsonArray positions || positions.Count != 1 + || positions[0]?["top_logprobs"] is not JsonArray probabilities) + throw new LocalScoringException("invalid_scoring_response", ProviderFailureClasses.Schema); + var entries = new List(); var seen = new HashSet(); + foreach (var probability in probabilities) + { + if (probability is not JsonObject p || p["id"] is not JsonValue idValue || !idValue.TryGetValue(out var id) || id < 0 + || !seen.Add(id) || p["logprob"] is not JsonValue logValue || !logValue.TryGetValue(out var logprob) + || !double.IsFinite(logprob) || logprob > 0 || p["bytes"] is not JsonArray bytes + || bytes.Any(b => b is not JsonValue v || !v.TryGetValue(out var n) || n is < 0 or > 255)) + throw new LocalScoringException("invalid_probability_entry", ProviderFailureClasses.Schema); + if (labels.Any(label => label.Id == id)) + entries.Add(new(id, bytes.Select(b => (byte)b!.GetValue()).ToArray(), logprob)); + } + return new(entries); + } +} diff --git a/src/Vyral.Providers.Local/LocalLlamaCppProviderTarget.cs b/src/Vyral.Providers.Local/LocalLlamaCppProviderTarget.cs new file mode 100644 index 0000000..0495716 --- /dev/null +++ b/src/Vyral.Providers.Local/LocalLlamaCppProviderTarget.cs @@ -0,0 +1,107 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Vyral.Providers.Abstractions; + +namespace Vyral.Providers.Local; + +/// Opt-in bounded native completion. No tools, process launch, downloads or source writes. +public sealed class LocalLlamaCppProviderTarget : IProviderTarget +{ + private readonly LlamaCppRuntimeSessionFactory _factory; + private readonly LlamaCppRuntimeOptions _options; + public LocalLlamaCppProviderTarget(LlamaCppRuntimeOptions options) + { + _options = options; _factory = new(options); + Profile = new() { Id = "local-llamacpp", DisplayName = "Local llama.cpp native completion", Family = "local", + Local = true, RequiresNetwork = true, Auth = "none", ConfigHash = _factory.BindingId }; + Capabilities = new[] { new ProviderCapabilityDescriptor { Id = ProviderCapabilityIds.AiChat, + Operations = new() { "run" }, ToolPolicy = ProviderToolPolicies.CallerOwned, + InputLimits = new() { ["maxContextTokens"] = options.ContextTokens, ["maxPayloadBytes"] = 70000 }, + OutputLimits = new() { ["maxOutputTokens"] = options.MaxOutputTokens, ["maxOutputBytes"] = 262144 }, + ModePolicies = new() { new() { Id = ProviderModes.Advisory, AllowNetwork = true, + AllowSourceWrites = false, MaxInputBytes = 70000, MaxOutputBytes = 262144, TimeoutSeconds = 120 } }, + UnsupportedFeatures = new() { "tools", "source_writes", "attachments", "multimodal", "internal_retries", "trusted_issuance" } } }; + } + public ProviderProfile Profile { get; } + public IReadOnlyList Capabilities { get; } + + public async Task RunAsync(ProviderRunRequest request, CancellationToken ct = default) + { + var clock = Stopwatch.StartNew(); int? input = null, generated = null; bool effectStarted = false; + var trace = new ProviderTraceEvent { Provider = Profile.Id, Capability = request.Capability, + Operation = request.Operation, Mode = request.Mode, ModelId = _options.ModelId, + AdapterId = "local-llamacpp-native-v1", ConfigHash = Profile.ConfigHash, + InputHash = ProviderHash.Sha256(request.Payload.ToJsonString(ProviderJson.Options)) }; + ProviderRunResult Result(ProviderRunStatus status, string reason, string? failure = null, JsonObject? output = null) + { + trace.DurationMs = clock.Elapsed.TotalMilliseconds; trace.FailureClass = failure; + trace.OutputHash = output is null ? null : ProviderHash.Sha256(output.ToJsonString(ProviderJson.Options)); + var result = new ProviderRunResult { Provider = Profile.Id, Capability = request.Capability, + Operation = request.Operation, Mode = request.Mode, Status = status, ProviderStatus = reason, + FailureClass = failure, Output = output ?? new(), Trace = trace }; + var usage = new JsonObject(); + if (input is not null) usage["input_tokens"] = input; + if (generated is not null) usage["output_tokens"] = generated; + result.MeteringMeasurements.AddRange(AiMeteringUsageNormalizer.Normalize(usage, sourceId: "llama.cpp.native")); + return result; + } + if (request.Capability != ProviderCapabilityIds.AiChat || request.Operation != "run" || request.Mode != ProviderModes.Advisory + || request.ModelId != _options.ModelId || request.ContextRefs.Count != 0 || request.ArtifactDirectory is not null) + return Result(ProviderRunStatus.Unsupported, "unsupported_request", ProviderFailureClasses.Unsupported); + if (request.TimeoutSeconds is <= 0 || request.MaxOutputBytes is <= 0) + return Result(ProviderRunStatus.Rejected, "invalid_request_limits", ProviderFailureClasses.Policy); + if (Encoding.UTF8.GetByteCount(request.Payload.ToJsonString(ProviderJson.Options)) > 70000) + return Result(ProviderRunStatus.Rejected, "input_limit", ProviderFailureClasses.Policy); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct); + deadline.CancelAfter(TimeSpan.FromSeconds(Math.Min(request.TimeoutSeconds ?? 120, 120))); + try + { + deadline.Token.ThrowIfCancellationRequested(); + if (request.Payload.Any(p => p.Key is not ("messages" or "maxOutputTokens" or "maxOutputChars"))) + throw new LocalScoringException("unsupported_payload", ProviderFailureClasses.Unsupported); + var chat = ProviderJson.DeserializePayload(request); + if (chat.Messages is null || chat.Messages.Count != 1 || chat.Messages[0] is null + || chat.Messages[0].Role != AiRoles.User || string.IsNullOrWhiteSpace(chat.Messages[0].Content) + || chat.Messages[0].ContentBlocks is not null) + throw new LocalScoringException("single_user_message_required", ProviderFailureClasses.Unsupported); + if (chat.MaxOutputTokens is not int ceiling || ceiling < 1 || ceiling > _options.MaxOutputTokens + || chat.MaxOutputChars is <= 0) + throw new LocalScoringException("explicit_token_ceiling_required", ProviderFailureClasses.Policy); + await using var session = (LlamaCppRuntimeSession)await _factory.OpenAsync(deadline.Token); + var prompt = await session.PrepareChatAsync(chat.Messages, deadline.Token); + if (prompt.TokenIds.Length > session.ContextTokens - ceiling) + throw new LocalScoringException("context_limit_exceeded", ProviderFailureClasses.Policy); + deadline.Token.ThrowIfCancellationRequested(); + effectStarted = true; + var response = await session.CompleteAsync(prompt, ceiling, false, deadline.Token); + input = Counter(response, "tokens_evaluated"); generated = Counter(response, "tokens_predicted"); + if (input is null || generated is null) + return Result(ProviderRunStatus.Failed, "invalid_native_usage", ProviderFailureClasses.Schema); + if (input != prompt.TokenIds.Length || generated > ceiling || response["truncated"]?.GetValue() != false) + return Result(ProviderRunStatus.Failed, "native_bound_violated", ProviderFailureClasses.Policy); + if (response["content"] is not JsonValue value || !value.TryGetValue(out var text)) + return Result(ProviderRunStatus.Failed, "invalid_native_content", ProviderFailureClasses.Schema); + if (chat.MaxOutputChars is int charLimit && text.Length > charLimit) + return Result(ProviderRunStatus.Failed, "output_character_limit", ProviderFailureClasses.Policy); + var output = ProviderJson.ToJsonObject(new AiChatResult { Message = new() { Role = AiRoles.Assistant, Content = text }, + StopReason = response["stop_type"]?.GetValue() ?? "unknown" }); + if (Encoding.UTF8.GetByteCount(output.ToJsonString(ProviderJson.Options)) > Math.Min(request.MaxOutputBytes ?? 262144, 262144)) + return Result(ProviderRunStatus.Failed, "output_limit", ProviderFailureClasses.Policy); + deadline.Token.ThrowIfCancellationRequested(); + return Result(ProviderRunStatus.Succeeded, "native_completion", output: output); + } + catch (LocalScoringException ex) { return Result(ProviderRunStatus.Rejected, ex.Reason, ex.FailureClass); } + catch (OperationCanceledException) + { + return Result(ct.IsCancellationRequested ? ProviderRunStatus.Cancelled : ProviderRunStatus.TimedOut, + effectStarted ? "native_effect_unresolved" : "no_native_completion_started", + ct.IsCancellationRequested ? ProviderFailureClasses.Cancelled : ProviderFailureClasses.Timeout); + } + catch (Exception ex) when (ex is HttpRequestException or IOException or InvalidOperationException or JsonException or ArgumentException) + { return Result(ProviderRunStatus.Failed, effectStarted ? "native_effect_unresolved" : "native_preflight_failed", ProviderFailureClasses.Schema); } + } + private static int? Counter(JsonObject response, string name) => + response[name] is JsonValue v && v.TryGetValue(out var value) && value >= 0 ? value : null; +} diff --git a/src/Vyral.Providers.Local/LocalLogprobJudgeProviderTarget.cs b/src/Vyral.Providers.Local/LocalLogprobJudgeProviderTarget.cs new file mode 100644 index 0000000..dfa4440 --- /dev/null +++ b/src/Vyral.Providers.Local/LocalLogprobJudgeProviderTarget.cs @@ -0,0 +1,198 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Vyral.Providers.Abstractions; + +namespace Vyral.Providers.Local; + +/// Portable Choice scoring mechanics over an explicitly supplied local runtime. +public sealed class LocalLogprobJudgeProviderTarget : IProviderTarget +{ + private readonly ILocalLogprobSessionFactory _factory; + private readonly LocalLogprobJudgeOptions _options; + public LocalLogprobJudgeProviderTarget(ILocalLogprobSessionFactory factory, LocalLogprobJudgeOptions? options = null) + { + _factory = factory ?? throw new ArgumentNullException(nameof(factory)); + _options = options ?? new(); + if (string.IsNullOrWhiteSpace(factory.BindingId) || string.IsNullOrWhiteSpace(factory.ModelId) + || _options.Rotations < 1 || _options.MaxQuestions < 1 || _options.MaxOptions is < 2 or > 26 + || _options.MaxScoringCalls < 1 || _options.MaxInputBytes < 1 || _options.MaxOutputBytes < 1 + || _options.TimeoutSeconds < 1) + throw new ArgumentException("A pinned runtime binding and positive bounded judge limits are required."); + Profile = new() { Id = "local-logprob-judge", DisplayName = "Local log-probability judge", + Family = "local", Local = true, RequiresNetwork = factory.RequiresNetwork, Auth = factory.Auth, + ConfigHash = ProviderHash.Sha256($"local-logprob-judge-v1|{factory.BindingId}|{JsonSerializer.Serialize(_options)}") }; + Capabilities = new[] { new ProviderCapabilityDescriptor { Id = ProviderCapabilityIds.AiJudge, + Operations = new() { "run" }, ToolPolicy = ProviderToolPolicies.CallerOwned, + InputLimits = new() { ["questionTypes"] = new[] { AiJudgeQuestionTypes.Choice }, + ["maxQuestions"] = _options.MaxQuestions, ["maxOptions"] = _options.MaxOptions, + ["maxScoringCalls"] = _options.MaxScoringCalls }, + OutputLimits = new() { ["maxOutputBytes"] = _options.MaxOutputBytes }, + ModePolicies = new() { new() { Id = ProviderModes.Mechanics, AllowNetwork = factory.RequiresNetwork, + AllowedOutputKinds = new() { ProviderOutputKinds.Judgment }, MaxInputBytes = _options.MaxInputBytes, + MaxOutputBytes = _options.MaxOutputBytes, TimeoutSeconds = _options.TimeoutSeconds } }, + UnsupportedFeatures = new() { "noul", "score", "calibrated_likelihoods", "tools", "source_writes" } } }; + } + + public ProviderProfile Profile { get; } + public IReadOnlyList Capabilities { get; } + + public async Task RunAsync(ProviderRunRequest request, CancellationToken ct = default) + { + var clock = Stopwatch.StartNew(); + var trace = new ProviderTraceEvent { Provider = Profile.Id, Capability = request.Capability, + Operation = request.Operation, Mode = request.Mode, ModelId = _factory.ModelId, + AdapterId = "local-logprob-judge-v1", ConfigHash = Profile.ConfigHash, + InputHash = ProviderHash.Sha256(request.Payload.ToJsonString(ProviderJson.Options)) }; + ProviderRunResult Result(ProviderRunStatus status, string reason, string? failure = null, JsonObject? output = null) + { + trace.DurationMs = clock.Elapsed.TotalMilliseconds; trace.FailureClass = failure; + trace.OutputHash = output is null ? null : ProviderHash.Sha256(output.ToJsonString(ProviderJson.Options)); + return new() { Provider = Profile.Id, Capability = request.Capability, Operation = request.Operation, + Mode = request.Mode, Status = status, ProviderStatus = reason, FailureClass = failure, + Output = output ?? new(), Trace = trace }; + } + if (request.Capability != ProviderCapabilityIds.AiJudge || request.Operation != "run" + || request.Mode != ProviderModes.Mechanics || request.ModelId is not null && request.ModelId != _factory.ModelId + || request.ContextRefs.Count != 0 || request.ArtifactDirectory is not null) + return Result(ProviderRunStatus.Unsupported, "unsupported_request", ProviderFailureClasses.Unsupported); + if (request.TimeoutSeconds is <= 0 || request.MaxOutputBytes is <= 0) + return Result(ProviderRunStatus.Rejected, "invalid_request_limits", ProviderFailureClasses.Policy); + if (Encoding.UTF8.GetByteCount(request.Payload.ToJsonString(ProviderJson.Options)) > _options.MaxInputBytes) + return Result(ProviderRunStatus.Rejected, "input_limit", ProviderFailureClasses.Policy); + using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct); + deadline.CancelAfter(TimeSpan.FromSeconds(Math.Min(request.TimeoutSeconds ?? _options.TimeoutSeconds, _options.TimeoutSeconds))); + ILocalLogprobSession? session = null; + ProviderRunResult result; + try + { + deadline.Token.ThrowIfCancellationRequested(); + if (request.Payload.Any(pair => pair.Key is not ("context" or "questions" or "references"))) + throw new LocalScoringException("unsupported_payload", ProviderFailureClasses.Unsupported); + var payload = ProviderJson.DeserializePayload(request); + Validate(payload); + var calls = payload.Questions.Sum(q => Math.Min(_options.Rotations, q.Options.Count)); + if (calls > _options.MaxScoringCalls) + throw new LocalScoringException("scoring_call_limit", ProviderFailureClasses.Policy); + session = await _factory.OpenAsync(deadline.Token).AsTask().WaitAsync(deadline.Token); + if (session.ContextTokens < 2) + throw new LocalScoringException("unknown_context_limit", ProviderFailureClasses.Unsupported); + var labels = new List(); + for (var index = 0; index < payload.Questions.Max(q => q.Options.Count); index++) + { + var label = ((char)('A' + index)).ToString(); + var token = await session.ResolveLabelAsync(label, deadline.Token).AsTask().WaitAsync(deadline.Token); + if (token is null || token.Bytes is null || token.Id < 0 || !token.Bytes.SequenceEqual(Encoding.UTF8.GetBytes(label)) + || labels.Any(previous => previous.Id == token.Id)) + throw new LocalScoringException("invalid_label_mapping", ProviderFailureClasses.Schema); + labels.Add(token); + } + // Prepare the entire batch before its first inference, including every rotation. + var prepared = new List<(AiJudgeQuestion Question, int Rotation, LocalScoringPrompt Prompt)>(); + foreach (var question in payload.Questions) + for (var rotation = 0; rotation < Math.Min(_options.Rotations, question.Options.Count); rotation++) + { + var prompt = await session.PrepareAsync(BuildPrompt(payload.Context, question, rotation), deadline.Token).AsTask().WaitAsync(deadline.Token); + if (prompt is null || prompt.TokenIds is null || prompt.TokenIds.Length == 0 || prompt.TokenIds.Any(id => id < 0)) + throw new LocalScoringException("invalid_prompt_tokens", ProviderFailureClasses.Schema); + if (prompt.TokenIds.Length >= session.ContextTokens) + throw new LocalScoringException("context_limit_exceeded", ProviderFailureClasses.Policy); + prepared.Add((question, rotation, prompt)); + } + var answers = new List(); + foreach (var question in payload.Questions) + { + var count = question.Options.Count; + var normalized = new double[count]; var raw = new double[count]; + var winners = new List(); var coverage = 1.0; + foreach (var item in prepared.Where(p => ReferenceEquals(p.Question, question))) + { + deadline.Token.ThrowIfCancellationRequested(); + var score = await session.ScoreAsync(item.Prompt, labels.Take(count).ToArray(), deadline.Token).AsTask().WaitAsync(deadline.Token); + if (score is null || score.Labels is null || !score.FullVocabulary || score.OutputPositions != 1) + throw new LocalScoringException("invalid_probability_basis", ProviderFailureClasses.Schema); + if (score.Labels.Count != count || score.Labels.Any(p => p is null) || score.Labels.Select(p => p.Id).Distinct().Count() != count) + throw new LocalScoringException("incomplete_label_probabilities", ProviderFailureClasses.Schema); + var probabilities = new double[count]; + for (var index = 0; index < count; index++) + { + var expected = labels[index]; + var probability = score.Labels.SingleOrDefault(p => p.Id == expected.Id); + if (probability is null || probability.Bytes is null || !probability.Bytes.SequenceEqual(expected.Bytes) + || !double.IsFinite(probability.LogProbability) || probability.LogProbability > 0) + throw new LocalScoringException("invalid_label_probability", ProviderFailureClasses.Schema); + probabilities[(index + item.Rotation) % count] = Math.Exp(probability.LogProbability); + } + var mass = probabilities.Sum(); + if (!double.IsFinite(mass) || mass <= 0 || mass > 1.0 + 1e-6) + throw new LocalScoringException("invalid_label_mass", ProviderFailureClasses.Schema); + coverage = Math.Min(coverage, Math.Min(1.0, mass)); + winners.Add(Winner(probabilities)); + for (var index = 0; index < count; index++) { raw[index] += probabilities[index]; normalized[index] += probabilities[index] / mass; } + } + var rotations = winners.Count; var winner = Winner(normalized); + answers.Add(new() { QuestionId = question.Id, Choice = question.Options[winner].Id, + Probabilities = Enumerable.Range(0, count).ToDictionary(i => question.Options[i].Id, i => normalized[i] / rotations), + RawLabelProbabilities = Enumerable.Range(0, count).ToDictionary(i => question.Options[i].Id, i => raw[i] / rotations), + LabelMassCoverage = coverage, RotationAgreement = winners.Count(i => i == winner) / (double)rotations, + Confidence = normalized[winner] / rotations, Calibrated = false }); + } + deadline.Token.ThrowIfCancellationRequested(); + var output = ProviderJson.ToJsonObject(new AiJudgeResult { Answers = answers, ModelId = _factory.ModelId }); + result = Encoding.UTF8.GetByteCount(output.ToJsonString(ProviderJson.Options)) > Math.Min(request.MaxOutputBytes ?? _options.MaxOutputBytes, _options.MaxOutputBytes) + ? Result(ProviderRunStatus.Failed, "output_limit", ProviderFailureClasses.Policy) + : Result(ProviderRunStatus.Succeeded, "ok", output: output); + } + catch (LocalScoringException ex) + { + result = Result(ex.FailureClass == ProviderFailureClasses.Unsupported ? ProviderRunStatus.Unsupported : ProviderRunStatus.Rejected, + ex.Reason, ex.FailureClass); + } + catch (OperationCanceledException) + { + result = Result(ct.IsCancellationRequested ? ProviderRunStatus.Cancelled : ProviderRunStatus.TimedOut, + ct.IsCancellationRequested ? "cancelled" : "deadline_exceeded", + ct.IsCancellationRequested ? ProviderFailureClasses.Cancelled : ProviderFailureClasses.Timeout); + } + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException or JsonException or IOException or HttpRequestException) + { + result = Result(ProviderRunStatus.Failed, "scoring_failed", ProviderFailureClasses.Schema); + } + if (session is not null) + { + // Session implementations must enforce their own bounded cleanup independently of caller cancellation. + try { await session.DisposeAsync().AsTask().WaitAsync(TimeSpan.FromSeconds(2)); } + catch (Exception) { if (result.Status == ProviderRunStatus.Succeeded) result = Result(ProviderRunStatus.Failed, "cleanup_failed", ProviderFailureClasses.ProviderUnavailable); } + } + return result; + } + + private void Validate(AiJudgeRequest payload) + { + if (payload.Questions is null || payload.Questions.Count == 0 || payload.Questions.Count > _options.MaxQuestions + || payload.Questions.Any(q => q is null || string.IsNullOrWhiteSpace(q.Id) || string.IsNullOrWhiteSpace(q.Prompt)) + || payload.Questions.Select(q => q.Id).Distinct(StringComparer.Ordinal).Count() != payload.Questions.Count) + throw new LocalScoringException("invalid_questions", ProviderFailureClasses.Schema); + if (payload.References is null || payload.References.Count != 0) + throw new LocalScoringException("references_unsupported", ProviderFailureClasses.Unsupported); + foreach (var question in payload.Questions) + { + if (question.Type != AiJudgeQuestionTypes.Choice) + throw new LocalScoringException("unsupported_question_type", ProviderFailureClasses.Unsupported); + if (question.Options is null || question.Options.Count is < 2 || question.Options.Count > _options.MaxOptions + || question.Options.Any(o => o is null || string.IsNullOrWhiteSpace(o.Id) || string.IsNullOrWhiteSpace(o.Label)) + || question.Options.Select(o => o.Id).Distinct(StringComparer.Ordinal).Count() != question.Options.Count) + throw new LocalScoringException("invalid_options", ProviderFailureClasses.Schema); + } + } + private static int Winner(IReadOnlyList values) => Enumerable.Range(0, values.Count).OrderByDescending(i => values[i]).ThenBy(i => i).First(); + private static string BuildPrompt(string? context, AiJudgeQuestion question, int rotation) + { + var text = new StringBuilder("Select the best option for the question using the supplied context. Reply with exactly one option letter.\nContext:\n"); + text.Append(context).Append("\nQuestion:\n").Append(question.Prompt).Append("\nOptions:\n"); + for (var index = 0; index < question.Options.Count; index++) + text.Append((char)('A' + index)).Append(": ").Append(question.Options[(index + rotation) % question.Options.Count].Label).Append('\n'); + return text.ToString(); + } +} diff --git a/src/Vyral.Providers.Local/LocalLogprobSession.cs b/src/Vyral.Providers.Local/LocalLogprobSession.cs new file mode 100644 index 0000000..2557204 --- /dev/null +++ b/src/Vyral.Providers.Local/LocalLogprobSession.cs @@ -0,0 +1,49 @@ +using Vyral.Providers.Abstractions; + +namespace Vyral.Providers.Local; + +/// A batch-scoped local scoring session. Host leasing and transport stay behind this interface. +public interface ILocalLogprobSessionFactory +{ + string BindingId { get; } + string ModelId { get; } + bool RequiresNetwork { get; } + string Auth => "session-managed"; + ValueTask OpenAsync(CancellationToken ct); +} + +public interface ILocalLogprobSession : IAsyncDisposable +{ + int ContextTokens { get; } + ValueTask ResolveLabelAsync(string label, CancellationToken ct); + /// Count the complete model-templated prompt, with no truncation or inference. + ValueTask PrepareAsync(string prompt, CancellationToken ct); + /// Return original full-vocabulary log probabilities at exactly the first output position. + /// Missing labels must remain missing. No partial-response retries or renormalization. + ValueTask ScoreAsync(LocalScoringPrompt prompt, + IReadOnlyList labels, CancellationToken ct); +} + +public sealed record LocalDecisionToken(int Id, byte[] Bytes); +public sealed record LocalScoringPrompt(int[] TokenIds); +public sealed record LocalLabelLogprob(int Id, byte[] Bytes, double LogProbability); +public sealed record LocalDecisionScore(IReadOnlyList Labels, + int OutputPositions = 1, bool FullVocabulary = true); + +/// Typed safe refusal; never embed credentials or raw backend errors in Message. +public sealed class LocalScoringException(string reason, string failureClass) : Exception(reason) +{ + public string Reason { get; } = reason; + public string FailureClass { get; } = failureClass; +} + +public sealed class LocalLogprobJudgeOptions +{ + public int Rotations { get; init; } = 3; + public int MaxQuestions { get; init; } = 32; + public int MaxOptions { get; init; } = 26; + public int MaxScoringCalls { get; init; } = 96; + public int MaxInputBytes { get; init; } = 65536; + public int MaxOutputBytes { get; init; } = 131072; + public int TimeoutSeconds { get; init; } = 120; +} diff --git a/src/Vyral.Server/Program.cs b/src/Vyral.Server/Program.cs index de12fa6..4a9cce5 100644 --- a/src/Vyral.Server/Program.cs +++ b/src/Vyral.Server/Program.cs @@ -4408,6 +4408,34 @@ static ProviderTargetRegistry CreateProviderTargetRegistry(IConfiguration config OnnxNliJudgeProviderTargets.CreateGpu(GetOnnxJudgeProviderOptions(configuration, "Providers:OnnxJudge:Gpu")) }; + if (ParseOptionalBool(configuration["Providers:WorkspaceAgent:Enabled"], "Providers:WorkspaceAgent:Enabled") == true) + { + if (!ServerAccessOptions.FromConfiguration(configuration).Enabled) + { + throw new InvalidOperationException("Providers:WorkspaceAgent:Enabled requires Server API-key authentication. Configure Server:RequireApiKey and Server:ApiKey before registering a source-writing target."); + } + + targets.Add(CliProviderTargets.CreateWorkspaceCodingAgent(GetCliWorkspaceCodingAgentOptions(configuration))); + } + + if (ParseOptionalBool(configuration["Providers:LocalRuntime:Enabled"], "Providers:LocalRuntime:Enabled") == true) + { + var runtime = new LlamaCppRuntimeOptions + { + Endpoint = new Uri(configuration["Providers:LocalRuntime:Endpoint"] ?? "http://127.0.0.1:8080/"), + ModelId = configuration["Providers:LocalRuntime:ModelId"] ?? "", + ModelPath = configuration["Providers:LocalRuntime:ModelPath"] ?? "", + ModelSha256 = configuration["Providers:LocalRuntime:ModelSha256"] ?? "", + BuildInfo = configuration["Providers:LocalRuntime:BuildInfo"] ?? "", + TemplateSha256 = configuration["Providers:LocalRuntime:TemplateSha256"] ?? "", + TemplateProbeSha256 = configuration["Providers:LocalRuntime:TemplateProbeSha256"] ?? "", + ContextTokens = ParseOptionalInt(configuration["Providers:LocalRuntime:ContextTokens"], "Providers:LocalRuntime:ContextTokens") ?? 2048, + MaxOutputTokens = ParseOptionalInt(configuration["Providers:LocalRuntime:MaxOutputTokens"], "Providers:LocalRuntime:MaxOutputTokens") ?? 512 + }; + targets.Add(new LocalLlamaCppProviderTarget(runtime)); + targets.Add(new LocalLogprobJudgeProviderTarget(new LlamaCppRuntimeSessionFactory(runtime))); + } + var enableLiveTargets = ParseOptionalBool(configuration["Providers:EnableLiveTargets"], "Providers:EnableLiveTargets") ?? false; if (!enableLiveTargets) { @@ -4420,16 +4448,6 @@ static ProviderTargetRegistry CreateProviderTargetRegistry(IConfiguration config targets.Add(CliProviderTargets.CreateAntigravity(overrides: GetCliProviderOptions(configuration, "Providers:Antigravity"))); targets.Add(CliProviderTargets.CreateGrokBuild(overrides: GetGrokBuildProviderOptions(configuration, "Providers:GrokBuild"))); - if (ParseOptionalBool(configuration["Providers:WorkspaceAgent:Enabled"], "Providers:WorkspaceAgent:Enabled") == true) - { - if (!ServerAccessOptions.FromConfiguration(configuration).Enabled) - { - throw new InvalidOperationException("Providers:WorkspaceAgent:Enabled requires Server API-key authentication. Configure Server:RequireApiKey and Server:ApiKey before registering a source-writing target."); - } - - targets.Add(CliProviderTargets.CreateWorkspaceCodingAgent(GetCliWorkspaceCodingAgentOptions(configuration))); - } - targets.Add(new JulesProviderTarget(new JulesProviderOptions { ApiKey = configuration["Providers:Jules:ApiKey"] ?? Environment.GetEnvironmentVariable("JULES_API_KEY"), diff --git a/src/Vyral.Server/ServerStorageFactory.cs b/src/Vyral.Server/ServerStorageFactory.cs index 9c179f2..72cea6b 100644 --- a/src/Vyral.Server/ServerStorageFactory.cs +++ b/src/Vyral.Server/ServerStorageFactory.cs @@ -67,6 +67,7 @@ public static async Task CreateTraceStoreAsync(ServerStorageOptions AccountId = options.CloudflareAccountId, AccessKeyId = options.CloudflareR2AccessKeyId, SecretAccessKey = options.CloudflareR2SecretAccessKey, + SessionToken = options.CloudflareR2SessionToken, ServiceUrl = options.CloudflareR2ServiceUrl }), _ => throw new InvalidOperationException($"Object store backend '{options.ObjectStore}' is not supported by this host.") diff --git a/src/Vyral.Server/ServerStorageOptions.cs b/src/Vyral.Server/ServerStorageOptions.cs index c2cc034..ed334a3 100644 --- a/src/Vyral.Server/ServerStorageOptions.cs +++ b/src/Vyral.Server/ServerStorageOptions.cs @@ -16,6 +16,7 @@ public sealed class ServerStorageOptions public string? CloudflareAccountId { get; init; } public string? CloudflareR2AccessKeyId { get; init; } public string? CloudflareR2SecretAccessKey { get; init; } + public string? CloudflareR2SessionToken { get; init; } public string? CloudflareR2ServiceUrl { get; init; } public string ObjectProbeContainer { get; init; } = "vyral-readiness"; @@ -100,6 +101,10 @@ public static ServerStorageOptions FromConfiguration(IConfiguration configuratio configuration["Cloudflare:R2:ServiceUrl"], configuration["VYRAL_R2_SERVICE_URL"], null), + CloudflareR2SessionToken = FirstNonEmpty( + configuration["Cloudflare:R2:SessionToken"], + configuration["VYRAL_R2_SESSION_TOKEN"], + null), ObjectProbeContainer = objectProbeContainer }; } diff --git a/src/Vyral.Server/contracts/vyral.openapi.json b/src/Vyral.Server/contracts/vyral.openapi.json index 8c3a7ae..388152d 100644 --- a/src/Vyral.Server/contracts/vyral.openapi.json +++ b/src/Vyral.Server/contracts/vyral.openapi.json @@ -16397,6 +16397,9 @@ { "$ref": "#/components/schemas/AiToolPlanPayload" }, + { + "$ref": "#/components/schemas/AiJudgePayload" + }, { "type": "object", "additionalProperties": true, @@ -16409,7 +16412,8 @@ "ai.rerank": "#/components/schemas/AiRerankPayload", "ai.review": "#/components/schemas/AiReviewPayload", "ai.scaffold": "#/components/schemas/AiScaffoldPayload", - "ai.toolPlan": "#/components/schemas/AiToolPlanPayload" + "ai.toolPlan": "#/components/schemas/AiToolPlanPayload", + "ai.judge": "#/components/schemas/AiJudgePayload" } }, "correlationId": { @@ -16514,6 +16518,9 @@ { "$ref": "#/components/schemas/AiToolPlanResult" }, + { + "$ref": "#/components/schemas/AiJudgeResult" + }, { "type": "object", "additionalProperties": true, @@ -16526,7 +16533,8 @@ "ai.rerank": "#/components/schemas/AiRerankResult", "ai.review": "#/components/schemas/AiReviewResult", "ai.scaffold": "#/components/schemas/AiScaffoldResult", - "ai.toolPlan": "#/components/schemas/AiToolPlanResult" + "ai.toolPlan": "#/components/schemas/AiToolPlanResult", + "ai.judge": "#/components/schemas/AiJudgeResult" } }, "failureClass": { @@ -17755,6 +17763,15 @@ "null" ], "format": "int32" + }, + "maxOutputTokens": { + "type": [ + "integer", + "null" + ], + "format": "int32", + "minimum": 1, + "description": "Positive ceiling on all generated tokens, including reasoning. Adapters that cannot enforce it reject before inference; character and byte limits remain separate." } }, "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiChatPayload" @@ -19317,6 +19334,211 @@ } }, "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/RagContextGraphEvaluationFailureModes" + }, + "AiJudgeOption": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgeOption" + }, + "AiJudgeQuestion": { + "type": "object", + "required": [ + "id", + "type", + "prompt" + ], + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "choice", + "noul", + "score" + ] + }, + "prompt": { + "type": "string" + }, + "options": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AiJudgeOption" + } + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgeQuestion" + }, + "AiJudgePayload": { + "type": "object", + "required": [ + "questions" + ], + "properties": { + "context": { + "type": [ + "string", + "null" + ] + }, + "questions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AiJudgeQuestion" + } + }, + "references": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AiReference" + } + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgePayload" + }, + "AiJudgeAnswer": { + "type": "object", + "required": [ + "questionId", + "calibrated" + ], + "properties": { + "questionId": { + "type": "string" + }, + "choice": { + "type": [ + "string", + "null" + ] + }, + "calibrated": { + "type": "boolean" + }, + "score": { + "type": [ + "number", + "null" + ], + "description": "Expected zero-based ordered level index." + }, + "legend": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "string" + } + }, + "probability": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "confidence": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "labelMassCoverage": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Minimum unconditioned decision-label mass over rotations; null means unavailable." + }, + "rotationAgreement": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Share of rotations whose argmax agrees with the final winner; null means unavailable." + }, + "probabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "rawLabelProbabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "description": "Mean unconditioned mass per original option; not a normalized probability distribution." + }, + "rawClassProbabilities": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgeAnswer" + }, + "AiJudgeResult": { + "type": "object", + "required": [ + "answers", + "validationStatus" + ], + "properties": { + "answers": { + "type": "array", + "items": { + "$ref": "#/components/schemas/AiJudgeAnswer" + } + }, + "validationStatus": { + "type": "string" + }, + "modelId": { + "type": [ + "string", + "null" + ] + } + }, + "$ref": "../../../contracts/schemas/vyral-public.schema.json#/$defs/AiJudgeResult" } } }, diff --git a/tests/Vyral.Tests.Aws/AwsConformanceTests.cs b/tests/Vyral.Tests.Aws/AwsConformanceTests.cs index d5089b4..367cacc 100644 --- a/tests/Vyral.Tests.Aws/AwsConformanceTests.cs +++ b/tests/Vyral.Tests.Aws/AwsConformanceTests.cs @@ -354,7 +354,7 @@ public async ValueTask DisposeAsync() ContinuationToken = continuationToken }); - foreach (var obj in listResponse.S3Objects) + foreach (var obj in listResponse.S3Objects ?? new List()) { try { diff --git a/tests/Vyral.Tests.Cloudflare/CloudflareR2ConformanceTests.cs b/tests/Vyral.Tests.Cloudflare/CloudflareR2ConformanceTests.cs index 60071de..5d8aff6 100644 --- a/tests/Vyral.Tests.Cloudflare/CloudflareR2ConformanceTests.cs +++ b/tests/Vyral.Tests.Cloudflare/CloudflareR2ConformanceTests.cs @@ -8,6 +8,7 @@ using Vyral.Abstractions.Models; using Vyral.Cloudflare; using Vyral.Tests.Conformance; +using Xunit; namespace Vyral.Tests.Cloudflare; @@ -30,8 +31,21 @@ public Task ObjectStore_RejectsNonPortableNames() => RunObjectStore_RejectsNonPortableNames(); [CloudflareR2LiveFact] - public Task ObjectStore_DeletesObjectsIdempotentlyAndEnforcesPreconditions() => - RunObjectStore_DeletesObjectsIdempotentlyAndEnforcesPreconditions(); + public async Task ObjectStore_DeletesIdempotentlyAndRefusesUnsupportedConditionalDelete() + { + await using var store = (ScopedR2ObjectStore)CreateObjectStore(); + var put = await store.PutObjectAsync(new() { Container = "objects", Key = "docs/delete.txt", + Content = new System.IO.MemoryStream(new byte[] { 1, 2, 3 }) }); + foreach (var etag in new[] { "\"missing\"", put.Etag }) + await Assert.ThrowsAsync(() => store.DeleteObjectAsync(new() { + Container = "objects", Key = "docs/delete.txt", IfMatch = etag })); + var retained = await store.GetObjectAsync(new() { Container = "objects", Key = "docs/delete.txt" }); + Assert.NotNull(retained); + await retained!.Content.DisposeAsync(); + await store.DeleteObjectAsync(new() { Container = "objects", Key = "docs/delete.txt" }); + await store.DeleteObjectAsync(new() { Container = "objects", Key = "docs/delete.txt" }); + Assert.Null(await store.GetObjectAsync(new() { Container = "objects", Key = "docs/delete.txt" })); + } [CloudflareR2LiveFact] public Task ObjectStore_ListsWithContinuationToken() => @@ -41,6 +55,16 @@ public Task ObjectStore_ListsWithContinuationToken() => public Task ObjectStore_RejectsInvalidListLimit() => RunObjectStore_RejectsInvalidListLimit(); + [CloudflareR2LiveFact] + public async Task ObjectStore_PreCancelledWriteLeavesNoObject() + { + await using var store = (ScopedR2ObjectStore)CreateObjectStore(); + using var cancellation = new CancellationTokenSource(); cancellation.Cancel(); + await Assert.ThrowsAnyAsync(() => store.PutObjectAsync(new() { + Container = "objects", Key = "cancelled.txt", Content = new System.IO.MemoryStream(new byte[] { 1 }) }, cancellation.Token)); + Assert.Null(await store.GetObjectAsync(new() { Container = "objects", Key = "cancelled.txt" })); + } + protected override IObjectStore CreateObjectStore() { var bucket = CloudflareR2LiveSettings.Bucket!; @@ -49,7 +73,8 @@ protected override IObjectStore CreateObjectStore() { AccountId = CloudflareR2LiveSettings.AccountId, AccessKeyId = CloudflareR2LiveSettings.AccessKeyId, - SecretAccessKey = CloudflareR2LiveSettings.SecretAccessKey + SecretAccessKey = CloudflareR2LiveSettings.SecretAccessKey, + SessionToken = CloudflareR2LiveSettings.SessionToken }); return new ScopedR2ObjectStore(store, bucket, keyPrefix); @@ -71,6 +96,7 @@ public ScopedR2ObjectStore(R2ObjectStore inner, string bucket, string keyPrefix) public async Task PutObjectAsync(ObjectWriteRequest request, CancellationToken ct = default) { + ObjectNameValidator.ValidateContainer(request.Container); var result = await _inner.PutObjectAsync(MapWrite(request), ct); result.Container = request.Container; result.Key = request.Key; @@ -79,6 +105,7 @@ public async Task PutObjectAsync(ObjectWriteRequest request, Cancell public async Task GetObjectAsync(ObjectReadRequest request, CancellationToken ct = default) { + ObjectNameValidator.ValidateContainer(request.Container); var result = await _inner.GetObjectAsync(new ObjectReadRequest { Container = _bucket, @@ -93,16 +120,22 @@ public async Task PutObjectAsync(ObjectWriteRequest request, Cancell return result; } - public Task DeleteObjectAsync(ObjectDeleteRequest request, CancellationToken ct = default) => - _inner.DeleteObjectAsync(new ObjectDeleteRequest + public Task DeleteObjectAsync(ObjectDeleteRequest request, CancellationToken ct = default) + { + ObjectNameValidator.ValidateContainer(request.Container); + return _inner.DeleteObjectAsync(new ObjectDeleteRequest { Container = _bucket, Key = ScopeKey(request.Key), IfMatch = request.IfMatch }, ct); + } public async Task ListObjectsAsync(ObjectListRequest request, CancellationToken ct = default) { + ObjectNameValidator.ValidateContainer(request.Container); + if (!string.IsNullOrEmpty(request.Prefix)) + ObjectNameValidator.NormalizeObjectKey(request.Prefix, allowTrailingSlash: true); var prefix = string.IsNullOrEmpty(request.Prefix) ? _keyPrefix + "/" : _keyPrefix + "/" + request.Prefix.TrimStart('/'); @@ -118,7 +151,7 @@ public async Task ListObjectsAsync(ObjectListRequest request, foreach (var item in result.Items) { item.Container = request.Container; - item.Key = item.Key[prefix.Length..].TrimStart('/'); + item.Key = item.Key[(_keyPrefix.Length + 1)..]; } return result; @@ -164,7 +197,7 @@ await _inner.DeleteObjectAsync(new ObjectDeleteRequest } } - private string ScopeKey(string key) => $"{_keyPrefix}/{key.TrimStart('/')}"; + private string ScopeKey(string key) => $"{_keyPrefix}/{ObjectNameValidator.NormalizeObjectKey(key)}"; private ObjectWriteRequest MapWrite(ObjectWriteRequest request) => new() diff --git a/tests/Vyral.Tests.Cloudflare/CloudflareR2LiveSettings.cs b/tests/Vyral.Tests.Cloudflare/CloudflareR2LiveSettings.cs index f87fbc2..6b54172 100644 --- a/tests/Vyral.Tests.Cloudflare/CloudflareR2LiveSettings.cs +++ b/tests/Vyral.Tests.Cloudflare/CloudflareR2LiveSettings.cs @@ -14,6 +14,9 @@ public static class CloudflareR2LiveSettings public static string? SecretAccessKey => Environment.GetEnvironmentVariable("VYRAL_R2_SECRET_ACCESS_KEY"); + public static string? SessionToken => + Environment.GetEnvironmentVariable("VYRAL_R2_SESSION_TOKEN"); + public static string? Bucket => Environment.GetEnvironmentVariable("VYRAL_R2_BUCKET"); diff --git a/tests/Vyral.Tests.Cloudflare/R2TransportTests.cs b/tests/Vyral.Tests.Cloudflare/R2TransportTests.cs new file mode 100644 index 0000000..71a0f84 --- /dev/null +++ b/tests/Vyral.Tests.Cloudflare/R2TransportTests.cs @@ -0,0 +1,59 @@ +using System.Net; +using System.Net.Sockets; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Vyral.Abstractions.Models; +using Vyral.Aws; +using Vyral.Cloudflare; +using Xunit; + +namespace Vyral.Tests.Cloudflare; + +public class R2TransportTests +{ + [Fact] + public async Task EmptySdkV4CollectionIsAnEmptyPageWithNoContinuation() + { + using var client = new EmptyClient(); + var result = await new S3ObjectStore(client).ListObjectsAsync(new() { Container = "objects" }); + Assert.Empty(result.Items); Assert.Null(result.ContinuationToken); + } + + [Fact] + public async Task DefaultTransportDoesNotRetryAServiceUnavailableResponse() + { + var socket = new TcpListener(IPAddress.Loopback, 0); socket.Start(); + var port = ((IPEndPoint)socket.LocalEndpoint).Port; socket.Stop(); + using var listener = new HttpListener(); listener.Prefixes.Add($"http://127.0.0.1:{port}/"); listener.Start(); + using var store = R2ObjectStore.Create(new() { ServiceUrl = $"http://127.0.0.1:{port}", AccessKeyId = "fixture", SecretAccessKey = "fixture" }); + var call = store.ListObjectsAsync(new() { Container = "objects" }); + var context = await listener.GetContextAsync().WaitAsync(TimeSpan.FromSeconds(5)); + context.Response.StatusCode = 503; + context.Response.Close(); + var exception = await Assert.ThrowsAsync(() => call); + Assert.Equal(HttpStatusCode.ServiceUnavailable, exception.StatusCode); + var secondRequest = listener.GetContextAsync(); + Assert.NotSame(secondRequest, await Task.WhenAny(secondRequest, Task.Delay(100))); + } + + [Fact] + public async Task PrecancelledPutPreservesCancellationAndSendsNoObject() + { + using var store = R2ObjectStore.Create(new() { ServiceUrl = "http://127.0.0.1:1", AccessKeyId = "fixture", SecretAccessKey = "fixture" }); + using var ct = new CancellationTokenSource(); ct.Cancel(); + await Assert.ThrowsAnyAsync(() => store.PutObjectAsync(new() { + Container = "objects", Key = "cancelled.txt", Content = new MemoryStream(new byte[] { 1,2,3 }) }, ct.Token)); + } + + [Theory] + [InlineData(-1)] [InlineData(4)] + public void RetryBudgetIsExplicitAndBounded(int retries) => + Assert.Throws(() => new CloudflareR2Options { AccessKeyId = "fixture", SecretAccessKey = "fixture", MaxErrorRetry = retries }.ValidateCredentials()); + + private sealed class EmptyClient() : AmazonS3Client(new AnonymousAWSCredentials(), new AmazonS3Config { ServiceURL = "http://127.0.0.1:1" }) + { + public override Task ListObjectsV2Async(ListObjectsV2Request request, CancellationToken ct = default) => + Task.FromResult(new ListObjectsV2Response { S3Objects = null, IsTruncated = false }); + } +} diff --git a/tests/Vyral.Tests.Local/ServerWorkflowTests.cs b/tests/Vyral.Tests.Local/ServerWorkflowTests.cs index aabe50c..acfcf97 100644 --- a/tests/Vyral.Tests.Local/ServerWorkflowTests.cs +++ b/tests/Vyral.Tests.Local/ServerWorkflowTests.cs @@ -4046,7 +4046,7 @@ public void Server_RejectsWorkspaceAgentRegistrationWithoutApiKeyAuthentication( using var factory = CreateFactory(dbPath, objectsPath, settings: new Dictionary { - ["Providers:EnableLiveTargets"] = "true", + ["Providers:EnableLiveTargets"] = "false", ["Providers:WorkspaceAgent:Enabled"] = "true" }); @@ -4069,7 +4069,7 @@ public async Task Server_RegistersWorkspaceAgentOnlyWithExplicitOptInAndAuthenti { await using var factory = CreateFactory(dbPath, objectsPath, settings: new Dictionary { - ["Providers:EnableLiveTargets"] = "true", + ["Providers:EnableLiveTargets"] = "false", ["Providers:WorkspaceAgent:Enabled"] = "true", ["Providers:WorkspaceAgent:AgentCommand"] = "/bin/true", ["Providers:WorkspaceAgent:AllowedWorkspaceRoots:0"] = workspaceRoot, @@ -4084,6 +4084,7 @@ public async Task Server_RegistersWorkspaceAgentOnlyWithExplicitOptInAndAuthenti Assert.NotNull(providers); Assert.Contains(providers, provider => provider.Id == "workspace-cli"); + Assert.DoesNotContain(providers, provider => provider.Id == "codex-cli"); } finally { @@ -4092,6 +4093,30 @@ public async Task Server_RegistersWorkspaceAgentOnlyWithExplicitOptInAndAuthenti } } + [Fact] + public async Task Server_RegistersPinnedLocalRuntimeWithoutRemoteCliTargets() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"vyral-runtime-{Guid.NewGuid():N}.sqlite"); + var objectsPath = Path.Combine(Path.GetTempPath(), $"vyral-runtime-{Guid.NewGuid():N}"); + await using var factory = CreateFactory(dbPath, objectsPath, settings: new Dictionary + { + ["Providers:EnableLiveTargets"] = "false", + ["Providers:LocalRuntime:Enabled"] = "true", + ["Providers:LocalRuntime:Endpoint"] = "http://127.0.0.1:18346/", + ["Providers:LocalRuntime:ModelId"] = "fixture", + ["Providers:LocalRuntime:ModelPath"] = Path.GetFullPath("fixture.gguf"), + ["Providers:LocalRuntime:ModelSha256"] = new string('a', 64), + ["Providers:LocalRuntime:BuildInfo"] = "fixture-build", + ["Providers:LocalRuntime:TemplateSha256"] = new string('b', 64), + ["Providers:LocalRuntime:TemplateProbeSha256"] = new string('c', 64) + }); + var providers = await factory.CreateClient().GetFromJsonAsync>("/providers"); + Assert.NotNull(providers); + Assert.Contains(providers, p => p.Id == "local-llamacpp"); + Assert.Contains(providers, p => p.Id == "local-logprob-judge"); + Assert.DoesNotContain(providers, p => p.Id == "codex-cli"); + } + [Fact] public async Task Server_DisabledLiveProviderReadinessReturnsCapabilityBlockers() { diff --git a/tests/Vyral.Tests.Providers/LocalLogprobJudgeProviderTests.cs b/tests/Vyral.Tests.Providers/LocalLogprobJudgeProviderTests.cs new file mode 100644 index 0000000..172b8c6 --- /dev/null +++ b/tests/Vyral.Tests.Providers/LocalLogprobJudgeProviderTests.cs @@ -0,0 +1,155 @@ +using System.Text; +using System.Text.Json.Nodes; +using Vyral.Providers.Abstractions; +using Vyral.Providers.Local; + +namespace Vyral.Tests.Providers; + +public class LocalLogprobJudgeProviderTests +{ + private static ProviderRunRequest Request(string type = AiJudgeQuestionTypes.Choice, int questions = 1) => + ProviderRunRequests.ForJudge(new() { Context = "Generic passage.", Questions = Enumerable.Range(0, questions).Select(i => new AiJudgeQuestion { + Id = $"q{i}", Type = type, Prompt = "Which option fits?", + Options = new() { new() { Id = "a", Label = "correct" }, new() { Id = "b", Label = "other" }, new() { Id = "c", Label = "third" } } + }).ToList() }); + + [Fact] + public async Task RotationsMapBackAndExposeRawMassAgreementAndUncalibratedScores() + { + var factory = new FakeFactory(); + var provider = new LocalLogprobJudgeProviderTarget(factory); + var result = await provider.RunAsync(Request(questions: 2)); + Assert.Equal(ProviderRunStatus.Succeeded, result.Status); + var answers = ProviderRunResults.GetJudge(result).Answers; + Assert.Equal(2, answers.Count); + foreach (var answer in answers) + { + Assert.Equal("a", answer.Choice); Assert.False(answer.Calibrated); + Assert.Equal(1, answer.RotationAgreement); Assert.Equal(0.3, answer.LabelMassCoverage!.Value, 10); + Assert.Equal(0.2, answer.RawLabelProbabilities!["a"], 10); + Assert.Equal(2.0 / 3, answer.Probabilities!["a"], 10); + Assert.Equal(1, answer.Probabilities.Values.Sum(), 10); + } + Assert.Equal(1, factory.Opens); Assert.Equal(1, factory.Session.Disposals); + Assert.Equal(6, factory.Session.Prepared); Assert.Equal(6, factory.Session.Scored); + Assert.True(factory.Session.FirstScoreSawCompletePreparation); + } + + [Theory] + [InlineData("missing")] [InlineData("bytes")] [InlineData("duplicate")] + [InlineData("nan")] [InlineData("positive")] [InlineData("mass")] + [InlineData("basis")] [InlineData("positions")] + public async Task InvalidProbabilityEvidenceFailsWithoutPartialAnswersAndDisposes(string fault) + { + var factory = new FakeFactory(); factory.Session.Fault = fault; + var result = await new LocalLogprobJudgeProviderTarget(factory).RunAsync(Request()); + Assert.NotEqual(ProviderRunStatus.Succeeded, result.Status); + Assert.Equal(ProviderFailureClasses.Schema, result.FailureClass); + Assert.Empty(result.Output); Assert.Equal(1, factory.Session.Disposals); + } + + [Theory] + [InlineData(AiJudgeQuestionTypes.Noul)] [InlineData(AiJudgeQuestionTypes.Score)] + public async Task UnsupportedTypesAreRejectedBeforeAcquisition(string type) + { + var factory = new FakeFactory(); + var result = await new LocalLogprobJudgeProviderTarget(factory).RunAsync(Request(type)); + Assert.Equal(ProviderRunStatus.Unsupported, result.Status); Assert.Equal(0, factory.Opens); + } + + [Fact] + public async Task EntireBatchIsPreflightedBeforeInferenceAndContextOverflowDoesNotTruncate() + { + var factory = new FakeFactory(); factory.Session.OverflowAt = 3; + var result = await new LocalLogprobJudgeProviderTarget(factory).RunAsync(Request()); + Assert.Equal("context_limit_exceeded", result.ProviderStatus); + Assert.Equal(ProviderFailureClasses.Policy, result.FailureClass); + Assert.Equal(0, factory.Session.Scored); Assert.Equal(1, factory.Session.Disposals); + } + + [Fact] + public async Task OutputLimitAndCallBudgetAreEnforced() + { + var factory = new FakeFactory(); var request = Request(); request.MaxOutputBytes = 1; + var result = await new LocalLogprobJudgeProviderTarget(factory).RunAsync(request); + Assert.Equal("output_limit", result.ProviderStatus); Assert.Empty(result.Output); + factory = new(); + result = await new LocalLogprobJudgeProviderTarget(factory, new() { MaxScoringCalls = 2 }).RunAsync(Request()); + Assert.Equal("scoring_call_limit", result.ProviderStatus); Assert.Equal(0, factory.Opens); + } + + [Fact] + public async Task CancellationBeforeAcquisitionHasNoEffects() + { + var factory = new FakeFactory(); using var cancellation = new CancellationTokenSource(); cancellation.Cancel(); + var result = await new LocalLogprobJudgeProviderTarget(factory).RunAsync(Request(), cancellation.Token); + Assert.Equal(ProviderRunStatus.Cancelled, result.Status); Assert.Equal(0, factory.Opens); + } + + [Fact] + public async Task CancellationDuringScoringDisposesAndNeverReturnsEarlierAnswers() + { + var factory = new FakeFactory(); using var cancellation = new CancellationTokenSource(); + factory.Session.Cancel = cancellation; + var result = await new LocalLogprobJudgeProviderTarget(factory).RunAsync(Request(questions: 2), cancellation.Token); + Assert.Equal(ProviderRunStatus.Cancelled, result.Status); Assert.Empty(result.Output); + Assert.Equal(1, factory.Session.Disposals); + } + + [Fact] + public async Task RotationDisagreementIsAvailableSeparatelyFromConfidence() + { + var factory = new FakeFactory(); factory.Session.Fault = "disagreement"; + var answer = ProviderRunResults.GetJudge(await new LocalLogprobJudgeProviderTarget(factory).RunAsync(Request())).Answers.Single(); + Assert.Equal("a", answer.Choice); Assert.Equal(2.0 / 3, answer.RotationAgreement!.Value, 10); + } + + [Fact] + public async Task TokenCeilingCannotBeSilentlyIgnoredByDeterministicProvider() + { + var result = await new DeterministicAiProviderTarget().RunAsync(ProviderRunRequests.ForChat( + new() { Messages = new() { new() { Role = AiRoles.User, Content = "Generic prompt" } }, MaxOutputTokens = 1 })); + Assert.Equal(ProviderRunStatus.Unsupported, result.Status); + Assert.Equal("token_ceiling_unsupported", result.ProviderStatus); + } + + private sealed class FakeFactory : ILocalLogprobSessionFactory + { + public int Opens; public FakeSession Session { get; } = new(); + public string BindingId => "fixture-v1"; public string ModelId => "fixture"; public bool RequiresNetwork => false; + public ValueTask OpenAsync(CancellationToken ct) { Opens++; return ValueTask.FromResult(Session); } + } + + private sealed class FakeSession : ILocalLogprobSession + { + public int ContextTokens => 100; + public int Prepared, Scored, Disposals, OverflowAt; + public bool FirstScoreSawCompletePreparation; + public string? Fault; + public CancellationTokenSource? Cancel; + private readonly Dictionary _winner = new(); + public ValueTask DisposeAsync() { Disposals++; return ValueTask.CompletedTask; } + public ValueTask ResolveLabelAsync(string label, CancellationToken ct) => + ValueTask.FromResult(new LocalDecisionToken(label[0], Encoding.UTF8.GetBytes(label))); + public ValueTask PrepareAsync(string prompt, CancellationToken ct) + { + Prepared++; _winner[Prepared] = Enumerable.Range(0, 3).Single(i => prompt.Contains($"{(char)('A' + i)}: correct")); + return ValueTask.FromResult(new LocalScoringPrompt(OverflowAt == Prepared ? new int[100] : new[] { Prepared })); + } + public ValueTask ScoreAsync(LocalScoringPrompt prompt, IReadOnlyList labels, CancellationToken ct) + { + Scored++; if (Scored == 1) FirstScoreSawCompletePreparation = Prepared >= 3; + if (Cancel is not null) { Cancel.Cancel(); ct.ThrowIfCancellationRequested(); } + var winner = _winner[prompt.TokenIds[0]]; + if (Fault == "disagreement" && Scored == 3) winner = (winner + 1) % 3; + var probabilities = labels.Select((label, i) => new LocalLabelLogprob(label.Id, label.Bytes, Math.Log(i == winner ? 0.2 : 0.05))).ToList(); + if (Fault == "missing") probabilities.RemoveAt(0); + if (Fault == "duplicate") probabilities[1] = probabilities[0]; + if (Fault == "bytes") probabilities[0] = probabilities[0] with { Bytes = new byte[] { 0 } }; + if (Fault == "nan") probabilities[0] = probabilities[0] with { LogProbability = double.NaN }; + if (Fault == "positive") probabilities[0] = probabilities[0] with { LogProbability = 1 }; + if (Fault == "mass") probabilities = probabilities.Select(p => p with { LogProbability = Math.Log(0.5) }).ToList(); + return ValueTask.FromResult(new LocalDecisionScore(probabilities, Fault == "positions" ? 2 : 1, Fault != "basis")); + } + } +} diff --git a/tests/Vyral.Tests.Providers/LocalNativeRuntimeTests.cs b/tests/Vyral.Tests.Providers/LocalNativeRuntimeTests.cs new file mode 100644 index 0000000..ddd96d0 --- /dev/null +++ b/tests/Vyral.Tests.Providers/LocalNativeRuntimeTests.cs @@ -0,0 +1,218 @@ +using System.Net; +using System.Net.Sockets; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Vyral.Providers.Abstractions; +using Vyral.Providers.Local; + +namespace Vyral.Tests.Providers; + +public class LocalNativeRuntimeTests +{ + [Fact] + public async Task ExactTokenCeilingTemplateAndModelBindingReachNativeAndTotalUsageIsDerived() + { + await using var server = new NativeFixture(); + var result = await new LocalLlamaCppProviderTarget(server.Options).RunAsync(Request(server.Options)); + Assert.Equal(ProviderRunStatus.Succeeded, result.Status); + Assert.Equal(new[] { "/props", "/apply-template", "/apply-template", "/tokenize", "/completion" }, server.Paths); + Assert.Equal(2, server.Completion!["n_predict"]!.GetValue()); + Assert.False(server.Completion["cache_prompt"]!.GetValue()); + Assert.Equal(new[] { 10, 11, 12 }, server.Completion["prompt"]!.AsArray().Select(n => n!.GetValue())); + var total = Assert.Single(result.MeteringMeasurements, m => m.Name == AiMeteringMeasurementNames.TotalTokens); + Assert.Equal(4, total.Value); + Assert.Equal(AiMeteringSources.ConsumerInference, total.Source); + Assert.Equal(AiMeteringQualities.Estimated, total.Quality); + } + + [Theory] + [InlineData("build")] [InlineData("template")] [InlineData("context")] + public async Task ChangedRuntimeBindingRefusesBeforeCompletion(string fault) + { + await using var server = new NativeFixture { Fault = fault }; + var result = await new LocalLlamaCppProviderTarget(server.Options).RunAsync(Request(server.Options)); + Assert.Equal("runtime_binding_mismatch", result.ProviderStatus); + Assert.DoesNotContain("/completion", server.Paths); + } + + [Fact] + public async Task ChangedEffectiveTemplateRefusesBeforeCompletion() + { + await using var server = new NativeFixture { Fault = "probe" }; + var result = await new LocalLlamaCppProviderTarget(server.Options).RunAsync(Request(server.Options)); + Assert.Equal("template_probe_mismatch", result.ProviderStatus); + Assert.DoesNotContain("/completion", server.Paths); + } + + [Theory] + [InlineData("negative")] [InlineData("missing")] [InlineData("truncated")] + [InlineData("overflow")] [InlineData("missingContent")] + public async Task InvalidOrMissingCountersAndContentNeverBecomeSuccessfulOrInventedZeroUsage(string fault) + { + await using var server = new NativeFixture { Fault = fault }; + var result = await new LocalLlamaCppProviderTarget(server.Options).RunAsync(Request(server.Options)); + Assert.NotEqual(ProviderRunStatus.Succeeded, result.Status); Assert.Empty(result.Output); + if (fault is "negative" or "missing") + { + Assert.DoesNotContain(result.MeteringMeasurements, m => m.Name == AiMeteringMeasurementNames.InputTokens); + Assert.DoesNotContain(result.MeteringMeasurements, m => m.Name == AiMeteringMeasurementNames.TotalTokens); + Assert.Single(result.MeteringMeasurements); + } + } + + [Fact] + public async Task ExplicitTokenCeilingAndOuterByteLimitRemainDifferentControls() + { + await using var server = new NativeFixture(); + var request = Request(server.Options); request.Payload.Remove("maxOutputTokens"); + var result = await new LocalLlamaCppProviderTarget(server.Options).RunAsync(request); + Assert.Equal("explicit_token_ceiling_required", result.ProviderStatus); Assert.Empty(server.Paths); + request = Request(server.Options); request.MaxOutputBytes = 1; + result = await new LocalLlamaCppProviderTarget(server.Options).RunAsync(request); + Assert.Equal("output_limit", result.ProviderStatus); + Assert.NotEmpty(result.MeteringMeasurements); + } + + [Fact] + public async Task ContextBudgetIsCheckedBeforeCompletion() + { + await using var server = new NativeFixture { Fault = "longPrompt" }; + var result = await new LocalLlamaCppProviderTarget(server.Options).RunAsync(Request(server.Options)); + Assert.Equal("context_limit_exceeded", result.ProviderStatus); + Assert.DoesNotContain("/completion", server.Paths); + } + + [Fact] + public async Task CallerCancellationBeforeEffectDoesNotInvokeRuntime() + { + await using var server = new NativeFixture(); using var ct = new CancellationTokenSource(); ct.Cancel(); + var result = await new LocalLlamaCppProviderTarget(server.Options).RunAsync(Request(server.Options), ct.Token); + Assert.Equal(ProviderRunStatus.Cancelled, result.Status); Assert.Empty(server.Paths); + } + + [Fact] + public async Task CancellationAfterDispatchLeavesNativeEffectUnresolved() + { + await using var server = new NativeFixture { Fault = "blockedCompletion" }; + using var cancellation = new CancellationTokenSource(); + var task = new LocalLlamaCppProviderTarget(server.Options).RunAsync(Request(server.Options), cancellation.Token); + await server.CompletionSeen.Task.WaitAsync(TimeSpan.FromSeconds(5)); + cancellation.Cancel(); + var result = await task.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal(ProviderRunStatus.Cancelled, result.Status); + Assert.Equal("native_effect_unresolved", result.ProviderStatus); + Assert.Empty(result.Output); + Assert.DoesNotContain(result.MeteringMeasurements, m => m.Name == AiMeteringMeasurementNames.TotalTokens); + Assert.Single(server.Paths, path => path == "/completion"); + } + + [LocalRuntimeLiveFact] + public async Task PinnedLocalRuntimeChatSmoke() + { + var options = ReadLiveOptions(); + var result = await new LocalLlamaCppProviderTarget(options).RunAsync(ProviderRunRequests.ForChat( + new() { Messages = new() { new() { Role = AiRoles.User, Content = "Reply with the single word OK." } }, MaxOutputTokens = 12 }, + modelId: options.ModelId)); + Assert.True(result.Status == ProviderRunStatus.Succeeded, result.ProviderStatus); + Assert.NotEmpty(ProviderRunResults.GetChat(result).Message.Content); + Assert.Contains(result.MeteringMeasurements, m => m.Name == AiMeteringMeasurementNames.TotalTokens); + } + + [LocalRuntimeLiveFact] + public async Task PinnedLocalRuntimeChoiceScoringSmoke() + { + var options = ReadLiveOptions(); + var result = await new LocalLogprobJudgeProviderTarget(new LlamaCppRuntimeSessionFactory(options)).RunAsync( + ProviderRunRequests.ForJudge(new() { Context = "A cat is an animal.", Questions = new() { new() { + Id = "q", Prompt = "What is a cat?", Options = new() { + new() { Id = "animal", Label = "An animal" }, new() { Id = "plant", Label = "A plant" } + } } } }, modelId: options.ModelId)); + Assert.True(result.Status == ProviderRunStatus.Succeeded, result.ProviderStatus); + var answer = ProviderRunResults.GetJudge(result).Answers.Single(); + Assert.False(answer.Calibrated); Assert.InRange(answer.LabelMassCoverage!.Value, 0, 1); + Assert.InRange(answer.RotationAgreement!.Value, 0, 1); Assert.Equal(1, answer.Probabilities!.Values.Sum(), 8); + } + + private static LlamaCppRuntimeOptions ReadLiveOptions() => JsonSerializer.Deserialize( + File.ReadAllText(Environment.GetEnvironmentVariable("VYRAL_LOCAL_RUNTIME_PROFILE")!), ProviderJson.Options)!; + private static ProviderRunRequest Request(LlamaCppRuntimeOptions options) => + ProviderRunRequests.ForChat(new() { Messages = new() { new() { Role = AiRoles.User, Content = "Generic fixture prompt." } }, + MaxOutputTokens = 2 }, modelId: options.ModelId); + + private sealed class NativeFixture : IAsyncDisposable + { + private readonly HttpListener _listener = new(); + private readonly Task _loop; + private readonly string _file = Path.GetTempFileName(); + public List Paths { get; } = new(); + public JsonObject? Completion; + public TaskCompletionSource CompletionSeen { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly CancellationTokenSource _shutdown = new(); + public string? Fault; + public LlamaCppRuntimeOptions Options { get; } + public NativeFixture() + { + File.WriteAllText(_file, "generic model fixture"); + var socket = new TcpListener(IPAddress.Loopback, 0); socket.Start(); + var port = ((IPEndPoint)socket.LocalEndpoint).Port; socket.Stop(); + var endpoint = new Uri($"http://127.0.0.1:{port}/"); + Options = new() { Endpoint = endpoint, ModelId = "fixture", ModelPath = Path.GetFullPath(_file), + ModelSha256 = Convert.ToHexStringLower(SHA256.HashData(File.ReadAllBytes(_file))), BuildInfo = "fixture-build", + TemplateSha256 = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("fixture-template"))), + TemplateProbeSha256 = Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes("templated fixture"))), + ContextTokens = 8, MaxOutputTokens = 4 }; + _listener.Prefixes.Add(endpoint.ToString()); _listener.Start(); + _loop = Task.Run(async () => + { + try + { + while (_listener.IsListening) + { + var context = await _listener.GetContextAsync(); var path = context.Request.Url!.AbsolutePath; + Paths.Add(path); + using var reader = new StreamReader(context.Request.InputStream); + var body = await reader.ReadToEndAsync(); + if (path == "/completion") + { + Completion = JsonNode.Parse(body)!.AsObject(); + CompletionSeen.TrySetResult(); + if (Fault == "blockedCompletion") await Task.Delay(Timeout.Infinite, _shutdown.Token); + } + JsonObject response = path switch { + "/props" => new() { ["model_path"] = Options.ModelPath, ["build_info"] = Fault == "build" ? "other" : "fixture-build", + ["chat_template"] = Fault == "template" ? "other" : "fixture-template", + ["default_generation_settings"] = new JsonObject { ["n_ctx"] = Fault == "context" ? 9 : 8 }, ["total_slots"] = 1 }, + "/apply-template" => new() { ["prompt"] = Fault == "probe" ? "changed renderer" : "templated fixture" }, + "/tokenize" => new() { ["tokens"] = Fault == "longPrompt" ? new JsonArray(1,2,3,4,5,6,7) : new JsonArray(10,11,12) }, + _ => new() { ["content"] = "OK", ["tokens_evaluated"] = 3, ["tokens_predicted"] = 1, ["truncated"] = false } + }; + if (path == "/completion") + { + if (Fault == "negative") response["tokens_evaluated"] = -1; + if (Fault == "missing") response.Remove("tokens_evaluated"); + if (Fault == "truncated") response["truncated"] = true; + if (Fault == "overflow") response["tokens_predicted"] = 3; + if (Fault == "missingContent") response.Remove("content"); + } + var bytes = Encoding.UTF8.GetBytes(response.ToJsonString()); + context.Response.ContentType = "application/json"; context.Response.ContentLength64 = bytes.Length; + await context.Response.OutputStream.WriteAsync(bytes); context.Response.Close(); + } + } + catch (Exception ex) when (ex is HttpListenerException or ObjectDisposedException or OperationCanceledException) { } + }); + } + public async ValueTask DisposeAsync() { _shutdown.Cancel(); _listener.Close(); await _loop.WaitAsync(TimeSpan.FromSeconds(2)); _shutdown.Dispose(); File.Delete(_file); } + } +} + +public sealed class LocalRuntimeLiveFactAttribute : FactAttribute +{ + public LocalRuntimeLiveFactAttribute() + { + if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("VYRAL_LOCAL_RUNTIME_PROFILE"))) + Skip = "Set VYRAL_LOCAL_RUNTIME_PROFILE to a private pinned local-runtime profile."; + } +}