From 01f3fbd0136d3cb70a8d83830e9f10edd1ef1ba8 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 03:27:44 -0700 Subject: [PATCH 01/20] Add canonical engine port contracts Define the schema-owned engine ports, runtime effect metadata, deterministic acceptance vector, and legacy harness preservation gate. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- schema/README.md | 5 + schema/model/main.tsp | 1 + schema/model/pipeline/engine-ports.tsp | 71 +++++++ schema/model/pipeline/executor.tsp | 9 +- schema/package.json | 3 +- schema/scripts/verify-engine-ports.mjs | 254 ++++++++++++++++++++++++ spec/turn-engine.md | 41 ++-- spec/vectors/engine/port_contracts.json | 95 +++++++++ 8 files changed, 458 insertions(+), 21 deletions(-) create mode 100644 schema/model/pipeline/engine-ports.tsp create mode 100644 schema/scripts/verify-engine-ports.mjs create mode 100644 spec/vectors/engine/port_contracts.json diff --git a/schema/README.md b/schema/README.md index 8d217526d..b1360c766 100644 --- a/schema/README.md +++ b/schema/README.md @@ -39,6 +39,7 @@ npm run build ``` This generates code into: + - `runtime/typescript/packages/core/src/model/` — TypeScript - `runtime/python/prompty/prompty/model/` — Python - `runtime/csharp/Prompty.Core/Model/` — C# @@ -52,6 +53,7 @@ This generates code into: Generated files are **committed to the repo**. The generator is a dev-time tool — consumers don't need TypeSpec installed. Generated files have a header: + ``` // WARNING: This is an auto-generated file. DO NOT EDIT THIS FILE DIRECTLY ``` @@ -78,6 +80,9 @@ The `scripts/` folder holds the supporting Node helpers: output deterministic — it normalizes the generation timestamp in the Typra manifest, collapses empty generated Python test files, and trims trailing whitespace in generated Go files. +- `verify-engine-ports.mjs` validates the canonical engine-port metadata and + native cancellation seams against `spec/vectors/engine/port_contracts.json`, + and protects the deterministic legacy `pipeline/harness.tsp` contract. - `verify-typra.mjs` backs `npm run verify:typra`, which compares the current Typra export surfaces, manifest, hydration seams, and JSON AST against the committed `HEAD` baseline to detect schema drift. diff --git a/schema/model/main.tsp b/schema/model/main.tsp index 591d08c54..7228f7d57 100644 --- a/schema/model/main.tsp +++ b/schema/model/main.tsp @@ -27,6 +27,7 @@ import "./pipeline/invocation.tsp"; import "./pipeline/engine-events.tsp"; import "./pipeline/checkpoint.tsp"; import "./pipeline/turn-engine.tsp"; +import "./pipeline/engine-ports.tsp"; import "./pipeline/policy.tsp"; import "./pipeline/context-planning.tsp"; import "./pipeline/turn.tsp"; diff --git a/schema/model/pipeline/engine-ports.tsp b/schema/model/pipeline/engine-ports.tsp new file mode 100644 index 000000000..0d1706a95 --- /dev/null +++ b/schema/model/pipeline/engine-ports.tsp @@ -0,0 +1,71 @@ +import "@typra/emitter"; +import "./checkpoint.tsp"; +import "./engine-events.tsp"; +import "./invocation.tsp"; +import "./turn-engine.tsp"; + +namespace Prompty; + +@@protocol(EnginePermissionPort); +@@method(EnginePermissionPort, + "authorize", + "EnginePermissionDecision", + "Authorize one model-requested tool before execution", + #{ request: "ModelToolRequest" }, + false, + false, + #{ runtimeCancellable: true } +); + +/** Authorizes model-requested tools at a runtime cancellation boundary. */ +model EnginePermissionPort {} + +@@protocol(EngineToolPort); +@@method(EngineToolPort, + "execute", + "ModelToolResult", + "Execute one authorized model-requested tool", + #{ request: "ModelToolRequest" }, + false, + false, + #{ runtimeCancellable: true } +); + +/** Executes authorized model-requested tools at a runtime cancellation boundary. */ +model EngineToolPort {} + +@@protocol(EngineDurabilityPort); +@@method(EngineDurabilityPort, + "append", + "void", + "Append one semantic engine event durably", + #{ event: "EngineEvent" }, + false, + false +); +@@method(EngineDurabilityPort, + "appendWithCheckpoint", + "void", + "Atomically append semantic engine events and persist the checkpoint that reflects them", + #{ events: "EngineEvent[]", checkpoint: "EngineCheckpoint" }, + false, + false, + #{ atomic: true } +); + +/** Persists semantic engine events and checkpoints without runtime cancellation. */ +model EngineDurabilityPort {} + +@@protocol(EnginePostCommitPort); +@@method(EnginePostCommitPort, + "afterCommit", + "void", + "Run one idempotent host effect after the turn is durably committed", + #{ effectId: "string", commit: "TurnCommit" }, + false, + false, + #{ runtimeCancellable: true, nonFatal: true } +); + +/** Runs non-fatal host effects after a turn is durably committed. */ +model EnginePostCommitPort {} diff --git a/schema/model/pipeline/executor.tsp b/schema/model/pipeline/executor.tsp index 357a09cb7..0780d7237 100644 --- a/schema/model/pipeline/executor.tsp +++ b/schema/model/pipeline/executor.tsp @@ -10,14 +10,19 @@ namespace Prompty; "execute", "unknown", "Call an LLM provider with messages and return the raw response", - #{ agent: "Prompty", messages: "Message[]" } + #{ agent: "Prompty", messages: "Message[]" }, + false, + false, + #{ runtimeCancellable: true } ); @@method(Executor, "executeStream", "unknown", "Call an LLM provider and return a streaming response. Returns a language-specific async iterable/stream of raw chunks. Not all providers support streaming; the default implementation should signal lack of support.", #{ agent: "Prompty", messages: "Message[]" }, - true + true, + false, + #{ runtimeCancellable: true } ); @@method(Executor, "formatToolMessages", diff --git a/schema/package.json b/schema/package.json index a0d593116..dbd093a50 100644 --- a/schema/package.json +++ b/schema/package.json @@ -7,8 +7,9 @@ "format:tsp:check": "npx tsp format \"model/**/*.tsp\" --check", "format:rust": "cargo fmt --all --manifest-path ../runtime/rust/prompty/Cargo.toml", "generate": "npx tsp compile model/main.tsp --config tspconfig.yaml && node scripts/normalize-typra-output.mjs", + "verify:engine-ports": "node scripts/verify-engine-ports.mjs", "verify:typra": "node scripts/verify-typra.mjs", - "build": "npm run format:tsp && npm run generate && npm run format:rust" + "build": "npm run format:tsp && npm run generate && npm run format:rust && npm run verify:engine-ports" }, "dependencies": { "@typespec/compiler": "1.10.0", diff --git a/schema/scripts/verify-engine-ports.mjs b/schema/scripts/verify-engine-ports.mjs new file mode 100644 index 000000000..32e6dc7fc --- /dev/null +++ b/schema/scripts/verify-engine-ports.mjs @@ -0,0 +1,254 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const repoRoot = execFileSync("git", ["rev-parse", "--show-toplevel"], { + encoding: "utf8", +}).trim(); +const vector = readJson( + join(repoRoot, "spec", "vectors", "engine", "port_contracts.json"), +); +const surfaces = readJson( + join( + repoRoot, + "schema", + "tsp-output", + ".typra-generated", + "export-surfaces.json", + ), +); + +verifyLegacyHarness(); +verifyExportSurfaces(); +verifyNativeSignatures(); + +console.log("Canonical engine port contracts verified."); + +function readJson(path) { + return JSON.parse(readFileSync(path, "utf8")); +} + +function verifyLegacyHarness() { + const harness = readFileSync( + join(repoRoot, "schema", "model", "pipeline", "harness.tsp"), + ); + const actual = createHash("sha256").update(harness).digest("hex"); + assertEqual( + actual, + vector.legacyHarnessSha256, + "pipeline/harness.tsp SHA-256", + ); +} + +function verifyExportSurfaces() { + for (const target of surfaces.targets) { + for (const [protocolName, expectedProtocol] of Object.entries( + vector.protocols, + )) { + const protocol = target.protocols.find( + (candidate) => candidate.name === protocolName, + ); + assert(protocol, `${target.target}: missing ${protocolName} protocol`); + assertExactKeys( + Object.fromEntries( + protocol.methods.map((method) => [method.name, true]), + ), + Object.fromEntries( + Object.keys(expectedProtocol.methods).map((methodName) => [ + methodName, + true, + ]), + ), + `${target.target}: ${protocolName} methods`, + ); + + for (const [methodName, expectedMethod] of Object.entries( + expectedProtocol.methods, + )) { + const method = protocol.methods.find( + (candidate) => candidate.name === methodName, + ); + assert( + method, + `${target.target}: missing ${protocolName}.${methodName}`, + ); + assertSubset( + method, + expectedMethod, + `${target.target}: ${protocolName}.${methodName}`, + ); + if (expectedMethod.params) { + assertExactKeys( + method.params, + expectedMethod.params, + `${target.target}: ${protocolName}.${methodName} params`, + ); + } + + for (const syntheticName of [ + "cancellation", + "cancellationToken", + "ctx", + "signal", + ]) { + assert( + !Object.hasOwn(method.params, syntheticName), + `${target.target}: ${protocolName}.${methodName} leaked runtime cancellation into schema params`, + ); + } + } + } + } +} + +function verifyNativeSignatures() { + const root = { + csharp: join( + repoRoot, + "runtime", + "csharp", + "Prompty.Core", + "Model", + "pipeline", + ), + go: join(repoRoot, "runtime", "go", "prompty", "model"), + python: join( + repoRoot, + "runtime", + "python", + "prompty", + "prompty", + "model", + "pipeline", + ), + rust: join( + repoRoot, + "runtime", + "rust", + "prompty", + "src", + "model", + "pipeline", + ), + typescript: join( + repoRoot, + "runtime", + "typescript", + "packages", + "core", + "src", + "model", + "pipeline", + ), + }; + + expectMatches( + join(root.csharp, "EnginePermissionPort.cs"), + /Task\s+AuthorizeAsync\(ModelToolRequest request,\s*CancellationToken cancellationToken = default\);/u, + ); + expectMatches( + join(root.go, "engine_permission_port.go"), + /Authorize\(ctx context\.Context,\s*request ModelToolRequest\)\s*\(EnginePermissionDecision, error\)/u, + ); + expectMatches( + join(root.typescript, "engine-permission-port.ts"), + /authorize\(request: ModelToolRequest,\s*signal\?: AbortSignal\): Promise;/u, + ); + expectMatches( + join(root.rust, "engine_permission_port.rs"), + /async fn authorize\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,\s*\)\s*-> Result EnginePermissionDecision:/u, + ); + + const durabilityFiles = [ + join(root.csharp, "EngineDurabilityPort.cs"), + join(root.go, "engine_durability_port.go"), + join(root.python, "_EngineDurabilityPort.py"), + join(root.rust, "engine_durability_port.rs"), + join(root.typescript, "engine-durability-port.ts"), + ]; + for (const file of durabilityFiles) { + const content = readFileSync(file, "utf8"); + for (const forbidden of [ + "CancellationToken", + "context.Context", + "AbortSignal", + ]) { + assert( + !content.includes(forbidden), + `${file}: durability protocol must remain non-cancellable`, + ); + } + } + + const executorFiles = [ + join(root.csharp, "Executor.cs"), + join(root.go, "executor.go"), + join(root.python, "_Executor.py"), + join(root.rust, "executor.rs"), + join(root.typescript, "executor.ts"), + ]; + for (const file of executorFiles) { + const content = readFileSync(file, "utf8"); + assert( + content.includes("formatToolMessages") || + content.includes("format_tool_messages") || + content.includes("FormatToolMessages"), + `${file}: missing synchronous formatToolMessages`, + ); + } +} + +function expectMatches(path, pattern) { + const content = readFileSync(path, "utf8"); + assert( + pattern.test(content), + `${path}: generated signature did not match ${pattern}`, + ); +} + +function assertSubset(actual, expected, label) { + for (const [key, expectedValue] of Object.entries(expected)) { + const actualValue = actual[key]; + if ( + expectedValue !== null && + typeof expectedValue === "object" && + !Array.isArray(expectedValue) + ) { + assert( + actualValue !== null && typeof actualValue === "object", + `${label}.${key}: expected an object`, + ); + assertSubset(actualValue, expectedValue, `${label}.${key}`); + } else { + assertEqual(actualValue, expectedValue, `${label}.${key}`); + } + } +} + +function assertExactKeys(actual, expected, label) { + const actualKeys = Object.keys(actual).sort(); + const expectedKeys = Object.keys(expected).sort(); + assertEqual( + JSON.stringify(actualKeys), + JSON.stringify(expectedKeys), + `${label} keys`, + ); +} + +function assertEqual(actual, expected, label) { + assert( + actual === expected, + `${label}: expected ${JSON.stringify(expected)}, received ${JSON.stringify(actual)}`, + ); +} + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} diff --git a/spec/turn-engine.md b/spec/turn-engine.md index 93bc65fa6..263874b6a 100644 --- a/spec/turn-engine.md +++ b/spec/turn-engine.md @@ -250,24 +250,29 @@ post-commit start event prevents the effect from running. Failure to persist com after the effect ran is returned as non-fatal recovery information and MUST NOT make the committed turn appear to have failed. -## Runtime-Local Ports - -Native runtime interfaces own async, streaming, cancellation, and SDK-specific behavior: - -- `ModelPort` -- `ContextSource` -- `ContextTransform` -- `ContextPackingStrategy` -- `PermissionPort` -- `ToolPort` -- `DurabilityPort` (atomic semantic-event and checkpoint persistence) -- `Clock` -- `IdGenerator` -- post-commit effect ports - -Portable TypeSpec models will be promoted only after the Rust state machine and -conformance vectors establish stable semantics. Native interfaces themselves are not -generated. +## Engine Ports + +The stable permission, tool, durability, and post-commit effect boundaries are canonical +TypeSpec protocols: + +- `EnginePermissionPort.authorize(ModelToolRequest) -> EnginePermissionDecision` is async + and runtime-cancellable. +- `EngineToolPort.execute(ModelToolRequest) -> ModelToolResult` is async and + runtime-cancellable. +- `EngineDurabilityPort.append(EngineEvent) -> void` is async and non-cancellable. +- `EngineDurabilityPort.appendWithCheckpoint(EngineEvent[], EngineCheckpoint) -> void` + is async, atomic, and non-cancellable. +- `EnginePostCommitPort.afterCommit(effectId, TurnCommit) -> void` is async, + runtime-cancellable, and non-fatal after the turn is committed. + +Runtime cancellation is projected as a native language seam rather than a wire or model +field: `CancellationToken` in C#, `context.Context` in Go, optional `AbortSignal` in +TypeScript, the runtime `CancellationToken` reference in Rust, and the runtime +`CancellationToken` signal in Python. Port failures remain native runtime errors such as +`PortError`; they are not portable wire models. + +Richer runtime interfaces continue to own SDK-specific streaming, context assembly, +host policy, retry, clocks, identifiers, and provider reconciliation behavior. ## Rust-First Conformance Gate diff --git a/spec/vectors/engine/port_contracts.json b/spec/vectors/engine/port_contracts.json new file mode 100644 index 000000000..ffefe7482 --- /dev/null +++ b/spec/vectors/engine/port_contracts.json @@ -0,0 +1,95 @@ +{ + "version": "1", + "legacyHarnessSha256": "c534140f1aae07034cedea13feec3d66a9acbc0df46869858753aef39e24fe69", + "protocols": { + "EnginePermissionPort": { + "methods": { + "authorize": { + "returns": "EnginePermissionDecision", + "params": { + "request": "ModelToolRequest" + }, + "optional": false, + "sync": false, + "runtimeCancellable": true, + "atomic": false, + "nonFatal": false + } + } + }, + "EngineToolPort": { + "methods": { + "execute": { + "returns": "ModelToolResult", + "params": { + "request": "ModelToolRequest" + }, + "optional": false, + "sync": false, + "runtimeCancellable": true, + "atomic": false, + "nonFatal": false + } + } + }, + "EngineDurabilityPort": { + "methods": { + "append": { + "returns": "void", + "params": { + "event": "EngineEvent" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + }, + "appendWithCheckpoint": { + "returns": "void", + "params": { + "events": "EngineEvent[]", + "checkpoint": "EngineCheckpoint" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": true, + "nonFatal": false + } + } + }, + "EnginePostCommitPort": { + "methods": { + "afterCommit": { + "returns": "void", + "params": { + "effectId": "string", + "commit": "TurnCommit" + }, + "optional": false, + "sync": false, + "runtimeCancellable": true, + "atomic": false, + "nonFatal": true + } + } + }, + "Executor": { + "methods": { + "execute": { + "runtimeCancellable": true, + "sync": false + }, + "executeStream": { + "runtimeCancellable": true, + "sync": false + }, + "formatToolMessages": { + "runtimeCancellable": false, + "sync": true + } + } + } + } +} From 1bb6f7b1e185824930f2d4aacc9ec0de2867483b Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 03:27:50 -0700 Subject: [PATCH 02/20] Propagate executor cancellation in C# Thread generated executor cancellation through the pipeline and provider SDK calls, with forwarding and pre-cancellation coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AnthropicExecutorTests.cs | 12 +++- .../Prompty.Anthropic/AnthropicExecutor.cs | 26 +++++--- .../AgentLoopIntegrationTests.cs | 5 +- .../Prompty.Core.Tests/PipelineTests.cs | 33 +++++++++- .../Prompty.Core.Tests/ResilienceTests.cs | 5 +- .../StructuredOutputPipelineTests.cs | 5 +- .../csharp/Prompty.Core.Tests/TracingTests.cs | 5 +- runtime/csharp/Prompty.Core/Pipeline.cs | 11 ++-- .../Prompty.OpenAI.Tests/AgentLoopTests.cs | 15 ++++- .../OpenAIExecutorTests.cs | 12 +++- .../SpecVectorAgentTests.cs | 5 +- .../csharp/Prompty.OpenAI/OpenAIExecutor.cs | 64 +++++++++++++------ 12 files changed, 154 insertions(+), 44 deletions(-) diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs index d73197cb3..63c0cdea5 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs @@ -9,6 +9,17 @@ namespace Prompty.Anthropic.Tests; /// public class AnthropicExecutorTests { + [Fact] + public async Task ExecuteAsync_Cancelled_ThrowsBeforeConnectionValidation() + { + var executor = new Anthropic.AnthropicExecutor(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => executor.ExecuteAsync(new Core.Prompty(), [], cancellation.Token)); + } + [Fact] public async Task ExecuteAsync_MissingApiKey_ThrowsInvalidOperationException() { @@ -156,4 +167,3 @@ public void FormatToolMessages_NoTextContent_OmitsTextBlock() Assert.Equal("tool_use", content[0]["type"]); } } - diff --git a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs index a64f6198d..5b6117e6a 100644 --- a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs +++ b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs @@ -19,30 +19,40 @@ public class AnthropicExecutor : IExecutor private const string ApiVersion = "2023-06-01"; private const int DefaultMaxTokens = 4096; - public async Task ExecuteAsync(Core.Prompty agent, List messages) + public async Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); var streaming = agent.Metadata?.TryGetValue("stream", out var streamVal) == true && streamVal is true; if (streaming) - return ExecuteStreamAsync(agent, messages); + return ExecuteStreamAsync(agent, messages, cancellationToken); - return await ExecuteNonStreamAsync(agent, messages); + return await ExecuteNonStreamAsync(agent, messages, cancellationToken); } - private async Task ExecuteNonStreamAsync(Core.Prompty agent, List messages) + private async Task ExecuteNonStreamAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var body = BuildRequestBody(agent, messages, stream: false); var (endpoint, apiKey) = GetConnectionInfo(agent); var request = CreateRequest(endpoint, apiKey, body); - var response = await _httpClient.SendAsync(request); + var response = await _httpClient.SendAsync(request, cancellationToken); response.EnsureSuccessStatusCode(); - var json = await response.Content.ReadFromJsonAsync(); + var json = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); return json; } - private PromptyStream ExecuteStreamAsync(Core.Prompty agent, List messages) + private PromptyStream ExecuteStreamAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var body = BuildRequestBody(agent, messages, stream: true); var (endpoint, apiKey) = GetConnectionInfo(agent); @@ -69,7 +79,7 @@ async IAsyncEnumerable StreamEvents([System.Runtime.CompilerServices.Enu } } - return new PromptyStream(StreamEvents()); + return new PromptyStream(StreamEvents(cancellationToken)); } internal Dictionary BuildRequestBody(Core.Prompty agent, List messages, bool stream) diff --git a/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs b/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs index ce0bcb19a..c84974bb7 100644 --- a/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs @@ -64,7 +64,10 @@ public Task> ParseAsync(Prompty agent, string rendered, Dictionary public void EnqueueResponse(object response) => _responses.Enqueue(response); - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) { // Snapshot the messages at call time Calls.Add(new List(messages)); diff --git a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs index 4c111edcb..332e79d79 100644 --- a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs @@ -335,6 +335,22 @@ public async Task ExecuteAsync_UsesProvider() Assert.Equal("mock-response", response); } + [Fact] + public async Task ExecuteAsync_ForwardsCancellationToken() + { + var agent = CreateAgent(); + var executor = new MockExecutor(); + InvokerRegistry.RegisterExecutor("openai", executor); + using var cancellation = new CancellationTokenSource(); + + await Pipeline.ExecuteAsync( + agent, + [new Message { Parts = [new TextPart { Value = "Hi" }] }], + cancellation.Token); + + Assert.Equal(cancellation.Token, executor.LastCancellationToken); + } + [Fact] public async Task ProcessAsync_UsesProvider() { @@ -601,8 +617,16 @@ public Task> ParseAsync(Prompty agent, string rendered, Dictionary internal class MockExecutor : IExecutor { - public Task ExecuteAsync(Prompty agent, List messages) - => Task.FromResult("mock-response"); + public CancellationToken LastCancellationToken { get; private set; } + + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) + { + LastCancellationToken = cancellationToken; + return Task.FromResult("mock-response"); + } public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) { @@ -629,7 +653,10 @@ internal class ToolCallingExecutor : IExecutor { private int _callCount; - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) { _callCount++; if (_callCount == 1) diff --git a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs index a972bcfb3..ea36025d6 100644 --- a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs @@ -22,7 +22,10 @@ namespace Prompty.Core.Tests; public void EnqueueResponse(object response) => _responses.Enqueue(response); public void EnqueueException(Exception ex) => _exceptions.Enqueue(ex); - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) { Calls.Add(new List(messages)); diff --git a/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs b/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs index 005771327..5430cff48 100644 --- a/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs @@ -261,7 +261,10 @@ internal class RawJsonExecutor : IExecutor public RawJsonExecutor(string rawJson) => _rawJson = rawJson; - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) => Task.FromResult(_rawJson); public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) diff --git a/runtime/csharp/Prompty.Core.Tests/TracingTests.cs b/runtime/csharp/Prompty.Core.Tests/TracingTests.cs index 7c588e007..e4bae9dcb 100644 --- a/runtime/csharp/Prompty.Core.Tests/TracingTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/TracingTests.cs @@ -577,7 +577,10 @@ public Task> ParseAsync(Prompty agent, string rendered, Dictionary private class MockExecutor(object response) : IExecutor { - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) => Task.FromResult(response); public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) diff --git a/runtime/csharp/Prompty.Core/Pipeline.cs b/runtime/csharp/Prompty.Core/Pipeline.cs index a476ca3a7..f3ce51bc7 100644 --- a/runtime/csharp/Prompty.Core/Pipeline.cs +++ b/runtime/csharp/Prompty.Core/Pipeline.cs @@ -104,14 +104,17 @@ public static async Task> ParseAsync(Prompty agent, string rendere /// /// Execute an LLM call with the given messages. /// - public static async Task ExecuteAsync(Prompty agent, List messages) + public static async Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) { return await Trace.TraceAsync("Prompty.Core.Pipeline.ExecuteAsync", async (emit) => { emit("inputs", new Dictionary { ["agent"] = agent.Name, ["message_count"] = messages.Count }); var provider = agent.Model?.Provider ?? "openai"; var executor = InvokerRegistry.GetExecutor(provider); - return await executor.ExecuteAsync(agent, messages); + return await executor.ExecuteAsync(agent, messages, cancellationToken); }); } @@ -280,7 +283,7 @@ public static async Task TurnAsync( object response; try { - response = await ExecuteAsync(agent, messages); + response = await ExecuteAsync(agent, messages, cancellationToken); } catch (Exception ex) { @@ -842,7 +845,7 @@ private static async Task InvokeWithRetryAsync( { try { - return await ExecuteAsync(agent, messages); + return await ExecuteAsync(agent, messages, cancellationToken); } catch (Exception ex) when (ex is not OperationCanceledException) { diff --git a/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs index 563759e14..cb54135a1 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs @@ -269,7 +269,10 @@ public Task> ParseAsync(Core.Prompty agent, string rendered, Dicti /// Executor that returns a fixed response. private class MockExecutor(object response) : IExecutor { - public Task ExecuteAsync(Core.Prompty agent, List messages) + public Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) => Task.FromResult(response); public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) @@ -296,7 +299,10 @@ private class SequenceExecutor(List responses) : IExecutor private int _index; public int CallCount => _index; - public Task ExecuteAsync(Core.Prompty agent, List messages) + public Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) { if (_index >= responses.Count) throw new InvalidOperationException("SequenceExecutor ran out of responses."); @@ -310,7 +316,10 @@ public List FormatToolMessages(object rawResponse, List toolC /// Executor that always returns a tool call — for testing max iterations. private class InfiniteToolCallExecutor : IExecutor { - public Task ExecuteAsync(Core.Prompty agent, List messages) + public Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) { return Task.FromResult(new ToolCallResult { diff --git a/runtime/csharp/Prompty.OpenAI.Tests/OpenAIExecutorTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/OpenAIExecutorTests.cs index 3e006632c..c340e0837 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/OpenAIExecutorTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/OpenAIExecutorTests.cs @@ -10,6 +10,17 @@ namespace Prompty.OpenAI.Tests; /// public class OpenAIExecutorTests { + [Fact] + public async Task ExecuteAsync_Cancelled_ThrowsBeforeConnectionValidation() + { + var executor = new OpenAI.OpenAIExecutor(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => executor.ExecuteAsync(new Core.Prompty(), [], cancellation.Token)); + } + [Fact] public async Task ExecuteAsync_MissingApiKey_ThrowsInvalidOperationException() { @@ -146,4 +157,3 @@ public void FormatToolMessages_CreatesIndividualToolMessages() Assert.Equal("call_2", messages[2].Metadata["tool_call_id"]); } } - diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs index a4a0af53d..26a4bc31a 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs @@ -715,7 +715,10 @@ public Task ProcessAsync(Core.Prompty agent, object response) private class LambdaExecutor(Func, object> fn) : IExecutor { - public Task ExecuteAsync(Core.Prompty agent, List messages) + public Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) => Task.FromResult(fn(messages)); public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) diff --git a/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs b/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs index 335b635bd..ae711ac0e 100644 --- a/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs +++ b/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs @@ -20,8 +20,12 @@ namespace Prompty.OpenAI; /// public class OpenAIExecutor : IExecutor { - public async Task ExecuteAsync(Core.Prompty agent, List messages) + public async Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); var apiType = agent.Model?.ApiType ?? "chat"; var model = agent.Model?.Id ?? "gpt-4"; var client = CreateClient(agent); @@ -29,12 +33,12 @@ public async Task ExecuteAsync(Core.Prompty agent, List message return apiType switch { - "chat" when streaming => ExecuteChatStreamAsync(client, model, agent, messages), - "chat" => await ExecuteChatAsync(client, model, agent, messages), - "responses" when streaming => ExecuteResponsesStreamAsync(client, model, agent, messages), - "responses" => await ExecuteResponsesAsync(client, model, agent, messages), - "embedding" => await ExecuteEmbeddingAsync(client, model, messages), - "image" => await ExecuteImageAsync(client, model, messages), + "chat" when streaming => ExecuteChatStreamAsync(client, model, agent, messages, cancellationToken), + "chat" => await ExecuteChatAsync(client, model, agent, messages, cancellationToken), + "responses" when streaming => ExecuteResponsesStreamAsync(client, model, agent, messages, cancellationToken), + "responses" => await ExecuteResponsesAsync(client, model, agent, messages, cancellationToken), + "embedding" => await ExecuteEmbeddingAsync(client, model, messages, cancellationToken), + "image" => await ExecuteImageAsync(client, model, messages, cancellationToken), _ => throw new InvalidOperationException($"Unsupported API type: {apiType}"), }; } @@ -83,18 +87,26 @@ protected virtual OpenAIClient CreateClient(Core.Prompty agent) } private static async Task ExecuteChatAsync( - OpenAIClient client, string model, Core.Prompty agent, List messages) + OpenAIClient client, + string model, + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var chatClient = client.GetChatClient(model); var chatMessages = messages.Select(WireFormat.MessageToWire).ToList(); var options = WireFormat.BuildOptions(agent); - var result = await chatClient.CompleteChatAsync(chatMessages, options); + var result = await chatClient.CompleteChatAsync(chatMessages, options, cancellationToken); return result.Value; } private static PromptyStream ExecuteChatStreamAsync( - OpenAIClient client, string model, Core.Prompty agent, List messages) + OpenAIClient client, + string model, + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var chatClient = client.GetChatClient(model); var chatMessages = messages.Select(WireFormat.MessageToWire).ToList(); @@ -109,7 +121,7 @@ async IAsyncEnumerable StreamChunks([EnumeratorCancellation] Cancellatio } } - return new PromptyStream(StreamChunks()); + return new PromptyStream(StreamChunks(cancellationToken)); } // ----------------------------------------------------------------------- @@ -117,16 +129,24 @@ async IAsyncEnumerable StreamChunks([EnumeratorCancellation] Cancellatio // ----------------------------------------------------------------------- private static async Task ExecuteResponsesAsync( - OpenAIClient client, string model, Core.Prompty agent, List messages) + OpenAIClient client, + string model, + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var responsesClient = client.GetResponsesClient(); var options = WireFormat.BuildResponsesOptions(model, agent, messages); - var result = await responsesClient.CreateResponseAsync(options); + var result = await responsesClient.CreateResponseAsync(options, cancellationToken); return result.Value; } private static PromptyStream ExecuteResponsesStreamAsync( - OpenAIClient client, string model, Core.Prompty agent, List messages) + OpenAIClient client, + string model, + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var responsesClient = client.GetResponsesClient(); var options = WireFormat.BuildResponsesOptions(model, agent, messages); @@ -141,7 +161,7 @@ async IAsyncEnumerable StreamChunks([EnumeratorCancellation] Cancellatio } } - return new PromptyStream(StreamChunks()); + return new PromptyStream(StreamChunks(cancellationToken)); } // ----------------------------------------------------------------------- @@ -149,20 +169,26 @@ async IAsyncEnumerable StreamChunks([EnumeratorCancellation] Cancellatio // ----------------------------------------------------------------------- private static async Task ExecuteEmbeddingAsync( - OpenAIClient client, string model, List messages) + OpenAIClient client, + string model, + List messages, + CancellationToken cancellationToken) { var embeddingClient = client.GetEmbeddingClient(model); var inputs = messages.Select(m => m.Text).ToList(); - var result = await embeddingClient.GenerateEmbeddingsAsync(inputs); + var result = await embeddingClient.GenerateEmbeddingsAsync(inputs, cancellationToken: cancellationToken); return result.Value; } private static async Task ExecuteImageAsync( - OpenAIClient client, string model, List messages) + OpenAIClient client, + string model, + List messages, + CancellationToken cancellationToken) { var imageClient = client.GetImageClient(model); var prompt = messages.LastOrDefault()?.Text ?? ""; - var result = await imageClient.GenerateImageAsync(prompt); + var result = await imageClient.GenerateImageAsync(prompt, cancellationToken: cancellationToken); return result.Value; } From bff0740431bb10c5ac8742927cc9e843b4c45135 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 03:38:21 -0700 Subject: [PATCH 03/20] Keep engine port errors runtime-native Extend the canonical acceptance vector so PortError cannot become a generated model or wire export. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- schema/scripts/verify-engine-ports.mjs | 7 +++++++ spec/vectors/engine/port_contracts.json | 3 +++ 2 files changed, 10 insertions(+) diff --git a/schema/scripts/verify-engine-ports.mjs b/schema/scripts/verify-engine-ports.mjs index 32e6dc7fc..541d14d47 100644 --- a/schema/scripts/verify-engine-ports.mjs +++ b/schema/scripts/verify-engine-ports.mjs @@ -43,6 +43,13 @@ function verifyLegacyHarness() { function verifyExportSurfaces() { for (const target of surfaces.targets) { + for (const nativeError of vector.nativeErrors) { + assert( + !target.rootExports.includes(nativeError), + `${target.target}: ${nativeError} must remain a native runtime error`, + ); + } + for (const [protocolName, expectedProtocol] of Object.entries( vector.protocols, )) { diff --git a/spec/vectors/engine/port_contracts.json b/spec/vectors/engine/port_contracts.json index ffefe7482..761d1b81f 100644 --- a/spec/vectors/engine/port_contracts.json +++ b/spec/vectors/engine/port_contracts.json @@ -1,6 +1,9 @@ { "version": "1", "legacyHarnessSha256": "c534140f1aae07034cedea13feec3d66a9acbc0df46869858753aef39e24fe69", + "nativeErrors": [ + "PortError" + ], "protocols": { "EnginePermissionPort": { "methods": { From 8f80e797d792db66403ff7e311a3c9e1d05681d6 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 04:16:44 -0700 Subject: [PATCH 04/20] Harden engine port acceptance probes Pin exact metadata, ordered parameters, wire exclusions, and complete native signatures for every configured target. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- schema/scripts/verify-engine-ports.mjs | 386 +++++++++++++++++++++--- spec/vectors/engine/port_contracts.json | 36 ++- 2 files changed, 378 insertions(+), 44 deletions(-) diff --git a/schema/scripts/verify-engine-ports.mjs b/schema/scripts/verify-engine-ports.mjs index 541d14d47..bfe482735 100644 --- a/schema/scripts/verify-engine-ports.mjs +++ b/schema/scripts/verify-engine-ports.mjs @@ -1,8 +1,16 @@ import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; +import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; +const EXPECTED_TARGETS = [ + "csharp", + "go", + "markdown", + "python", + "rust", + "typescript", +]; const repoRoot = execFileSync("git", ["rev-parse", "--show-toplevel"], { encoding: "utf8", }).trim(); @@ -21,6 +29,7 @@ const surfaces = readJson( verifyLegacyHarness(); verifyExportSurfaces(); +verifyNoWireCancellation(); verifyNativeSignatures(); console.log("Canonical engine port contracts verified."); @@ -42,11 +51,25 @@ function verifyLegacyHarness() { } function verifyExportSurfaces() { + const targetNames = surfaces.targets.map((target) => target.target); + assertEqual( + new Set(targetNames).size, + targetNames.length, + "Typra targets: duplicate names are not allowed", + ); + assertEqual( + JSON.stringify(targetNames.sort()), + JSON.stringify(EXPECTED_TARGETS), + "Typra targets", + ); + for (const target of surfaces.targets) { + assertUniqueNames(target.protocols, `${target.target}: protocols`); + for (const nativeError of vector.nativeErrors) { assert( - !target.rootExports.includes(nativeError), - `${target.target}: ${nativeError} must remain a native runtime error`, + !JSON.stringify(target).includes(`"${nativeError}"`), + `${target.target}: ${nativeError} must not appear in generated exports`, ); } @@ -57,6 +80,10 @@ function verifyExportSurfaces() { (candidate) => candidate.name === protocolName, ); assert(protocol, `${target.target}: missing ${protocolName} protocol`); + assertUniqueNames( + protocol.methods, + `${target.target}: ${protocolName} methods`, + ); assertExactKeys( Object.fromEntries( protocol.methods.map((method) => [method.name, true]), @@ -80,13 +107,20 @@ function verifyExportSurfaces() { method, `${target.target}: missing ${protocolName}.${methodName}`, ); + assertExactKeys( + method, + Object.fromEntries( + ["name", ...Object.keys(expectedMethod)].map((key) => [key, true]), + ), + `${target.target}: ${protocolName}.${methodName} metadata`, + ); assertSubset( method, expectedMethod, `${target.target}: ${protocolName}.${methodName}`, ); if (expectedMethod.params) { - assertExactKeys( + assertExactKeyOrder( method.params, expectedMethod.params, `${target.target}: ${protocolName}.${methodName} params`, @@ -109,6 +143,54 @@ function verifyExportSurfaces() { } } +function verifyNoWireCancellation() { + const schemaRoot = join(repoRoot, "vscode", "prompty", "schemas"); + const forbiddenFields = ["ctx", "runtimeCancellable", "atomic", "nonFatal"]; + + for (const file of collectFiles(schemaRoot)) { + const content = readFileSync(file, "utf8"); + for (const field of forbiddenFields) { + const propertyPattern = new RegExp( + `^\\s*(?:${field}|["']${field}["'])\\s*:\\s*`, + "mi", + ); + assert( + !propertyPattern.test(content), + `${file}: ${field} must not be emitted as a portable model field`, + ); + } + assert( + !/^\s*(?:["']?[\w]*(?:cancel|abort|signal)[\w]*["']?)\s*:\s*/imu.test( + content, + ), + `${file}: runtime cancellation must not be emitted as a portable model field`, + ); + assert( + !/(?:cancel|abort|signal)[^\\/]*\.ya?ml$/iu.test(file), + `${file}: runtime cancellation types must not be portable models`, + ); + } + + for (const protocol of [ + "EnginePermissionPort", + "EngineToolPort", + "EngineDurabilityPort", + "EnginePostCommitPort", + "Executor", + ]) { + const file = join(schemaRoot, `${protocol}.yaml`); + const content = readFileSync(file, "utf8"); + assert( + /^properties:\s*\{\}\s*$/mu.test(content), + `${file}: protocol schemas must not expose properties`, + ); + assert( + !/^required:/mu.test(content), + `${file}: protocol schemas must not expose required wire fields`, + ); + } +} + function verifyNativeSignatures() { const root = { csharp: join( @@ -150,26 +232,11 @@ function verifyNativeSignatures() { ), }; - expectMatches( - join(root.csharp, "EnginePermissionPort.cs"), - /Task\s+AuthorizeAsync\(ModelToolRequest request,\s*CancellationToken cancellationToken = default\);/u, - ); - expectMatches( - join(root.go, "engine_permission_port.go"), - /Authorize\(ctx context\.Context,\s*request ModelToolRequest\)\s*\(EnginePermissionDecision, error\)/u, - ); - expectMatches( - join(root.typescript, "engine-permission-port.ts"), - /authorize\(request: ModelToolRequest,\s*signal\?: AbortSignal\): Promise;/u, - ); - expectMatches( - join(root.rust, "engine_permission_port.rs"), - /async fn authorize\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,\s*\)\s*-> Result EnginePermissionDecision:/u, - ); + verifyCSharpSignatures(root.csharp); + verifyGoSignatures(root.go); + verifyTypeScriptSignatures(root.typescript); + verifyRustSignatures(root.rust); + verifyPythonSignatures(root.python); const durabilityFiles = [ join(root.csharp, "EngineDurabilityPort.cs"), @@ -191,21 +258,218 @@ function verifyNativeSignatures() { ); } } +} - const executorFiles = [ - join(root.csharp, "Executor.cs"), - join(root.go, "executor.go"), - join(root.python, "_Executor.py"), - join(root.rust, "executor.rs"), - join(root.typescript, "executor.ts"), - ]; - for (const file of executorFiles) { - const content = readFileSync(file, "utf8"); - assert( - content.includes("formatToolMessages") || - content.includes("format_tool_messages") || - content.includes("FormatToolMessages"), - `${file}: missing synchronous formatToolMessages`, +function verifyCSharpSignatures(root) { + expectMatches( + join(root, "EnginePermissionPort.cs"), + /^\s*Task\s+AuthorizeAsync\(\s*ModelToolRequest request,\s*CancellationToken cancellationToken = default\s*\);/mu, + ); + expectMatches( + join(root, "EngineToolPort.cs"), + /^\s*Task\s+ExecuteAsync\(\s*ModelToolRequest request,\s*CancellationToken cancellationToken = default\s*\);/mu, + ); + expectAllMatches(join(root, "EngineDurabilityPort.cs"), [ + /^\s*Task\s+AppendAsync\(\s*EngineEvent @?event\s*\);/mu, + /^\s*Task\s+AppendWithCheckpointAsync\(\s*List events,\s*EngineCheckpoint checkpoint\s*\);/mu, + ]); + expectMatches( + join(root, "EnginePostCommitPort.cs"), + /^\s*Task\s+AfterCommitAsync\(\s*string effectId,\s*TurnCommit commit,\s*CancellationToken cancellationToken = default\s*\);/mu, + ); + expectAllMatches(join(root, "Executor.cs"), [ + /^\s*Task\s+ExecuteAsync\(\s*Prompty agent,\s*List messages,\s*CancellationToken cancellationToken = default\s*\);/mu, + /^\s*Task\s+ExecuteStreamAsync\(\s*Prompty agent,\s*List messages,\s*CancellationToken cancellationToken = default\s*\)/mu, + /^\s*List\s+FormatToolMessages\(\s*object rawResponse,\s*List toolCalls,\s*List toolResults,\s*string\? textContent\s*\);/mu, + ]); + assertDeclarationNames( + join(root, "EnginePermissionPort.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["AuthorizeAsync"], + ); + assertDeclarationNames( + join(root, "EngineToolPort.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["ExecuteAsync"], + ); + assertDeclarationNames( + join(root, "EngineDurabilityPort.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["AppendAsync", "AppendWithCheckpointAsync"], + ); + assertDeclarationNames( + join(root, "EnginePostCommitPort.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["AfterCommitAsync"], + ); + assertDeclarationNames( + join(root, "Executor.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["ExecuteAsync", "ExecuteStreamAsync", "FormatToolMessages"], + ); +} + +function verifyGoSignatures(root) { + expectMatches( + join(root, "engine_permission_port.go"), + /^\s*Authorize\(ctx context\.Context,\s*request ModelToolRequest\)\s*\(EnginePermissionDecision, error\)/mu, + ); + expectMatches( + join(root, "engine_tool_port.go"), + /^\s*Execute\(ctx context\.Context,\s*request ModelToolRequest\)\s*\(ModelToolResult, error\)/mu, + ); + expectAllMatches(join(root, "engine_durability_port.go"), [ + /^\s*Append\(event EngineEvent\)\s*error/mu, + /^\s*AppendWithCheckpoint\(events \[\]EngineEvent,\s*checkpoint EngineCheckpoint\)\s*error/mu, + ]); + expectMatches( + join(root, "engine_post_commit_port.go"), + /^\s*AfterCommit\(ctx context\.Context,\s*effectId string,\s*commit TurnCommit\)\s*error/mu, + ); + expectAllMatches(join(root, "executor.go"), [ + /^\s*Execute\(ctx context\.Context,\s*agent Prompty,\s*messages \[\]Message\)\s*\(interface\{\}, error\)/mu, + /^\s*ExecuteStream\(ctx context\.Context,\s*agent Prompty,\s*messages \[\]Message\)\s*\(interface\{\}, error\)/mu, + /^\s*FormatToolMessages\(rawResponse interface\{\},\s*toolCalls \[\]ToolCall,\s*toolResults \[\]string,\s*textContent \*string\)\s*\(\[\]Message, error\)/mu, + ]); + for (const [file, expected] of [ + ["engine_permission_port.go", ["Authorize"]], + ["engine_tool_port.go", ["Execute"]], + ["engine_durability_port.go", ["Append", "AppendWithCheckpoint"]], + ["engine_post_commit_port.go", ["AfterCommit"]], + ["executor.go", ["Execute", "ExecuteStream", "FormatToolMessages"]], + ]) { + assertDeclarationNames(join(root, file), /^\s*([A-Z]\w*)\(/gmu, expected); + } +} + +function verifyTypeScriptSignatures(root) { + expectMatches( + join(root, "engine-permission-port.ts"), + /^\s{2}authorize\(\s*request: ModelToolRequest,\s*signal\?: AbortSignal\s*\): Promise;/mu, + ); + expectMatches( + join(root, "engine-tool-port.ts"), + /^\s{2}execute\(\s*request: ModelToolRequest,\s*signal\?: AbortSignal\s*\): Promise;/mu, + ); + expectAllMatches(join(root, "engine-durability-port.ts"), [ + /^\s{2}append\(\s*event: EngineEvent\s*\): Promise;/mu, + /^\s{2}appendWithCheckpoint\(\s*events: EngineEvent\[\],\s*checkpoint: EngineCheckpoint\s*\): Promise;/mu, + ]); + expectMatches( + join(root, "engine-post-commit-port.ts"), + /^\s{2}afterCommit\(\s*effectId: string,\s*commit: TurnCommit,\s*signal\?: AbortSignal\s*\): Promise;/mu, + ); + expectAllMatches(join(root, "executor.ts"), [ + /^\s{2}execute\(\s*agent: Prompty,\s*messages: Message\[\],\s*signal\?: AbortSignal\s*\): Promise;/mu, + /^\s{2}executeStream\?\(\s*agent: Prompty,\s*messages: Message\[\],\s*signal\?: AbortSignal\s*\): Promise;/mu, + /^\s{2}formatToolMessages\(\s*rawResponse: unknown,\s*toolCalls: ToolCall\[\],\s*toolResults: string\[\],\s*textContent: string \| null,\s*\): Message\[\];/mu, + ]); + for (const [file, expected] of [ + ["engine-permission-port.ts", ["authorize"]], + ["engine-tool-port.ts", ["execute"]], + ["engine-durability-port.ts", ["append", "appendWithCheckpoint"]], + ["engine-post-commit-port.ts", ["afterCommit"]], + ["executor.ts", ["execute", "executeStream", "formatToolMessages"]], + ]) { + assertDeclarationNames( + join(root, file), + /^\s{2}([a-z]\w*)\??\(/gmu, + expected, + ); + } +} + +function verifyRustSignatures(root) { + expectMatches( + join(root, "engine_permission_port.rs"), + /^\s{4}async fn authorize\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,\s*\)\s*-> Result>;/mu, + ); + expectMatches( + join(root, "engine_tool_port.rs"), + /^\s{4}async fn execute\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,\s*\)\s*-> Result>;/mu, + ); + expectAllMatches(join(root, "engine_durability_port.rs"), [ + /^\s{4}async fn append\(\s*&self,\s*event: &EngineEvent,\s*\)\s*-> Result<\(\),\s*Box>;/mu, + /^\s{4}async fn append_with_checkpoint\(\s*&self,\s*events: &Vec,\s*checkpoint: &EngineCheckpoint,\s*\)\s*-> Result<\(\),\s*Box>;/mu, + ]); + expectMatches( + join(root, "engine_post_commit_port.rs"), + /^\s{4}async fn after_commit\(\s*&self,\s*effect_id: &String,\s*commit: &TurnCommit,\s*cancellation: &CancellationToken,\s*\)\s*-> Result<\(\),\s*Box>;/mu, + ); + expectAllMatches(join(root, "executor.rs"), [ + /^\s{4}async fn execute\(\s*&self,\s*agent: &Prompty,\s*messages: &Vec,\s*cancellation: &CancellationToken,\s*\)\s*-> Result>;/mu, + /^\s{4}async fn execute_stream\(\s*&self,\s*agent: &Prompty,\s*messages: &Vec,\s*cancellation: &CancellationToken,\s*\)\s*-> Result>/mu, + /^\s{4}fn format_tool_messages\(\s*&self,\s*raw_response: &serde_json::Value,\s*tool_calls: &Vec,\s*tool_results: &Vec,\s*text_content: &Option,\s*\)\s*-> Vec;/mu, + ]); + for (const [file, expected] of [ + ["engine_permission_port.rs", ["authorize"]], + ["engine_tool_port.rs", ["execute"]], + ["engine_durability_port.rs", ["append", "append_with_checkpoint"]], + ["engine_post_commit_port.rs", ["after_commit"]], + ["executor.rs", ["execute", "execute_stream", "format_tool_messages"]], + ]) { + assertDeclarationNames( + join(root, file), + /^\s{4}(?:async\s+)?fn\s+([a-z]\w*)\(/gmu, + expected, + ); + } +} + +function verifyPythonSignatures(root) { + expectAllMatches(join(root, "_EnginePermissionPort.py"), [ + /^\s{4}def authorize\(\s*self,\s*request: ModelToolRequest,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> EnginePermissionDecision:/mu, + /^\s{4}async def authorize_async\(\s*self,\s*request: ModelToolRequest,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> EnginePermissionDecision:/mu, + ]); + expectAllMatches(join(root, "_EngineToolPort.py"), [ + /^\s{4}def execute\(\s*self,\s*request: ModelToolRequest,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> ModelToolResult:/mu, + /^\s{4}async def execute_async\(\s*self,\s*request: ModelToolRequest,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> ModelToolResult:/mu, + ]); + expectAllMatches(join(root, "_EngineDurabilityPort.py"), [ + /^\s{4}def append\(\s*self,\s*event: EngineEvent\s*\)\s*-> None:/mu, + /^\s{4}async def append_async\(\s*self,\s*event: EngineEvent\s*\)\s*-> None:/mu, + /^\s{4}def append_with_checkpoint\(\s*self,\s*events: list\[EngineEvent\],\s*checkpoint: EngineCheckpoint\s*\)\s*-> None:/mu, + /^\s{4}async def append_with_checkpoint_async\(\s*self,\s*events: list\[EngineEvent\],\s*checkpoint: EngineCheckpoint\s*\)\s*-> None:/mu, + ]); + expectAllMatches(join(root, "_EnginePostCommitPort.py"), [ + /^\s{4}def after_commit\(\s*self,\s*effect_id: str,\s*commit: TurnCommit,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> None:/mu, + /^\s{4}async def after_commit_async\(\s*self,\s*effect_id: str,\s*commit: TurnCommit,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> None:/mu, + ]); + expectAllMatches(join(root, "_Executor.py"), [ + /^\s{4}def execute\(\s*self,\s*agent: Prompty,\s*messages: list\[Message\],\s*cancellation: CancellationToken \| None = None\s*\)\s*-> Any:/mu, + /^\s{4}async def execute_async\(\s*self,\s*agent: Prompty,\s*messages: list\[Message\],\s*cancellation: CancellationToken \| None = None\s*\)\s*-> Any:/mu, + /^\s{4}def execute_stream\(\s*self,\s*agent: Prompty,\s*messages: list\[Message\],\s*cancellation: CancellationToken \| None = None\s*\)\s*-> Any:/mu, + /^\s{4}async def execute_stream_async\(\s*self,\s*agent: Prompty,\s*messages: list\[Message\],\s*cancellation: CancellationToken \| None = None\s*\)\s*-> Any:/mu, + /^\s{4}def format_tool_messages\(\s*self,\s*raw_response: Any,\s*tool_calls: list\[ToolCall\],\s*tool_results: list\[str\],\s*text_content: str \| None\s*\)\s*-> list\[Message\]:/mu, + ]); + for (const [file, expected] of [ + ["_EnginePermissionPort.py", ["authorize", "authorize_async"]], + ["_EngineToolPort.py", ["execute", "execute_async"]], + [ + "_EngineDurabilityPort.py", + [ + "append", + "append_async", + "append_with_checkpoint", + "append_with_checkpoint_async", + ], + ], + ["_EnginePostCommitPort.py", ["after_commit", "after_commit_async"]], + [ + "_Executor.py", + [ + "execute", + "execute_async", + "execute_stream", + "execute_stream_async", + "format_tool_messages", + ], + ], + ]) { + assertDeclarationNames( + join(root, file), + /^\s{4}(?:async\s+)?def\s+([a-z]\w*)\(/gmu, + expected, ); } } @@ -218,6 +482,35 @@ function expectMatches(path, pattern) { ); } +function expectAllMatches(path, patterns) { + for (const pattern of patterns) { + expectMatches(path, pattern); + } +} + +function assertDeclarationNames(path, pattern, expected) { + const content = readFileSync(path, "utf8"); + const actual = [...content.matchAll(pattern)].map((match) => match[1]).sort(); + assertEqual( + JSON.stringify(actual), + JSON.stringify([...expected].sort()), + `${path}: native method declarations`, + ); +} + +function collectFiles(root) { + const files = []; + for (const entry of readdirSync(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory()) { + files.push(...collectFiles(path)); + } else if (entry.isFile()) { + files.push(path); + } + } + return files; +} + function assertSubset(actual, expected, label) { for (const [key, expectedValue] of Object.entries(expected)) { const actualValue = actual[key]; @@ -247,6 +540,23 @@ function assertExactKeys(actual, expected, label) { ); } +function assertExactKeyOrder(actual, expected, label) { + assertEqual( + JSON.stringify(Object.keys(actual)), + JSON.stringify(Object.keys(expected)), + `${label} order`, + ); +} + +function assertUniqueNames(items, label) { + const names = items.map((item) => item.name); + assertEqual( + new Set(names).size, + names.length, + `${label}: duplicate names are not allowed`, + ); +} + function assertEqual(actual, expected, label) { assert( actual === expected, diff --git a/spec/vectors/engine/port_contracts.json b/spec/vectors/engine/port_contracts.json index 761d1b81f..0ed4a1cc5 100644 --- a/spec/vectors/engine/port_contracts.json +++ b/spec/vectors/engine/port_contracts.json @@ -1,9 +1,7 @@ { "version": "1", "legacyHarnessSha256": "c534140f1aae07034cedea13feec3d66a9acbc0df46869858753aef39e24fe69", - "nativeErrors": [ - "PortError" - ], + "nativeErrors": ["PortError"], "protocols": { "EnginePermissionPort": { "methods": { @@ -81,16 +79,42 @@ "Executor": { "methods": { "execute": { + "returns": "unknown", + "params": { + "agent": "Prompty", + "messages": "Message[]" + }, + "optional": false, "runtimeCancellable": true, - "sync": false + "sync": false, + "atomic": false, + "nonFatal": false }, "executeStream": { + "returns": "unknown", + "params": { + "agent": "Prompty", + "messages": "Message[]" + }, + "optional": true, "runtimeCancellable": true, - "sync": false + "sync": false, + "atomic": false, + "nonFatal": false }, "formatToolMessages": { + "returns": "Message[]", + "params": { + "rawResponse": "unknown", + "toolCalls": "ToolCall[]", + "toolResults": "string[]", + "textContent": "string?" + }, + "optional": false, "runtimeCancellable": false, - "sync": true + "sync": true, + "atomic": false, + "nonFatal": false } } } From aa157714c0afacf04acd4659aacbea85332a2bb6 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 11:39:34 -0700 Subject: [PATCH 05/20] Define unknown connection roundtrip contract Open the Connection discriminator and pin exact forward-compatible payload preservation with shared known, unknown, and case-collision vectors. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- schema/model/connection/connection.tsp | 3 +- spec/spec.md | 14 +++ .../model/connection_roundtrip_vectors.json | 106 ++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 spec/vectors/model/connection_roundtrip_vectors.json diff --git a/schema/model/connection/connection.tsp b/schema/model/connection/connection.tsp index e74898086..2d6659213 100644 --- a/schema/model/connection/connection.tsp +++ b/schema/model/connection/connection.tsp @@ -9,7 +9,8 @@ alias ConnectionType = | "key" | "anonymous" | "foundry" - | "oauth"; + | "oauth" + | string; alias AuthenticationMode = "user" | "system"; /** diff --git a/spec/spec.md b/spec/spec.md index 67c554f69..36fea1a1f 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -285,6 +285,20 @@ Connection types are discriminated by the `kind` field: | `foundry` | `endpoint` | Microsoft Foundry connection | | `oauth` | `endpoint`, `authenticationMode` | OAuth-based authentication | +The `kind` discriminator is open for forward compatibility. Known-kind matching is +exact and case-sensitive, so a value such as `Reference` is an unknown kind rather than +the known `reference` kind. When a runtime loads a connection whose string `kind` is +not listed above, it MUST preserve that exact discriminator and every JSON-compatible +property in the connection payload through a load → save → reload cycle. +Implementations MUST NOT coerce an unknown connection to a known/default connection +kind or discard its additional payload. This passthrough requirement applies to unknown +connection kinds; known connection kinds retain their schema-defined fields. The shared +acceptance vector is `spec/vectors/model/connection_roundtrip_vectors.json`. + +This contract is independent of tool dispatch. An unknown tool `kind` continues to load +as `CustomTool` under §2.9; an unknown connection remains an unknown `Connection` and +does not imply `CustomTool`. + ### §2.6 ModelOptions | Property | Type | Description | diff --git a/spec/vectors/model/connection_roundtrip_vectors.json b/spec/vectors/model/connection_roundtrip_vectors.json new file mode 100644 index 000000000..0336c31ad --- /dev/null +++ b/spec/vectors/model/connection_roundtrip_vectors.json @@ -0,0 +1,106 @@ +{ + "version": "1", + "description": "Cross-runtime Connection load/save/reload contracts. Unknown string discriminators are forward-compatible Connection values, not CustomTool values: runtimes must preserve the exact kind and complete JSON-compatible payload without coercing to a known/default connection kind.", + "vectors": [ + { + "name": "known_reference_connection_roundtrip_unchanged", + "operation": "load-save-reload", + "input": { + "kind": "reference", + "authenticationMode": "system", + "usageDescription": "Exercise the known discriminator control", + "name": "shared-connection", + "target": "model-service" + }, + "expected": { + "kind": "reference", + "authenticationMode": "system", + "usageDescription": "Exercise the known discriminator control", + "name": "shared-connection", + "target": "model-service" + } + }, + { + "name": "unknown_connection_kind_preserves_payload", + "operation": "load-save-reload", + "input": { + "kind": "future-auth", + "authenticationMode": "system", + "usageDescription": "Exercise forward-compatible connection preservation", + "endpoint": "https://future.example.test", + "tenant": "example-tenant", + "priority": 5, + "enabled": false, + "weight": 0.1, + "regions": [ + "west", + "east" + ], + "notes": null, + "providerOptions": { + "audience": "prompty", + "features": [ + "delegation", + { + "name": "nested-option", + "enabled": true + } + ], + "retry": { + "maxAttempts": 3, + "backoffSeconds": 0.1 + }, + "nullable": null + } + }, + "expected": { + "kind": "future-auth", + "authenticationMode": "system", + "usageDescription": "Exercise forward-compatible connection preservation", + "endpoint": "https://future.example.test", + "tenant": "example-tenant", + "priority": 5, + "enabled": false, + "weight": 0.1, + "regions": [ + "west", + "east" + ], + "notes": null, + "providerOptions": { + "audience": "prompty", + "features": [ + "delegation", + { + "name": "nested-option", + "enabled": true + } + ], + "retry": { + "maxAttempts": 3, + "backoffSeconds": 0.1 + }, + "nullable": null + } + } + }, + { + "name": "unknown_connection_case_collision_preserves_payload", + "operation": "load-save-reload", + "input": { + "kind": "Reference", + "name": "case-sensitive-unknown", + "payload": { + "mode": "future" + } + }, + "expected": { + "kind": "Reference", + "name": "case-sensitive-unknown", + "payload": { + "mode": "future" + } + } + } + ] +} From 244f1865b529710ad906ddc666d1f226ac78d88f Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 11:39:42 -0700 Subject: [PATCH 06/20] Add Rust connection roundtrip gate Exercise generated Connection load/save APIs against the shared forward-compatibility vectors without editing generated runtime code. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../tests/connection_roundtrip_vectors.rs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 runtime/rust/prompty/tests/connection_roundtrip_vectors.rs diff --git a/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs b/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs new file mode 100644 index 000000000..b4f84c13a --- /dev/null +++ b/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs @@ -0,0 +1,70 @@ +//! Cross-runtime Connection roundtrip tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::{LoadContext, SaveContext}; +use prompty::model::Connection; +use serde_json::Value; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("connection_roundtrip_vectors.json") +} + +#[test] +fn connection_roundtrip_vectors() { + let raw = std::fs::read_to_string(vectors_path()) + .expect("failed to read Connection roundtrip vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse Connection roundtrip vectors"); + let vectors = document["vectors"] + .as_array() + .expect("Connection roundtrip vectors must contain a vectors array"); + + for vector in vectors { + let name = vector["name"] + .as_str() + .expect("vector name must be a string"); + assert_eq!( + vector["operation"], "load-save-reload", + "[{name}] unsupported vector operation" + ); + + let input = &vector["input"]; + let expected = &vector["expected"]; + let kind = input["kind"] + .as_str() + .expect("Connection kind must be a string"); + let load_context = LoadContext::default(); + let save_context = SaveContext::default(); + + let loaded = Connection::load_from_value(input, &load_context); + assert_eq!( + loaded.kind_str(), + kind, + "[{name}] load changed the discriminator" + ); + + let saved = loaded.to_value(&save_context); + assert_eq!( + saved, *expected, + "[{name}] save changed the Connection payload" + ); + + let reloaded = Connection::load_from_value(&saved, &load_context); + let resaved = reloaded.to_value(&save_context); + assert_eq!( + resaved, *expected, + "[{name}] reload changed the Connection payload" + ); + } +} From 5e418b831f97974eae1bd5dbc83e5d06baf4e26d Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 12:01:46 -0700 Subject: [PATCH 07/20] Align message text joining across runtimes Join TextPart values with newlines in Python and TypeScript and return an empty string for empty TypeScript messages, matching the canonical contract and Rust behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/python/prompty/prompty/core/types.py | 4 ++-- runtime/python/prompty/tests/test_types.py | 21 +++++++++++++++++++ .../packages/core/src/core/types.ts | 8 +++---- .../packages/core/tests/types.test.ts | 7 +++++-- 4 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 runtime/python/prompty/tests/test_types.py diff --git a/runtime/python/prompty/prompty/core/types.py b/runtime/python/prompty/prompty/core/types.py index b5ae4ee5c..d0719039d 100644 --- a/runtime/python/prompty/prompty/core/types.py +++ b/runtime/python/prompty/prompty/core/types.py @@ -59,8 +59,8 @@ def _message_text(self: Message) -> str: - """Concatenate all TextPart values into a single string.""" - return "".join(p.value for p in self.parts if isinstance(p, TextPart)) + """Concatenate all TextPart values joined by newline.""" + return "\n".join(p.value for p in self.parts if isinstance(p, TextPart)) def _message_to_text_content(self: Message) -> str | list[dict[str, Any]]: diff --git a/runtime/python/prompty/tests/test_types.py b/runtime/python/prompty/tests/test_types.py new file mode 100644 index 000000000..0e020afd8 --- /dev/null +++ b/runtime/python/prompty/tests/test_types.py @@ -0,0 +1,21 @@ +"""Verify handwritten runtime extensions for generated conversation types.""" + +from __future__ import annotations + +from prompty.core.types import Message, TextPart + + +def test_message_text_parts_are_joined_by_newline() -> None: + """Join multiple text parts according to the canonical method contract.""" + message = Message(role="user", parts=[TextPart(value="first"), TextPart(value="second")]) + + assert message.text == "first\nsecond" + assert message.to_text_content() == "first\nsecond" + + +def test_empty_message_text_content_is_empty_string() -> None: + """Represent an empty all-text message as an empty string.""" + message = Message(role="user", parts=[]) + + assert message.text == "" + assert message.to_text_content() == "" diff --git a/runtime/typescript/packages/core/src/core/types.ts b/runtime/typescript/packages/core/src/core/types.ts index 0f398f5d9..b9f772b8c 100644 --- a/runtime/typescript/packages/core/src/core/types.ts +++ b/runtime/typescript/packages/core/src/core/types.ts @@ -80,12 +80,12 @@ export class Message implements MessageHelpers { this.metadata = init?.metadata ?? {}; } - /** Concatenate all TextPart values into a single string. */ + /** Concatenate all TextPart values joined by newline. */ get text(): string { return this.parts .filter((p): p is TextPart => p.kind === "text") .map((p) => p.value) - .join(""); + .join("\n"); } /** @@ -94,8 +94,8 @@ export class Message implements MessageHelpers { * - If multimodal, return an array of content objects. */ toTextContent(): string | Record[] { - if (this.parts.length === 1 && this.parts[0].kind === "text") { - return (this.parts[0] as TextPart).value; + if (this.parts.every((part) => part.kind === "text")) { + return this.text; } return this.parts.map(partToWireContent); } diff --git a/runtime/typescript/packages/core/tests/types.test.ts b/runtime/typescript/packages/core/tests/types.test.ts index 77ab20dab..3018dda72 100644 --- a/runtime/typescript/packages/core/tests/types.test.ts +++ b/runtime/typescript/packages/core/tests/types.test.ts @@ -20,10 +20,11 @@ describe("Message", () => { it("concatenates multiple text parts", () => { const msg = new Message({ role: "user", parts: [ - { kind: "text", value: "Hello " }, + { kind: "text", value: "Hello" }, { kind: "text", value: "world" }, ] }); - expect(msg.text).toBe("Hello world"); + expect(msg.text).toBe("Hello\nworld"); + expect(msg.toTextContent()).toBe("Hello\nworld"); }); it("returns string for single text part in toTextContent", () => { @@ -45,6 +46,8 @@ describe("Message", () => { const msg = new Message({ role: "system" }); expect(msg.parts).toEqual([]); expect(msg.metadata).toEqual({}); + expect(msg.text).toBe(""); + expect(msg.toTextContent()).toBe(""); }); }); From b820d7850da35d667928194194917e895750d436 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 12:23:14 -0700 Subject: [PATCH 08/20] Define strict content part discriminators Document ContentPart as a closed, case-sensitive union and add shared known, unknown, and wrong-case acceptance vectors. Add a non-generated Rust gate that exposes the current generated fallback-to-text defect without modifying generated code. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../content_part_discriminator_vectors.rs | 80 +++++++++++++++++++ spec/spec.md | 14 ++++ .../content_part_discriminator_vectors.json | 45 +++++++++++ 3 files changed, 139 insertions(+) create mode 100644 runtime/rust/prompty/tests/content_part_discriminator_vectors.rs create mode 100644 spec/vectors/model/content_part_discriminator_vectors.json diff --git a/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs b/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs new file mode 100644 index 000000000..e4a2758e1 --- /dev/null +++ b/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs @@ -0,0 +1,80 @@ +//! Closed ContentPart discriminator tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::{LoadContext, SaveContext}; +use prompty::model::ContentPart; +use serde_json::Value; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("content_part_discriminator_vectors.json") +} + +#[test] +fn content_part_discriminator_vectors() { + let raw = std::fs::read_to_string(vectors_path()) + .expect("failed to read ContentPart discriminator vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse ContentPart discriminator vectors"); + let vectors = document["vectors"] + .as_array() + .expect("ContentPart discriminator vectors must contain a vectors array"); + let context = LoadContext::default(); + + for vector in vectors { + let name = vector["name"] + .as_str() + .expect("vector name must be a string"); + let input = &vector["input"]; + let json = serde_json::to_string(input).expect("vector input must be JSON-compatible"); + let result = ContentPart::from_json(&json, &context); + + match vector["operation"].as_str() { + Some("load") => { + let content_part = result.unwrap_or_else(|error| { + panic!("[{name}] known ContentPart failed to load: {error}") + }); + assert_eq!( + content_part.to_value(&SaveContext::default()), + vector["expected"], + "[{name}] known ContentPart payload changed during load/save" + ); + } + Some("load-error") => { + let error = match result { + Ok(_) => panic!( + "[{name}] closed ContentPart accepted unknown discriminator {:?}", + input["kind"] + ), + Err(error) => error, + }; + let diagnostic = error.to_string(); + let discriminator = vector["expected"]["discriminator"] + .as_str() + .expect("error vector must declare the discriminator field"); + let value = vector["expected"]["value"] + .as_str() + .expect("error vector must declare the exact discriminator value"); + assert!( + diagnostic.contains(discriminator), + "[{name}] error diagnostic did not identify discriminator {discriminator:?}: {diagnostic}" + ); + assert!( + diagnostic.contains(value), + "[{name}] error diagnostic did not preserve discriminator value {value:?}: {diagnostic}" + ); + } + operation => panic!("[{name}] unsupported vector operation: {operation:?}"), + } + } +} diff --git a/spec/spec.md b/spec/spec.md index 36fea1a1f..f728f88d5 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -1285,6 +1285,20 @@ ToolResult: parts: ContentPart[] // Rich content from tool execution ``` +`ContentPart` is a closed, exact, case-sensitive discriminated union. A typed +`ContentPart` load MUST reject any `kind` other than `text`, `image`, `audio`, or +`file`, including case-only variants such as `Text`. Implementations MUST NOT create an +unknown fallback variant, preserve the unknown payload as a `ContentPart`, or coerce it +to a known/default kind. The rejection MUST be classified as an unknown-discriminator +load error and report both the discriminator field (`kind`) and the exact offending +string value. Runtimes MAY use their native load-error type, but its diagnostic or +structured data MUST expose equivalent information. + +This strict contract is distinct from the intentionally open discriminator contracts: +an unknown tool kind loads as `CustomTool` under §2.9, and an unknown connection kind +is preserved under §2.5. The shared acceptance vectors are in +`spec/vectors/model/content_part_discriminator_vectors.json`. + **ToolResult** enables tools to return rich content (text, images, files, audio) rather than plain strings. Implementations MUST support conversion from a plain string to a `ToolResult` containing a single `TextPart` for backward compatibility diff --git a/spec/vectors/model/content_part_discriminator_vectors.json b/spec/vectors/model/content_part_discriminator_vectors.json new file mode 100644 index 000000000..41c237dbd --- /dev/null +++ b/spec/vectors/model/content_part_discriminator_vectors.json @@ -0,0 +1,45 @@ +{ + "version": "1", + "description": "Cross-runtime strict ContentPart discriminator contracts. ContentPart is closed and case-sensitive: unknown kinds are rejected rather than preserved like unknown Connection values or dispatched like unknown Tool values.", + "vectors": [ + { + "name": "known_text_content_part_loads", + "operation": "load", + "input": { + "kind": "text", + "value": "hello" + }, + "expected": { + "kind": "text", + "value": "hello" + } + }, + { + "name": "unknown_content_part_kind_is_rejected", + "operation": "load-error", + "input": { + "kind": "video", + "source": "https://example.test/video.mp4", + "durationSeconds": 3 + }, + "expected": { + "error": "unknown-discriminator", + "discriminator": "kind", + "value": "video" + } + }, + { + "name": "content_part_case_collision_is_rejected", + "operation": "load-error", + "input": { + "kind": "Text", + "value": "case-sensitive" + }, + "expected": { + "error": "unknown-discriminator", + "discriminator": "kind", + "value": "Text" + } + } + ] +} From 31ff17de96338c52352fa3f141fe02706669db5b Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 12:37:42 -0700 Subject: [PATCH 09/20] Define canonical named collection contracts Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../prompty/tests/named_collection_vectors.rs | 222 ++++++++++++++ spec/spec.md | 48 +++ .../model/named_collection_vectors.json | 287 ++++++++++++++++++ 3 files changed, 557 insertions(+) create mode 100644 runtime/rust/prompty/tests/named_collection_vectors.rs create mode 100644 spec/vectors/model/named_collection_vectors.json diff --git a/runtime/rust/prompty/tests/named_collection_vectors.rs b/runtime/rust/prompty/tests/named_collection_vectors.rs new file mode 100644 index 000000000..4f7c4849d --- /dev/null +++ b/runtime/rust/prompty/tests/named_collection_vectors.rs @@ -0,0 +1,222 @@ +//! Named-collection roundtrip tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::{LoadContext, SaveContext}; +use prompty::model::Prompty; +use serde_json::{Map, Value}; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("named_collection_vectors.json") +} + +fn semantic_entries(collection: &Value) -> Vec { + match collection { + Value::Array(entries) => entries + .iter() + .map(|entry| { + let mut entry = entry + .as_object() + .expect("array-form named collection entries must be objects") + .clone(); + entry + .entry("name".to_string()) + .or_insert_with(|| Value::String(String::new())); + Value::Object(entry) + }) + .collect(), + Value::Object(entries) => entries + .iter() + .map(|(name, entry)| { + let mut entry = entry + .as_object() + .expect("object-form named collection entries must be objects") + .clone(); + entry.insert("name".to_string(), Value::String(name.clone())); + Value::Object(entry) + }) + .collect(), + value => panic!("named collection must be an array or object, got {value:?}"), + } +} + +fn assert_subset(actual: &Value, expected: &Value, path: &str) { + match expected { + Value::Object(expected) => { + let actual = actual + .as_object() + .unwrap_or_else(|| panic!("[{path}] expected object, got {actual:?}")); + for (key, expected_value) in expected { + let actual_value = actual + .get(key) + .unwrap_or_else(|| panic!("[{path}] missing expected key {key:?}")); + assert_subset(actual_value, expected_value, &format!("{path}.{key}")); + } + } + Value::Array(expected) => { + let actual = actual + .as_array() + .unwrap_or_else(|| panic!("[{path}] expected array, got {actual:?}")); + assert_eq!( + actual.len(), + expected.len(), + "[{path}] array length changed" + ); + for (index, expected_value) in expected.iter().enumerate() { + assert_subset(&actual[index], expected_value, &format!("{path}[{index}]")); + } + } + expected => assert_eq!(actual, expected, "[{path}] value changed"), + } +} + +fn assert_collection(vector_name: &str, collection: &Value, expected: &Value) { + let expected_format = expected["collectionFormat"] + .as_str() + .expect("roundtrip vector must declare collectionFormat"); + assert_eq!( + if collection.is_array() { + "array" + } else if collection.is_object() { + "object" + } else { + "invalid" + }, + expected_format, + "[{vector_name}] canonical collection format changed" + ); + + let actual_entries = semantic_entries(collection); + let expected_entries = expected["entries"] + .as_array() + .expect("roundtrip vector must declare entries"); + assert_eq!( + actual_entries.len(), + expected_entries.len(), + "[{vector_name}] named collection entry count changed" + ); + + if expected["preserveOrder"].as_bool() == Some(true) { + for (index, expected_entry) in expected_entries.iter().enumerate() { + assert_subset( + &actual_entries[index], + expected_entry, + &format!("{vector_name}.entries[{index}]"), + ); + } + } else { + let actual_by_name: Map = actual_entries + .into_iter() + .map(|entry| { + let name = entry["name"] + .as_str() + .expect("semantic entry name must be a string") + .to_string(); + (name, entry) + }) + .collect(); + for expected_entry in expected_entries { + let name = expected_entry["name"] + .as_str() + .expect("expected entry name must be a string"); + let actual_entry = actual_by_name + .get(name) + .unwrap_or_else(|| panic!("[{vector_name}] missing named entry {name:?}")); + assert_subset( + actual_entry, + expected_entry, + &format!("{vector_name}.entries.{name}"), + ); + } + } +} + +fn vectors() -> Vec { + let raw = + std::fs::read_to_string(vectors_path()).expect("failed to read named collection vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse named collection vectors"); + document["vectors"] + .as_array() + .expect("named collection vectors must contain a vectors array") + .clone() +} + +#[test] +fn named_collection_roundtrip_vectors() { + for vector in vectors() + .into_iter() + .filter(|vector| vector["operation"] == "load-save-reload") + { + let name = vector["name"] + .as_str() + .expect("vector name must be a string"); + let json = serde_json::to_string(&vector["input"]) + .expect("named collection vector input must be JSON-compatible"); + let result = Prompty::from_json(&json, &LoadContext::default()); + + let loaded = + result.unwrap_or_else(|error| panic!("[{name}] valid collection failed: {error}")); + let saved = loaded.to_value(&SaveContext::default()); + let collection_path = vector["collectionPath"] + .as_str() + .expect("roundtrip vector must declare collectionPath"); + let collection = saved + .get(collection_path) + .unwrap_or_else(|| panic!("[{name}] missing collection {collection_path:?}")); + assert_collection(name, collection, &vector["expected"]); + + let saved_json = + serde_json::to_string(&saved).expect("saved named collection must be JSON-compatible"); + let reloaded = Prompty::from_json(&saved_json, &LoadContext::default()) + .unwrap_or_else(|error| panic!("[{name}] saved collection failed: {error}")); + let resaved = reloaded.to_value(&SaveContext::default()); + let reloaded_collection = resaved + .get(collection_path) + .unwrap_or_else(|| panic!("[{name}] reload lost collection {collection_path:?}")); + assert_collection(name, reloaded_collection, &vector["expected"]); + } +} + +#[test] +fn named_collection_rejection_vectors() { + for vector in vectors() + .into_iter() + .filter(|vector| vector["operation"] == "load-error") + { + let name = vector["name"] + .as_str() + .expect("vector name must be a string"); + let json = serde_json::to_string(&vector["input"]) + .expect("named collection vector input must be JSON-compatible"); + let error = match Prompty::from_json(&json, &LoadContext::default()) { + Ok(_) => panic!("[{name}] invalid nested array was accepted"), + Err(error) => error, + }; + let diagnostic = error.to_string(); + let expected_path = vector["expected"]["path"] + .as_str() + .expect("error vector must declare path"); + let value_category = vector["expected"]["valueCategory"] + .as_str() + .expect("error vector must declare valueCategory"); + assert!( + diagnostic.contains(expected_path), + "[{name}] diagnostic did not identify path {expected_path:?}: {diagnostic}" + ); + assert!( + diagnostic.contains(value_category), + "[{name}] diagnostic did not identify category {value_category:?}: {diagnostic}" + ); + } +} diff --git a/spec/spec.md b/spec/spec.md index f728f88d5..591c25d5c 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -346,6 +346,53 @@ schema is a `Property` object: Rich kinds (`thread`, `image`, `file`, `audio`) receive special handling during rendering — see §5 for details. +#### Named collection encoding + +Named collections such as `Properties`, `Tools`, `Bindings`, and `Connections` accept +either a flat array of entries or a name-keyed object. Names are opaque parsed strings: +implementations MUST NOT trim, case-fold, or Unicode-normalize them before comparison. +Missing `name` and an explicit empty `name` are the same unnamed state because `name` +defaults to `""`. + +The canonical serialization is a name-keyed object when every entry has a non-empty +name and all names are unique by exact parsed-string comparison. Otherwise, the +serializer MUST encode the entire collection as an array, preserving entry order and +every entry's payload. The canonical array form MUST omit an empty `name`. An explicit +array-format option MAY force array encoding for a losslessly object-encodable +collection, but an object-format option MUST NOT override the lossless fallback. + +Implementations MUST NOT omit unnamed entries, overwrite duplicate names, or invent +synthetic keys such as `_unnamed`, indexes, or suffixed names. Loading either canonical +form and then saving and reloading it MUST preserve the same entries and payloads. +Array fallback MUST preserve model entry order; object ordering is not semantically +significant. + +At every named-collection boundary, recursively: + +- The array form is the collection itself: a flat array of entries. +- In the name-keyed object form, each key maps to exactly one entry. +- An array used as the immediate value of a name-keyed entry is structurally invalid. + It MUST be rejected at the first invalid value and MUST NOT be skipped, flattened, + stringified, or coerced into a default entry. +- The native load error MUST identify the full collection path, including the offending + key, and identify the invalid value category as `array`. + +This validation is schema-aware and applies after JSON/YAML parsing and reference +resolution. It does not reject the outer flat array form or arrays in declared fields +inside a valid entry, such as `Property.default`. In particular, list shorthand is not +available as the immediate value of a name-keyed `Property` entry because it is +ambiguous with an invalid nested collection. Use the expanded form instead: + +```yaml +inputs: + aliases: + kind: array + default: [Ada, Grace] +``` + +The normative load/save/reload and rejection cases are +`spec/vectors/model/named_collection_vectors.json`. + **Scalar shorthand**: When a property value is a plain scalar instead of a `Property` object, it MUST be interpreted as `Property(kind: , default: )`. @@ -363,6 +410,7 @@ inputs: Kind inference from scalar type: string → `"string"`, integer → `"integer"`, float → `"float"`, boolean → `"boolean"`, list → `"array"`, dict → `"object"`. +The named-collection object-form exception for list values is defined above. ### §2.8 Template diff --git a/spec/vectors/model/named_collection_vectors.json b/spec/vectors/model/named_collection_vectors.json new file mode 100644 index 000000000..6864949fe --- /dev/null +++ b/spec/vectors/model/named_collection_vectors.json @@ -0,0 +1,287 @@ +{ + "version": "1", + "description": "Cross-runtime named-collection load/save/reload contracts. Canonical serialization uses a name-keyed object only when every parsed name is non-empty and unique; otherwise it uses a whole-collection array fallback without omission, collision, or synthetic names. Array-valued entries in name-keyed object form are rejected recursively, while arrays in declared entry fields remain valid.", + "vectors": [ + { + "name": "unique_names_use_canonical_object_form", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "unique-names", + "inputs": [ + { + "name": "alpha", + "kind": "string", + "description": "first entry" + }, + { + "name": "beta", + "kind": "boolean", + "required": true + } + ] + }, + "expected": { + "collectionFormat": "object", + "entries": [ + { + "name": "alpha", + "kind": "string", + "description": "first entry" + }, + { + "name": "beta", + "kind": "boolean", + "required": true + } + ] + } + }, + { + "name": "missing_and_empty_names_use_lossless_array_fallback", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "unnamed-inputs", + "inputs": [ + { + "name": "head", + "kind": "string", + "description": "preserve the named head" + }, + { + "kind": "integer", + "default": 7 + }, + { + "name": "", + "kind": "boolean", + "example": false + }, + { + "name": "tail", + "kind": "array", + "default": [ + 1, + null, + { + "nested": [ + "x", + 2 + ] + } + ], + "items": { + "kind": "object" + } + } + ] + }, + "expected": { + "collectionFormat": "array", + "preserveOrder": true, + "entries": [ + { + "name": "head", + "kind": "string", + "description": "preserve the named head" + }, + { + "name": "", + "kind": "integer", + "default": 7 + }, + { + "name": "", + "kind": "boolean", + "example": false + }, + { + "name": "tail", + "kind": "array", + "default": [ + 1, + null, + { + "nested": [ + "x", + 2 + ] + } + ], + "items": { + "kind": "object" + } + } + ] + } + }, + { + "name": "duplicate_names_use_lossless_array_fallback", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "duplicate-inputs", + "inputs": [ + { + "name": "same", + "kind": "string", + "description": "first duplicate" + }, + { + "name": "same", + "kind": "integer", + "default": 2 + } + ] + }, + "expected": { + "collectionFormat": "array", + "preserveOrder": true, + "entries": [ + { + "name": "same", + "kind": "string", + "description": "first duplicate" + }, + { + "name": "same", + "kind": "integer", + "default": 2 + } + ] + } + }, + { + "name": "empty_object_key_reloads_as_unnamed_array_entry", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "empty-object-key", + "inputs": { + "": { + "kind": "string", + "description": "preserve empty key payload" + }, + "tail": { + "kind": "boolean" + } + } + }, + "expected": { + "collectionFormat": "array", + "entries": [ + { + "name": "", + "kind": "string", + "description": "preserve empty key payload" + }, + { + "name": "tail", + "kind": "boolean" + } + ] + } + }, + { + "name": "array_in_declared_property_field_remains_valid", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "declared-array-field", + "inputs": { + "aliases": { + "kind": "array", + "default": [ + "Ada", + "Grace", + null + ], + "items": { + "kind": "string" + } + } + } + }, + "expected": { + "collectionFormat": "object", + "entries": [ + { + "name": "aliases", + "kind": "array", + "default": [ + "Ada", + "Grace", + null + ], + "items": { + "kind": "string" + } + } + ] + } + }, + { + "name": "scalar_array_shorthand_in_name_keyed_inputs_is_rejected", + "operation": "load-error", + "input": { + "name": "invalid-scalar-array-shorthand", + "inputs": { + "arrayDefault": [ + 1, + "two", + null + ] + } + }, + "expected": { + "error": "invalid-named-collection-entry", + "path": "inputs.arrayDefault", + "valueCategory": "array" + } + }, + { + "name": "array_value_in_name_keyed_inputs_is_rejected", + "operation": "load-error", + "input": { + "name": "invalid-top-level-array", + "inputs": { + "arrayEntry": [ + { + "kind": "string" + } + ] + } + }, + "expected": { + "error": "invalid-named-collection-entry", + "path": "inputs.arrayEntry", + "valueCategory": "array" + } + }, + { + "name": "array_value_in_nested_properties_is_rejected", + "operation": "load-error", + "input": { + "name": "invalid-recursive-array", + "inputs": { + "profile": { + "kind": "object", + "properties": { + "arrayEntry": [ + { + "kind": "string" + } + ] + } + } + } + }, + "expected": { + "error": "invalid-named-collection-entry", + "path": "inputs.profile.properties.arrayEntry", + "valueCategory": "array" + } + } + ] +} From d02c0b018825c2a3a58cfceef6b0070e36074307 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 13:03:40 -0700 Subject: [PATCH 10/20] Define Record unknown nullability contract Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../record_unknown_nullability_vectors.rs | 87 +++++ schema/model/agent/agent.tsp | 2 +- schema/model/conversation/message.tsp | 2 +- schema/model/events/payloads.tsp | 4 +- schema/model/events/session.tsp | 2 +- schema/model/model/discovery.tsp | 2 +- schema/model/pipeline/turn.tsp | 6 +- spec/spec.md | 41 ++ .../record_unknown_nullability_vectors.json | 364 ++++++++++++++++++ 9 files changed, 501 insertions(+), 9 deletions(-) create mode 100644 runtime/rust/prompty/tests/record_unknown_nullability_vectors.rs create mode 100644 spec/vectors/model/record_unknown_nullability_vectors.json diff --git a/runtime/rust/prompty/tests/record_unknown_nullability_vectors.rs b/runtime/rust/prompty/tests/record_unknown_nullability_vectors.rs new file mode 100644 index 000000000..c5f775e43 --- /dev/null +++ b/runtime/rust/prompty/tests/record_unknown_nullability_vectors.rs @@ -0,0 +1,87 @@ +//! Record nullability tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::{LoadContext, SaveContext}; +use prompty::model::{ + HostToolRequest, Message, ModelInfo, Prompty, RunTurnRequest, SessionEvent, TurnEvent, + TurnModelRequest, TurnModelResponse, +}; +use serde_json::Value; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("record_unknown_nullability_vectors.json") +} + +macro_rules! roundtrip { + ($model:ty, $json:expr) => {{ + let loaded = + <$model>::from_json($json, &LoadContext::default()).expect("vector input must load"); + let saved = loaded.to_value(&SaveContext::default()); + let saved_json = + serde_json::to_string(&saved).expect("saved model must be JSON-compatible"); + let reloaded = <$model>::from_json(&saved_json, &LoadContext::default()) + .expect("saved model must reload"); + reloaded.to_value(&SaveContext::default()) + }}; +} + +#[test] +fn record_unknown_nullability_vectors() { + let raw = std::fs::read_to_string(vectors_path()) + .expect("failed to read Record nullability vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse Record nullability vectors"); + let vectors = document["vectors"] + .as_array() + .expect("Record nullability vectors must contain a vectors array"); + + for vector in vectors { + let name = vector["name"] + .as_str() + .expect("vector name must be a string"); + assert_eq!( + vector["operation"], "load-save-reload", + "[{name}] unsupported vector operation" + ); + let model = vector["model"] + .as_str() + .expect("vector model must be a string"); + let field_path = vector["fieldPath"] + .as_str() + .expect("vector fieldPath must be a string"); + let json = + serde_json::to_string(&vector["input"]).expect("vector input must be JSON-compatible"); + + let resaved = match model { + "Message" => roundtrip!(Message, &json), + "Prompty" => roundtrip!(Prompty, &json), + "ModelInfo" => roundtrip!(ModelInfo, &json), + "TurnModelRequest" => roundtrip!(TurnModelRequest, &json), + "RunTurnRequest" => roundtrip!(RunTurnRequest, &json), + "TurnModelResponse" => roundtrip!(TurnModelResponse, &json), + "HostToolRequest" => roundtrip!(HostToolRequest, &json), + "TurnEvent" => roundtrip!(TurnEvent, &json), + "SessionEvent" => roundtrip!(SessionEvent, &json), + _ => panic!("[{name}] unsupported model {model:?}"), + }; + + let actual = resaved + .get(field_path) + .unwrap_or_else(|| panic!("[{name}] reload lost field {field_path:?}")); + assert_eq!( + actual, &vector["expected"], + "[{name}] reload changed null-valued record entries" + ); + } +} diff --git a/schema/model/agent/agent.tsp b/schema/model/agent/agent.tsp index 0a7094e12..30e459fdc 100644 --- a/schema/model/agent/agent.tsp +++ b/schema/model/agent/agent.tsp @@ -45,7 +45,7 @@ model Prompty { }) description?: string = ""; - @doc("Additional metadata including authors, tags, and other arbitrary properties") + @doc("Additional metadata including authors, tags, and other arbitrary properties. Values may be explicit null.") @sample(#{ metadata: #{ authors: #["sethjuarez", "jietong"], diff --git a/schema/model/conversation/message.tsp b/schema/model/conversation/message.tsp index ae8fdf083..38855c072 100644 --- a/schema/model/conversation/message.tsp +++ b/schema/model/conversation/message.tsp @@ -42,7 +42,7 @@ model Message { @sample(#{ parts: #[#{ kind: "text", value: "Hello!" }] }) parts: ContentPart[]; - @doc("Optional metadata associated with the message") + @doc("Optional metadata associated with the message. Values may be explicit null.") @sample(#{ metadata: #{ source: "user-input" } }) metadata: Record = #{}; } diff --git a/schema/model/events/payloads.tsp b/schema/model/events/payloads.tsp index ef113549d..5a44c543a 100644 --- a/schema/model/events/payloads.tsp +++ b/schema/model/events/payloads.tsp @@ -87,7 +87,7 @@ model TurnEvent { @sample(#{ spanId: "span_tool_001" }) spanId?: string; - @doc("Event-specific payload. Use the typed payload model matching 'type'.") + @doc("Event-specific payload. Values may be explicit null. Use the typed payload model matching 'type'.") payload: Record = #{}; } @@ -468,7 +468,7 @@ model HostToolRequest { @sample(#{ toolName: "powershell" }) toolName: string; - @doc("Tool arguments after host-side sanitization") + @doc("Tool arguments after host-side sanitization. Values may be explicit null.") arguments?: Record; @doc("Working directory or execution scope for the tool") diff --git a/schema/model/events/session.tsp b/schema/model/events/session.tsp index c502d9680..8cccc77f8 100644 --- a/schema/model/events/session.tsp +++ b/schema/model/events/session.tsp @@ -157,7 +157,7 @@ model SessionEvent { @sample(#{ spanId: "span_hook_001" }) spanId?: string; - @doc("Event-specific payload. Use the typed payload model matching 'type'.") + @doc("Event-specific payload. Values may be explicit null. Use the typed payload model matching 'type'.") payload: Record = #{}; @doc("Redaction state for sensitive payload fields") diff --git a/schema/model/model/discovery.tsp b/schema/model/model/discovery.tsp index 118716343..78f4b1fc0 100644 --- a/schema/model/model/discovery.tsp +++ b/schema/model/model/discovery.tsp @@ -46,7 +46,7 @@ model ModelInfo { @sample(#{ outputModalities: #["text"] }) outputModalities?: string[]; - @doc("Additional provider-specific properties") + @doc("Additional provider-specific properties. Values may be explicit null.") @sample(#{ additionalProperties: #{ supportsStreaming: true } }) additionalProperties?: Record; } diff --git a/schema/model/pipeline/turn.tsp b/schema/model/pipeline/turn.tsp index 1a260b7f4..5d795a4b8 100644 --- a/schema/model/pipeline/turn.tsp +++ b/schema/model/pipeline/turn.tsp @@ -81,7 +81,7 @@ model TurnModelRequest { @sample(#{ iteration: 0 }) iteration: int32; - @doc("Inputs supplied to the deterministic single-turn run") + @doc("Inputs supplied to the deterministic single-turn run. Values may be explicit null.") inputs?: Record = #{}; @doc("Canonical turn execution options") @@ -104,7 +104,7 @@ model TurnModelResponse { @doc("Host tool execution requests emitted by the model callback") toolRequests?: HostToolRequest[] = #[]; - @doc("Additional deterministic state to merge into the iteration checkpoint") + @doc("Additional deterministic state to merge into the iteration checkpoint. Values may be explicit null.") checkpointState?: Record = #{}; } @@ -120,7 +120,7 @@ model RunTurnRequest { @sample(#{ turnId: "turn_abc123" }) turnId: string; - @doc("Inputs supplied to the deterministic single-turn run") + @doc("Inputs supplied to the deterministic single-turn run. Values may be explicit null.") inputs?: Record = #{}; @doc("Canonical turn execution options") diff --git a/spec/spec.md b/spec/spec.md index 591c25d5c..3ede96860 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -246,6 +246,47 @@ correspond to the TypeSpec-generated data model. Unknown top-level properties SHOULD be preserved in `metadata` or ignored. Implementations MUST NOT raise an error for unknown properties. +#### `Record` value nullability + +`Record` has two independent nullability axes: + +1. The model property's optionality controls whether the record itself may be absent. +2. The `unknown` value type permits every JSON-compatible value, including explicit + `null`, at any nesting depth. + +After YAML parsing, YAML null forms have the same explicit-null semantics as JSON +`null`. + +An implementation MUST preserve null-valued keys through load → save → reload. It MUST +NOT drop a key whose value is `null`, coerce that value to an empty object or another +sentinel, or conflate the present-null state with an absent key. Nested objects and arrays +MUST apply the same rule recursively. + +This contract applies to every `Record` surface. The following generated fields +are specifically covered by the shared acceptance vectors because they cross canonical +runtime boundaries: + +| Model | Field | Record presence | +| ----- | ----- | --------------- | +| `Message` | `metadata` | Required | +| `Prompty` | `metadata` | Optional | +| `ModelInfo` | `additionalProperties` | Optional | +| `TurnModelRequest` | `inputs` | Optional | +| `RunTurnRequest` | `inputs` | Optional | +| `TurnModelResponse` | `checkpointState` | Optional | +| `HostToolRequest` | `arguments` | Optional | +| `TurnEvent` | `payload` | Required | +| `SessionEvent` | `payload` | Required | + +Language bindings MUST retain both axes. For example, the conforming C# mapping is +`IDictionary` for a required record and +`IDictionary?` for an optional record. Mapping a +`Record` value as non-null `object` changes the schema contract and is +non-conformant. Nullable annotations do not add a wire or model field. + +The normative shared vectors are +`spec/vectors/model/record_unknown_nullability_vectors.json`. + ### §2.4 Model The `model` property configures the LLM provider and parameters. diff --git a/spec/vectors/model/record_unknown_nullability_vectors.json b/spec/vectors/model/record_unknown_nullability_vectors.json new file mode 100644 index 000000000..32f9bec1f --- /dev/null +++ b/spec/vectors/model/record_unknown_nullability_vectors.json @@ -0,0 +1,364 @@ +{ + "version": "1", + "description": "Cross-runtime Record nullability contracts. Optionality controls whether the record itself may be absent; present records permit explicit null values at every nesting depth. Load/save/reload must preserve null-valued keys and must not conflate present-null with absence.", + "vectors": [ + { + "name": "message_metadata_preserves_null_values", + "operation": "load-save-reload", + "model": "Message", + "fieldPath": "metadata", + "input": { + "role": "user", + "parts": [], + "metadata": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "prompty_metadata_preserves_null_values", + "operation": "load-save-reload", + "model": "Prompty", + "fieldPath": "metadata", + "input": { + "name": "nullable-prompty-metadata", + "metadata": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "model_info_additional_properties_preserve_null_values", + "operation": "load-save-reload", + "model": "ModelInfo", + "fieldPath": "additionalProperties", + "input": { + "id": "nullable-provider-model", + "additionalProperties": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "turn_model_request_inputs_preserve_null_values", + "operation": "load-save-reload", + "model": "TurnModelRequest", + "fieldPath": "inputs", + "input": { + "sessionId": "sess_nullable", + "turnId": "turn_nullable", + "iteration": 0, + "inputs": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "run_turn_request_inputs_preserve_null_values", + "operation": "load-save-reload", + "model": "RunTurnRequest", + "fieldPath": "inputs", + "input": { + "sessionId": "sess_nullable", + "turnId": "turn_nullable", + "inputs": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "turn_model_response_checkpoint_state_preserves_null_values", + "operation": "load-save-reload", + "model": "TurnModelResponse", + "fieldPath": "checkpointState", + "input": { + "checkpointState": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "host_tool_request_arguments_preserve_null_values", + "operation": "load-save-reload", + "model": "HostToolRequest", + "fieldPath": "arguments", + "input": { + "toolName": "nullable-tool", + "arguments": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "turn_event_payload_preserves_null_values", + "operation": "load-save-reload", + "model": "TurnEvent", + "fieldPath": "payload", + "input": { + "id": "evt_turn_nullable", + "type": "turn_start", + "timestamp": "2026-07-01T00:00:00Z", + "payload": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "session_event_payload_preserves_null_values", + "operation": "load-save-reload", + "model": "SessionEvent", + "fieldPath": "payload", + "input": { + "id": "evt_session_nullable", + "type": "session_start", + "timestamp": "2026-07-01T00:00:00Z", + "payload": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + } + ] +} From 0277feb635a675a424c60abac0bb9603d61ab1b4 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 13:25:07 -0700 Subject: [PATCH 11/20] Harden engine port output verification Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- schema/scripts/verify-engine-ports.mjs | 143 ++++++++++++++++++++----- 1 file changed, 118 insertions(+), 25 deletions(-) diff --git a/schema/scripts/verify-engine-ports.mjs b/schema/scripts/verify-engine-ports.mjs index bfe482735..b75198d68 100644 --- a/schema/scripts/verify-engine-ports.mjs +++ b/schema/scripts/verify-engine-ports.mjs @@ -31,6 +31,7 @@ verifyLegacyHarness(); verifyExportSurfaces(); verifyNoWireCancellation(); verifyNativeSignatures(); +verifyMarkdownSemantics(); console.log("Canonical engine port contracts verified."); @@ -120,7 +121,7 @@ function verifyExportSurfaces() { `${target.target}: ${protocolName}.${methodName}`, ); if (expectedMethod.params) { - assertExactKeyOrder( + assertExactKeys( method.params, expectedMethod.params, `${target.target}: ${protocolName}.${methodName} params`, @@ -345,24 +346,24 @@ function verifyGoSignatures(root) { function verifyTypeScriptSignatures(root) { expectMatches( join(root, "engine-permission-port.ts"), - /^\s{2}authorize\(\s*request: ModelToolRequest,\s*signal\?: AbortSignal\s*\): Promise;/mu, + /^\s{2}authorize\(\s*request: ModelToolRequest,\s*signal\?: AbortSignal,?\s*\): Promise;/mu, ); expectMatches( join(root, "engine-tool-port.ts"), - /^\s{2}execute\(\s*request: ModelToolRequest,\s*signal\?: AbortSignal\s*\): Promise;/mu, + /^\s{2}execute\(\s*request: ModelToolRequest,\s*signal\?: AbortSignal,?\s*\): Promise;/mu, ); expectAllMatches(join(root, "engine-durability-port.ts"), [ - /^\s{2}append\(\s*event: EngineEvent\s*\): Promise;/mu, - /^\s{2}appendWithCheckpoint\(\s*events: EngineEvent\[\],\s*checkpoint: EngineCheckpoint\s*\): Promise;/mu, + /^\s{2}append\(\s*event: EngineEvent,?\s*\): Promise;/mu, + /^\s{2}appendWithCheckpoint\(\s*events: EngineEvent\[\],\s*checkpoint: EngineCheckpoint,?\s*\): Promise;/mu, ]); expectMatches( join(root, "engine-post-commit-port.ts"), - /^\s{2}afterCommit\(\s*effectId: string,\s*commit: TurnCommit,\s*signal\?: AbortSignal\s*\): Promise;/mu, + /^\s{2}afterCommit\(\s*effectId: string,\s*commit: TurnCommit,\s*signal\?: AbortSignal,?\s*\): Promise;/mu, ); expectAllMatches(join(root, "executor.ts"), [ - /^\s{2}execute\(\s*agent: Prompty,\s*messages: Message\[\],\s*signal\?: AbortSignal\s*\): Promise;/mu, - /^\s{2}executeStream\?\(\s*agent: Prompty,\s*messages: Message\[\],\s*signal\?: AbortSignal\s*\): Promise;/mu, - /^\s{2}formatToolMessages\(\s*rawResponse: unknown,\s*toolCalls: ToolCall\[\],\s*toolResults: string\[\],\s*textContent: string \| null,\s*\): Message\[\];/mu, + /^\s{2}execute\(\s*agent: Prompty,\s*messages: Message\[\],\s*signal\?: AbortSignal,?\s*\): Promise;/mu, + /^\s{2}executeStream\?\(\s*agent: Prompty,\s*messages: Message\[\],\s*signal\?: AbortSignal,?\s*\): Promise;/mu, + /^\s{2}formatToolMessages\(\s*rawResponse: unknown,\s*toolCalls: ToolCall\[\],\s*toolResults: string\[\],\s*textContent: string \| null,?\s*\): Message\[\];/mu, ]); for (const [file, expected] of [ ["engine-permission-port.ts", ["authorize"]], @@ -382,24 +383,24 @@ function verifyTypeScriptSignatures(root) { function verifyRustSignatures(root) { expectMatches( join(root, "engine_permission_port.rs"), - /^\s{4}async fn authorize\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,\s*\)\s*-> Result>;/mu, + /^\s{4}async fn authorize\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result>;/mu, ); expectMatches( join(root, "engine_tool_port.rs"), - /^\s{4}async fn execute\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,\s*\)\s*-> Result>;/mu, + /^\s{4}async fn execute\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result>;/mu, ); expectAllMatches(join(root, "engine_durability_port.rs"), [ - /^\s{4}async fn append\(\s*&self,\s*event: &EngineEvent,\s*\)\s*-> Result<\(\),\s*Box>;/mu, - /^\s{4}async fn append_with_checkpoint\(\s*&self,\s*events: &Vec,\s*checkpoint: &EngineCheckpoint,\s*\)\s*-> Result<\(\),\s*Box>;/mu, + /^\s{4}async fn append\(\s*&self,\s*event: &EngineEvent,?\s*\)\s*-> Result<\(\),\s*Box>;/mu, + /^\s{4}async fn append_with_checkpoint\(\s*&self,\s*events: &Vec,\s*checkpoint: &EngineCheckpoint,?\s*\)\s*-> Result<\(\),\s*Box>;/mu, ]); expectMatches( join(root, "engine_post_commit_port.rs"), - /^\s{4}async fn after_commit\(\s*&self,\s*effect_id: &String,\s*commit: &TurnCommit,\s*cancellation: &CancellationToken,\s*\)\s*-> Result<\(\),\s*Box>;/mu, + /^\s{4}async fn after_commit\(\s*&self,\s*effect_id: &String,\s*commit: &TurnCommit,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result<\(\),\s*Box>;/mu, ); expectAllMatches(join(root, "executor.rs"), [ - /^\s{4}async fn execute\(\s*&self,\s*agent: &Prompty,\s*messages: &Vec,\s*cancellation: &CancellationToken,\s*\)\s*-> Result>;/mu, - /^\s{4}async fn execute_stream\(\s*&self,\s*agent: &Prompty,\s*messages: &Vec,\s*cancellation: &CancellationToken,\s*\)\s*-> Result>/mu, - /^\s{4}fn format_tool_messages\(\s*&self,\s*raw_response: &serde_json::Value,\s*tool_calls: &Vec,\s*tool_results: &Vec,\s*text_content: &Option,\s*\)\s*-> Vec;/mu, + /^\s{4}async fn execute\(\s*&self,\s*agent: &Prompty,\s*messages: &Vec,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result>;/mu, + /^\s{4}async fn execute_stream\(\s*&self,\s*agent: &Prompty,\s*messages: &Vec,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result>/mu, + /^\s{4}fn format_tool_messages\(\s*&self,\s*raw_response: &serde_json::Value,\s*tool_calls: &Vec,\s*tool_results: &Vec,\s*text_content: &Option,?\s*\)\s*-> Vec;/mu, ]); for (const [file, expected] of [ ["engine_permission_port.rs", ["authorize"]], @@ -474,6 +475,106 @@ function verifyPythonSignatures(root) { } } +function verifyMarkdownSemantics() { + const root = join(repoRoot, "web", "src", "content", "docs", "reference"); + expectMarkdownMethod( + join(root, "EnginePermissionPort.md"), + "authorize", + "authorize(request: ModelToolRequest) -> EnginePermissionDecision", + ["async-capable", "runtime-cancellable"], + ["atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "EngineToolPort.md"), + "execute", + "execute(request: ModelToolRequest) -> ModelToolResult", + ["async-capable", "runtime-cancellable"], + ["atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "EngineDurabilityPort.md"), + "append", + "append(event: EngineEvent) -> void", + ["async-capable"], + ["runtime-cancellable", "atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "EngineDurabilityPort.md"), + "appendWithCheckpoint", + "appendWithCheckpoint(events: EngineEvent[], checkpoint: EngineCheckpoint) -> void", + ["async-capable", "atomic"], + ["runtime-cancellable", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "EnginePostCommitPort.md"), + "afterCommit", + "afterCommit(effectId: string, commit: TurnCommit) -> void", + ["async-capable", "runtime-cancellable", "non-fatal"], + ["atomic", "sync"], + ); + expectMarkdownMethod( + join(root, "Executor.md"), + "execute", + "execute(agent: Prompty, messages: Message[]) -> unknown", + ["async-capable", "runtime-cancellable"], + ["atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "Executor.md"), + "executeStream", + "executeStream(agent: Prompty, messages: Message[]) -> unknown", + ["async-capable", "runtime-cancellable"], + ["atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "Executor.md"), + "formatToolMessages", + "formatToolMessages(rawResponse: unknown, toolCalls: ToolCall[], toolResults: string[], textContent: string?) -> Message[]", + ["sync"], + ["async-capable", "runtime-cancellable", "atomic", "non-fatal"], + ); +} + +function expectMarkdownMethod( + path, + methodName, + signature, + requiredEffects, + forbiddenEffects, +) { + const content = readFileSync(path, "utf8"); + const row = content + .split(/\r?\n/u) + .find((line) => line.startsWith(`| \`${methodName}\` |`)); + assert(row, `${path}: missing ${methodName} helper-method row`); + + const columns = row.split("|"); + assertEqual( + columns[2].trim(), + `\`${signature}\``, + `${path}: ${methodName} signature`, + ); + const runtimeShape = columns[3].trim().toLowerCase(); + for (const effect of requiredEffects) { + assert( + hasRuntimeEffect(runtimeShape, effect), + `${path}: ${methodName} runtime shape must include ${effect}`, + ); + } + for (const effect of forbiddenEffects) { + assert( + !hasRuntimeEffect(runtimeShape, effect), + `${path}: ${methodName} runtime shape must not include ${effect}`, + ); + } +} + +function hasRuntimeEffect(runtimeShape, effect) { + const escaped = effect.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const normalized = runtimeShape.replace(/[_()]/gu, " "); + return new RegExp(`(?:^|[,\\s])${escaped}(?=$|[,\\s])`, "u").test(normalized); +} + function expectMatches(path, pattern) { const content = readFileSync(path, "utf8"); assert( @@ -540,14 +641,6 @@ function assertExactKeys(actual, expected, label) { ); } -function assertExactKeyOrder(actual, expected, label) { - assertEqual( - JSON.stringify(Object.keys(actual)), - JSON.stringify(Object.keys(expected)), - `${label} order`, - ); -} - function assertUniqueNames(items, label) { const names = items.map((item) => item.name); assertEqual( From ec3b1be20cf1c3f75742a5d179f40756a3e913aa Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 13:36:18 -0700 Subject: [PATCH 12/20] test: enforce unknown connection preservation Add native public-API gates for the shared Connection load-save-reload vectors across C#, Go, Python, Rust, and TypeScript. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../ConnectionRoundtripVectorTests.cs | 78 +++++++++++++++ .../connection_roundtrip_vectors_test.go | 94 +++++++++++++++++++ .../test_connection_roundtrip_vectors.py | 41 ++++++++ .../tests/connection_roundtrip_vectors.rs | 85 +++++++++-------- .../connection-roundtrip-vectors.test.ts | 51 ++++++++++ 5 files changed, 312 insertions(+), 37 deletions(-) create mode 100644 runtime/csharp/Prompty.Core.Tests/ConnectionRoundtripVectorTests.cs create mode 100644 runtime/go/prompty/model/connection_roundtrip_vectors_test.go create mode 100644 runtime/python/prompty/tests/test_connection_roundtrip_vectors.py create mode 100644 runtime/typescript/packages/core/tests/connection-roundtrip-vectors.test.ts diff --git a/runtime/csharp/Prompty.Core.Tests/ConnectionRoundtripVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/ConnectionRoundtripVectorTests.cs new file mode 100644 index 000000000..166739bbe --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/ConnectionRoundtripVectorTests.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class ConnectionRoundtripVectorTests +{ + private static readonly string VectorsPath = FindVectorsPath(); + + [Theory] + [InlineData("known_reference_connection_roundtrip_unchanged")] + [InlineData("unknown_connection_kind_preserves_payload")] + [InlineData("unknown_connection_case_collision_preserves_payload")] + public void ConnectionRoundtripVectors_PreserveExactDiscriminatorAndPayload(string vectorName) + { + using var document = JsonDocument.Parse(File.ReadAllText(VectorsPath)); + var vector = document.RootElement + .GetProperty("vectors") + .EnumerateArray() + .Single(candidate => candidate.GetProperty("name").GetString() == vectorName); + + var input = vector.GetProperty("input"); + var expected = vector.GetProperty("expected"); + var expectedKind = expected.GetProperty("kind").GetString()!; + var data = JsonSerializer.Deserialize>(input.GetRawText())!; + + var loaded = Connection.Load(data); + if (expectedKind == "reference") + Assert.IsType(loaded); + else + Assert.IsNotType(loaded); + + var saved = loaded.Save(); + Assert.Equal(expectedKind, saved["kind"]); + AssertJsonEqual(vectorName, "save", expected, saved); + + var reloaded = Connection.Load(saved); + var resaved = reloaded.Save(); + Assert.Equal(expectedKind, resaved["kind"]); + AssertJsonEqual(vectorName, "reload", expected, resaved); + } + + private static void AssertJsonEqual( + string vectorName, + string operation, + JsonElement expected, + Dictionary actual) + { + var expectedNode = JsonNode.Parse(expected.GetRawText()); + var actualNode = JsonNode.Parse(JsonSerializer.Serialize(actual)); + Assert.True( + JsonNode.DeepEquals(expectedNode, actualNode), + $"[{vectorName}] {operation} changed the Connection payload.\nExpected: {expectedNode}\nActual: {actualNode}"); + } + + private static string FindVectorsPath() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine( + directory, + "spec", + "vectors", + "model", + "connection_roundtrip_vectors.json"); + if (File.Exists(candidate)) + return candidate; + + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new FileNotFoundException("Could not locate the shared Connection roundtrip vectors."); + } +} diff --git a/runtime/go/prompty/model/connection_roundtrip_vectors_test.go b/runtime/go/prompty/model/connection_roundtrip_vectors_test.go new file mode 100644 index 000000000..74f38d617 --- /dev/null +++ b/runtime/go/prompty/model/connection_roundtrip_vectors_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft. All rights reserved. + +package prompty_test + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" + + "prompty/model" +) + +type connectionRoundtripVectorDocument struct { + Vectors []connectionRoundtripVector `json:"vectors"` +} + +type connectionRoundtripVector struct { + Name string `json:"name"` + Operation string `json:"operation"` + Input map[string]interface{} `json:"input"` + Expected map[string]interface{} `json:"expected"` +} + +func TestConnectionRoundtripVectorsPreserveExactDiscriminatorAndPayload(t *testing.T) { + path := filepath.Join( + "..", + "..", + "..", + "..", + "spec", + "vectors", + "model", + "connection_roundtrip_vectors.json", + ) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read Connection roundtrip vectors: %v", err) + } + + var document connectionRoundtripVectorDocument + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatalf("failed to parse Connection roundtrip vectors: %v", err) + } + + for _, vector := range document.Vectors { + t.Run(vector.Name, func(t *testing.T) { + if vector.Operation != "load-save-reload" { + t.Fatalf("unsupported vector operation %q", vector.Operation) + } + + loaded, err := prompty.LoadConnection(vector.Input, prompty.NewLoadContext()) + if err != nil { + t.Fatalf("load failed: %v", err) + } + + saved := saveConnection(t, loaded) + if saved["kind"] != vector.Expected["kind"] { + t.Fatalf("save changed discriminator: expected %v, got %v", vector.Expected["kind"], saved["kind"]) + } + if !reflect.DeepEqual(saved, vector.Expected) { + t.Fatalf("save changed Connection payload:\nexpected: %#v\nactual: %#v", vector.Expected, saved) + } + + reloaded, err := prompty.LoadConnection(saved, prompty.NewLoadContext()) + if err != nil { + t.Fatalf("reload failed: %v", err) + } + resaved := saveConnection(t, reloaded) + if !reflect.DeepEqual(resaved, vector.Expected) { + t.Fatalf("reload changed Connection payload:\nexpected: %#v\nactual: %#v", vector.Expected, resaved) + } + }) + } +} + +func saveConnection(t *testing.T, connection interface{}) map[string]interface{} { + t.Helper() + + method := reflect.ValueOf(connection).MethodByName("Save") + if !method.IsValid() { + t.Fatalf("loaded Connection type %T does not expose Save", connection) + } + results := method.Call([]reflect.Value{reflect.ValueOf(prompty.NewSaveContext())}) + if len(results) != 1 { + t.Fatalf("loaded Connection type %T returned %d Save results", connection, len(results)) + } + saved, ok := results[0].Interface().(map[string]interface{}) + if !ok { + t.Fatalf("loaded Connection type %T returned unexpected Save result %T", connection, results[0].Interface()) + } + return saved +} diff --git a/runtime/python/prompty/tests/test_connection_roundtrip_vectors.py b/runtime/python/prompty/tests/test_connection_roundtrip_vectors.py new file mode 100644 index 000000000..afafde526 --- /dev/null +++ b/runtime/python/prompty/tests/test_connection_roundtrip_vectors.py @@ -0,0 +1,41 @@ +"""Validate the canonical forward-compatible Connection roundtrip contract.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from prompty.model import Connection, ReferenceConnection + +_VECTORS_PATH = Path(__file__).resolve().parents[4] / "spec" / "vectors" / "model" / "connection_roundtrip_vectors.json" + + +def _load_vectors() -> list[dict[str, Any]]: + document: dict[str, Any] = json.loads(_VECTORS_PATH.read_text(encoding="utf-8")) + return document["vectors"] + + +@pytest.mark.parametrize("vector", _load_vectors(), ids=lambda vector: vector["name"]) +def test_connection_roundtrip_vectors_preserve_exact_discriminator_and_payload(vector: dict[str, Any]) -> None: + """Preserve known and unknown Connection values through load, save, and reload.""" + + assert vector["operation"] == "load-save-reload" + expected = vector["expected"] + + loaded = Connection.load(vector["input"]) + if expected["kind"] == "reference": + assert isinstance(loaded, ReferenceConnection), vector["name"] + else: + assert not isinstance(loaded, ReferenceConnection), vector["name"] + + saved = loaded.save() + assert saved["kind"] == expected["kind"], vector["name"] + assert saved == expected, vector["name"] + + reloaded = Connection.load(saved) + resaved = reloaded.save() + assert resaved["kind"] == expected["kind"], vector["name"] + assert resaved == expected, vector["name"] diff --git a/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs b/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs index b4f84c13a..3ab0de197 100644 --- a/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs +++ b/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs @@ -21,50 +21,61 @@ fn vectors_path() -> PathBuf { } #[test] -fn connection_roundtrip_vectors() { +fn known_reference_connection_roundtrip_unchanged() { + assert_connection_roundtrip_vector("known_reference_connection_roundtrip_unchanged"); +} + +#[test] +fn unknown_connection_kind_preserves_payload() { + assert_connection_roundtrip_vector("unknown_connection_kind_preserves_payload"); +} + +#[test] +fn unknown_connection_case_collision_preserves_payload() { + assert_connection_roundtrip_vector("unknown_connection_case_collision_preserves_payload"); +} + +fn assert_connection_roundtrip_vector(vector_name: &str) { let raw = std::fs::read_to_string(vectors_path()) .expect("failed to read Connection roundtrip vectors"); let document: Value = serde_json::from_str(&raw).expect("failed to parse Connection roundtrip vectors"); - let vectors = document["vectors"] + let vector = document["vectors"] .as_array() - .expect("Connection roundtrip vectors must contain a vectors array"); - - for vector in vectors { - let name = vector["name"] - .as_str() - .expect("vector name must be a string"); - assert_eq!( - vector["operation"], "load-save-reload", - "[{name}] unsupported vector operation" - ); + .expect("Connection roundtrip vectors must contain a vectors array") + .iter() + .find(|candidate| candidate["name"] == vector_name) + .unwrap_or_else(|| panic!("missing Connection roundtrip vector {vector_name}")); + assert_eq!( + vector["operation"], "load-save-reload", + "[{vector_name}] unsupported vector operation" + ); - let input = &vector["input"]; - let expected = &vector["expected"]; - let kind = input["kind"] - .as_str() - .expect("Connection kind must be a string"); - let load_context = LoadContext::default(); - let save_context = SaveContext::default(); + let input = &vector["input"]; + let expected = &vector["expected"]; + let kind = input["kind"] + .as_str() + .expect("Connection kind must be a string"); + let load_context = LoadContext::default(); + let save_context = SaveContext::default(); - let loaded = Connection::load_from_value(input, &load_context); - assert_eq!( - loaded.kind_str(), - kind, - "[{name}] load changed the discriminator" - ); + let loaded = Connection::load_from_value(input, &load_context); + assert_eq!( + loaded.kind_str(), + kind, + "[{vector_name}] load changed the discriminator" + ); - let saved = loaded.to_value(&save_context); - assert_eq!( - saved, *expected, - "[{name}] save changed the Connection payload" - ); + let saved = loaded.to_value(&save_context); + assert_eq!( + saved, *expected, + "[{vector_name}] save changed the Connection payload" + ); - let reloaded = Connection::load_from_value(&saved, &load_context); - let resaved = reloaded.to_value(&save_context); - assert_eq!( - resaved, *expected, - "[{name}] reload changed the Connection payload" - ); - } + let reloaded = Connection::load_from_value(&saved, &load_context); + let resaved = reloaded.to_value(&save_context); + assert_eq!( + resaved, *expected, + "[{vector_name}] reload changed the Connection payload" + ); } diff --git a/runtime/typescript/packages/core/tests/connection-roundtrip-vectors.test.ts b/runtime/typescript/packages/core/tests/connection-roundtrip-vectors.test.ts new file mode 100644 index 000000000..af1f844d4 --- /dev/null +++ b/runtime/typescript/packages/core/tests/connection-roundtrip-vectors.test.ts @@ -0,0 +1,51 @@ +/** + * Canonical forward-compatibility tests for open Connection discriminators. + */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { Connection, ReferenceConnection } from "../src/index.js"; + +interface ConnectionRoundtripVector { + name: string; + operation: "load-save-reload"; + input: Record; + expected: Record; +} + +const vectorsPath = resolve( + import.meta.dirname, + "../../../../../spec/vectors/model/connection_roundtrip_vectors.json", +); +const vectors = ( + JSON.parse(readFileSync(vectorsPath, "utf8")) as { + vectors: ConnectionRoundtripVector[]; + } +).vectors; + +describe("Connection roundtrip vectors", () => { + it.each(vectors)( + "$name preserves the exact discriminator and payload", + (vector) => { + expect(vector.operation).toBe("load-save-reload"); + + const loaded = Connection.load(vector.input); + if (vector.expected.kind === "reference") { + expect(loaded).toBeInstanceOf(ReferenceConnection); + } else { + expect(loaded).not.toBeInstanceOf(ReferenceConnection); + } + + const saved = loaded.save(); + expect(saved.kind).toBe(vector.expected.kind); + expect(saved).toEqual(vector.expected); + + const reloaded = Connection.load(saved); + const resaved = reloaded.save(); + expect(resaved.kind).toBe(vector.expected.kind); + expect(resaved).toEqual(vector.expected); + }, + ); +}); From 0c64ef33646f19c7d467e80b8e641a3ada3bb183 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 13:43:03 -0700 Subject: [PATCH 13/20] test: enforce closed content part discriminators Add native public-API gates for the shared strict ContentPart vectors across C#, Go, Python, Rust, and TypeScript. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../ContentPartDiscriminatorVectorTests.cs | 76 +++++++++++++ ...content_part_discriminator_vectors_test.go | 97 ++++++++++++++++ ...test_content_part_discriminator_vectors.py | 38 +++++++ .../content_part_discriminator_vectors.rs | 104 ++++++++++-------- ...content-part-discriminator-vectors.test.ts | 51 +++++++++ 5 files changed, 320 insertions(+), 46 deletions(-) create mode 100644 runtime/csharp/Prompty.Core.Tests/ContentPartDiscriminatorVectorTests.cs create mode 100644 runtime/go/prompty/model/content_part_discriminator_vectors_test.go create mode 100644 runtime/python/prompty/tests/test_content_part_discriminator_vectors.py create mode 100644 runtime/typescript/packages/core/tests/content-part-discriminator-vectors.test.ts diff --git a/runtime/csharp/Prompty.Core.Tests/ContentPartDiscriminatorVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/ContentPartDiscriminatorVectorTests.cs new file mode 100644 index 000000000..1f25ec2ee --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/ContentPartDiscriminatorVectorTests.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class ContentPartDiscriminatorVectorTests +{ + private static readonly string VectorsPath = FindVectorsPath(); + + [Theory] + [InlineData("known_text_content_part_loads")] + [InlineData("unknown_content_part_kind_is_rejected")] + [InlineData("content_part_case_collision_is_rejected")] + public void ContentPartDiscriminatorVectors_EnforceClosedCaseSensitiveKinds(string vectorName) + { + using var document = JsonDocument.Parse(File.ReadAllText(VectorsPath)); + var vector = document.RootElement + .GetProperty("vectors") + .EnumerateArray() + .Single(candidate => candidate.GetProperty("name").GetString() == vectorName); + var input = vector.GetProperty("input"); + var expected = vector.GetProperty("expected"); + var data = JsonSerializer.Deserialize>(input.GetRawText())!; + + switch (vector.GetProperty("operation").GetString()) + { + case "load": + var loaded = ContentPart.Load(data); + Assert.IsType(loaded); + AssertJsonEqual(vectorName, expected, loaded.Save()); + break; + case "load-error": + var exception = Assert.ThrowsAny(() => ContentPart.Load(data)); + Assert.Contains(expected.GetProperty("discriminator").GetString()!, exception.Message); + Assert.Contains(expected.GetProperty("value").GetString()!, exception.Message); + break; + default: + throw new InvalidOperationException($"[{vectorName}] unsupported vector operation."); + } + } + + private static void AssertJsonEqual( + string vectorName, + JsonElement expected, + Dictionary actual) + { + var expectedNode = JsonNode.Parse(expected.GetRawText()); + var actualNode = JsonNode.Parse(JsonSerializer.Serialize(actual)); + Assert.True( + JsonNode.DeepEquals(expectedNode, actualNode), + $"[{vectorName}] load/save changed the ContentPart payload.\nExpected: {expectedNode}\nActual: {actualNode}"); + } + + private static string FindVectorsPath() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine( + directory, + "spec", + "vectors", + "model", + "content_part_discriminator_vectors.json"); + if (File.Exists(candidate)) + return candidate; + + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new FileNotFoundException("Could not locate the shared ContentPart discriminator vectors."); + } +} diff --git a/runtime/go/prompty/model/content_part_discriminator_vectors_test.go b/runtime/go/prompty/model/content_part_discriminator_vectors_test.go new file mode 100644 index 000000000..a8b95c9fb --- /dev/null +++ b/runtime/go/prompty/model/content_part_discriminator_vectors_test.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft. All rights reserved. + +package prompty_test + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "prompty/model" +) + +type contentPartDiscriminatorVectorDocument struct { + Vectors []contentPartDiscriminatorVector `json:"vectors"` +} + +type contentPartDiscriminatorVector struct { + Name string `json:"name"` + Operation string `json:"operation"` + Input map[string]interface{} `json:"input"` + Expected map[string]interface{} `json:"expected"` +} + +func TestContentPartDiscriminatorVectorsEnforceClosedCaseSensitiveKinds(t *testing.T) { + path := filepath.Join( + "..", + "..", + "..", + "..", + "spec", + "vectors", + "model", + "content_part_discriminator_vectors.json", + ) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read ContentPart discriminator vectors: %v", err) + } + + var document contentPartDiscriminatorVectorDocument + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatalf("failed to parse ContentPart discriminator vectors: %v", err) + } + + for _, vector := range document.Vectors { + t.Run(vector.Name, func(t *testing.T) { + loaded, err := prompty.LoadContentPart(vector.Input, prompty.NewLoadContext()) + + switch vector.Operation { + case "load": + if err != nil { + t.Fatalf("known ContentPart failed to load: %v", err) + } + saved := saveContentPart(t, loaded) + if !reflect.DeepEqual(saved, vector.Expected) { + t.Fatalf("load/save changed ContentPart payload:\nexpected: %#v\nactual: %#v", vector.Expected, saved) + } + case "load-error": + if err == nil { + t.Fatalf("closed ContentPart accepted unknown discriminator %v", vector.Input["kind"]) + } + diagnostic := err.Error() + discriminator := vector.Expected["discriminator"].(string) + value := vector.Expected["value"].(string) + if !strings.Contains(diagnostic, discriminator) { + t.Fatalf("error did not identify discriminator %q: %s", discriminator, diagnostic) + } + if !strings.Contains(diagnostic, value) { + t.Fatalf("error did not preserve discriminator value %q: %s", value, diagnostic) + } + default: + t.Fatalf("unsupported vector operation %q", vector.Operation) + } + }) + } +} + +func saveContentPart(t *testing.T, contentPart interface{}) map[string]interface{} { + t.Helper() + + method := reflect.ValueOf(contentPart).MethodByName("Save") + if !method.IsValid() { + t.Fatalf("loaded ContentPart type %T does not expose Save", contentPart) + } + results := method.Call([]reflect.Value{reflect.ValueOf(prompty.NewSaveContext())}) + if len(results) != 1 { + t.Fatalf("loaded ContentPart type %T returned %d Save results", contentPart, len(results)) + } + saved, ok := results[0].Interface().(map[string]interface{}) + if !ok { + t.Fatalf("loaded ContentPart type %T returned unexpected Save result %T", contentPart, results[0].Interface()) + } + return saved +} diff --git a/runtime/python/prompty/tests/test_content_part_discriminator_vectors.py b/runtime/python/prompty/tests/test_content_part_discriminator_vectors.py new file mode 100644 index 000000000..8c3377518 --- /dev/null +++ b/runtime/python/prompty/tests/test_content_part_discriminator_vectors.py @@ -0,0 +1,38 @@ +"""Validate the canonical closed ContentPart discriminator contract.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from prompty.model import ContentPart, TextPart + +_VECTORS_PATH = ( + Path(__file__).resolve().parents[4] / "spec" / "vectors" / "model" / "content_part_discriminator_vectors.json" +) + + +def _load_vectors() -> list[dict[str, Any]]: + document: dict[str, Any] = json.loads(_VECTORS_PATH.read_text(encoding="utf-8")) + return document["vectors"] + + +@pytest.mark.parametrize("vector", _load_vectors(), ids=lambda vector: vector["name"]) +def test_content_part_discriminator_vectors_enforce_closed_case_sensitive_kinds(vector: dict[str, Any]) -> None: + """Load known kinds and reject unknown or case-colliding discriminator values.""" + + if vector["operation"] == "load": + loaded = ContentPart.load(vector["input"]) + assert isinstance(loaded, TextPart), vector["name"] + assert loaded.save() == vector["expected"], vector["name"] + return + + with pytest.raises(ValueError) as error: + ContentPart.load(vector["input"]) + + diagnostic = str(error.value) + assert vector["expected"]["discriminator"] in diagnostic, vector["name"] + assert vector["expected"]["value"] in diagnostic, vector["name"] diff --git a/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs b/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs index e4a2758e1..f587cb49b 100644 --- a/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs +++ b/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs @@ -21,60 +21,72 @@ fn vectors_path() -> PathBuf { } #[test] -fn content_part_discriminator_vectors() { +fn known_text_content_part_loads() { + assert_content_part_discriminator_vector("known_text_content_part_loads"); +} + +#[test] +fn unknown_content_part_kind_is_rejected() { + assert_content_part_discriminator_vector("unknown_content_part_kind_is_rejected"); +} + +#[test] +fn content_part_case_collision_is_rejected() { + assert_content_part_discriminator_vector("content_part_case_collision_is_rejected"); +} + +fn assert_content_part_discriminator_vector(vector_name: &str) { let raw = std::fs::read_to_string(vectors_path()) .expect("failed to read ContentPart discriminator vectors"); let document: Value = serde_json::from_str(&raw).expect("failed to parse ContentPart discriminator vectors"); - let vectors = document["vectors"] + let vector = document["vectors"] .as_array() - .expect("ContentPart discriminator vectors must contain a vectors array"); + .expect("ContentPart discriminator vectors must contain a vectors array") + .iter() + .find(|candidate| candidate["name"] == vector_name) + .unwrap_or_else(|| panic!("missing ContentPart discriminator vector {vector_name}")); let context = LoadContext::default(); - for vector in vectors { - let name = vector["name"] - .as_str() - .expect("vector name must be a string"); - let input = &vector["input"]; - let json = serde_json::to_string(input).expect("vector input must be JSON-compatible"); - let result = ContentPart::from_json(&json, &context); + let input = &vector["input"]; + let json = serde_json::to_string(input).expect("vector input must be JSON-compatible"); + let result = ContentPart::from_json(&json, &context); - match vector["operation"].as_str() { - Some("load") => { - let content_part = result.unwrap_or_else(|error| { - panic!("[{name}] known ContentPart failed to load: {error}") - }); - assert_eq!( - content_part.to_value(&SaveContext::default()), - vector["expected"], - "[{name}] known ContentPart payload changed during load/save" - ); - } - Some("load-error") => { - let error = match result { - Ok(_) => panic!( - "[{name}] closed ContentPart accepted unknown discriminator {:?}", - input["kind"] - ), - Err(error) => error, - }; - let diagnostic = error.to_string(); - let discriminator = vector["expected"]["discriminator"] - .as_str() - .expect("error vector must declare the discriminator field"); - let value = vector["expected"]["value"] - .as_str() - .expect("error vector must declare the exact discriminator value"); - assert!( - diagnostic.contains(discriminator), - "[{name}] error diagnostic did not identify discriminator {discriminator:?}: {diagnostic}" - ); - assert!( - diagnostic.contains(value), - "[{name}] error diagnostic did not preserve discriminator value {value:?}: {diagnostic}" - ); - } - operation => panic!("[{name}] unsupported vector operation: {operation:?}"), + match vector["operation"].as_str() { + Some("load") => { + let content_part = result.unwrap_or_else(|error| { + panic!("[{vector_name}] known ContentPart failed to load: {error}") + }); + assert_eq!( + content_part.to_value(&SaveContext::default()), + vector["expected"], + "[{vector_name}] known ContentPart payload changed during load/save" + ); + } + Some("load-error") => { + let error = match result { + Ok(_) => panic!( + "[{vector_name}] closed ContentPart accepted unknown discriminator {:?}", + input["kind"] + ), + Err(error) => error, + }; + let diagnostic = error.to_string(); + let discriminator = vector["expected"]["discriminator"] + .as_str() + .expect("error vector must declare the discriminator field"); + let value = vector["expected"]["value"] + .as_str() + .expect("error vector must declare the exact discriminator value"); + assert!( + diagnostic.contains(discriminator), + "[{vector_name}] error diagnostic did not identify discriminator {discriminator:?}: {diagnostic}" + ); + assert!( + diagnostic.contains(value), + "[{vector_name}] error diagnostic did not preserve discriminator value {value:?}: {diagnostic}" + ); } + operation => panic!("[{vector_name}] unsupported vector operation: {operation:?}"), } } diff --git a/runtime/typescript/packages/core/tests/content-part-discriminator-vectors.test.ts b/runtime/typescript/packages/core/tests/content-part-discriminator-vectors.test.ts new file mode 100644 index 000000000..754f88279 --- /dev/null +++ b/runtime/typescript/packages/core/tests/content-part-discriminator-vectors.test.ts @@ -0,0 +1,51 @@ +/** + * Canonical strict-discriminator tests for the closed ContentPart hierarchy. + */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + ContentPart, + TextPart, +} from "../src/model/conversation/content-part.js"; + +interface ContentPartDiscriminatorVector { + name: string; + operation: "load" | "load-error"; + input: Record; + expected: Record; +} + +const vectorsPath = resolve( + import.meta.dirname, + "../../../../../spec/vectors/model/content_part_discriminator_vectors.json", +); +const vectors = ( + JSON.parse(readFileSync(vectorsPath, "utf8")) as { + vectors: ContentPartDiscriminatorVector[]; + } +).vectors; + +describe("ContentPart discriminator vectors", () => { + it.each(vectors)("$name enforces closed, case-sensitive kinds", (vector) => { + if (vector.operation === "load") { + const loaded = ContentPart.load(vector.input); + expect(loaded).toBeInstanceOf(TextPart); + expect(loaded.save()).toEqual(vector.expected); + return; + } + + let diagnostic = ""; + try { + ContentPart.load(vector.input); + } catch (error) { + diagnostic = String(error); + } + + expect(diagnostic).not.toBe(""); + expect(diagnostic).toContain(String(vector.expected.discriminator)); + expect(diagnostic).toContain(String(vector.expected.value)); + }); +}); From b633e77724b836b590cd1d46a7fabcaed3bce35f Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 13:53:03 -0700 Subject: [PATCH 14/20] test: validate function tool load bindings Make the canonical tools_function_load vector assert exact binding map keys and input values across C#, Go, Python, Rust, and TypeScript. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Prompty.Core.Tests/SpecVectorTests.cs | 48 +++++++++ ...function_tool_bindings_load_vector_test.go | 97 +++++++++++++++++++ .../python/prompty/tests/test_spec_vectors.py | 2 + runtime/rust/prompty/tests/loader_test.rs | 36 +++++++ .../packages/core/tests/spec-vectors.test.ts | 9 +- 5 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 runtime/go/prompty/model/function_tool_bindings_load_vector_test.go diff --git a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs index 91519b0a8..aa278a3eb 100644 --- a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs @@ -460,6 +460,54 @@ private static List CompareAgentToExpected(Prompty agent, JsonElement ex } } + if (expected.TryGetProperty("tools", out var toolsEl) && toolsEl.ValueKind == JsonValueKind.Array) + { + var expectedTools = toolsEl.EnumerateArray().ToList(); + var actualTools = agent.Tools ?? []; + if (actualTools.Count != expectedTools.Count) + { + errors.Add($"tools: expected {expectedTools.Count}, got {actualTools.Count}"); + } + + for (var i = 0; i < Math.Min(actualTools.Count, expectedTools.Count); i++) + { + var expectedTool = expectedTools[i]; + if (!expectedTool.TryGetProperty("bindings", out var bindingsEl) || + bindingsEl.ValueKind != JsonValueKind.Object) + { + continue; + } + + var expectedBindings = bindingsEl.EnumerateObject().ToList(); + var actualBindings = actualTools[i].Bindings ?? []; + if (actualBindings.Count != expectedBindings.Count) + { + errors.Add( + $"tools[{i}].bindings: expected {expectedBindings.Count}, got {actualBindings.Count}"); + } + + foreach (var expectedBinding in expectedBindings) + { + var actualBinding = actualBindings.FirstOrDefault(binding => binding.Name == expectedBinding.Name); + if (actualBinding is null) + { + errors.Add($"tools[{i}].bindings: missing binding '{expectedBinding.Name}'"); + continue; + } + + var expectedInput = expectedBinding.Value.ValueKind == JsonValueKind.Object + ? expectedBinding.Value.GetProperty("input").GetString() + : expectedBinding.Value.GetString(); + if (actualBinding.Input != expectedInput) + { + errors.Add( + $"tools[{i}].bindings.{expectedBinding.Name}.input: " + + $"expected '{expectedInput}', got '{actualBinding.Input}'"); + } + } + } + } + return errors; } diff --git a/runtime/go/prompty/model/function_tool_bindings_load_vector_test.go b/runtime/go/prompty/model/function_tool_bindings_load_vector_test.go new file mode 100644 index 000000000..fe94e07c9 --- /dev/null +++ b/runtime/go/prompty/model/function_tool_bindings_load_vector_test.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft. All rights reserved. + +package prompty_test + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" + + "prompty/model" +) + +type functionToolLoadVector struct { + Name string `json:"name"` + Expected struct { + Tools []struct { + Bindings map[string]struct { + Input string `json:"input"` + } `json:"bindings"` + } `json:"tools"` + } `json:"expected"` +} + +func TestFunctionToolBindingsLoadVector(t *testing.T) { + repositoryRoot := filepath.Join("..", "..", "..", "..") + vectorsRaw, err := os.ReadFile(filepath.Join(repositoryRoot, "spec", "vectors", "load", "load_vectors.json")) + if err != nil { + t.Fatalf("failed to read load vectors: %v", err) + } + + var vectors []functionToolLoadVector + if err := json.Unmarshal(vectorsRaw, &vectors); err != nil { + t.Fatalf("failed to parse load vectors: %v", err) + } + + var expectedBindings map[string]struct { + Input string `json:"input"` + } + for _, vector := range vectors { + if vector.Name == "tools_function_load" { + if len(vector.Expected.Tools) != 1 { + t.Fatalf("tools_function_load expected one tool, got %d", len(vector.Expected.Tools)) + } + expectedBindings = vector.Expected.Tools[0].Bindings + break + } + } + if expectedBindings == nil { + t.Fatal("tools_function_load vector is missing expected bindings") + } + + fixtureRaw, err := os.ReadFile(filepath.Join(repositoryRoot, "spec", "fixtures", "tools_function.prompty")) + if err != nil { + t.Fatalf("failed to read tools_function.prompty: %v", err) + } + sections := strings.SplitN(string(fixtureRaw), "---", 3) + if len(sections) != 3 { + t.Fatal("tools_function.prompty must contain YAML frontmatter") + } + + var frontmatter struct { + Tools []map[string]interface{} `yaml:"tools"` + } + if err := yaml.Unmarshal([]byte(sections[1]), &frontmatter); err != nil { + t.Fatalf("failed to parse tools_function.prompty frontmatter: %v", err) + } + if len(frontmatter.Tools) != 1 { + t.Fatalf("tools_function.prompty contains %d tools, expected one", len(frontmatter.Tools)) + } + + tool, err := prompty.LoadFunctionTool(frontmatter.Tools[0], prompty.NewLoadContext()) + if err != nil { + t.Fatalf("failed to load FunctionTool: %v", err) + } + for name, expected := range expectedBindings { + var actual *prompty.Binding + for index := range tool.Bindings { + if tool.Bindings[index].Name == name { + actual = &tool.Bindings[index] + break + } + } + if actual == nil { + t.Fatalf("missing binding %q", name) + } + if actual.Input != expected.Input { + t.Fatalf("binding %q input: expected %q, got %q", name, expected.Input, actual.Input) + } + } + if len(tool.Bindings) != len(expectedBindings) { + t.Fatalf("expected %d bindings, got %d", len(expectedBindings), len(tool.Bindings)) + } +} diff --git a/runtime/python/prompty/tests/test_spec_vectors.py b/runtime/python/prompty/tests/test_spec_vectors.py index 29eba1df6..973d25205 100644 --- a/runtime/python/prompty/tests/test_spec_vectors.py +++ b/runtime/python/prompty/tests/test_spec_vectors.py @@ -497,6 +497,8 @@ def _check_tools(actual: list, expected: list[dict], errors: list[str]): act_bindings = getattr(act, "bindings", []) or [] exp_bindings = exp["bindings"] if isinstance(exp_bindings, dict): + if len(act_bindings) != len(exp_bindings): + errors.append(f" {prefix}.bindings: count {len(act_bindings)} != expected {len(exp_bindings)}") for bname, bval in exp_bindings.items(): found = [b for b in act_bindings if b.name == bname] if not found: diff --git a/runtime/rust/prompty/tests/loader_test.rs b/runtime/rust/prompty/tests/loader_test.rs index 07ffeb161..e1af167df 100644 --- a/runtime/rust/prompty/tests/loader_test.rs +++ b/runtime/rust/prompty/tests/loader_test.rs @@ -20,6 +20,16 @@ fn fixtures_dir() -> PathBuf { .join("fixtures") } +/// Path to the canonical load vectors. +fn load_vectors_path() -> PathBuf { + fixtures_dir() + .parent() + .expect("spec/fixtures must have a spec parent") + .join("vectors") + .join("load") + .join("load_vectors.json") +} + /// Load a fixture `.prompty` file with optional env vars set. fn load_fixture( name: &str, @@ -260,6 +270,32 @@ fn test_tools_function_load() { assert_eq!(tools.len(), 1); assert_eq!(tools[0].name, "get_weather"); assert_eq!(tools[0].kind_str(), "function"); + + let raw = std::fs::read_to_string(load_vectors_path()).unwrap(); + let vectors: serde_json::Value = serde_json::from_str(&raw).unwrap(); + let vector = vectors + .as_array() + .unwrap() + .iter() + .find(|vector| vector["name"] == "tools_function_load") + .unwrap(); + let expected_bindings = vector["expected"]["tools"][0]["bindings"] + .as_object() + .unwrap(); + + assert_eq!(tools[0].bindings.len(), expected_bindings.len()); + for (name, expected) in expected_bindings { + let actual = tools[0] + .bindings + .iter() + .find(|binding| binding.name == *name) + .unwrap_or_else(|| panic!("missing binding {name:?}")); + assert_eq!( + actual.input, + expected["input"].as_str().unwrap(), + "binding {name:?} input mismatch" + ); + } } #[test] diff --git a/runtime/typescript/packages/core/tests/spec-vectors.test.ts b/runtime/typescript/packages/core/tests/spec-vectors.test.ts index fdc30b66c..d369cedb3 100644 --- a/runtime/typescript/packages/core/tests/spec-vectors.test.ts +++ b/runtime/typescript/packages/core/tests/spec-vectors.test.ts @@ -430,11 +430,12 @@ function validateAgentFields(agent: Prompty, expected: any, vecName: string): vo } } if (et.bindings !== undefined) { - const atBindings = (at as any).bindings as Array<{name: string; input: string}>; + const atBindings = (at as FunctionTool).bindings ?? []; + const expectedBindings = Object.entries(et.bindings as Record); expect(atBindings).toBeDefined(); - expect(atBindings.length).toBeGreaterThan(0); - for (const [bk, bv] of Object.entries(et.bindings as Record)) { - const found = atBindings.find((b: any) => b.name === bk); + expect(atBindings).toHaveLength(expectedBindings.length); + for (const [bk, bv] of expectedBindings) { + const found = atBindings.find((binding) => binding.name === bk); expect(found).toBeDefined(); if (bv.input !== undefined) { expect(found!.input).toBe(bv.input); From 70c98437f98533687f040de2fb22fbc65aa17506 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 14:10:42 -0700 Subject: [PATCH 15/20] test(model): pin scalar property coercion Define direct Property scalar coercion separately from named input shorthand and add one atomic shared vector across C#, Go, Python, Rust, and TypeScript. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../PropertyScalarCoercionVectorTests.cs | 63 +++++++++++ .../property_scalar_coercion_vectors_test.go | 101 ++++++++++++++++++ .../test_property_scalar_coercion_vectors.py | 29 +++++ .../tests/property_scalar_coercion_vectors.rs | 68 ++++++++++++ .../property-scalar-coercion-vectors.test.ts | 55 ++++++++++ spec/spec.md | 9 ++ .../property_scalar_coercion_vectors.json | 44 ++++++++ 7 files changed, 369 insertions(+) create mode 100644 runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs create mode 100644 runtime/go/prompty/model/property_scalar_coercion_vectors_test.go create mode 100644 runtime/python/prompty/tests/test_property_scalar_coercion_vectors.py create mode 100644 runtime/rust/prompty/tests/property_scalar_coercion_vectors.rs create mode 100644 runtime/typescript/packages/core/tests/property-scalar-coercion-vectors.test.ts create mode 100644 spec/vectors/model/property_scalar_coercion_vectors.json diff --git a/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs new file mode 100644 index 000000000..2bec2a037 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class PropertyScalarCoercionVectorTests +{ + private static readonly string VectorsPath = FindVectorsPath(); + + [Fact] + public void AllPrimitivePropertyScalarsCoerceAtomically() + { + using var document = JsonDocument.Parse(File.ReadAllText(VectorsPath)); + var vectors = document.RootElement.GetProperty("vectors").EnumerateArray().ToArray(); + Assert.Single(vectors); + + var vector = vectors[0]; + Assert.Equal("all_primitive_property_scalars_coerce_atomically", vector.GetProperty("name").GetString()); + Assert.Equal("load", vector.GetProperty("operation").GetString()); + + var cases = vector.GetProperty("cases").EnumerateArray().ToArray(); + Assert.Equal( + ["string", "integer", "float", "boolean"], + cases.Select(candidate => candidate.GetProperty("name").GetString()).ToArray()); + + foreach (var scalarCase in cases) + { + var name = scalarCase.GetProperty("name").GetString(); + var expected = scalarCase.GetProperty("expected"); + var loaded = Property.FromJson(scalarCase.GetProperty("input").GetRawText()); + + Assert.Equal(expected.GetProperty("kind").GetString(), loaded.Kind); + Assert.True( + JsonNode.DeepEquals( + JsonNode.Parse(expected.GetProperty("example").GetRawText()), + JsonNode.Parse(JsonSerializer.Serialize(loaded.Example))), + $"[{name}] changed or dropped the Property example."); + } + } + + private static string FindVectorsPath() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine( + directory, + "spec", + "vectors", + "model", + "property_scalar_coercion_vectors.json"); + if (File.Exists(candidate)) + return candidate; + + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new FileNotFoundException("Could not locate the shared Property scalar coercion vectors."); + } +} diff --git a/runtime/go/prompty/model/property_scalar_coercion_vectors_test.go b/runtime/go/prompty/model/property_scalar_coercion_vectors_test.go new file mode 100644 index 000000000..a50c22f61 --- /dev/null +++ b/runtime/go/prompty/model/property_scalar_coercion_vectors_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft. All rights reserved. + +package prompty_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "prompty/model" +) + +type propertyScalarCoercionVectorDocument struct { + Vectors []propertyScalarCoercionVector `json:"vectors"` +} + +type propertyScalarCoercionVector struct { + Name string `json:"name"` + Operation string `json:"operation"` + Cases []propertyScalarCoercionCase `json:"cases"` +} + +type propertyScalarCoercionCase struct { + Name string `json:"name"` + Input json.RawMessage `json:"input"` + Expected struct { + Kind string `json:"kind"` + Example interface{} `json:"example"` + } `json:"expected"` +} + +func TestAllPrimitivePropertyScalarsCoerceAtomically(t *testing.T) { + path := filepath.Join( + "..", + "..", + "..", + "..", + "spec", + "vectors", + "model", + "property_scalar_coercion_vectors.json", + ) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read Property scalar coercion vectors: %v", err) + } + + var document propertyScalarCoercionVectorDocument + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatalf("failed to parse Property scalar coercion vectors: %v", err) + } + if len(document.Vectors) != 1 { + t.Fatalf("expected one atomic Property scalar coercion vector, got %d", len(document.Vectors)) + } + vector := document.Vectors[0] + if vector.Name != "all_primitive_property_scalars_coerce_atomically" || vector.Operation != "load" { + t.Fatalf("unexpected Property scalar coercion vector %q operation %q", vector.Name, vector.Operation) + } + expectedNames := []string{"string", "integer", "float", "boolean"} + if len(vector.Cases) != len(expectedNames) { + t.Fatalf("expected all four primitive scalar cases, got %d", len(vector.Cases)) + } + + for index, scalarCase := range vector.Cases { + if scalarCase.Name != expectedNames[index] { + t.Fatalf("scalar case %d: expected %q, got %q", index, expectedNames[index], scalarCase.Name) + } + loaded, err := prompty.PropertyFromJSON(string(scalarCase.Input)) + if err != nil { + t.Errorf("[%s] load failed: %v", scalarCase.Name, err) + continue + } + property, ok := loaded.(prompty.Property) + if !ok { + t.Errorf("[%s] expected Property, got %T", scalarCase.Name, loaded) + continue + } + if property.Kind != scalarCase.Expected.Kind { + t.Errorf("[%s] expected kind %q, got %q", scalarCase.Name, scalarCase.Expected.Kind, property.Kind) + continue + } + if property.Example == nil { + t.Errorf("[%s] expected example, got nil", scalarCase.Name) + continue + } + actualJSON, err := json.Marshal(*property.Example) + if err != nil { + t.Errorf("[%s] failed to encode actual example: %v", scalarCase.Name, err) + continue + } + expectedJSON, err := json.Marshal(scalarCase.Expected.Example) + if err != nil { + t.Errorf("[%s] failed to encode expected example: %v", scalarCase.Name, err) + continue + } + if string(actualJSON) != string(expectedJSON) { + t.Errorf("[%s] expected example %s, got %s", scalarCase.Name, expectedJSON, actualJSON) + } + } +} diff --git a/runtime/python/prompty/tests/test_property_scalar_coercion_vectors.py b/runtime/python/prompty/tests/test_property_scalar_coercion_vectors.py new file mode 100644 index 000000000..d5565a0c5 --- /dev/null +++ b/runtime/python/prompty/tests/test_property_scalar_coercion_vectors.py @@ -0,0 +1,29 @@ +"""Validate the canonical atomic Property scalar coercion contract.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from prompty.model import Property + +_VECTORS_PATH = ( + Path(__file__).resolve().parents[4] / "spec" / "vectors" / "model" / "property_scalar_coercion_vectors.json" +) + + +def test_all_primitive_property_scalars_coerce_atomically() -> None: + """Infer and preserve every primitive scalar coercion branch.""" + + document: dict[str, Any] = json.loads(_VECTORS_PATH.read_text(encoding="utf-8")) + vector = document["vectors"][0] + assert vector["name"] == "all_primitive_property_scalars_coerce_atomically" + assert vector["operation"] == "load" + assert [case["name"] for case in vector["cases"]] == ["string", "integer", "float", "boolean"] + + for case in vector["cases"]: + loaded = Property.load(case["input"]) + assert loaded.kind == case["expected"]["kind"], case["name"] + assert type(loaded.example) is type(case["expected"]["example"]), case["name"] + assert loaded.example == case["expected"]["example"], case["name"] diff --git a/runtime/rust/prompty/tests/property_scalar_coercion_vectors.rs b/runtime/rust/prompty/tests/property_scalar_coercion_vectors.rs new file mode 100644 index 000000000..2caf44422 --- /dev/null +++ b/runtime/rust/prompty/tests/property_scalar_coercion_vectors.rs @@ -0,0 +1,68 @@ +//! Cross-runtime Property scalar coercion tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::LoadContext; +use prompty::model::Property; +use serde_json::Value; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("property_scalar_coercion_vectors.json") +} + +#[test] +fn all_primitive_property_scalars_coerce_atomically() { + let raw = std::fs::read_to_string(vectors_path()) + .expect("failed to read Property scalar coercion vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse Property scalar coercion vectors"); + let vector = &document["vectors"][0]; + assert_eq!( + vector["name"], "all_primitive_property_scalars_coerce_atomically", + "unexpected Property scalar coercion vector" + ); + assert_eq!(vector["operation"], "load"); + + let cases = vector["cases"] + .as_array() + .expect("Property scalar coercion vector must contain cases"); + let case_names: Vec<&str> = cases + .iter() + .map(|case| case["name"].as_str().expect("scalar case must have a name")) + .collect(); + assert_eq!(case_names, ["string", "integer", "float", "boolean"]); + + let context = LoadContext::default(); + let mut failures = Vec::new(); + for case in cases { + let case_name = case["name"].as_str().expect("scalar case must have a name"); + let loaded = Property::load_from_value(&case["input"], &context); + let expected_kind = case["expected"]["kind"] + .as_str() + .expect("expected kind must be a string"); + if loaded.kind_str() != expected_kind { + failures.push(format!( + "[{case_name}] expected kind {expected_kind:?}, got {:?}", + loaded.kind_str() + )); + continue; + } + if loaded.example.as_ref() != Some(&case["expected"]["example"]) { + failures.push(format!( + "[{case_name}] expected example {}, got {:?}", + case["expected"]["example"], loaded.example + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/runtime/typescript/packages/core/tests/property-scalar-coercion-vectors.test.ts b/runtime/typescript/packages/core/tests/property-scalar-coercion-vectors.test.ts new file mode 100644 index 000000000..92190b5c9 --- /dev/null +++ b/runtime/typescript/packages/core/tests/property-scalar-coercion-vectors.test.ts @@ -0,0 +1,55 @@ +/** + * Canonical atomic Property scalar coercion tests backed by shared model vectors. + */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { Property } from "../src/index.js"; + +interface PropertyScalarCase { + name: string; + input: string | number | boolean; + expected: { + kind: string; + example: string | number | boolean; + }; +} + +interface PropertyScalarVector { + name: string; + operation: "load"; + cases: PropertyScalarCase[]; +} + +const vectorsPath = resolve( + import.meta.dirname, + "../../../../../spec/vectors/model/property_scalar_coercion_vectors.json", +); +const vector = ( + JSON.parse(readFileSync(vectorsPath, "utf8")) as { + vectors: PropertyScalarVector[]; + } +).vectors[0]; + +describe("Property scalar coercion vectors", () => { + it("coerces all primitive scalar branches atomically", () => { + expect(vector.name).toBe("all_primitive_property_scalars_coerce_atomically"); + expect(vector.operation).toBe("load"); + expect(vector.cases.map((candidate) => candidate.name)).toEqual([ + "string", + "integer", + "float", + "boolean", + ]); + + for (const scalarCase of vector.cases) { + const loaded = Property.fromJson(JSON.stringify(scalarCase.input)); + expect(loaded.kind, scalarCase.name).toBe(scalarCase.expected.kind); + expect(loaded.example, scalarCase.name).toEqual( + scalarCase.expected.example, + ); + } + }); +}); diff --git a/spec/spec.md b/spec/spec.md index 3ede96860..86d1badcd 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -453,6 +453,15 @@ Kind inference from scalar type: string → `"string"`, integer → `"integer"`, float → `"float"`, boolean → `"boolean"`, list → `"array"`, dict → `"object"`. The named-collection object-form exception for list values is defined above. +This named-collection shorthand is distinct from direct generated-model coercion. +When a generated `Property` loader receives a scalar as its complete input, the +TypeSpec `@coerce` contract MUST infer the same scalar kind and store the scalar in +`example`; it MUST NOT drop or coerce the value. JSON integer and fractional number +inputs MUST remain distinguishable as `"integer"` and `"float"` respectively. The +four primitive scalar branches are an atomic contract: string, integer, float, and +boolean MUST all be supported. The normative acceptance vector is +`spec/vectors/model/property_scalar_coercion_vectors.json`. + ### §2.8 Template The `template` property selects the template engine and parser. diff --git a/spec/vectors/model/property_scalar_coercion_vectors.json b/spec/vectors/model/property_scalar_coercion_vectors.json new file mode 100644 index 000000000..90b622d0b --- /dev/null +++ b/spec/vectors/model/property_scalar_coercion_vectors.json @@ -0,0 +1,44 @@ +{ + "version": "1", + "description": "Atomic cross-runtime Property scalar coercion contract. Direct generated-model JSON loading infers the exact primitive kind and stores the unmodified scalar in example. All four cases are required together.", + "vectors": [ + { + "name": "all_primitive_property_scalars_coerce_atomically", + "operation": "load", + "cases": [ + { + "name": "string", + "input": "example", + "expected": { + "kind": "string", + "example": "example" + } + }, + { + "name": "integer", + "input": 4, + "expected": { + "kind": "integer", + "example": 4 + } + }, + { + "name": "float", + "input": 3.14, + "expected": { + "kind": "float", + "example": 3.14 + } + }, + { + "name": "boolean", + "input": false, + "expected": { + "kind": "boolean", + "example": false + } + } + ] + } + ] +} From 400ad4ec736a69844ceae1eacfca938210d9fa68 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 14:22:09 -0700 Subject: [PATCH 16/20] test(model): pin named scalar properties Define collection-context scalar shorthand precedence and add focused string, integer, float, and boolean vectors with a Rust no-degradation gate. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../prompty/tests/named_collection_vectors.rs | 94 ++++++++++++++++++ spec/spec.md | 7 ++ .../model/named_collection_vectors.json | 98 ++++++++++++++++++- 3 files changed, 198 insertions(+), 1 deletion(-) diff --git a/runtime/rust/prompty/tests/named_collection_vectors.rs b/runtime/rust/prompty/tests/named_collection_vectors.rs index 4f7c4849d..e6df171cd 100644 --- a/runtime/rust/prompty/tests/named_collection_vectors.rs +++ b/runtime/rust/prompty/tests/named_collection_vectors.rs @@ -105,6 +105,18 @@ fn assert_collection(vector_name: &str, collection: &Value, expected: &Value) { expected_entries.len(), "[{vector_name}] named collection entry count changed" ); + if let Some(absent_fields) = expected["absentEntryFields"].as_array() { + for entry in &actual_entries { + for field in absent_fields { + let field = field.as_str().expect("absent entry field must be a string"); + assert!( + entry.get(field).is_none(), + "[{vector_name}] entry {:?} unexpectedly populated field {field:?}", + entry["name"] + ); + } + } + } if expected["preserveOrder"].as_bool() == Some(true) { for (index, expected_entry) in expected_entries.iter().enumerate() { @@ -188,6 +200,88 @@ fn named_collection_roundtrip_vectors() { } } +#[test] +fn name_keyed_property_scalars_infer_kind_and_default_without_degradation() { + let vector_names = [ + "string_scalar_in_name_keyed_inputs_infers_property", + "integer_scalar_in_name_keyed_inputs_infers_property", + "float_scalar_in_name_keyed_inputs_infers_property", + "boolean_scalar_in_name_keyed_inputs_infers_property", + ]; + let all_vectors = vectors(); + let mut failures = Vec::new(); + + for vector_name in vector_names { + let vector = all_vectors + .iter() + .find(|candidate| candidate["name"] == vector_name) + .unwrap_or_else(|| panic!("missing named collection vector {vector_name}")); + let json = serde_json::to_string(&vector["input"]) + .expect("named collection vector input must be JSON-compatible"); + let loaded = match Prompty::from_json(&json, &LoadContext::default()) { + Ok(loaded) => loaded, + Err(error) => { + failures.push(format!("[{vector_name}] valid scalar failed: {error}")); + continue; + } + }; + let saved = loaded.to_value(&SaveContext::default()); + let collection_path = vector["collectionPath"] + .as_str() + .expect("scalar vector must declare collectionPath"); + let collection = match saved.get(collection_path) { + Some(collection) => collection, + None => { + failures.push(format!( + "[{vector_name}] missing saved collection {collection_path:?}" + )); + continue; + } + }; + let actual_entries = semantic_entries(collection); + let expected_entry = &vector["expected"]["entries"][0]; + let expected_name = expected_entry["name"] + .as_str() + .expect("expected scalar entry name must be a string"); + let actual_entry = match actual_entries + .iter() + .find(|entry| entry["name"] == expected_name) + { + Some(entry) => entry, + None => { + failures.push(format!( + "[{vector_name}] missing scalar entry {expected_name:?}" + )); + continue; + } + }; + + let expected_kind = &expected_entry["kind"]; + if actual_entry["kind"].as_str().unwrap_or_default().is_empty() { + failures.push(format!("[{vector_name}] silently produced an empty kind")); + } else if actual_entry["kind"] != *expected_kind { + failures.push(format!( + "[{vector_name}] expected kind {expected_kind}, got {}", + actual_entry["kind"] + )); + } + if actual_entry["default"] != expected_entry["default"] { + failures.push(format!( + "[{vector_name}] expected default {}, got {}", + expected_entry["default"], actual_entry["default"] + )); + } + if let Some(example) = actual_entry.get("example") { + failures.push(format!( + "[{vector_name}] collection shorthand unexpectedly populated example {}", + example + )); + } + } + + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + #[test] fn named_collection_rejection_vectors() { for vector in vectors() diff --git a/spec/spec.md b/spec/spec.md index 86d1badcd..b6235b69b 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -436,6 +436,13 @@ The normative load/save/reload and rejection cases are **Scalar shorthand**: When a property value is a plain scalar instead of a `Property` object, it MUST be interpreted as `Property(kind: , default: )`. +This applies to every immediate string, integer, float, or boolean value in the +name-keyed object form of a `Property` collection. The object key supplies `name`. +Implementations MUST NOT reinterpret the scalar as the `kind` field, reject a valid +primitive scalar, or silently produce an empty `kind`. At a name-keyed collection +boundary, this normalization to `default` MUST occur before direct generated-model +`@coerce` handling; the direct-coercion `example` behavior MUST NOT apply to the +collection entry, and `example` MUST remain unset. ```yaml # Shorthand diff --git a/spec/vectors/model/named_collection_vectors.json b/spec/vectors/model/named_collection_vectors.json index 6864949fe..7de39756f 100644 --- a/spec/vectors/model/named_collection_vectors.json +++ b/spec/vectors/model/named_collection_vectors.json @@ -1,6 +1,6 @@ { "version": "1", - "description": "Cross-runtime named-collection load/save/reload contracts. Canonical serialization uses a name-keyed object only when every parsed name is non-empty and unique; otherwise it uses a whole-collection array fallback without omission, collision, or synthetic names. Array-valued entries in name-keyed object form are rejected recursively, while arrays in declared entry fields remain valid.", + "description": "Cross-runtime named-collection load/save/reload contracts. Canonical serialization uses a name-keyed object only when every parsed name is non-empty and unique; otherwise it uses a whole-collection array fallback without omission, collision, or synthetic names. Immediate primitive Property values infer kind and default without leaking direct-coercion example semantics. Array-valued entries in name-keyed object form are rejected recursively, while arrays in declared entry fields remain valid.", "vectors": [ { "name": "unique_names_use_canonical_object_form", @@ -221,6 +221,102 @@ ] } }, + { + "name": "string_scalar_in_name_keyed_inputs_infers_property", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "string-scalar-input", + "inputs": { + "city": "Seattle" + } + }, + "expected": { + "collectionFormat": "object", + "absentEntryFields": [ + "example" + ], + "entries": [ + { + "name": "city", + "kind": "string", + "default": "Seattle" + } + ] + } + }, + { + "name": "integer_scalar_in_name_keyed_inputs_infers_property", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "integer-scalar-input", + "inputs": { + "city": 3 + } + }, + "expected": { + "collectionFormat": "object", + "absentEntryFields": [ + "example" + ], + "entries": [ + { + "name": "city", + "kind": "integer", + "default": 3 + } + ] + } + }, + { + "name": "float_scalar_in_name_keyed_inputs_infers_property", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "float-scalar-input", + "inputs": { + "city": 1.5 + } + }, + "expected": { + "collectionFormat": "object", + "absentEntryFields": [ + "example" + ], + "entries": [ + { + "name": "city", + "kind": "float", + "default": 1.5 + } + ] + } + }, + { + "name": "boolean_scalar_in_name_keyed_inputs_infers_property", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "boolean-scalar-input", + "inputs": { + "city": true + } + }, + "expected": { + "collectionFormat": "object", + "absentEntryFields": [ + "example" + ], + "entries": [ + { + "name": "city", + "kind": "boolean", + "default": true + } + ] + } + }, { "name": "scalar_array_shorthand_in_name_keyed_inputs_is_rejected", "operation": "load-error", From ec875140a5ba8412f4e77133b11d9afcc67d0d73 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 14:32:25 -0700 Subject: [PATCH 17/20] test(csharp): enforce null-valued records Consume the shared Record vectors in C# and assert exact model-field coverage plus load/save/reload preservation of direct, nested, and list nulls. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../PropertyScalarCoercionVectorTests.cs | 2 +- .../RecordUnknownNullabilityVectorTests.cs | 120 ++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilityVectorTests.cs diff --git a/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs index 2bec2a037..ebe2e480d 100644 --- a/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs @@ -24,7 +24,7 @@ public void AllPrimitivePropertyScalarsCoerceAtomically() var cases = vector.GetProperty("cases").EnumerateArray().ToArray(); Assert.Equal( ["string", "integer", "float", "boolean"], - cases.Select(candidate => candidate.GetProperty("name").GetString()).ToArray()); + cases.Select(candidate => candidate.GetProperty("name").GetString()!).ToArray()); foreach (var scalarCase in cases) { diff --git a/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilityVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilityVectorTests.cs new file mode 100644 index 000000000..bdb54f058 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilityVectorTests.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class RecordUnknownNullabilityVectorTests +{ + private static readonly string VectorsPath = FindVectorsPath(); + + [Fact] + public void RecordUnknownNullabilityVectors_PreserveExplicitNullValues() + { + using var document = JsonDocument.Parse(File.ReadAllText(VectorsPath)); + var vectors = document.RootElement.GetProperty("vectors").EnumerateArray().ToArray(); + Assert.Equal(9, vectors.Length); + var expectedCoverage = new HashSet(StringComparer.Ordinal) + { + "Message:metadata", + "Prompty:metadata", + "ModelInfo:additionalProperties", + "TurnModelRequest:inputs", + "RunTurnRequest:inputs", + "TurnModelResponse:checkpointState", + "HostToolRequest:arguments", + "TurnEvent:payload", + "SessionEvent:payload", + }; + var actualCoverage = vectors + .Select(vector => $"{vector.GetProperty("model").GetString()}:{vector.GetProperty("fieldPath").GetString()}") + .ToHashSet(StringComparer.Ordinal); + Assert.True( + expectedCoverage.SetEquals(actualCoverage), + $"Record vector coverage changed.\nExpected: {string.Join(", ", expectedCoverage)}\n" + + $"Actual: {string.Join(", ", actualCoverage)}"); + + foreach (var vector in vectors) + { + var name = vector.GetProperty("name").GetString()!; + Assert.Equal("load-save-reload", vector.GetProperty("operation").GetString()); + + var model = vector.GetProperty("model").GetString()!; + var fieldPath = vector.GetProperty("fieldPath").GetString()!; + var resaved = Roundtrip(model, vector.GetProperty("input").GetRawText()); + var actual = resaved.GetProperty(fieldPath); + var expected = vector.GetProperty("expected"); + + Assert.True( + actual.TryGetProperty("direct", out var direct) && direct.ValueKind == JsonValueKind.Null, + $"[{name}] direct null-valued key was lost or changed."); + Assert.True( + JsonNode.DeepEquals(JsonNode.Parse(expected.GetRawText()), JsonNode.Parse(actual.GetRawText())), + $"[{name}] load/save/reload changed null-valued record entries.\nExpected: {expected}\nActual: {actual}"); + } + } + + private static JsonElement Roundtrip(string model, string input) + { + var resaved = model switch + { + "Message" => Roundtrip(input, json => Message.FromJson(json), value => value.ToJson()), + "Prompty" => Roundtrip(input, json => Prompty.FromJson(json), value => value.ToJson()), + "ModelInfo" => Roundtrip(input, json => ModelInfo.FromJson(json), value => value.ToJson()), + "TurnModelRequest" => Roundtrip( + input, + json => TurnModelRequest.FromJson(json), + value => value.ToJson()), + "RunTurnRequest" => Roundtrip( + input, + json => RunTurnRequest.FromJson(json), + value => value.ToJson()), + "TurnModelResponse" => Roundtrip( + input, + json => TurnModelResponse.FromJson(json), + value => value.ToJson()), + "HostToolRequest" => Roundtrip( + input, + json => HostToolRequest.FromJson(json), + value => value.ToJson()), + "TurnEvent" => Roundtrip(input, json => TurnEvent.FromJson(json), value => value.ToJson()), + "SessionEvent" => Roundtrip( + input, + json => SessionEvent.FromJson(json), + value => value.ToJson()), + _ => throw new InvalidOperationException($"Unsupported vector model '{model}'."), + }; + using var document = JsonDocument.Parse(resaved); + return document.RootElement.Clone(); + } + + private static string Roundtrip(string input, Func load, Func save) + { + var loaded = load(input); + var saved = save(loaded); + var reloaded = load(saved); + return save(reloaded); + } + + private static string FindVectorsPath() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine( + directory, + "spec", + "vectors", + "model", + "record_unknown_nullability_vectors.json"); + if (File.Exists(candidate)) + return candidate; + + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new FileNotFoundException("Could not locate the shared Record nullability vectors."); + } +} From c6bdfb29e904a3dfc2b60b40544bdc620b77f36f Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 14:38:25 -0700 Subject: [PATCH 18/20] test(csharp): pin nullable record signatures Assert the canonical IDictionary mapping and outer optionality for all nine shared Record vector surfaces. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../RecordUnknownNullabilitySignatureTests.cs | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilitySignatureTests.cs diff --git a/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilitySignatureTests.cs b/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilitySignatureTests.cs new file mode 100644 index 000000000..6594b74e1 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilitySignatureTests.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Reflection; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class RecordUnknownNullabilitySignatureTests +{ + [Fact] + public void RecordUnknownProperties_ExposeCanonicalNullableValueSignatures() + { + var cases = new (Type Model, string Property, NullabilityState Presence)[] + { + (typeof(Message), nameof(Message.Metadata), NullabilityState.NotNull), + (typeof(Prompty), nameof(Prompty.Metadata), NullabilityState.Nullable), + (typeof(ModelInfo), nameof(ModelInfo.AdditionalProperties), NullabilityState.Nullable), + (typeof(TurnModelRequest), nameof(TurnModelRequest.Inputs), NullabilityState.Nullable), + (typeof(RunTurnRequest), nameof(RunTurnRequest.Inputs), NullabilityState.Nullable), + (typeof(TurnModelResponse), nameof(TurnModelResponse.CheckpointState), NullabilityState.Nullable), + (typeof(HostToolRequest), nameof(HostToolRequest.Arguments), NullabilityState.Nullable), + (typeof(TurnEvent), nameof(TurnEvent.Payload), NullabilityState.NotNull), + (typeof(SessionEvent), nameof(SessionEvent.Payload), NullabilityState.NotNull), + }; + var context = new NullabilityInfoContext(); + + foreach (var (model, propertyName, expectedPresence) in cases) + { + var property = model.GetProperty(propertyName) + ?? throw new InvalidOperationException($"{model.Name}.{propertyName} does not exist."); + Assert.Equal(typeof(IDictionary<,>), property.PropertyType.GetGenericTypeDefinition()); + + var nullability = context.Create(property); + Assert.Equal(expectedPresence, nullability.ReadState); + Assert.Equal(typeof(string), nullability.GenericTypeArguments[0].Type); + Assert.Equal(NullabilityState.NotNull, nullability.GenericTypeArguments[0].ReadState); + Assert.Equal(typeof(object), nullability.GenericTypeArguments[1].Type); + Assert.Equal(NullabilityState.Nullable, nullability.GenericTypeArguments[1].ReadState); + } + } +} From 772e9e91c125071ec71d0893fb30d4f951e62722 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 15:09:47 -0700 Subject: [PATCH 19/20] Fix FunctionTool parameter example Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- spec/spec.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/spec/spec.md b/spec/spec.md index b6235b69b..76fef3c8c 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -534,13 +534,12 @@ tools: kind: function description: Get orders for a user parameters: - properties: - - name: user_id - kind: string - required: true - - name: limit - kind: integer - default: 10 + - name: user_id + kind: string + required: true + - name: limit + kind: integer + default: 10 bindings: user_id: ${env:CURRENT_USER_ID} ``` From 7837c4f21197fea86745093f50814c96a72100c4 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 15:30:19 -0700 Subject: [PATCH 20/20] test(model): gate unnamed composite fallback Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../prompty/tests/named_collection_vectors.rs | 71 +++++++++++++++++++ .../model/named_collection_vectors.json | 45 ++++++++++++ 2 files changed, 116 insertions(+) diff --git a/runtime/rust/prompty/tests/named_collection_vectors.rs b/runtime/rust/prompty/tests/named_collection_vectors.rs index e6df171cd..2b1174f3e 100644 --- a/runtime/rust/prompty/tests/named_collection_vectors.rs +++ b/runtime/rust/prompty/tests/named_collection_vectors.rs @@ -96,6 +96,35 @@ fn assert_collection(vector_name: &str, collection: &Value, expected: &Value) { "[{vector_name}] canonical collection format changed" ); + if let Some(wire_entries) = expected["wireEntries"].as_array() { + let entries = collection + .as_array() + .unwrap_or_else(|| panic!("[{vector_name}] wire entry assertions require array form")); + for assertion in wire_entries { + let index = assertion["index"] + .as_u64() + .expect("wire entry assertion must declare an index") + as usize; + let entry = entries + .get(index) + .unwrap_or_else(|| panic!("[{vector_name}] missing wire entry at index {index}")) + .as_object() + .unwrap_or_else(|| { + panic!("[{vector_name}] wire entry at index {index} must be an object") + }); + for field in assertion["absentFields"] + .as_array() + .expect("wire entry assertion must declare absentFields") + { + let field = field.as_str().expect("wire absent field must be a string"); + assert!( + !entry.contains_key(field), + "[{vector_name}] wire entry {index} unexpectedly serialized field {field:?}" + ); + } + } + } + let actual_entries = semantic_entries(collection); let expected_entries = expected["entries"] .as_array() @@ -200,6 +229,48 @@ fn named_collection_roundtrip_vectors() { } } +#[test] +fn unnamed_composite_omits_empty_name_stably() { + let vector_name = "unnamed_composite_omits_empty_name_stably"; + let vector = vectors() + .into_iter() + .find(|vector| vector["name"] == vector_name) + .expect("missing unnamed composite vector"); + let json = serde_json::to_string(&vector["input"]) + .expect("unnamed composite vector input must be JSON-compatible"); + let loaded = Prompty::from_json(&json, &LoadContext::default()) + .unwrap_or_else(|error| panic!("[{vector_name}] valid collection failed: {error}")); + + assert_eq!( + loaded.inputs.len(), + 1, + "[{vector_name}] load changed the entry count" + ); + assert_eq!( + loaded.inputs[0].name, "", + "[{vector_name}] absent wire name did not materialize as an empty in-memory name" + ); + + let saved = loaded.to_value(&SaveContext::default()); + let collection = saved + .get("inputs") + .expect("[unnamed_composite_omits_empty_name_stably] first save lost inputs"); + assert_collection(vector_name, collection, &vector["expected"]); + + let saved_json = serde_json::to_string(&saved).expect("first save must remain JSON-compatible"); + let reloaded = Prompty::from_json(&saved_json, &LoadContext::default()) + .unwrap_or_else(|error| panic!("[{vector_name}] first save failed to reload: {error}")); + assert_eq!( + reloaded.inputs[0].name, "", + "[{vector_name}] reload changed the unnamed in-memory state" + ); + let resaved = reloaded.to_value(&SaveContext::default()); + let reloaded_collection = resaved + .get("inputs") + .expect("[unnamed_composite_omits_empty_name_stably] reload/save lost inputs"); + assert_collection(vector_name, reloaded_collection, &vector["expected"]); +} + #[test] fn name_keyed_property_scalars_infer_kind_and_default_without_degradation() { let vector_names = [ diff --git a/spec/vectors/model/named_collection_vectors.json b/spec/vectors/model/named_collection_vectors.json index 7de39756f..9c69f9804 100644 --- a/spec/vectors/model/named_collection_vectors.json +++ b/spec/vectors/model/named_collection_vectors.json @@ -152,6 +152,51 @@ ] } }, + { + "name": "unnamed_composite_omits_empty_name_stably", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "unnamed-composite", + "inputs": [ + { + "kind": "object", + "description": "preserve unnamed composite payload", + "properties": { + "nested": { + "kind": "string", + "default": "kept" + } + } + } + ] + }, + "expected": { + "collectionFormat": "array", + "preserveOrder": true, + "wireEntries": [ + { + "index": 0, + "absentFields": [ + "name" + ] + } + ], + "entries": [ + { + "name": "", + "kind": "object", + "description": "preserve unnamed composite payload", + "properties": { + "nested": { + "kind": "string", + "default": "kept" + } + } + } + ] + } + }, { "name": "empty_object_key_reloads_as_unnamed_array_entry", "operation": "load-save-reload",