diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs index d73197cb3..63c0cdea5 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs @@ -9,6 +9,17 @@ namespace Prompty.Anthropic.Tests; /// public class AnthropicExecutorTests { + [Fact] + public async Task ExecuteAsync_Cancelled_ThrowsBeforeConnectionValidation() + { + var executor = new Anthropic.AnthropicExecutor(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => executor.ExecuteAsync(new Core.Prompty(), [], cancellation.Token)); + } + [Fact] public async Task ExecuteAsync_MissingApiKey_ThrowsInvalidOperationException() { @@ -156,4 +167,3 @@ public void FormatToolMessages_NoTextContent_OmitsTextBlock() Assert.Equal("tool_use", content[0]["type"]); } } - diff --git a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs index a64f6198d..5b6117e6a 100644 --- a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs +++ b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs @@ -19,30 +19,40 @@ public class AnthropicExecutor : IExecutor private const string ApiVersion = "2023-06-01"; private const int DefaultMaxTokens = 4096; - public async Task ExecuteAsync(Core.Prompty agent, List messages) + public async Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); var streaming = agent.Metadata?.TryGetValue("stream", out var streamVal) == true && streamVal is true; if (streaming) - return ExecuteStreamAsync(agent, messages); + return ExecuteStreamAsync(agent, messages, cancellationToken); - return await ExecuteNonStreamAsync(agent, messages); + return await ExecuteNonStreamAsync(agent, messages, cancellationToken); } - private async Task ExecuteNonStreamAsync(Core.Prompty agent, List messages) + private async Task ExecuteNonStreamAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var body = BuildRequestBody(agent, messages, stream: false); var (endpoint, apiKey) = GetConnectionInfo(agent); var request = CreateRequest(endpoint, apiKey, body); - var response = await _httpClient.SendAsync(request); + var response = await _httpClient.SendAsync(request, cancellationToken); response.EnsureSuccessStatusCode(); - var json = await response.Content.ReadFromJsonAsync(); + var json = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken); return json; } - private PromptyStream ExecuteStreamAsync(Core.Prompty agent, List messages) + private PromptyStream ExecuteStreamAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var body = BuildRequestBody(agent, messages, stream: true); var (endpoint, apiKey) = GetConnectionInfo(agent); @@ -69,7 +79,7 @@ async IAsyncEnumerable StreamEvents([System.Runtime.CompilerServices.Enu } } - return new PromptyStream(StreamEvents()); + return new PromptyStream(StreamEvents(cancellationToken)); } internal Dictionary BuildRequestBody(Core.Prompty agent, List messages, bool stream) diff --git a/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs b/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs index ce0bcb19a..c84974bb7 100644 --- a/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs @@ -64,7 +64,10 @@ public Task> ParseAsync(Prompty agent, string rendered, Dictionary public void EnqueueResponse(object response) => _responses.Enqueue(response); - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) { // Snapshot the messages at call time Calls.Add(new List(messages)); diff --git a/runtime/csharp/Prompty.Core.Tests/ConnectionRoundtripVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/ConnectionRoundtripVectorTests.cs new file mode 100644 index 000000000..166739bbe --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/ConnectionRoundtripVectorTests.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class ConnectionRoundtripVectorTests +{ + private static readonly string VectorsPath = FindVectorsPath(); + + [Theory] + [InlineData("known_reference_connection_roundtrip_unchanged")] + [InlineData("unknown_connection_kind_preserves_payload")] + [InlineData("unknown_connection_case_collision_preserves_payload")] + public void ConnectionRoundtripVectors_PreserveExactDiscriminatorAndPayload(string vectorName) + { + using var document = JsonDocument.Parse(File.ReadAllText(VectorsPath)); + var vector = document.RootElement + .GetProperty("vectors") + .EnumerateArray() + .Single(candidate => candidate.GetProperty("name").GetString() == vectorName); + + var input = vector.GetProperty("input"); + var expected = vector.GetProperty("expected"); + var expectedKind = expected.GetProperty("kind").GetString()!; + var data = JsonSerializer.Deserialize>(input.GetRawText())!; + + var loaded = Connection.Load(data); + if (expectedKind == "reference") + Assert.IsType(loaded); + else + Assert.IsNotType(loaded); + + var saved = loaded.Save(); + Assert.Equal(expectedKind, saved["kind"]); + AssertJsonEqual(vectorName, "save", expected, saved); + + var reloaded = Connection.Load(saved); + var resaved = reloaded.Save(); + Assert.Equal(expectedKind, resaved["kind"]); + AssertJsonEqual(vectorName, "reload", expected, resaved); + } + + private static void AssertJsonEqual( + string vectorName, + string operation, + JsonElement expected, + Dictionary actual) + { + var expectedNode = JsonNode.Parse(expected.GetRawText()); + var actualNode = JsonNode.Parse(JsonSerializer.Serialize(actual)); + Assert.True( + JsonNode.DeepEquals(expectedNode, actualNode), + $"[{vectorName}] {operation} changed the Connection payload.\nExpected: {expectedNode}\nActual: {actualNode}"); + } + + private static string FindVectorsPath() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine( + directory, + "spec", + "vectors", + "model", + "connection_roundtrip_vectors.json"); + if (File.Exists(candidate)) + return candidate; + + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new FileNotFoundException("Could not locate the shared Connection roundtrip vectors."); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/ContentPartDiscriminatorVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/ContentPartDiscriminatorVectorTests.cs new file mode 100644 index 000000000..1f25ec2ee --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/ContentPartDiscriminatorVectorTests.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class ContentPartDiscriminatorVectorTests +{ + private static readonly string VectorsPath = FindVectorsPath(); + + [Theory] + [InlineData("known_text_content_part_loads")] + [InlineData("unknown_content_part_kind_is_rejected")] + [InlineData("content_part_case_collision_is_rejected")] + public void ContentPartDiscriminatorVectors_EnforceClosedCaseSensitiveKinds(string vectorName) + { + using var document = JsonDocument.Parse(File.ReadAllText(VectorsPath)); + var vector = document.RootElement + .GetProperty("vectors") + .EnumerateArray() + .Single(candidate => candidate.GetProperty("name").GetString() == vectorName); + var input = vector.GetProperty("input"); + var expected = vector.GetProperty("expected"); + var data = JsonSerializer.Deserialize>(input.GetRawText())!; + + switch (vector.GetProperty("operation").GetString()) + { + case "load": + var loaded = ContentPart.Load(data); + Assert.IsType(loaded); + AssertJsonEqual(vectorName, expected, loaded.Save()); + break; + case "load-error": + var exception = Assert.ThrowsAny(() => ContentPart.Load(data)); + Assert.Contains(expected.GetProperty("discriminator").GetString()!, exception.Message); + Assert.Contains(expected.GetProperty("value").GetString()!, exception.Message); + break; + default: + throw new InvalidOperationException($"[{vectorName}] unsupported vector operation."); + } + } + + private static void AssertJsonEqual( + string vectorName, + JsonElement expected, + Dictionary actual) + { + var expectedNode = JsonNode.Parse(expected.GetRawText()); + var actualNode = JsonNode.Parse(JsonSerializer.Serialize(actual)); + Assert.True( + JsonNode.DeepEquals(expectedNode, actualNode), + $"[{vectorName}] load/save changed the ContentPart payload.\nExpected: {expectedNode}\nActual: {actualNode}"); + } + + private static string FindVectorsPath() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine( + directory, + "spec", + "vectors", + "model", + "content_part_discriminator_vectors.json"); + if (File.Exists(candidate)) + return candidate; + + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new FileNotFoundException("Could not locate the shared ContentPart discriminator vectors."); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs index 4c111edcb..332e79d79 100644 --- a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs @@ -335,6 +335,22 @@ public async Task ExecuteAsync_UsesProvider() Assert.Equal("mock-response", response); } + [Fact] + public async Task ExecuteAsync_ForwardsCancellationToken() + { + var agent = CreateAgent(); + var executor = new MockExecutor(); + InvokerRegistry.RegisterExecutor("openai", executor); + using var cancellation = new CancellationTokenSource(); + + await Pipeline.ExecuteAsync( + agent, + [new Message { Parts = [new TextPart { Value = "Hi" }] }], + cancellation.Token); + + Assert.Equal(cancellation.Token, executor.LastCancellationToken); + } + [Fact] public async Task ProcessAsync_UsesProvider() { @@ -601,8 +617,16 @@ public Task> ParseAsync(Prompty agent, string rendered, Dictionary internal class MockExecutor : IExecutor { - public Task ExecuteAsync(Prompty agent, List messages) - => Task.FromResult("mock-response"); + public CancellationToken LastCancellationToken { get; private set; } + + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) + { + LastCancellationToken = cancellationToken; + return Task.FromResult("mock-response"); + } public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) { @@ -629,7 +653,10 @@ internal class ToolCallingExecutor : IExecutor { private int _callCount; - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) { _callCount++; if (_callCount == 1) diff --git a/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs new file mode 100644 index 000000000..ebe2e480d --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/PropertyScalarCoercionVectorTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class PropertyScalarCoercionVectorTests +{ + private static readonly string VectorsPath = FindVectorsPath(); + + [Fact] + public void AllPrimitivePropertyScalarsCoerceAtomically() + { + using var document = JsonDocument.Parse(File.ReadAllText(VectorsPath)); + var vectors = document.RootElement.GetProperty("vectors").EnumerateArray().ToArray(); + Assert.Single(vectors); + + var vector = vectors[0]; + Assert.Equal("all_primitive_property_scalars_coerce_atomically", vector.GetProperty("name").GetString()); + Assert.Equal("load", vector.GetProperty("operation").GetString()); + + var cases = vector.GetProperty("cases").EnumerateArray().ToArray(); + Assert.Equal( + ["string", "integer", "float", "boolean"], + cases.Select(candidate => candidate.GetProperty("name").GetString()!).ToArray()); + + foreach (var scalarCase in cases) + { + var name = scalarCase.GetProperty("name").GetString(); + var expected = scalarCase.GetProperty("expected"); + var loaded = Property.FromJson(scalarCase.GetProperty("input").GetRawText()); + + Assert.Equal(expected.GetProperty("kind").GetString(), loaded.Kind); + Assert.True( + JsonNode.DeepEquals( + JsonNode.Parse(expected.GetProperty("example").GetRawText()), + JsonNode.Parse(JsonSerializer.Serialize(loaded.Example))), + $"[{name}] changed or dropped the Property example."); + } + } + + private static string FindVectorsPath() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine( + directory, + "spec", + "vectors", + "model", + "property_scalar_coercion_vectors.json"); + if (File.Exists(candidate)) + return candidate; + + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new FileNotFoundException("Could not locate the shared Property scalar coercion vectors."); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilitySignatureTests.cs b/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilitySignatureTests.cs new file mode 100644 index 000000000..6594b74e1 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilitySignatureTests.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Reflection; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class RecordUnknownNullabilitySignatureTests +{ + [Fact] + public void RecordUnknownProperties_ExposeCanonicalNullableValueSignatures() + { + var cases = new (Type Model, string Property, NullabilityState Presence)[] + { + (typeof(Message), nameof(Message.Metadata), NullabilityState.NotNull), + (typeof(Prompty), nameof(Prompty.Metadata), NullabilityState.Nullable), + (typeof(ModelInfo), nameof(ModelInfo.AdditionalProperties), NullabilityState.Nullable), + (typeof(TurnModelRequest), nameof(TurnModelRequest.Inputs), NullabilityState.Nullable), + (typeof(RunTurnRequest), nameof(RunTurnRequest.Inputs), NullabilityState.Nullable), + (typeof(TurnModelResponse), nameof(TurnModelResponse.CheckpointState), NullabilityState.Nullable), + (typeof(HostToolRequest), nameof(HostToolRequest.Arguments), NullabilityState.Nullable), + (typeof(TurnEvent), nameof(TurnEvent.Payload), NullabilityState.NotNull), + (typeof(SessionEvent), nameof(SessionEvent.Payload), NullabilityState.NotNull), + }; + var context = new NullabilityInfoContext(); + + foreach (var (model, propertyName, expectedPresence) in cases) + { + var property = model.GetProperty(propertyName) + ?? throw new InvalidOperationException($"{model.Name}.{propertyName} does not exist."); + Assert.Equal(typeof(IDictionary<,>), property.PropertyType.GetGenericTypeDefinition()); + + var nullability = context.Create(property); + Assert.Equal(expectedPresence, nullability.ReadState); + Assert.Equal(typeof(string), nullability.GenericTypeArguments[0].Type); + Assert.Equal(NullabilityState.NotNull, nullability.GenericTypeArguments[0].ReadState); + Assert.Equal(typeof(object), nullability.GenericTypeArguments[1].Type); + Assert.Equal(NullabilityState.Nullable, nullability.GenericTypeArguments[1].ReadState); + } + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilityVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilityVectorTests.cs new file mode 100644 index 000000000..bdb54f058 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/RecordUnknownNullabilityVectorTests.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +public class RecordUnknownNullabilityVectorTests +{ + private static readonly string VectorsPath = FindVectorsPath(); + + [Fact] + public void RecordUnknownNullabilityVectors_PreserveExplicitNullValues() + { + using var document = JsonDocument.Parse(File.ReadAllText(VectorsPath)); + var vectors = document.RootElement.GetProperty("vectors").EnumerateArray().ToArray(); + Assert.Equal(9, vectors.Length); + var expectedCoverage = new HashSet(StringComparer.Ordinal) + { + "Message:metadata", + "Prompty:metadata", + "ModelInfo:additionalProperties", + "TurnModelRequest:inputs", + "RunTurnRequest:inputs", + "TurnModelResponse:checkpointState", + "HostToolRequest:arguments", + "TurnEvent:payload", + "SessionEvent:payload", + }; + var actualCoverage = vectors + .Select(vector => $"{vector.GetProperty("model").GetString()}:{vector.GetProperty("fieldPath").GetString()}") + .ToHashSet(StringComparer.Ordinal); + Assert.True( + expectedCoverage.SetEquals(actualCoverage), + $"Record vector coverage changed.\nExpected: {string.Join(", ", expectedCoverage)}\n" + + $"Actual: {string.Join(", ", actualCoverage)}"); + + foreach (var vector in vectors) + { + var name = vector.GetProperty("name").GetString()!; + Assert.Equal("load-save-reload", vector.GetProperty("operation").GetString()); + + var model = vector.GetProperty("model").GetString()!; + var fieldPath = vector.GetProperty("fieldPath").GetString()!; + var resaved = Roundtrip(model, vector.GetProperty("input").GetRawText()); + var actual = resaved.GetProperty(fieldPath); + var expected = vector.GetProperty("expected"); + + Assert.True( + actual.TryGetProperty("direct", out var direct) && direct.ValueKind == JsonValueKind.Null, + $"[{name}] direct null-valued key was lost or changed."); + Assert.True( + JsonNode.DeepEquals(JsonNode.Parse(expected.GetRawText()), JsonNode.Parse(actual.GetRawText())), + $"[{name}] load/save/reload changed null-valued record entries.\nExpected: {expected}\nActual: {actual}"); + } + } + + private static JsonElement Roundtrip(string model, string input) + { + var resaved = model switch + { + "Message" => Roundtrip(input, json => Message.FromJson(json), value => value.ToJson()), + "Prompty" => Roundtrip(input, json => Prompty.FromJson(json), value => value.ToJson()), + "ModelInfo" => Roundtrip(input, json => ModelInfo.FromJson(json), value => value.ToJson()), + "TurnModelRequest" => Roundtrip( + input, + json => TurnModelRequest.FromJson(json), + value => value.ToJson()), + "RunTurnRequest" => Roundtrip( + input, + json => RunTurnRequest.FromJson(json), + value => value.ToJson()), + "TurnModelResponse" => Roundtrip( + input, + json => TurnModelResponse.FromJson(json), + value => value.ToJson()), + "HostToolRequest" => Roundtrip( + input, + json => HostToolRequest.FromJson(json), + value => value.ToJson()), + "TurnEvent" => Roundtrip(input, json => TurnEvent.FromJson(json), value => value.ToJson()), + "SessionEvent" => Roundtrip( + input, + json => SessionEvent.FromJson(json), + value => value.ToJson()), + _ => throw new InvalidOperationException($"Unsupported vector model '{model}'."), + }; + using var document = JsonDocument.Parse(resaved); + return document.RootElement.Clone(); + } + + private static string Roundtrip(string input, Func load, Func save) + { + var loaded = load(input); + var saved = save(loaded); + var reloaded = load(saved); + return save(reloaded); + } + + private static string FindVectorsPath() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine( + directory, + "spec", + "vectors", + "model", + "record_unknown_nullability_vectors.json"); + if (File.Exists(candidate)) + return candidate; + + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new FileNotFoundException("Could not locate the shared Record nullability vectors."); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs index a972bcfb3..ea36025d6 100644 --- a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs @@ -22,7 +22,10 @@ namespace Prompty.Core.Tests; public void EnqueueResponse(object response) => _responses.Enqueue(response); public void EnqueueException(Exception ex) => _exceptions.Enqueue(ex); - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) { Calls.Add(new List(messages)); diff --git a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs index 91519b0a8..aa278a3eb 100644 --- a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs @@ -460,6 +460,54 @@ private static List CompareAgentToExpected(Prompty agent, JsonElement ex } } + if (expected.TryGetProperty("tools", out var toolsEl) && toolsEl.ValueKind == JsonValueKind.Array) + { + var expectedTools = toolsEl.EnumerateArray().ToList(); + var actualTools = agent.Tools ?? []; + if (actualTools.Count != expectedTools.Count) + { + errors.Add($"tools: expected {expectedTools.Count}, got {actualTools.Count}"); + } + + for (var i = 0; i < Math.Min(actualTools.Count, expectedTools.Count); i++) + { + var expectedTool = expectedTools[i]; + if (!expectedTool.TryGetProperty("bindings", out var bindingsEl) || + bindingsEl.ValueKind != JsonValueKind.Object) + { + continue; + } + + var expectedBindings = bindingsEl.EnumerateObject().ToList(); + var actualBindings = actualTools[i].Bindings ?? []; + if (actualBindings.Count != expectedBindings.Count) + { + errors.Add( + $"tools[{i}].bindings: expected {expectedBindings.Count}, got {actualBindings.Count}"); + } + + foreach (var expectedBinding in expectedBindings) + { + var actualBinding = actualBindings.FirstOrDefault(binding => binding.Name == expectedBinding.Name); + if (actualBinding is null) + { + errors.Add($"tools[{i}].bindings: missing binding '{expectedBinding.Name}'"); + continue; + } + + var expectedInput = expectedBinding.Value.ValueKind == JsonValueKind.Object + ? expectedBinding.Value.GetProperty("input").GetString() + : expectedBinding.Value.GetString(); + if (actualBinding.Input != expectedInput) + { + errors.Add( + $"tools[{i}].bindings.{expectedBinding.Name}.input: " + + $"expected '{expectedInput}', got '{actualBinding.Input}'"); + } + } + } + } + return errors; } diff --git a/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs b/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs index 005771327..5430cff48 100644 --- a/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/StructuredOutputPipelineTests.cs @@ -261,7 +261,10 @@ internal class RawJsonExecutor : IExecutor public RawJsonExecutor(string rawJson) => _rawJson = rawJson; - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) => Task.FromResult(_rawJson); public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) diff --git a/runtime/csharp/Prompty.Core.Tests/TracingTests.cs b/runtime/csharp/Prompty.Core.Tests/TracingTests.cs index 7c588e007..e4bae9dcb 100644 --- a/runtime/csharp/Prompty.Core.Tests/TracingTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/TracingTests.cs @@ -577,7 +577,10 @@ public Task> ParseAsync(Prompty agent, string rendered, Dictionary private class MockExecutor(object response) : IExecutor { - public Task ExecuteAsync(Prompty agent, List messages) + public Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) => Task.FromResult(response); public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) diff --git a/runtime/csharp/Prompty.Core/Pipeline.cs b/runtime/csharp/Prompty.Core/Pipeline.cs index a476ca3a7..f3ce51bc7 100644 --- a/runtime/csharp/Prompty.Core/Pipeline.cs +++ b/runtime/csharp/Prompty.Core/Pipeline.cs @@ -104,14 +104,17 @@ public static async Task> ParseAsync(Prompty agent, string rendere /// /// Execute an LLM call with the given messages. /// - public static async Task ExecuteAsync(Prompty agent, List messages) + public static async Task ExecuteAsync( + Prompty agent, + List messages, + CancellationToken cancellationToken = default) { return await Trace.TraceAsync("Prompty.Core.Pipeline.ExecuteAsync", async (emit) => { emit("inputs", new Dictionary { ["agent"] = agent.Name, ["message_count"] = messages.Count }); var provider = agent.Model?.Provider ?? "openai"; var executor = InvokerRegistry.GetExecutor(provider); - return await executor.ExecuteAsync(agent, messages); + return await executor.ExecuteAsync(agent, messages, cancellationToken); }); } @@ -280,7 +283,7 @@ public static async Task TurnAsync( object response; try { - response = await ExecuteAsync(agent, messages); + response = await ExecuteAsync(agent, messages, cancellationToken); } catch (Exception ex) { @@ -842,7 +845,7 @@ private static async Task InvokeWithRetryAsync( { try { - return await ExecuteAsync(agent, messages); + return await ExecuteAsync(agent, messages, cancellationToken); } catch (Exception ex) when (ex is not OperationCanceledException) { diff --git a/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs index 563759e14..cb54135a1 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs @@ -269,7 +269,10 @@ public Task> ParseAsync(Core.Prompty agent, string rendered, Dicti /// Executor that returns a fixed response. private class MockExecutor(object response) : IExecutor { - public Task ExecuteAsync(Core.Prompty agent, List messages) + public Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) => Task.FromResult(response); public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) @@ -296,7 +299,10 @@ private class SequenceExecutor(List responses) : IExecutor private int _index; public int CallCount => _index; - public Task ExecuteAsync(Core.Prompty agent, List messages) + public Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) { if (_index >= responses.Count) throw new InvalidOperationException("SequenceExecutor ran out of responses."); @@ -310,7 +316,10 @@ public List FormatToolMessages(object rawResponse, List toolC /// Executor that always returns a tool call — for testing max iterations. private class InfiniteToolCallExecutor : IExecutor { - public Task ExecuteAsync(Core.Prompty agent, List messages) + public Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) { return Task.FromResult(new ToolCallResult { diff --git a/runtime/csharp/Prompty.OpenAI.Tests/OpenAIExecutorTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/OpenAIExecutorTests.cs index 3e006632c..c340e0837 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/OpenAIExecutorTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/OpenAIExecutorTests.cs @@ -10,6 +10,17 @@ namespace Prompty.OpenAI.Tests; /// public class OpenAIExecutorTests { + [Fact] + public async Task ExecuteAsync_Cancelled_ThrowsBeforeConnectionValidation() + { + var executor = new OpenAI.OpenAIExecutor(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync( + () => executor.ExecuteAsync(new Core.Prompty(), [], cancellation.Token)); + } + [Fact] public async Task ExecuteAsync_MissingApiKey_ThrowsInvalidOperationException() { @@ -146,4 +157,3 @@ public void FormatToolMessages_CreatesIndividualToolMessages() Assert.Equal("call_2", messages[2].Metadata["tool_call_id"]); } } - diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs index a4a0af53d..26a4bc31a 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs @@ -715,7 +715,10 @@ public Task ProcessAsync(Core.Prompty agent, object response) private class LambdaExecutor(Func, object> fn) : IExecutor { - public Task ExecuteAsync(Core.Prompty agent, List messages) + public Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) => Task.FromResult(fn(messages)); public List FormatToolMessages(object rawResponse, List toolCalls, List toolResults, string? textContent = null) diff --git a/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs b/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs index 335b635bd..ae711ac0e 100644 --- a/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs +++ b/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs @@ -20,8 +20,12 @@ namespace Prompty.OpenAI; /// public class OpenAIExecutor : IExecutor { - public async Task ExecuteAsync(Core.Prompty agent, List messages) + public async Task ExecuteAsync( + Core.Prompty agent, + List messages, + CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); var apiType = agent.Model?.ApiType ?? "chat"; var model = agent.Model?.Id ?? "gpt-4"; var client = CreateClient(agent); @@ -29,12 +33,12 @@ public async Task ExecuteAsync(Core.Prompty agent, List message return apiType switch { - "chat" when streaming => ExecuteChatStreamAsync(client, model, agent, messages), - "chat" => await ExecuteChatAsync(client, model, agent, messages), - "responses" when streaming => ExecuteResponsesStreamAsync(client, model, agent, messages), - "responses" => await ExecuteResponsesAsync(client, model, agent, messages), - "embedding" => await ExecuteEmbeddingAsync(client, model, messages), - "image" => await ExecuteImageAsync(client, model, messages), + "chat" when streaming => ExecuteChatStreamAsync(client, model, agent, messages, cancellationToken), + "chat" => await ExecuteChatAsync(client, model, agent, messages, cancellationToken), + "responses" when streaming => ExecuteResponsesStreamAsync(client, model, agent, messages, cancellationToken), + "responses" => await ExecuteResponsesAsync(client, model, agent, messages, cancellationToken), + "embedding" => await ExecuteEmbeddingAsync(client, model, messages, cancellationToken), + "image" => await ExecuteImageAsync(client, model, messages, cancellationToken), _ => throw new InvalidOperationException($"Unsupported API type: {apiType}"), }; } @@ -83,18 +87,26 @@ protected virtual OpenAIClient CreateClient(Core.Prompty agent) } private static async Task ExecuteChatAsync( - OpenAIClient client, string model, Core.Prompty agent, List messages) + OpenAIClient client, + string model, + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var chatClient = client.GetChatClient(model); var chatMessages = messages.Select(WireFormat.MessageToWire).ToList(); var options = WireFormat.BuildOptions(agent); - var result = await chatClient.CompleteChatAsync(chatMessages, options); + var result = await chatClient.CompleteChatAsync(chatMessages, options, cancellationToken); return result.Value; } private static PromptyStream ExecuteChatStreamAsync( - OpenAIClient client, string model, Core.Prompty agent, List messages) + OpenAIClient client, + string model, + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var chatClient = client.GetChatClient(model); var chatMessages = messages.Select(WireFormat.MessageToWire).ToList(); @@ -109,7 +121,7 @@ async IAsyncEnumerable StreamChunks([EnumeratorCancellation] Cancellatio } } - return new PromptyStream(StreamChunks()); + return new PromptyStream(StreamChunks(cancellationToken)); } // ----------------------------------------------------------------------- @@ -117,16 +129,24 @@ async IAsyncEnumerable StreamChunks([EnumeratorCancellation] Cancellatio // ----------------------------------------------------------------------- private static async Task ExecuteResponsesAsync( - OpenAIClient client, string model, Core.Prompty agent, List messages) + OpenAIClient client, + string model, + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var responsesClient = client.GetResponsesClient(); var options = WireFormat.BuildResponsesOptions(model, agent, messages); - var result = await responsesClient.CreateResponseAsync(options); + var result = await responsesClient.CreateResponseAsync(options, cancellationToken); return result.Value; } private static PromptyStream ExecuteResponsesStreamAsync( - OpenAIClient client, string model, Core.Prompty agent, List messages) + OpenAIClient client, + string model, + Core.Prompty agent, + List messages, + CancellationToken cancellationToken) { var responsesClient = client.GetResponsesClient(); var options = WireFormat.BuildResponsesOptions(model, agent, messages); @@ -141,7 +161,7 @@ async IAsyncEnumerable StreamChunks([EnumeratorCancellation] Cancellatio } } - return new PromptyStream(StreamChunks()); + return new PromptyStream(StreamChunks(cancellationToken)); } // ----------------------------------------------------------------------- @@ -149,20 +169,26 @@ async IAsyncEnumerable StreamChunks([EnumeratorCancellation] Cancellatio // ----------------------------------------------------------------------- private static async Task ExecuteEmbeddingAsync( - OpenAIClient client, string model, List messages) + OpenAIClient client, + string model, + List messages, + CancellationToken cancellationToken) { var embeddingClient = client.GetEmbeddingClient(model); var inputs = messages.Select(m => m.Text).ToList(); - var result = await embeddingClient.GenerateEmbeddingsAsync(inputs); + var result = await embeddingClient.GenerateEmbeddingsAsync(inputs, cancellationToken: cancellationToken); return result.Value; } private static async Task ExecuteImageAsync( - OpenAIClient client, string model, List messages) + OpenAIClient client, + string model, + List messages, + CancellationToken cancellationToken) { var imageClient = client.GetImageClient(model); var prompt = messages.LastOrDefault()?.Text ?? ""; - var result = await imageClient.GenerateImageAsync(prompt); + var result = await imageClient.GenerateImageAsync(prompt, cancellationToken: cancellationToken); return result.Value; } diff --git a/runtime/go/prompty/model/connection_roundtrip_vectors_test.go b/runtime/go/prompty/model/connection_roundtrip_vectors_test.go new file mode 100644 index 000000000..74f38d617 --- /dev/null +++ b/runtime/go/prompty/model/connection_roundtrip_vectors_test.go @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft. All rights reserved. + +package prompty_test + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "testing" + + "prompty/model" +) + +type connectionRoundtripVectorDocument struct { + Vectors []connectionRoundtripVector `json:"vectors"` +} + +type connectionRoundtripVector struct { + Name string `json:"name"` + Operation string `json:"operation"` + Input map[string]interface{} `json:"input"` + Expected map[string]interface{} `json:"expected"` +} + +func TestConnectionRoundtripVectorsPreserveExactDiscriminatorAndPayload(t *testing.T) { + path := filepath.Join( + "..", + "..", + "..", + "..", + "spec", + "vectors", + "model", + "connection_roundtrip_vectors.json", + ) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read Connection roundtrip vectors: %v", err) + } + + var document connectionRoundtripVectorDocument + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatalf("failed to parse Connection roundtrip vectors: %v", err) + } + + for _, vector := range document.Vectors { + t.Run(vector.Name, func(t *testing.T) { + if vector.Operation != "load-save-reload" { + t.Fatalf("unsupported vector operation %q", vector.Operation) + } + + loaded, err := prompty.LoadConnection(vector.Input, prompty.NewLoadContext()) + if err != nil { + t.Fatalf("load failed: %v", err) + } + + saved := saveConnection(t, loaded) + if saved["kind"] != vector.Expected["kind"] { + t.Fatalf("save changed discriminator: expected %v, got %v", vector.Expected["kind"], saved["kind"]) + } + if !reflect.DeepEqual(saved, vector.Expected) { + t.Fatalf("save changed Connection payload:\nexpected: %#v\nactual: %#v", vector.Expected, saved) + } + + reloaded, err := prompty.LoadConnection(saved, prompty.NewLoadContext()) + if err != nil { + t.Fatalf("reload failed: %v", err) + } + resaved := saveConnection(t, reloaded) + if !reflect.DeepEqual(resaved, vector.Expected) { + t.Fatalf("reload changed Connection payload:\nexpected: %#v\nactual: %#v", vector.Expected, resaved) + } + }) + } +} + +func saveConnection(t *testing.T, connection interface{}) map[string]interface{} { + t.Helper() + + method := reflect.ValueOf(connection).MethodByName("Save") + if !method.IsValid() { + t.Fatalf("loaded Connection type %T does not expose Save", connection) + } + results := method.Call([]reflect.Value{reflect.ValueOf(prompty.NewSaveContext())}) + if len(results) != 1 { + t.Fatalf("loaded Connection type %T returned %d Save results", connection, len(results)) + } + saved, ok := results[0].Interface().(map[string]interface{}) + if !ok { + t.Fatalf("loaded Connection type %T returned unexpected Save result %T", connection, results[0].Interface()) + } + return saved +} diff --git a/runtime/go/prompty/model/content_part_discriminator_vectors_test.go b/runtime/go/prompty/model/content_part_discriminator_vectors_test.go new file mode 100644 index 000000000..a8b95c9fb --- /dev/null +++ b/runtime/go/prompty/model/content_part_discriminator_vectors_test.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft. All rights reserved. + +package prompty_test + +import ( + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "prompty/model" +) + +type contentPartDiscriminatorVectorDocument struct { + Vectors []contentPartDiscriminatorVector `json:"vectors"` +} + +type contentPartDiscriminatorVector struct { + Name string `json:"name"` + Operation string `json:"operation"` + Input map[string]interface{} `json:"input"` + Expected map[string]interface{} `json:"expected"` +} + +func TestContentPartDiscriminatorVectorsEnforceClosedCaseSensitiveKinds(t *testing.T) { + path := filepath.Join( + "..", + "..", + "..", + "..", + "spec", + "vectors", + "model", + "content_part_discriminator_vectors.json", + ) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read ContentPart discriminator vectors: %v", err) + } + + var document contentPartDiscriminatorVectorDocument + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatalf("failed to parse ContentPart discriminator vectors: %v", err) + } + + for _, vector := range document.Vectors { + t.Run(vector.Name, func(t *testing.T) { + loaded, err := prompty.LoadContentPart(vector.Input, prompty.NewLoadContext()) + + switch vector.Operation { + case "load": + if err != nil { + t.Fatalf("known ContentPart failed to load: %v", err) + } + saved := saveContentPart(t, loaded) + if !reflect.DeepEqual(saved, vector.Expected) { + t.Fatalf("load/save changed ContentPart payload:\nexpected: %#v\nactual: %#v", vector.Expected, saved) + } + case "load-error": + if err == nil { + t.Fatalf("closed ContentPart accepted unknown discriminator %v", vector.Input["kind"]) + } + diagnostic := err.Error() + discriminator := vector.Expected["discriminator"].(string) + value := vector.Expected["value"].(string) + if !strings.Contains(diagnostic, discriminator) { + t.Fatalf("error did not identify discriminator %q: %s", discriminator, diagnostic) + } + if !strings.Contains(diagnostic, value) { + t.Fatalf("error did not preserve discriminator value %q: %s", value, diagnostic) + } + default: + t.Fatalf("unsupported vector operation %q", vector.Operation) + } + }) + } +} + +func saveContentPart(t *testing.T, contentPart interface{}) map[string]interface{} { + t.Helper() + + method := reflect.ValueOf(contentPart).MethodByName("Save") + if !method.IsValid() { + t.Fatalf("loaded ContentPart type %T does not expose Save", contentPart) + } + results := method.Call([]reflect.Value{reflect.ValueOf(prompty.NewSaveContext())}) + if len(results) != 1 { + t.Fatalf("loaded ContentPart type %T returned %d Save results", contentPart, len(results)) + } + saved, ok := results[0].Interface().(map[string]interface{}) + if !ok { + t.Fatalf("loaded ContentPart type %T returned unexpected Save result %T", contentPart, results[0].Interface()) + } + return saved +} diff --git a/runtime/go/prompty/model/function_tool_bindings_load_vector_test.go b/runtime/go/prompty/model/function_tool_bindings_load_vector_test.go new file mode 100644 index 000000000..fe94e07c9 --- /dev/null +++ b/runtime/go/prompty/model/function_tool_bindings_load_vector_test.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft. All rights reserved. + +package prompty_test + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" + + "prompty/model" +) + +type functionToolLoadVector struct { + Name string `json:"name"` + Expected struct { + Tools []struct { + Bindings map[string]struct { + Input string `json:"input"` + } `json:"bindings"` + } `json:"tools"` + } `json:"expected"` +} + +func TestFunctionToolBindingsLoadVector(t *testing.T) { + repositoryRoot := filepath.Join("..", "..", "..", "..") + vectorsRaw, err := os.ReadFile(filepath.Join(repositoryRoot, "spec", "vectors", "load", "load_vectors.json")) + if err != nil { + t.Fatalf("failed to read load vectors: %v", err) + } + + var vectors []functionToolLoadVector + if err := json.Unmarshal(vectorsRaw, &vectors); err != nil { + t.Fatalf("failed to parse load vectors: %v", err) + } + + var expectedBindings map[string]struct { + Input string `json:"input"` + } + for _, vector := range vectors { + if vector.Name == "tools_function_load" { + if len(vector.Expected.Tools) != 1 { + t.Fatalf("tools_function_load expected one tool, got %d", len(vector.Expected.Tools)) + } + expectedBindings = vector.Expected.Tools[0].Bindings + break + } + } + if expectedBindings == nil { + t.Fatal("tools_function_load vector is missing expected bindings") + } + + fixtureRaw, err := os.ReadFile(filepath.Join(repositoryRoot, "spec", "fixtures", "tools_function.prompty")) + if err != nil { + t.Fatalf("failed to read tools_function.prompty: %v", err) + } + sections := strings.SplitN(string(fixtureRaw), "---", 3) + if len(sections) != 3 { + t.Fatal("tools_function.prompty must contain YAML frontmatter") + } + + var frontmatter struct { + Tools []map[string]interface{} `yaml:"tools"` + } + if err := yaml.Unmarshal([]byte(sections[1]), &frontmatter); err != nil { + t.Fatalf("failed to parse tools_function.prompty frontmatter: %v", err) + } + if len(frontmatter.Tools) != 1 { + t.Fatalf("tools_function.prompty contains %d tools, expected one", len(frontmatter.Tools)) + } + + tool, err := prompty.LoadFunctionTool(frontmatter.Tools[0], prompty.NewLoadContext()) + if err != nil { + t.Fatalf("failed to load FunctionTool: %v", err) + } + for name, expected := range expectedBindings { + var actual *prompty.Binding + for index := range tool.Bindings { + if tool.Bindings[index].Name == name { + actual = &tool.Bindings[index] + break + } + } + if actual == nil { + t.Fatalf("missing binding %q", name) + } + if actual.Input != expected.Input { + t.Fatalf("binding %q input: expected %q, got %q", name, expected.Input, actual.Input) + } + } + if len(tool.Bindings) != len(expectedBindings) { + t.Fatalf("expected %d bindings, got %d", len(expectedBindings), len(tool.Bindings)) + } +} diff --git a/runtime/go/prompty/model/property_scalar_coercion_vectors_test.go b/runtime/go/prompty/model/property_scalar_coercion_vectors_test.go new file mode 100644 index 000000000..a50c22f61 --- /dev/null +++ b/runtime/go/prompty/model/property_scalar_coercion_vectors_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft. All rights reserved. + +package prompty_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "prompty/model" +) + +type propertyScalarCoercionVectorDocument struct { + Vectors []propertyScalarCoercionVector `json:"vectors"` +} + +type propertyScalarCoercionVector struct { + Name string `json:"name"` + Operation string `json:"operation"` + Cases []propertyScalarCoercionCase `json:"cases"` +} + +type propertyScalarCoercionCase struct { + Name string `json:"name"` + Input json.RawMessage `json:"input"` + Expected struct { + Kind string `json:"kind"` + Example interface{} `json:"example"` + } `json:"expected"` +} + +func TestAllPrimitivePropertyScalarsCoerceAtomically(t *testing.T) { + path := filepath.Join( + "..", + "..", + "..", + "..", + "spec", + "vectors", + "model", + "property_scalar_coercion_vectors.json", + ) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read Property scalar coercion vectors: %v", err) + } + + var document propertyScalarCoercionVectorDocument + if err := json.Unmarshal(raw, &document); err != nil { + t.Fatalf("failed to parse Property scalar coercion vectors: %v", err) + } + if len(document.Vectors) != 1 { + t.Fatalf("expected one atomic Property scalar coercion vector, got %d", len(document.Vectors)) + } + vector := document.Vectors[0] + if vector.Name != "all_primitive_property_scalars_coerce_atomically" || vector.Operation != "load" { + t.Fatalf("unexpected Property scalar coercion vector %q operation %q", vector.Name, vector.Operation) + } + expectedNames := []string{"string", "integer", "float", "boolean"} + if len(vector.Cases) != len(expectedNames) { + t.Fatalf("expected all four primitive scalar cases, got %d", len(vector.Cases)) + } + + for index, scalarCase := range vector.Cases { + if scalarCase.Name != expectedNames[index] { + t.Fatalf("scalar case %d: expected %q, got %q", index, expectedNames[index], scalarCase.Name) + } + loaded, err := prompty.PropertyFromJSON(string(scalarCase.Input)) + if err != nil { + t.Errorf("[%s] load failed: %v", scalarCase.Name, err) + continue + } + property, ok := loaded.(prompty.Property) + if !ok { + t.Errorf("[%s] expected Property, got %T", scalarCase.Name, loaded) + continue + } + if property.Kind != scalarCase.Expected.Kind { + t.Errorf("[%s] expected kind %q, got %q", scalarCase.Name, scalarCase.Expected.Kind, property.Kind) + continue + } + if property.Example == nil { + t.Errorf("[%s] expected example, got nil", scalarCase.Name) + continue + } + actualJSON, err := json.Marshal(*property.Example) + if err != nil { + t.Errorf("[%s] failed to encode actual example: %v", scalarCase.Name, err) + continue + } + expectedJSON, err := json.Marshal(scalarCase.Expected.Example) + if err != nil { + t.Errorf("[%s] failed to encode expected example: %v", scalarCase.Name, err) + continue + } + if string(actualJSON) != string(expectedJSON) { + t.Errorf("[%s] expected example %s, got %s", scalarCase.Name, expectedJSON, actualJSON) + } + } +} diff --git a/runtime/python/prompty/prompty/core/types.py b/runtime/python/prompty/prompty/core/types.py index b5ae4ee5c..d0719039d 100644 --- a/runtime/python/prompty/prompty/core/types.py +++ b/runtime/python/prompty/prompty/core/types.py @@ -59,8 +59,8 @@ def _message_text(self: Message) -> str: - """Concatenate all TextPart values into a single string.""" - return "".join(p.value for p in self.parts if isinstance(p, TextPart)) + """Concatenate all TextPart values joined by newline.""" + return "\n".join(p.value for p in self.parts if isinstance(p, TextPart)) def _message_to_text_content(self: Message) -> str | list[dict[str, Any]]: diff --git a/runtime/python/prompty/tests/test_connection_roundtrip_vectors.py b/runtime/python/prompty/tests/test_connection_roundtrip_vectors.py new file mode 100644 index 000000000..afafde526 --- /dev/null +++ b/runtime/python/prompty/tests/test_connection_roundtrip_vectors.py @@ -0,0 +1,41 @@ +"""Validate the canonical forward-compatible Connection roundtrip contract.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from prompty.model import Connection, ReferenceConnection + +_VECTORS_PATH = Path(__file__).resolve().parents[4] / "spec" / "vectors" / "model" / "connection_roundtrip_vectors.json" + + +def _load_vectors() -> list[dict[str, Any]]: + document: dict[str, Any] = json.loads(_VECTORS_PATH.read_text(encoding="utf-8")) + return document["vectors"] + + +@pytest.mark.parametrize("vector", _load_vectors(), ids=lambda vector: vector["name"]) +def test_connection_roundtrip_vectors_preserve_exact_discriminator_and_payload(vector: dict[str, Any]) -> None: + """Preserve known and unknown Connection values through load, save, and reload.""" + + assert vector["operation"] == "load-save-reload" + expected = vector["expected"] + + loaded = Connection.load(vector["input"]) + if expected["kind"] == "reference": + assert isinstance(loaded, ReferenceConnection), vector["name"] + else: + assert not isinstance(loaded, ReferenceConnection), vector["name"] + + saved = loaded.save() + assert saved["kind"] == expected["kind"], vector["name"] + assert saved == expected, vector["name"] + + reloaded = Connection.load(saved) + resaved = reloaded.save() + assert resaved["kind"] == expected["kind"], vector["name"] + assert resaved == expected, vector["name"] diff --git a/runtime/python/prompty/tests/test_content_part_discriminator_vectors.py b/runtime/python/prompty/tests/test_content_part_discriminator_vectors.py new file mode 100644 index 000000000..8c3377518 --- /dev/null +++ b/runtime/python/prompty/tests/test_content_part_discriminator_vectors.py @@ -0,0 +1,38 @@ +"""Validate the canonical closed ContentPart discriminator contract.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from prompty.model import ContentPart, TextPart + +_VECTORS_PATH = ( + Path(__file__).resolve().parents[4] / "spec" / "vectors" / "model" / "content_part_discriminator_vectors.json" +) + + +def _load_vectors() -> list[dict[str, Any]]: + document: dict[str, Any] = json.loads(_VECTORS_PATH.read_text(encoding="utf-8")) + return document["vectors"] + + +@pytest.mark.parametrize("vector", _load_vectors(), ids=lambda vector: vector["name"]) +def test_content_part_discriminator_vectors_enforce_closed_case_sensitive_kinds(vector: dict[str, Any]) -> None: + """Load known kinds and reject unknown or case-colliding discriminator values.""" + + if vector["operation"] == "load": + loaded = ContentPart.load(vector["input"]) + assert isinstance(loaded, TextPart), vector["name"] + assert loaded.save() == vector["expected"], vector["name"] + return + + with pytest.raises(ValueError) as error: + ContentPart.load(vector["input"]) + + diagnostic = str(error.value) + assert vector["expected"]["discriminator"] in diagnostic, vector["name"] + assert vector["expected"]["value"] in diagnostic, vector["name"] diff --git a/runtime/python/prompty/tests/test_property_scalar_coercion_vectors.py b/runtime/python/prompty/tests/test_property_scalar_coercion_vectors.py new file mode 100644 index 000000000..d5565a0c5 --- /dev/null +++ b/runtime/python/prompty/tests/test_property_scalar_coercion_vectors.py @@ -0,0 +1,29 @@ +"""Validate the canonical atomic Property scalar coercion contract.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from prompty.model import Property + +_VECTORS_PATH = ( + Path(__file__).resolve().parents[4] / "spec" / "vectors" / "model" / "property_scalar_coercion_vectors.json" +) + + +def test_all_primitive_property_scalars_coerce_atomically() -> None: + """Infer and preserve every primitive scalar coercion branch.""" + + document: dict[str, Any] = json.loads(_VECTORS_PATH.read_text(encoding="utf-8")) + vector = document["vectors"][0] + assert vector["name"] == "all_primitive_property_scalars_coerce_atomically" + assert vector["operation"] == "load" + assert [case["name"] for case in vector["cases"]] == ["string", "integer", "float", "boolean"] + + for case in vector["cases"]: + loaded = Property.load(case["input"]) + assert loaded.kind == case["expected"]["kind"], case["name"] + assert type(loaded.example) is type(case["expected"]["example"]), case["name"] + assert loaded.example == case["expected"]["example"], case["name"] diff --git a/runtime/python/prompty/tests/test_spec_vectors.py b/runtime/python/prompty/tests/test_spec_vectors.py index 29eba1df6..973d25205 100644 --- a/runtime/python/prompty/tests/test_spec_vectors.py +++ b/runtime/python/prompty/tests/test_spec_vectors.py @@ -497,6 +497,8 @@ def _check_tools(actual: list, expected: list[dict], errors: list[str]): act_bindings = getattr(act, "bindings", []) or [] exp_bindings = exp["bindings"] if isinstance(exp_bindings, dict): + if len(act_bindings) != len(exp_bindings): + errors.append(f" {prefix}.bindings: count {len(act_bindings)} != expected {len(exp_bindings)}") for bname, bval in exp_bindings.items(): found = [b for b in act_bindings if b.name == bname] if not found: diff --git a/runtime/python/prompty/tests/test_types.py b/runtime/python/prompty/tests/test_types.py new file mode 100644 index 000000000..0e020afd8 --- /dev/null +++ b/runtime/python/prompty/tests/test_types.py @@ -0,0 +1,21 @@ +"""Verify handwritten runtime extensions for generated conversation types.""" + +from __future__ import annotations + +from prompty.core.types import Message, TextPart + + +def test_message_text_parts_are_joined_by_newline() -> None: + """Join multiple text parts according to the canonical method contract.""" + message = Message(role="user", parts=[TextPart(value="first"), TextPart(value="second")]) + + assert message.text == "first\nsecond" + assert message.to_text_content() == "first\nsecond" + + +def test_empty_message_text_content_is_empty_string() -> None: + """Represent an empty all-text message as an empty string.""" + message = Message(role="user", parts=[]) + + assert message.text == "" + assert message.to_text_content() == "" diff --git a/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs b/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs new file mode 100644 index 000000000..3ab0de197 --- /dev/null +++ b/runtime/rust/prompty/tests/connection_roundtrip_vectors.rs @@ -0,0 +1,81 @@ +//! Cross-runtime Connection roundtrip tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::{LoadContext, SaveContext}; +use prompty::model::Connection; +use serde_json::Value; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("connection_roundtrip_vectors.json") +} + +#[test] +fn known_reference_connection_roundtrip_unchanged() { + assert_connection_roundtrip_vector("known_reference_connection_roundtrip_unchanged"); +} + +#[test] +fn unknown_connection_kind_preserves_payload() { + assert_connection_roundtrip_vector("unknown_connection_kind_preserves_payload"); +} + +#[test] +fn unknown_connection_case_collision_preserves_payload() { + assert_connection_roundtrip_vector("unknown_connection_case_collision_preserves_payload"); +} + +fn assert_connection_roundtrip_vector(vector_name: &str) { + let raw = std::fs::read_to_string(vectors_path()) + .expect("failed to read Connection roundtrip vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse Connection roundtrip vectors"); + let vector = document["vectors"] + .as_array() + .expect("Connection roundtrip vectors must contain a vectors array") + .iter() + .find(|candidate| candidate["name"] == vector_name) + .unwrap_or_else(|| panic!("missing Connection roundtrip vector {vector_name}")); + assert_eq!( + vector["operation"], "load-save-reload", + "[{vector_name}] unsupported vector operation" + ); + + let input = &vector["input"]; + let expected = &vector["expected"]; + let kind = input["kind"] + .as_str() + .expect("Connection kind must be a string"); + let load_context = LoadContext::default(); + let save_context = SaveContext::default(); + + let loaded = Connection::load_from_value(input, &load_context); + assert_eq!( + loaded.kind_str(), + kind, + "[{vector_name}] load changed the discriminator" + ); + + let saved = loaded.to_value(&save_context); + assert_eq!( + saved, *expected, + "[{vector_name}] save changed the Connection payload" + ); + + let reloaded = Connection::load_from_value(&saved, &load_context); + let resaved = reloaded.to_value(&save_context); + assert_eq!( + resaved, *expected, + "[{vector_name}] reload changed the Connection payload" + ); +} diff --git a/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs b/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs new file mode 100644 index 000000000..f587cb49b --- /dev/null +++ b/runtime/rust/prompty/tests/content_part_discriminator_vectors.rs @@ -0,0 +1,92 @@ +//! Closed ContentPart discriminator tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::{LoadContext, SaveContext}; +use prompty::model::ContentPart; +use serde_json::Value; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("content_part_discriminator_vectors.json") +} + +#[test] +fn known_text_content_part_loads() { + assert_content_part_discriminator_vector("known_text_content_part_loads"); +} + +#[test] +fn unknown_content_part_kind_is_rejected() { + assert_content_part_discriminator_vector("unknown_content_part_kind_is_rejected"); +} + +#[test] +fn content_part_case_collision_is_rejected() { + assert_content_part_discriminator_vector("content_part_case_collision_is_rejected"); +} + +fn assert_content_part_discriminator_vector(vector_name: &str) { + let raw = std::fs::read_to_string(vectors_path()) + .expect("failed to read ContentPart discriminator vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse ContentPart discriminator vectors"); + let vector = document["vectors"] + .as_array() + .expect("ContentPart discriminator vectors must contain a vectors array") + .iter() + .find(|candidate| candidate["name"] == vector_name) + .unwrap_or_else(|| panic!("missing ContentPart discriminator vector {vector_name}")); + let context = LoadContext::default(); + + let input = &vector["input"]; + let json = serde_json::to_string(input).expect("vector input must be JSON-compatible"); + let result = ContentPart::from_json(&json, &context); + + match vector["operation"].as_str() { + Some("load") => { + let content_part = result.unwrap_or_else(|error| { + panic!("[{vector_name}] known ContentPart failed to load: {error}") + }); + assert_eq!( + content_part.to_value(&SaveContext::default()), + vector["expected"], + "[{vector_name}] known ContentPart payload changed during load/save" + ); + } + Some("load-error") => { + let error = match result { + Ok(_) => panic!( + "[{vector_name}] closed ContentPart accepted unknown discriminator {:?}", + input["kind"] + ), + Err(error) => error, + }; + let diagnostic = error.to_string(); + let discriminator = vector["expected"]["discriminator"] + .as_str() + .expect("error vector must declare the discriminator field"); + let value = vector["expected"]["value"] + .as_str() + .expect("error vector must declare the exact discriminator value"); + assert!( + diagnostic.contains(discriminator), + "[{vector_name}] error diagnostic did not identify discriminator {discriminator:?}: {diagnostic}" + ); + assert!( + diagnostic.contains(value), + "[{vector_name}] error diagnostic did not preserve discriminator value {value:?}: {diagnostic}" + ); + } + operation => panic!("[{vector_name}] unsupported vector operation: {operation:?}"), + } +} diff --git a/runtime/rust/prompty/tests/loader_test.rs b/runtime/rust/prompty/tests/loader_test.rs index 07ffeb161..e1af167df 100644 --- a/runtime/rust/prompty/tests/loader_test.rs +++ b/runtime/rust/prompty/tests/loader_test.rs @@ -20,6 +20,16 @@ fn fixtures_dir() -> PathBuf { .join("fixtures") } +/// Path to the canonical load vectors. +fn load_vectors_path() -> PathBuf { + fixtures_dir() + .parent() + .expect("spec/fixtures must have a spec parent") + .join("vectors") + .join("load") + .join("load_vectors.json") +} + /// Load a fixture `.prompty` file with optional env vars set. fn load_fixture( name: &str, @@ -260,6 +270,32 @@ fn test_tools_function_load() { assert_eq!(tools.len(), 1); assert_eq!(tools[0].name, "get_weather"); assert_eq!(tools[0].kind_str(), "function"); + + let raw = std::fs::read_to_string(load_vectors_path()).unwrap(); + let vectors: serde_json::Value = serde_json::from_str(&raw).unwrap(); + let vector = vectors + .as_array() + .unwrap() + .iter() + .find(|vector| vector["name"] == "tools_function_load") + .unwrap(); + let expected_bindings = vector["expected"]["tools"][0]["bindings"] + .as_object() + .unwrap(); + + assert_eq!(tools[0].bindings.len(), expected_bindings.len()); + for (name, expected) in expected_bindings { + let actual = tools[0] + .bindings + .iter() + .find(|binding| binding.name == *name) + .unwrap_or_else(|| panic!("missing binding {name:?}")); + assert_eq!( + actual.input, + expected["input"].as_str().unwrap(), + "binding {name:?} input mismatch" + ); + } } #[test] diff --git a/runtime/rust/prompty/tests/named_collection_vectors.rs b/runtime/rust/prompty/tests/named_collection_vectors.rs new file mode 100644 index 000000000..2b1174f3e --- /dev/null +++ b/runtime/rust/prompty/tests/named_collection_vectors.rs @@ -0,0 +1,387 @@ +//! Named-collection roundtrip tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::{LoadContext, SaveContext}; +use prompty::model::Prompty; +use serde_json::{Map, Value}; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("named_collection_vectors.json") +} + +fn semantic_entries(collection: &Value) -> Vec { + match collection { + Value::Array(entries) => entries + .iter() + .map(|entry| { + let mut entry = entry + .as_object() + .expect("array-form named collection entries must be objects") + .clone(); + entry + .entry("name".to_string()) + .or_insert_with(|| Value::String(String::new())); + Value::Object(entry) + }) + .collect(), + Value::Object(entries) => entries + .iter() + .map(|(name, entry)| { + let mut entry = entry + .as_object() + .expect("object-form named collection entries must be objects") + .clone(); + entry.insert("name".to_string(), Value::String(name.clone())); + Value::Object(entry) + }) + .collect(), + value => panic!("named collection must be an array or object, got {value:?}"), + } +} + +fn assert_subset(actual: &Value, expected: &Value, path: &str) { + match expected { + Value::Object(expected) => { + let actual = actual + .as_object() + .unwrap_or_else(|| panic!("[{path}] expected object, got {actual:?}")); + for (key, expected_value) in expected { + let actual_value = actual + .get(key) + .unwrap_or_else(|| panic!("[{path}] missing expected key {key:?}")); + assert_subset(actual_value, expected_value, &format!("{path}.{key}")); + } + } + Value::Array(expected) => { + let actual = actual + .as_array() + .unwrap_or_else(|| panic!("[{path}] expected array, got {actual:?}")); + assert_eq!( + actual.len(), + expected.len(), + "[{path}] array length changed" + ); + for (index, expected_value) in expected.iter().enumerate() { + assert_subset(&actual[index], expected_value, &format!("{path}[{index}]")); + } + } + expected => assert_eq!(actual, expected, "[{path}] value changed"), + } +} + +fn assert_collection(vector_name: &str, collection: &Value, expected: &Value) { + let expected_format = expected["collectionFormat"] + .as_str() + .expect("roundtrip vector must declare collectionFormat"); + assert_eq!( + if collection.is_array() { + "array" + } else if collection.is_object() { + "object" + } else { + "invalid" + }, + expected_format, + "[{vector_name}] canonical collection format changed" + ); + + if let Some(wire_entries) = expected["wireEntries"].as_array() { + let entries = collection + .as_array() + .unwrap_or_else(|| panic!("[{vector_name}] wire entry assertions require array form")); + for assertion in wire_entries { + let index = assertion["index"] + .as_u64() + .expect("wire entry assertion must declare an index") + as usize; + let entry = entries + .get(index) + .unwrap_or_else(|| panic!("[{vector_name}] missing wire entry at index {index}")) + .as_object() + .unwrap_or_else(|| { + panic!("[{vector_name}] wire entry at index {index} must be an object") + }); + for field in assertion["absentFields"] + .as_array() + .expect("wire entry assertion must declare absentFields") + { + let field = field.as_str().expect("wire absent field must be a string"); + assert!( + !entry.contains_key(field), + "[{vector_name}] wire entry {index} unexpectedly serialized field {field:?}" + ); + } + } + } + + let actual_entries = semantic_entries(collection); + let expected_entries = expected["entries"] + .as_array() + .expect("roundtrip vector must declare entries"); + assert_eq!( + actual_entries.len(), + expected_entries.len(), + "[{vector_name}] named collection entry count changed" + ); + if let Some(absent_fields) = expected["absentEntryFields"].as_array() { + for entry in &actual_entries { + for field in absent_fields { + let field = field.as_str().expect("absent entry field must be a string"); + assert!( + entry.get(field).is_none(), + "[{vector_name}] entry {:?} unexpectedly populated field {field:?}", + entry["name"] + ); + } + } + } + + if expected["preserveOrder"].as_bool() == Some(true) { + for (index, expected_entry) in expected_entries.iter().enumerate() { + assert_subset( + &actual_entries[index], + expected_entry, + &format!("{vector_name}.entries[{index}]"), + ); + } + } else { + let actual_by_name: Map = actual_entries + .into_iter() + .map(|entry| { + let name = entry["name"] + .as_str() + .expect("semantic entry name must be a string") + .to_string(); + (name, entry) + }) + .collect(); + for expected_entry in expected_entries { + let name = expected_entry["name"] + .as_str() + .expect("expected entry name must be a string"); + let actual_entry = actual_by_name + .get(name) + .unwrap_or_else(|| panic!("[{vector_name}] missing named entry {name:?}")); + assert_subset( + actual_entry, + expected_entry, + &format!("{vector_name}.entries.{name}"), + ); + } + } +} + +fn vectors() -> Vec { + let raw = + std::fs::read_to_string(vectors_path()).expect("failed to read named collection vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse named collection vectors"); + document["vectors"] + .as_array() + .expect("named collection vectors must contain a vectors array") + .clone() +} + +#[test] +fn named_collection_roundtrip_vectors() { + for vector in vectors() + .into_iter() + .filter(|vector| vector["operation"] == "load-save-reload") + { + let name = vector["name"] + .as_str() + .expect("vector name must be a string"); + let json = serde_json::to_string(&vector["input"]) + .expect("named collection vector input must be JSON-compatible"); + let result = Prompty::from_json(&json, &LoadContext::default()); + + let loaded = + result.unwrap_or_else(|error| panic!("[{name}] valid collection failed: {error}")); + let saved = loaded.to_value(&SaveContext::default()); + let collection_path = vector["collectionPath"] + .as_str() + .expect("roundtrip vector must declare collectionPath"); + let collection = saved + .get(collection_path) + .unwrap_or_else(|| panic!("[{name}] missing collection {collection_path:?}")); + assert_collection(name, collection, &vector["expected"]); + + let saved_json = + serde_json::to_string(&saved).expect("saved named collection must be JSON-compatible"); + let reloaded = Prompty::from_json(&saved_json, &LoadContext::default()) + .unwrap_or_else(|error| panic!("[{name}] saved collection failed: {error}")); + let resaved = reloaded.to_value(&SaveContext::default()); + let reloaded_collection = resaved + .get(collection_path) + .unwrap_or_else(|| panic!("[{name}] reload lost collection {collection_path:?}")); + assert_collection(name, reloaded_collection, &vector["expected"]); + } +} + +#[test] +fn unnamed_composite_omits_empty_name_stably() { + let vector_name = "unnamed_composite_omits_empty_name_stably"; + let vector = vectors() + .into_iter() + .find(|vector| vector["name"] == vector_name) + .expect("missing unnamed composite vector"); + let json = serde_json::to_string(&vector["input"]) + .expect("unnamed composite vector input must be JSON-compatible"); + let loaded = Prompty::from_json(&json, &LoadContext::default()) + .unwrap_or_else(|error| panic!("[{vector_name}] valid collection failed: {error}")); + + assert_eq!( + loaded.inputs.len(), + 1, + "[{vector_name}] load changed the entry count" + ); + assert_eq!( + loaded.inputs[0].name, "", + "[{vector_name}] absent wire name did not materialize as an empty in-memory name" + ); + + let saved = loaded.to_value(&SaveContext::default()); + let collection = saved + .get("inputs") + .expect("[unnamed_composite_omits_empty_name_stably] first save lost inputs"); + assert_collection(vector_name, collection, &vector["expected"]); + + let saved_json = serde_json::to_string(&saved).expect("first save must remain JSON-compatible"); + let reloaded = Prompty::from_json(&saved_json, &LoadContext::default()) + .unwrap_or_else(|error| panic!("[{vector_name}] first save failed to reload: {error}")); + assert_eq!( + reloaded.inputs[0].name, "", + "[{vector_name}] reload changed the unnamed in-memory state" + ); + let resaved = reloaded.to_value(&SaveContext::default()); + let reloaded_collection = resaved + .get("inputs") + .expect("[unnamed_composite_omits_empty_name_stably] reload/save lost inputs"); + assert_collection(vector_name, reloaded_collection, &vector["expected"]); +} + +#[test] +fn name_keyed_property_scalars_infer_kind_and_default_without_degradation() { + let vector_names = [ + "string_scalar_in_name_keyed_inputs_infers_property", + "integer_scalar_in_name_keyed_inputs_infers_property", + "float_scalar_in_name_keyed_inputs_infers_property", + "boolean_scalar_in_name_keyed_inputs_infers_property", + ]; + let all_vectors = vectors(); + let mut failures = Vec::new(); + + for vector_name in vector_names { + let vector = all_vectors + .iter() + .find(|candidate| candidate["name"] == vector_name) + .unwrap_or_else(|| panic!("missing named collection vector {vector_name}")); + let json = serde_json::to_string(&vector["input"]) + .expect("named collection vector input must be JSON-compatible"); + let loaded = match Prompty::from_json(&json, &LoadContext::default()) { + Ok(loaded) => loaded, + Err(error) => { + failures.push(format!("[{vector_name}] valid scalar failed: {error}")); + continue; + } + }; + let saved = loaded.to_value(&SaveContext::default()); + let collection_path = vector["collectionPath"] + .as_str() + .expect("scalar vector must declare collectionPath"); + let collection = match saved.get(collection_path) { + Some(collection) => collection, + None => { + failures.push(format!( + "[{vector_name}] missing saved collection {collection_path:?}" + )); + continue; + } + }; + let actual_entries = semantic_entries(collection); + let expected_entry = &vector["expected"]["entries"][0]; + let expected_name = expected_entry["name"] + .as_str() + .expect("expected scalar entry name must be a string"); + let actual_entry = match actual_entries + .iter() + .find(|entry| entry["name"] == expected_name) + { + Some(entry) => entry, + None => { + failures.push(format!( + "[{vector_name}] missing scalar entry {expected_name:?}" + )); + continue; + } + }; + + let expected_kind = &expected_entry["kind"]; + if actual_entry["kind"].as_str().unwrap_or_default().is_empty() { + failures.push(format!("[{vector_name}] silently produced an empty kind")); + } else if actual_entry["kind"] != *expected_kind { + failures.push(format!( + "[{vector_name}] expected kind {expected_kind}, got {}", + actual_entry["kind"] + )); + } + if actual_entry["default"] != expected_entry["default"] { + failures.push(format!( + "[{vector_name}] expected default {}, got {}", + expected_entry["default"], actual_entry["default"] + )); + } + if let Some(example) = actual_entry.get("example") { + failures.push(format!( + "[{vector_name}] collection shorthand unexpectedly populated example {}", + example + )); + } + } + + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + +#[test] +fn named_collection_rejection_vectors() { + for vector in vectors() + .into_iter() + .filter(|vector| vector["operation"] == "load-error") + { + let name = vector["name"] + .as_str() + .expect("vector name must be a string"); + let json = serde_json::to_string(&vector["input"]) + .expect("named collection vector input must be JSON-compatible"); + let error = match Prompty::from_json(&json, &LoadContext::default()) { + Ok(_) => panic!("[{name}] invalid nested array was accepted"), + Err(error) => error, + }; + let diagnostic = error.to_string(); + let expected_path = vector["expected"]["path"] + .as_str() + .expect("error vector must declare path"); + let value_category = vector["expected"]["valueCategory"] + .as_str() + .expect("error vector must declare valueCategory"); + assert!( + diagnostic.contains(expected_path), + "[{name}] diagnostic did not identify path {expected_path:?}: {diagnostic}" + ); + assert!( + diagnostic.contains(value_category), + "[{name}] diagnostic did not identify category {value_category:?}: {diagnostic}" + ); + } +} diff --git a/runtime/rust/prompty/tests/property_scalar_coercion_vectors.rs b/runtime/rust/prompty/tests/property_scalar_coercion_vectors.rs new file mode 100644 index 000000000..2caf44422 --- /dev/null +++ b/runtime/rust/prompty/tests/property_scalar_coercion_vectors.rs @@ -0,0 +1,68 @@ +//! Cross-runtime Property scalar coercion tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::LoadContext; +use prompty::model::Property; +use serde_json::Value; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("property_scalar_coercion_vectors.json") +} + +#[test] +fn all_primitive_property_scalars_coerce_atomically() { + let raw = std::fs::read_to_string(vectors_path()) + .expect("failed to read Property scalar coercion vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse Property scalar coercion vectors"); + let vector = &document["vectors"][0]; + assert_eq!( + vector["name"], "all_primitive_property_scalars_coerce_atomically", + "unexpected Property scalar coercion vector" + ); + assert_eq!(vector["operation"], "load"); + + let cases = vector["cases"] + .as_array() + .expect("Property scalar coercion vector must contain cases"); + let case_names: Vec<&str> = cases + .iter() + .map(|case| case["name"].as_str().expect("scalar case must have a name")) + .collect(); + assert_eq!(case_names, ["string", "integer", "float", "boolean"]); + + let context = LoadContext::default(); + let mut failures = Vec::new(); + for case in cases { + let case_name = case["name"].as_str().expect("scalar case must have a name"); + let loaded = Property::load_from_value(&case["input"], &context); + let expected_kind = case["expected"]["kind"] + .as_str() + .expect("expected kind must be a string"); + if loaded.kind_str() != expected_kind { + failures.push(format!( + "[{case_name}] expected kind {expected_kind:?}, got {:?}", + loaded.kind_str() + )); + continue; + } + if loaded.example.as_ref() != Some(&case["expected"]["example"]) { + failures.push(format!( + "[{case_name}] expected example {}, got {:?}", + case["expected"]["example"], loaded.example + )); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/runtime/rust/prompty/tests/record_unknown_nullability_vectors.rs b/runtime/rust/prompty/tests/record_unknown_nullability_vectors.rs new file mode 100644 index 000000000..c5f775e43 --- /dev/null +++ b/runtime/rust/prompty/tests/record_unknown_nullability_vectors.rs @@ -0,0 +1,87 @@ +//! Record nullability tests backed by the shared model vectors. + +use std::path::PathBuf; + +use prompty::model::context::{LoadContext, SaveContext}; +use prompty::model::{ + HostToolRequest, Message, ModelInfo, Prompty, RunTurnRequest, SessionEvent, TurnEvent, + TurnModelRequest, TurnModelResponse, +}; +use serde_json::Value; + +fn vectors_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("runtime/rust/prompty must have a rust parent") + .parent() + .expect("runtime/rust must have a runtime parent") + .parent() + .expect("runtime must have a repository parent") + .join("spec") + .join("vectors") + .join("model") + .join("record_unknown_nullability_vectors.json") +} + +macro_rules! roundtrip { + ($model:ty, $json:expr) => {{ + let loaded = + <$model>::from_json($json, &LoadContext::default()).expect("vector input must load"); + let saved = loaded.to_value(&SaveContext::default()); + let saved_json = + serde_json::to_string(&saved).expect("saved model must be JSON-compatible"); + let reloaded = <$model>::from_json(&saved_json, &LoadContext::default()) + .expect("saved model must reload"); + reloaded.to_value(&SaveContext::default()) + }}; +} + +#[test] +fn record_unknown_nullability_vectors() { + let raw = std::fs::read_to_string(vectors_path()) + .expect("failed to read Record nullability vectors"); + let document: Value = + serde_json::from_str(&raw).expect("failed to parse Record nullability vectors"); + let vectors = document["vectors"] + .as_array() + .expect("Record nullability vectors must contain a vectors array"); + + for vector in vectors { + let name = vector["name"] + .as_str() + .expect("vector name must be a string"); + assert_eq!( + vector["operation"], "load-save-reload", + "[{name}] unsupported vector operation" + ); + let model = vector["model"] + .as_str() + .expect("vector model must be a string"); + let field_path = vector["fieldPath"] + .as_str() + .expect("vector fieldPath must be a string"); + let json = + serde_json::to_string(&vector["input"]).expect("vector input must be JSON-compatible"); + + let resaved = match model { + "Message" => roundtrip!(Message, &json), + "Prompty" => roundtrip!(Prompty, &json), + "ModelInfo" => roundtrip!(ModelInfo, &json), + "TurnModelRequest" => roundtrip!(TurnModelRequest, &json), + "RunTurnRequest" => roundtrip!(RunTurnRequest, &json), + "TurnModelResponse" => roundtrip!(TurnModelResponse, &json), + "HostToolRequest" => roundtrip!(HostToolRequest, &json), + "TurnEvent" => roundtrip!(TurnEvent, &json), + "SessionEvent" => roundtrip!(SessionEvent, &json), + _ => panic!("[{name}] unsupported model {model:?}"), + }; + + let actual = resaved + .get(field_path) + .unwrap_or_else(|| panic!("[{name}] reload lost field {field_path:?}")); + assert_eq!( + actual, &vector["expected"], + "[{name}] reload changed null-valued record entries" + ); + } +} diff --git a/runtime/typescript/packages/core/src/core/types.ts b/runtime/typescript/packages/core/src/core/types.ts index 0f398f5d9..b9f772b8c 100644 --- a/runtime/typescript/packages/core/src/core/types.ts +++ b/runtime/typescript/packages/core/src/core/types.ts @@ -80,12 +80,12 @@ export class Message implements MessageHelpers { this.metadata = init?.metadata ?? {}; } - /** Concatenate all TextPart values into a single string. */ + /** Concatenate all TextPart values joined by newline. */ get text(): string { return this.parts .filter((p): p is TextPart => p.kind === "text") .map((p) => p.value) - .join(""); + .join("\n"); } /** @@ -94,8 +94,8 @@ export class Message implements MessageHelpers { * - If multimodal, return an array of content objects. */ toTextContent(): string | Record[] { - if (this.parts.length === 1 && this.parts[0].kind === "text") { - return (this.parts[0] as TextPart).value; + if (this.parts.every((part) => part.kind === "text")) { + return this.text; } return this.parts.map(partToWireContent); } diff --git a/runtime/typescript/packages/core/tests/connection-roundtrip-vectors.test.ts b/runtime/typescript/packages/core/tests/connection-roundtrip-vectors.test.ts new file mode 100644 index 000000000..af1f844d4 --- /dev/null +++ b/runtime/typescript/packages/core/tests/connection-roundtrip-vectors.test.ts @@ -0,0 +1,51 @@ +/** + * Canonical forward-compatibility tests for open Connection discriminators. + */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { Connection, ReferenceConnection } from "../src/index.js"; + +interface ConnectionRoundtripVector { + name: string; + operation: "load-save-reload"; + input: Record; + expected: Record; +} + +const vectorsPath = resolve( + import.meta.dirname, + "../../../../../spec/vectors/model/connection_roundtrip_vectors.json", +); +const vectors = ( + JSON.parse(readFileSync(vectorsPath, "utf8")) as { + vectors: ConnectionRoundtripVector[]; + } +).vectors; + +describe("Connection roundtrip vectors", () => { + it.each(vectors)( + "$name preserves the exact discriminator and payload", + (vector) => { + expect(vector.operation).toBe("load-save-reload"); + + const loaded = Connection.load(vector.input); + if (vector.expected.kind === "reference") { + expect(loaded).toBeInstanceOf(ReferenceConnection); + } else { + expect(loaded).not.toBeInstanceOf(ReferenceConnection); + } + + const saved = loaded.save(); + expect(saved.kind).toBe(vector.expected.kind); + expect(saved).toEqual(vector.expected); + + const reloaded = Connection.load(saved); + const resaved = reloaded.save(); + expect(resaved.kind).toBe(vector.expected.kind); + expect(resaved).toEqual(vector.expected); + }, + ); +}); diff --git a/runtime/typescript/packages/core/tests/content-part-discriminator-vectors.test.ts b/runtime/typescript/packages/core/tests/content-part-discriminator-vectors.test.ts new file mode 100644 index 000000000..754f88279 --- /dev/null +++ b/runtime/typescript/packages/core/tests/content-part-discriminator-vectors.test.ts @@ -0,0 +1,51 @@ +/** + * Canonical strict-discriminator tests for the closed ContentPart hierarchy. + */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { + ContentPart, + TextPart, +} from "../src/model/conversation/content-part.js"; + +interface ContentPartDiscriminatorVector { + name: string; + operation: "load" | "load-error"; + input: Record; + expected: Record; +} + +const vectorsPath = resolve( + import.meta.dirname, + "../../../../../spec/vectors/model/content_part_discriminator_vectors.json", +); +const vectors = ( + JSON.parse(readFileSync(vectorsPath, "utf8")) as { + vectors: ContentPartDiscriminatorVector[]; + } +).vectors; + +describe("ContentPart discriminator vectors", () => { + it.each(vectors)("$name enforces closed, case-sensitive kinds", (vector) => { + if (vector.operation === "load") { + const loaded = ContentPart.load(vector.input); + expect(loaded).toBeInstanceOf(TextPart); + expect(loaded.save()).toEqual(vector.expected); + return; + } + + let diagnostic = ""; + try { + ContentPart.load(vector.input); + } catch (error) { + diagnostic = String(error); + } + + expect(diagnostic).not.toBe(""); + expect(diagnostic).toContain(String(vector.expected.discriminator)); + expect(diagnostic).toContain(String(vector.expected.value)); + }); +}); diff --git a/runtime/typescript/packages/core/tests/property-scalar-coercion-vectors.test.ts b/runtime/typescript/packages/core/tests/property-scalar-coercion-vectors.test.ts new file mode 100644 index 000000000..92190b5c9 --- /dev/null +++ b/runtime/typescript/packages/core/tests/property-scalar-coercion-vectors.test.ts @@ -0,0 +1,55 @@ +/** + * Canonical atomic Property scalar coercion tests backed by shared model vectors. + */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { Property } from "../src/index.js"; + +interface PropertyScalarCase { + name: string; + input: string | number | boolean; + expected: { + kind: string; + example: string | number | boolean; + }; +} + +interface PropertyScalarVector { + name: string; + operation: "load"; + cases: PropertyScalarCase[]; +} + +const vectorsPath = resolve( + import.meta.dirname, + "../../../../../spec/vectors/model/property_scalar_coercion_vectors.json", +); +const vector = ( + JSON.parse(readFileSync(vectorsPath, "utf8")) as { + vectors: PropertyScalarVector[]; + } +).vectors[0]; + +describe("Property scalar coercion vectors", () => { + it("coerces all primitive scalar branches atomically", () => { + expect(vector.name).toBe("all_primitive_property_scalars_coerce_atomically"); + expect(vector.operation).toBe("load"); + expect(vector.cases.map((candidate) => candidate.name)).toEqual([ + "string", + "integer", + "float", + "boolean", + ]); + + for (const scalarCase of vector.cases) { + const loaded = Property.fromJson(JSON.stringify(scalarCase.input)); + expect(loaded.kind, scalarCase.name).toBe(scalarCase.expected.kind); + expect(loaded.example, scalarCase.name).toEqual( + scalarCase.expected.example, + ); + } + }); +}); diff --git a/runtime/typescript/packages/core/tests/spec-vectors.test.ts b/runtime/typescript/packages/core/tests/spec-vectors.test.ts index fdc30b66c..d369cedb3 100644 --- a/runtime/typescript/packages/core/tests/spec-vectors.test.ts +++ b/runtime/typescript/packages/core/tests/spec-vectors.test.ts @@ -430,11 +430,12 @@ function validateAgentFields(agent: Prompty, expected: any, vecName: string): vo } } if (et.bindings !== undefined) { - const atBindings = (at as any).bindings as Array<{name: string; input: string}>; + const atBindings = (at as FunctionTool).bindings ?? []; + const expectedBindings = Object.entries(et.bindings as Record); expect(atBindings).toBeDefined(); - expect(atBindings.length).toBeGreaterThan(0); - for (const [bk, bv] of Object.entries(et.bindings as Record)) { - const found = atBindings.find((b: any) => b.name === bk); + expect(atBindings).toHaveLength(expectedBindings.length); + for (const [bk, bv] of expectedBindings) { + const found = atBindings.find((binding) => binding.name === bk); expect(found).toBeDefined(); if (bv.input !== undefined) { expect(found!.input).toBe(bv.input); diff --git a/runtime/typescript/packages/core/tests/types.test.ts b/runtime/typescript/packages/core/tests/types.test.ts index 77ab20dab..3018dda72 100644 --- a/runtime/typescript/packages/core/tests/types.test.ts +++ b/runtime/typescript/packages/core/tests/types.test.ts @@ -20,10 +20,11 @@ describe("Message", () => { it("concatenates multiple text parts", () => { const msg = new Message({ role: "user", parts: [ - { kind: "text", value: "Hello " }, + { kind: "text", value: "Hello" }, { kind: "text", value: "world" }, ] }); - expect(msg.text).toBe("Hello world"); + expect(msg.text).toBe("Hello\nworld"); + expect(msg.toTextContent()).toBe("Hello\nworld"); }); it("returns string for single text part in toTextContent", () => { @@ -45,6 +46,8 @@ describe("Message", () => { const msg = new Message({ role: "system" }); expect(msg.parts).toEqual([]); expect(msg.metadata).toEqual({}); + expect(msg.text).toBe(""); + expect(msg.toTextContent()).toBe(""); }); }); diff --git a/schema/README.md b/schema/README.md index 8d217526d..b1360c766 100644 --- a/schema/README.md +++ b/schema/README.md @@ -39,6 +39,7 @@ npm run build ``` This generates code into: + - `runtime/typescript/packages/core/src/model/` — TypeScript - `runtime/python/prompty/prompty/model/` — Python - `runtime/csharp/Prompty.Core/Model/` — C# @@ -52,6 +53,7 @@ This generates code into: Generated files are **committed to the repo**. The generator is a dev-time tool — consumers don't need TypeSpec installed. Generated files have a header: + ``` // WARNING: This is an auto-generated file. DO NOT EDIT THIS FILE DIRECTLY ``` @@ -78,6 +80,9 @@ The `scripts/` folder holds the supporting Node helpers: output deterministic — it normalizes the generation timestamp in the Typra manifest, collapses empty generated Python test files, and trims trailing whitespace in generated Go files. +- `verify-engine-ports.mjs` validates the canonical engine-port metadata and + native cancellation seams against `spec/vectors/engine/port_contracts.json`, + and protects the deterministic legacy `pipeline/harness.tsp` contract. - `verify-typra.mjs` backs `npm run verify:typra`, which compares the current Typra export surfaces, manifest, hydration seams, and JSON AST against the committed `HEAD` baseline to detect schema drift. diff --git a/schema/model/agent/agent.tsp b/schema/model/agent/agent.tsp index 0a7094e12..30e459fdc 100644 --- a/schema/model/agent/agent.tsp +++ b/schema/model/agent/agent.tsp @@ -45,7 +45,7 @@ model Prompty { }) description?: string = ""; - @doc("Additional metadata including authors, tags, and other arbitrary properties") + @doc("Additional metadata including authors, tags, and other arbitrary properties. Values may be explicit null.") @sample(#{ metadata: #{ authors: #["sethjuarez", "jietong"], diff --git a/schema/model/connection/connection.tsp b/schema/model/connection/connection.tsp index e74898086..2d6659213 100644 --- a/schema/model/connection/connection.tsp +++ b/schema/model/connection/connection.tsp @@ -9,7 +9,8 @@ alias ConnectionType = | "key" | "anonymous" | "foundry" - | "oauth"; + | "oauth" + | string; alias AuthenticationMode = "user" | "system"; /** diff --git a/schema/model/conversation/message.tsp b/schema/model/conversation/message.tsp index ae8fdf083..38855c072 100644 --- a/schema/model/conversation/message.tsp +++ b/schema/model/conversation/message.tsp @@ -42,7 +42,7 @@ model Message { @sample(#{ parts: #[#{ kind: "text", value: "Hello!" }] }) parts: ContentPart[]; - @doc("Optional metadata associated with the message") + @doc("Optional metadata associated with the message. Values may be explicit null.") @sample(#{ metadata: #{ source: "user-input" } }) metadata: Record = #{}; } diff --git a/schema/model/events/payloads.tsp b/schema/model/events/payloads.tsp index ef113549d..5a44c543a 100644 --- a/schema/model/events/payloads.tsp +++ b/schema/model/events/payloads.tsp @@ -87,7 +87,7 @@ model TurnEvent { @sample(#{ spanId: "span_tool_001" }) spanId?: string; - @doc("Event-specific payload. Use the typed payload model matching 'type'.") + @doc("Event-specific payload. Values may be explicit null. Use the typed payload model matching 'type'.") payload: Record = #{}; } @@ -468,7 +468,7 @@ model HostToolRequest { @sample(#{ toolName: "powershell" }) toolName: string; - @doc("Tool arguments after host-side sanitization") + @doc("Tool arguments after host-side sanitization. Values may be explicit null.") arguments?: Record; @doc("Working directory or execution scope for the tool") diff --git a/schema/model/events/session.tsp b/schema/model/events/session.tsp index c502d9680..8cccc77f8 100644 --- a/schema/model/events/session.tsp +++ b/schema/model/events/session.tsp @@ -157,7 +157,7 @@ model SessionEvent { @sample(#{ spanId: "span_hook_001" }) spanId?: string; - @doc("Event-specific payload. Use the typed payload model matching 'type'.") + @doc("Event-specific payload. Values may be explicit null. Use the typed payload model matching 'type'.") payload: Record = #{}; @doc("Redaction state for sensitive payload fields") diff --git a/schema/model/main.tsp b/schema/model/main.tsp index 591d08c54..7228f7d57 100644 --- a/schema/model/main.tsp +++ b/schema/model/main.tsp @@ -27,6 +27,7 @@ import "./pipeline/invocation.tsp"; import "./pipeline/engine-events.tsp"; import "./pipeline/checkpoint.tsp"; import "./pipeline/turn-engine.tsp"; +import "./pipeline/engine-ports.tsp"; import "./pipeline/policy.tsp"; import "./pipeline/context-planning.tsp"; import "./pipeline/turn.tsp"; diff --git a/schema/model/model/discovery.tsp b/schema/model/model/discovery.tsp index 118716343..78f4b1fc0 100644 --- a/schema/model/model/discovery.tsp +++ b/schema/model/model/discovery.tsp @@ -46,7 +46,7 @@ model ModelInfo { @sample(#{ outputModalities: #["text"] }) outputModalities?: string[]; - @doc("Additional provider-specific properties") + @doc("Additional provider-specific properties. Values may be explicit null.") @sample(#{ additionalProperties: #{ supportsStreaming: true } }) additionalProperties?: Record; } diff --git a/schema/model/pipeline/engine-ports.tsp b/schema/model/pipeline/engine-ports.tsp new file mode 100644 index 000000000..0d1706a95 --- /dev/null +++ b/schema/model/pipeline/engine-ports.tsp @@ -0,0 +1,71 @@ +import "@typra/emitter"; +import "./checkpoint.tsp"; +import "./engine-events.tsp"; +import "./invocation.tsp"; +import "./turn-engine.tsp"; + +namespace Prompty; + +@@protocol(EnginePermissionPort); +@@method(EnginePermissionPort, + "authorize", + "EnginePermissionDecision", + "Authorize one model-requested tool before execution", + #{ request: "ModelToolRequest" }, + false, + false, + #{ runtimeCancellable: true } +); + +/** Authorizes model-requested tools at a runtime cancellation boundary. */ +model EnginePermissionPort {} + +@@protocol(EngineToolPort); +@@method(EngineToolPort, + "execute", + "ModelToolResult", + "Execute one authorized model-requested tool", + #{ request: "ModelToolRequest" }, + false, + false, + #{ runtimeCancellable: true } +); + +/** Executes authorized model-requested tools at a runtime cancellation boundary. */ +model EngineToolPort {} + +@@protocol(EngineDurabilityPort); +@@method(EngineDurabilityPort, + "append", + "void", + "Append one semantic engine event durably", + #{ event: "EngineEvent" }, + false, + false +); +@@method(EngineDurabilityPort, + "appendWithCheckpoint", + "void", + "Atomically append semantic engine events and persist the checkpoint that reflects them", + #{ events: "EngineEvent[]", checkpoint: "EngineCheckpoint" }, + false, + false, + #{ atomic: true } +); + +/** Persists semantic engine events and checkpoints without runtime cancellation. */ +model EngineDurabilityPort {} + +@@protocol(EnginePostCommitPort); +@@method(EnginePostCommitPort, + "afterCommit", + "void", + "Run one idempotent host effect after the turn is durably committed", + #{ effectId: "string", commit: "TurnCommit" }, + false, + false, + #{ runtimeCancellable: true, nonFatal: true } +); + +/** Runs non-fatal host effects after a turn is durably committed. */ +model EnginePostCommitPort {} diff --git a/schema/model/pipeline/executor.tsp b/schema/model/pipeline/executor.tsp index 357a09cb7..0780d7237 100644 --- a/schema/model/pipeline/executor.tsp +++ b/schema/model/pipeline/executor.tsp @@ -10,14 +10,19 @@ namespace Prompty; "execute", "unknown", "Call an LLM provider with messages and return the raw response", - #{ agent: "Prompty", messages: "Message[]" } + #{ agent: "Prompty", messages: "Message[]" }, + false, + false, + #{ runtimeCancellable: true } ); @@method(Executor, "executeStream", "unknown", "Call an LLM provider and return a streaming response. Returns a language-specific async iterable/stream of raw chunks. Not all providers support streaming; the default implementation should signal lack of support.", #{ agent: "Prompty", messages: "Message[]" }, - true + true, + false, + #{ runtimeCancellable: true } ); @@method(Executor, "formatToolMessages", diff --git a/schema/model/pipeline/turn.tsp b/schema/model/pipeline/turn.tsp index 1a260b7f4..5d795a4b8 100644 --- a/schema/model/pipeline/turn.tsp +++ b/schema/model/pipeline/turn.tsp @@ -81,7 +81,7 @@ model TurnModelRequest { @sample(#{ iteration: 0 }) iteration: int32; - @doc("Inputs supplied to the deterministic single-turn run") + @doc("Inputs supplied to the deterministic single-turn run. Values may be explicit null.") inputs?: Record = #{}; @doc("Canonical turn execution options") @@ -104,7 +104,7 @@ model TurnModelResponse { @doc("Host tool execution requests emitted by the model callback") toolRequests?: HostToolRequest[] = #[]; - @doc("Additional deterministic state to merge into the iteration checkpoint") + @doc("Additional deterministic state to merge into the iteration checkpoint. Values may be explicit null.") checkpointState?: Record = #{}; } @@ -120,7 +120,7 @@ model RunTurnRequest { @sample(#{ turnId: "turn_abc123" }) turnId: string; - @doc("Inputs supplied to the deterministic single-turn run") + @doc("Inputs supplied to the deterministic single-turn run. Values may be explicit null.") inputs?: Record = #{}; @doc("Canonical turn execution options") diff --git a/schema/package.json b/schema/package.json index a0d593116..dbd093a50 100644 --- a/schema/package.json +++ b/schema/package.json @@ -7,8 +7,9 @@ "format:tsp:check": "npx tsp format \"model/**/*.tsp\" --check", "format:rust": "cargo fmt --all --manifest-path ../runtime/rust/prompty/Cargo.toml", "generate": "npx tsp compile model/main.tsp --config tspconfig.yaml && node scripts/normalize-typra-output.mjs", + "verify:engine-ports": "node scripts/verify-engine-ports.mjs", "verify:typra": "node scripts/verify-typra.mjs", - "build": "npm run format:tsp && npm run generate && npm run format:rust" + "build": "npm run format:tsp && npm run generate && npm run format:rust && npm run verify:engine-ports" }, "dependencies": { "@typespec/compiler": "1.10.0", diff --git a/schema/scripts/verify-engine-ports.mjs b/schema/scripts/verify-engine-ports.mjs new file mode 100644 index 000000000..b75198d68 --- /dev/null +++ b/schema/scripts/verify-engine-ports.mjs @@ -0,0 +1,664 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +const EXPECTED_TARGETS = [ + "csharp", + "go", + "markdown", + "python", + "rust", + "typescript", +]; +const repoRoot = execFileSync("git", ["rev-parse", "--show-toplevel"], { + encoding: "utf8", +}).trim(); +const vector = readJson( + join(repoRoot, "spec", "vectors", "engine", "port_contracts.json"), +); +const surfaces = readJson( + join( + repoRoot, + "schema", + "tsp-output", + ".typra-generated", + "export-surfaces.json", + ), +); + +verifyLegacyHarness(); +verifyExportSurfaces(); +verifyNoWireCancellation(); +verifyNativeSignatures(); +verifyMarkdownSemantics(); + +console.log("Canonical engine port contracts verified."); + +function readJson(path) { + return JSON.parse(readFileSync(path, "utf8")); +} + +function verifyLegacyHarness() { + const harness = readFileSync( + join(repoRoot, "schema", "model", "pipeline", "harness.tsp"), + ); + const actual = createHash("sha256").update(harness).digest("hex"); + assertEqual( + actual, + vector.legacyHarnessSha256, + "pipeline/harness.tsp SHA-256", + ); +} + +function verifyExportSurfaces() { + const targetNames = surfaces.targets.map((target) => target.target); + assertEqual( + new Set(targetNames).size, + targetNames.length, + "Typra targets: duplicate names are not allowed", + ); + assertEqual( + JSON.stringify(targetNames.sort()), + JSON.stringify(EXPECTED_TARGETS), + "Typra targets", + ); + + for (const target of surfaces.targets) { + assertUniqueNames(target.protocols, `${target.target}: protocols`); + + for (const nativeError of vector.nativeErrors) { + assert( + !JSON.stringify(target).includes(`"${nativeError}"`), + `${target.target}: ${nativeError} must not appear in generated exports`, + ); + } + + for (const [protocolName, expectedProtocol] of Object.entries( + vector.protocols, + )) { + const protocol = target.protocols.find( + (candidate) => candidate.name === protocolName, + ); + assert(protocol, `${target.target}: missing ${protocolName} protocol`); + assertUniqueNames( + protocol.methods, + `${target.target}: ${protocolName} methods`, + ); + assertExactKeys( + Object.fromEntries( + protocol.methods.map((method) => [method.name, true]), + ), + Object.fromEntries( + Object.keys(expectedProtocol.methods).map((methodName) => [ + methodName, + true, + ]), + ), + `${target.target}: ${protocolName} methods`, + ); + + for (const [methodName, expectedMethod] of Object.entries( + expectedProtocol.methods, + )) { + const method = protocol.methods.find( + (candidate) => candidate.name === methodName, + ); + assert( + method, + `${target.target}: missing ${protocolName}.${methodName}`, + ); + assertExactKeys( + method, + Object.fromEntries( + ["name", ...Object.keys(expectedMethod)].map((key) => [key, true]), + ), + `${target.target}: ${protocolName}.${methodName} metadata`, + ); + assertSubset( + method, + expectedMethod, + `${target.target}: ${protocolName}.${methodName}`, + ); + if (expectedMethod.params) { + assertExactKeys( + method.params, + expectedMethod.params, + `${target.target}: ${protocolName}.${methodName} params`, + ); + } + + for (const syntheticName of [ + "cancellation", + "cancellationToken", + "ctx", + "signal", + ]) { + assert( + !Object.hasOwn(method.params, syntheticName), + `${target.target}: ${protocolName}.${methodName} leaked runtime cancellation into schema params`, + ); + } + } + } + } +} + +function verifyNoWireCancellation() { + const schemaRoot = join(repoRoot, "vscode", "prompty", "schemas"); + const forbiddenFields = ["ctx", "runtimeCancellable", "atomic", "nonFatal"]; + + for (const file of collectFiles(schemaRoot)) { + const content = readFileSync(file, "utf8"); + for (const field of forbiddenFields) { + const propertyPattern = new RegExp( + `^\\s*(?:${field}|["']${field}["'])\\s*:\\s*`, + "mi", + ); + assert( + !propertyPattern.test(content), + `${file}: ${field} must not be emitted as a portable model field`, + ); + } + assert( + !/^\s*(?:["']?[\w]*(?:cancel|abort|signal)[\w]*["']?)\s*:\s*/imu.test( + content, + ), + `${file}: runtime cancellation must not be emitted as a portable model field`, + ); + assert( + !/(?:cancel|abort|signal)[^\\/]*\.ya?ml$/iu.test(file), + `${file}: runtime cancellation types must not be portable models`, + ); + } + + for (const protocol of [ + "EnginePermissionPort", + "EngineToolPort", + "EngineDurabilityPort", + "EnginePostCommitPort", + "Executor", + ]) { + const file = join(schemaRoot, `${protocol}.yaml`); + const content = readFileSync(file, "utf8"); + assert( + /^properties:\s*\{\}\s*$/mu.test(content), + `${file}: protocol schemas must not expose properties`, + ); + assert( + !/^required:/mu.test(content), + `${file}: protocol schemas must not expose required wire fields`, + ); + } +} + +function verifyNativeSignatures() { + const root = { + csharp: join( + repoRoot, + "runtime", + "csharp", + "Prompty.Core", + "Model", + "pipeline", + ), + go: join(repoRoot, "runtime", "go", "prompty", "model"), + python: join( + repoRoot, + "runtime", + "python", + "prompty", + "prompty", + "model", + "pipeline", + ), + rust: join( + repoRoot, + "runtime", + "rust", + "prompty", + "src", + "model", + "pipeline", + ), + typescript: join( + repoRoot, + "runtime", + "typescript", + "packages", + "core", + "src", + "model", + "pipeline", + ), + }; + + verifyCSharpSignatures(root.csharp); + verifyGoSignatures(root.go); + verifyTypeScriptSignatures(root.typescript); + verifyRustSignatures(root.rust); + verifyPythonSignatures(root.python); + + const durabilityFiles = [ + join(root.csharp, "EngineDurabilityPort.cs"), + join(root.go, "engine_durability_port.go"), + join(root.python, "_EngineDurabilityPort.py"), + join(root.rust, "engine_durability_port.rs"), + join(root.typescript, "engine-durability-port.ts"), + ]; + for (const file of durabilityFiles) { + const content = readFileSync(file, "utf8"); + for (const forbidden of [ + "CancellationToken", + "context.Context", + "AbortSignal", + ]) { + assert( + !content.includes(forbidden), + `${file}: durability protocol must remain non-cancellable`, + ); + } + } +} + +function verifyCSharpSignatures(root) { + expectMatches( + join(root, "EnginePermissionPort.cs"), + /^\s*Task\s+AuthorizeAsync\(\s*ModelToolRequest request,\s*CancellationToken cancellationToken = default\s*\);/mu, + ); + expectMatches( + join(root, "EngineToolPort.cs"), + /^\s*Task\s+ExecuteAsync\(\s*ModelToolRequest request,\s*CancellationToken cancellationToken = default\s*\);/mu, + ); + expectAllMatches(join(root, "EngineDurabilityPort.cs"), [ + /^\s*Task\s+AppendAsync\(\s*EngineEvent @?event\s*\);/mu, + /^\s*Task\s+AppendWithCheckpointAsync\(\s*List events,\s*EngineCheckpoint checkpoint\s*\);/mu, + ]); + expectMatches( + join(root, "EnginePostCommitPort.cs"), + /^\s*Task\s+AfterCommitAsync\(\s*string effectId,\s*TurnCommit commit,\s*CancellationToken cancellationToken = default\s*\);/mu, + ); + expectAllMatches(join(root, "Executor.cs"), [ + /^\s*Task\s+ExecuteAsync\(\s*Prompty agent,\s*List messages,\s*CancellationToken cancellationToken = default\s*\);/mu, + /^\s*Task\s+ExecuteStreamAsync\(\s*Prompty agent,\s*List messages,\s*CancellationToken cancellationToken = default\s*\)/mu, + /^\s*List\s+FormatToolMessages\(\s*object rawResponse,\s*List toolCalls,\s*List toolResults,\s*string\? textContent\s*\);/mu, + ]); + assertDeclarationNames( + join(root, "EnginePermissionPort.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["AuthorizeAsync"], + ); + assertDeclarationNames( + join(root, "EngineToolPort.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["ExecuteAsync"], + ); + assertDeclarationNames( + join(root, "EngineDurabilityPort.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["AppendAsync", "AppendWithCheckpointAsync"], + ); + assertDeclarationNames( + join(root, "EnginePostCommitPort.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["AfterCommitAsync"], + ); + assertDeclarationNames( + join(root, "Executor.cs"), + /^\s*[A-Za-z_][\w<>,? .]*\s+([A-Z]\w*)\(/gmu, + ["ExecuteAsync", "ExecuteStreamAsync", "FormatToolMessages"], + ); +} + +function verifyGoSignatures(root) { + expectMatches( + join(root, "engine_permission_port.go"), + /^\s*Authorize\(ctx context\.Context,\s*request ModelToolRequest\)\s*\(EnginePermissionDecision, error\)/mu, + ); + expectMatches( + join(root, "engine_tool_port.go"), + /^\s*Execute\(ctx context\.Context,\s*request ModelToolRequest\)\s*\(ModelToolResult, error\)/mu, + ); + expectAllMatches(join(root, "engine_durability_port.go"), [ + /^\s*Append\(event EngineEvent\)\s*error/mu, + /^\s*AppendWithCheckpoint\(events \[\]EngineEvent,\s*checkpoint EngineCheckpoint\)\s*error/mu, + ]); + expectMatches( + join(root, "engine_post_commit_port.go"), + /^\s*AfterCommit\(ctx context\.Context,\s*effectId string,\s*commit TurnCommit\)\s*error/mu, + ); + expectAllMatches(join(root, "executor.go"), [ + /^\s*Execute\(ctx context\.Context,\s*agent Prompty,\s*messages \[\]Message\)\s*\(interface\{\}, error\)/mu, + /^\s*ExecuteStream\(ctx context\.Context,\s*agent Prompty,\s*messages \[\]Message\)\s*\(interface\{\}, error\)/mu, + /^\s*FormatToolMessages\(rawResponse interface\{\},\s*toolCalls \[\]ToolCall,\s*toolResults \[\]string,\s*textContent \*string\)\s*\(\[\]Message, error\)/mu, + ]); + for (const [file, expected] of [ + ["engine_permission_port.go", ["Authorize"]], + ["engine_tool_port.go", ["Execute"]], + ["engine_durability_port.go", ["Append", "AppendWithCheckpoint"]], + ["engine_post_commit_port.go", ["AfterCommit"]], + ["executor.go", ["Execute", "ExecuteStream", "FormatToolMessages"]], + ]) { + assertDeclarationNames(join(root, file), /^\s*([A-Z]\w*)\(/gmu, expected); + } +} + +function verifyTypeScriptSignatures(root) { + expectMatches( + join(root, "engine-permission-port.ts"), + /^\s{2}authorize\(\s*request: ModelToolRequest,\s*signal\?: AbortSignal,?\s*\): Promise;/mu, + ); + expectMatches( + join(root, "engine-tool-port.ts"), + /^\s{2}execute\(\s*request: ModelToolRequest,\s*signal\?: AbortSignal,?\s*\): Promise;/mu, + ); + expectAllMatches(join(root, "engine-durability-port.ts"), [ + /^\s{2}append\(\s*event: EngineEvent,?\s*\): Promise;/mu, + /^\s{2}appendWithCheckpoint\(\s*events: EngineEvent\[\],\s*checkpoint: EngineCheckpoint,?\s*\): Promise;/mu, + ]); + expectMatches( + join(root, "engine-post-commit-port.ts"), + /^\s{2}afterCommit\(\s*effectId: string,\s*commit: TurnCommit,\s*signal\?: AbortSignal,?\s*\): Promise;/mu, + ); + expectAllMatches(join(root, "executor.ts"), [ + /^\s{2}execute\(\s*agent: Prompty,\s*messages: Message\[\],\s*signal\?: AbortSignal,?\s*\): Promise;/mu, + /^\s{2}executeStream\?\(\s*agent: Prompty,\s*messages: Message\[\],\s*signal\?: AbortSignal,?\s*\): Promise;/mu, + /^\s{2}formatToolMessages\(\s*rawResponse: unknown,\s*toolCalls: ToolCall\[\],\s*toolResults: string\[\],\s*textContent: string \| null,?\s*\): Message\[\];/mu, + ]); + for (const [file, expected] of [ + ["engine-permission-port.ts", ["authorize"]], + ["engine-tool-port.ts", ["execute"]], + ["engine-durability-port.ts", ["append", "appendWithCheckpoint"]], + ["engine-post-commit-port.ts", ["afterCommit"]], + ["executor.ts", ["execute", "executeStream", "formatToolMessages"]], + ]) { + assertDeclarationNames( + join(root, file), + /^\s{2}([a-z]\w*)\??\(/gmu, + expected, + ); + } +} + +function verifyRustSignatures(root) { + expectMatches( + join(root, "engine_permission_port.rs"), + /^\s{4}async fn authorize\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result>;/mu, + ); + expectMatches( + join(root, "engine_tool_port.rs"), + /^\s{4}async fn execute\(\s*&self,\s*request: &ModelToolRequest,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result>;/mu, + ); + expectAllMatches(join(root, "engine_durability_port.rs"), [ + /^\s{4}async fn append\(\s*&self,\s*event: &EngineEvent,?\s*\)\s*-> Result<\(\),\s*Box>;/mu, + /^\s{4}async fn append_with_checkpoint\(\s*&self,\s*events: &Vec,\s*checkpoint: &EngineCheckpoint,?\s*\)\s*-> Result<\(\),\s*Box>;/mu, + ]); + expectMatches( + join(root, "engine_post_commit_port.rs"), + /^\s{4}async fn after_commit\(\s*&self,\s*effect_id: &String,\s*commit: &TurnCommit,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result<\(\),\s*Box>;/mu, + ); + expectAllMatches(join(root, "executor.rs"), [ + /^\s{4}async fn execute\(\s*&self,\s*agent: &Prompty,\s*messages: &Vec,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result>;/mu, + /^\s{4}async fn execute_stream\(\s*&self,\s*agent: &Prompty,\s*messages: &Vec,\s*cancellation: &CancellationToken,?\s*\)\s*-> Result>/mu, + /^\s{4}fn format_tool_messages\(\s*&self,\s*raw_response: &serde_json::Value,\s*tool_calls: &Vec,\s*tool_results: &Vec,\s*text_content: &Option,?\s*\)\s*-> Vec;/mu, + ]); + for (const [file, expected] of [ + ["engine_permission_port.rs", ["authorize"]], + ["engine_tool_port.rs", ["execute"]], + ["engine_durability_port.rs", ["append", "append_with_checkpoint"]], + ["engine_post_commit_port.rs", ["after_commit"]], + ["executor.rs", ["execute", "execute_stream", "format_tool_messages"]], + ]) { + assertDeclarationNames( + join(root, file), + /^\s{4}(?:async\s+)?fn\s+([a-z]\w*)\(/gmu, + expected, + ); + } +} + +function verifyPythonSignatures(root) { + expectAllMatches(join(root, "_EnginePermissionPort.py"), [ + /^\s{4}def authorize\(\s*self,\s*request: ModelToolRequest,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> EnginePermissionDecision:/mu, + /^\s{4}async def authorize_async\(\s*self,\s*request: ModelToolRequest,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> EnginePermissionDecision:/mu, + ]); + expectAllMatches(join(root, "_EngineToolPort.py"), [ + /^\s{4}def execute\(\s*self,\s*request: ModelToolRequest,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> ModelToolResult:/mu, + /^\s{4}async def execute_async\(\s*self,\s*request: ModelToolRequest,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> ModelToolResult:/mu, + ]); + expectAllMatches(join(root, "_EngineDurabilityPort.py"), [ + /^\s{4}def append\(\s*self,\s*event: EngineEvent\s*\)\s*-> None:/mu, + /^\s{4}async def append_async\(\s*self,\s*event: EngineEvent\s*\)\s*-> None:/mu, + /^\s{4}def append_with_checkpoint\(\s*self,\s*events: list\[EngineEvent\],\s*checkpoint: EngineCheckpoint\s*\)\s*-> None:/mu, + /^\s{4}async def append_with_checkpoint_async\(\s*self,\s*events: list\[EngineEvent\],\s*checkpoint: EngineCheckpoint\s*\)\s*-> None:/mu, + ]); + expectAllMatches(join(root, "_EnginePostCommitPort.py"), [ + /^\s{4}def after_commit\(\s*self,\s*effect_id: str,\s*commit: TurnCommit,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> None:/mu, + /^\s{4}async def after_commit_async\(\s*self,\s*effect_id: str,\s*commit: TurnCommit,\s*cancellation: CancellationToken \| None = None\s*\)\s*-> None:/mu, + ]); + expectAllMatches(join(root, "_Executor.py"), [ + /^\s{4}def execute\(\s*self,\s*agent: Prompty,\s*messages: list\[Message\],\s*cancellation: CancellationToken \| None = None\s*\)\s*-> Any:/mu, + /^\s{4}async def execute_async\(\s*self,\s*agent: Prompty,\s*messages: list\[Message\],\s*cancellation: CancellationToken \| None = None\s*\)\s*-> Any:/mu, + /^\s{4}def execute_stream\(\s*self,\s*agent: Prompty,\s*messages: list\[Message\],\s*cancellation: CancellationToken \| None = None\s*\)\s*-> Any:/mu, + /^\s{4}async def execute_stream_async\(\s*self,\s*agent: Prompty,\s*messages: list\[Message\],\s*cancellation: CancellationToken \| None = None\s*\)\s*-> Any:/mu, + /^\s{4}def format_tool_messages\(\s*self,\s*raw_response: Any,\s*tool_calls: list\[ToolCall\],\s*tool_results: list\[str\],\s*text_content: str \| None\s*\)\s*-> list\[Message\]:/mu, + ]); + for (const [file, expected] of [ + ["_EnginePermissionPort.py", ["authorize", "authorize_async"]], + ["_EngineToolPort.py", ["execute", "execute_async"]], + [ + "_EngineDurabilityPort.py", + [ + "append", + "append_async", + "append_with_checkpoint", + "append_with_checkpoint_async", + ], + ], + ["_EnginePostCommitPort.py", ["after_commit", "after_commit_async"]], + [ + "_Executor.py", + [ + "execute", + "execute_async", + "execute_stream", + "execute_stream_async", + "format_tool_messages", + ], + ], + ]) { + assertDeclarationNames( + join(root, file), + /^\s{4}(?:async\s+)?def\s+([a-z]\w*)\(/gmu, + expected, + ); + } +} + +function verifyMarkdownSemantics() { + const root = join(repoRoot, "web", "src", "content", "docs", "reference"); + expectMarkdownMethod( + join(root, "EnginePermissionPort.md"), + "authorize", + "authorize(request: ModelToolRequest) -> EnginePermissionDecision", + ["async-capable", "runtime-cancellable"], + ["atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "EngineToolPort.md"), + "execute", + "execute(request: ModelToolRequest) -> ModelToolResult", + ["async-capable", "runtime-cancellable"], + ["atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "EngineDurabilityPort.md"), + "append", + "append(event: EngineEvent) -> void", + ["async-capable"], + ["runtime-cancellable", "atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "EngineDurabilityPort.md"), + "appendWithCheckpoint", + "appendWithCheckpoint(events: EngineEvent[], checkpoint: EngineCheckpoint) -> void", + ["async-capable", "atomic"], + ["runtime-cancellable", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "EnginePostCommitPort.md"), + "afterCommit", + "afterCommit(effectId: string, commit: TurnCommit) -> void", + ["async-capable", "runtime-cancellable", "non-fatal"], + ["atomic", "sync"], + ); + expectMarkdownMethod( + join(root, "Executor.md"), + "execute", + "execute(agent: Prompty, messages: Message[]) -> unknown", + ["async-capable", "runtime-cancellable"], + ["atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "Executor.md"), + "executeStream", + "executeStream(agent: Prompty, messages: Message[]) -> unknown", + ["async-capable", "runtime-cancellable"], + ["atomic", "non-fatal", "sync"], + ); + expectMarkdownMethod( + join(root, "Executor.md"), + "formatToolMessages", + "formatToolMessages(rawResponse: unknown, toolCalls: ToolCall[], toolResults: string[], textContent: string?) -> Message[]", + ["sync"], + ["async-capable", "runtime-cancellable", "atomic", "non-fatal"], + ); +} + +function expectMarkdownMethod( + path, + methodName, + signature, + requiredEffects, + forbiddenEffects, +) { + const content = readFileSync(path, "utf8"); + const row = content + .split(/\r?\n/u) + .find((line) => line.startsWith(`| \`${methodName}\` |`)); + assert(row, `${path}: missing ${methodName} helper-method row`); + + const columns = row.split("|"); + assertEqual( + columns[2].trim(), + `\`${signature}\``, + `${path}: ${methodName} signature`, + ); + const runtimeShape = columns[3].trim().toLowerCase(); + for (const effect of requiredEffects) { + assert( + hasRuntimeEffect(runtimeShape, effect), + `${path}: ${methodName} runtime shape must include ${effect}`, + ); + } + for (const effect of forbiddenEffects) { + assert( + !hasRuntimeEffect(runtimeShape, effect), + `${path}: ${methodName} runtime shape must not include ${effect}`, + ); + } +} + +function hasRuntimeEffect(runtimeShape, effect) { + const escaped = effect.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const normalized = runtimeShape.replace(/[_()]/gu, " "); + return new RegExp(`(?:^|[,\\s])${escaped}(?=$|[,\\s])`, "u").test(normalized); +} + +function expectMatches(path, pattern) { + const content = readFileSync(path, "utf8"); + assert( + pattern.test(content), + `${path}: generated signature did not match ${pattern}`, + ); +} + +function expectAllMatches(path, patterns) { + for (const pattern of patterns) { + expectMatches(path, pattern); + } +} + +function assertDeclarationNames(path, pattern, expected) { + const content = readFileSync(path, "utf8"); + const actual = [...content.matchAll(pattern)].map((match) => match[1]).sort(); + assertEqual( + JSON.stringify(actual), + JSON.stringify([...expected].sort()), + `${path}: native method declarations`, + ); +} + +function collectFiles(root) { + const files = []; + for (const entry of readdirSync(root, { withFileTypes: true })) { + const path = join(root, entry.name); + if (entry.isDirectory()) { + files.push(...collectFiles(path)); + } else if (entry.isFile()) { + files.push(path); + } + } + return files; +} + +function assertSubset(actual, expected, label) { + for (const [key, expectedValue] of Object.entries(expected)) { + const actualValue = actual[key]; + if ( + expectedValue !== null && + typeof expectedValue === "object" && + !Array.isArray(expectedValue) + ) { + assert( + actualValue !== null && typeof actualValue === "object", + `${label}.${key}: expected an object`, + ); + assertSubset(actualValue, expectedValue, `${label}.${key}`); + } else { + assertEqual(actualValue, expectedValue, `${label}.${key}`); + } + } +} + +function assertExactKeys(actual, expected, label) { + const actualKeys = Object.keys(actual).sort(); + const expectedKeys = Object.keys(expected).sort(); + assertEqual( + JSON.stringify(actualKeys), + JSON.stringify(expectedKeys), + `${label} keys`, + ); +} + +function assertUniqueNames(items, label) { + const names = items.map((item) => item.name); + assertEqual( + new Set(names).size, + names.length, + `${label}: duplicate names are not allowed`, + ); +} + +function assertEqual(actual, expected, label) { + assert( + actual === expected, + `${label}: expected ${JSON.stringify(expected)}, received ${JSON.stringify(actual)}`, + ); +} + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} diff --git a/spec/spec.md b/spec/spec.md index 67c554f69..76fef3c8c 100644 --- a/spec/spec.md +++ b/spec/spec.md @@ -246,6 +246,47 @@ correspond to the TypeSpec-generated data model. Unknown top-level properties SHOULD be preserved in `metadata` or ignored. Implementations MUST NOT raise an error for unknown properties. +#### `Record` value nullability + +`Record` has two independent nullability axes: + +1. The model property's optionality controls whether the record itself may be absent. +2. The `unknown` value type permits every JSON-compatible value, including explicit + `null`, at any nesting depth. + +After YAML parsing, YAML null forms have the same explicit-null semantics as JSON +`null`. + +An implementation MUST preserve null-valued keys through load → save → reload. It MUST +NOT drop a key whose value is `null`, coerce that value to an empty object or another +sentinel, or conflate the present-null state with an absent key. Nested objects and arrays +MUST apply the same rule recursively. + +This contract applies to every `Record` surface. The following generated fields +are specifically covered by the shared acceptance vectors because they cross canonical +runtime boundaries: + +| Model | Field | Record presence | +| ----- | ----- | --------------- | +| `Message` | `metadata` | Required | +| `Prompty` | `metadata` | Optional | +| `ModelInfo` | `additionalProperties` | Optional | +| `TurnModelRequest` | `inputs` | Optional | +| `RunTurnRequest` | `inputs` | Optional | +| `TurnModelResponse` | `checkpointState` | Optional | +| `HostToolRequest` | `arguments` | Optional | +| `TurnEvent` | `payload` | Required | +| `SessionEvent` | `payload` | Required | + +Language bindings MUST retain both axes. For example, the conforming C# mapping is +`IDictionary` for a required record and +`IDictionary?` for an optional record. Mapping a +`Record` value as non-null `object` changes the schema contract and is +non-conformant. Nullable annotations do not add a wire or model field. + +The normative shared vectors are +`spec/vectors/model/record_unknown_nullability_vectors.json`. + ### §2.4 Model The `model` property configures the LLM provider and parameters. @@ -285,6 +326,20 @@ Connection types are discriminated by the `kind` field: | `foundry` | `endpoint` | Microsoft Foundry connection | | `oauth` | `endpoint`, `authenticationMode` | OAuth-based authentication | +The `kind` discriminator is open for forward compatibility. Known-kind matching is +exact and case-sensitive, so a value such as `Reference` is an unknown kind rather than +the known `reference` kind. When a runtime loads a connection whose string `kind` is +not listed above, it MUST preserve that exact discriminator and every JSON-compatible +property in the connection payload through a load → save → reload cycle. +Implementations MUST NOT coerce an unknown connection to a known/default connection +kind or discard its additional payload. This passthrough requirement applies to unknown +connection kinds; known connection kinds retain their schema-defined fields. The shared +acceptance vector is `spec/vectors/model/connection_roundtrip_vectors.json`. + +This contract is independent of tool dispatch. An unknown tool `kind` continues to load +as `CustomTool` under §2.9; an unknown connection remains an unknown `Connection` and +does not imply `CustomTool`. + ### §2.6 ModelOptions | Property | Type | Description | @@ -332,8 +387,62 @@ schema is a `Property` object: Rich kinds (`thread`, `image`, `file`, `audio`) receive special handling during rendering — see §5 for details. +#### Named collection encoding + +Named collections such as `Properties`, `Tools`, `Bindings`, and `Connections` accept +either a flat array of entries or a name-keyed object. Names are opaque parsed strings: +implementations MUST NOT trim, case-fold, or Unicode-normalize them before comparison. +Missing `name` and an explicit empty `name` are the same unnamed state because `name` +defaults to `""`. + +The canonical serialization is a name-keyed object when every entry has a non-empty +name and all names are unique by exact parsed-string comparison. Otherwise, the +serializer MUST encode the entire collection as an array, preserving entry order and +every entry's payload. The canonical array form MUST omit an empty `name`. An explicit +array-format option MAY force array encoding for a losslessly object-encodable +collection, but an object-format option MUST NOT override the lossless fallback. + +Implementations MUST NOT omit unnamed entries, overwrite duplicate names, or invent +synthetic keys such as `_unnamed`, indexes, or suffixed names. Loading either canonical +form and then saving and reloading it MUST preserve the same entries and payloads. +Array fallback MUST preserve model entry order; object ordering is not semantically +significant. + +At every named-collection boundary, recursively: + +- The array form is the collection itself: a flat array of entries. +- In the name-keyed object form, each key maps to exactly one entry. +- An array used as the immediate value of a name-keyed entry is structurally invalid. + It MUST be rejected at the first invalid value and MUST NOT be skipped, flattened, + stringified, or coerced into a default entry. +- The native load error MUST identify the full collection path, including the offending + key, and identify the invalid value category as `array`. + +This validation is schema-aware and applies after JSON/YAML parsing and reference +resolution. It does not reject the outer flat array form or arrays in declared fields +inside a valid entry, such as `Property.default`. In particular, list shorthand is not +available as the immediate value of a name-keyed `Property` entry because it is +ambiguous with an invalid nested collection. Use the expanded form instead: + +```yaml +inputs: + aliases: + kind: array + default: [Ada, Grace] +``` + +The normative load/save/reload and rejection cases are +`spec/vectors/model/named_collection_vectors.json`. + **Scalar shorthand**: When a property value is a plain scalar instead of a `Property` object, it MUST be interpreted as `Property(kind: , default: )`. +This applies to every immediate string, integer, float, or boolean value in the +name-keyed object form of a `Property` collection. The object key supplies `name`. +Implementations MUST NOT reinterpret the scalar as the `kind` field, reject a valid +primitive scalar, or silently produce an empty `kind`. At a name-keyed collection +boundary, this normalization to `default` MUST occur before direct generated-model +`@coerce` handling; the direct-coercion `example` behavior MUST NOT apply to the +collection entry, and `example` MUST remain unset. ```yaml # Shorthand @@ -349,6 +458,16 @@ inputs: Kind inference from scalar type: string → `"string"`, integer → `"integer"`, float → `"float"`, boolean → `"boolean"`, list → `"array"`, dict → `"object"`. +The named-collection object-form exception for list values is defined above. + +This named-collection shorthand is distinct from direct generated-model coercion. +When a generated `Property` loader receives a scalar as its complete input, the +TypeSpec `@coerce` contract MUST infer the same scalar kind and store the scalar in +`example`; it MUST NOT drop or coerce the value. JSON integer and fractional number +inputs MUST remain distinguishable as `"integer"` and `"float"` respectively. The +four primitive scalar branches are an atomic contract: string, integer, float, and +boolean MUST all be supported. The normative acceptance vector is +`spec/vectors/model/property_scalar_coercion_vectors.json`. ### §2.8 Template @@ -415,13 +534,12 @@ tools: kind: function description: Get orders for a user parameters: - properties: - - name: user_id - kind: string - required: true - - name: limit - kind: integer - default: 10 + - name: user_id + kind: string + required: true + - name: limit + kind: integer + default: 10 bindings: user_id: ${env:CURRENT_USER_ID} ``` @@ -1271,6 +1389,20 @@ ToolResult: parts: ContentPart[] // Rich content from tool execution ``` +`ContentPart` is a closed, exact, case-sensitive discriminated union. A typed +`ContentPart` load MUST reject any `kind` other than `text`, `image`, `audio`, or +`file`, including case-only variants such as `Text`. Implementations MUST NOT create an +unknown fallback variant, preserve the unknown payload as a `ContentPart`, or coerce it +to a known/default kind. The rejection MUST be classified as an unknown-discriminator +load error and report both the discriminator field (`kind`) and the exact offending +string value. Runtimes MAY use their native load-error type, but its diagnostic or +structured data MUST expose equivalent information. + +This strict contract is distinct from the intentionally open discriminator contracts: +an unknown tool kind loads as `CustomTool` under §2.9, and an unknown connection kind +is preserved under §2.5. The shared acceptance vectors are in +`spec/vectors/model/content_part_discriminator_vectors.json`. + **ToolResult** enables tools to return rich content (text, images, files, audio) rather than plain strings. Implementations MUST support conversion from a plain string to a `ToolResult` containing a single `TextPart` for backward compatibility diff --git a/spec/turn-engine.md b/spec/turn-engine.md index 93bc65fa6..263874b6a 100644 --- a/spec/turn-engine.md +++ b/spec/turn-engine.md @@ -250,24 +250,29 @@ post-commit start event prevents the effect from running. Failure to persist com after the effect ran is returned as non-fatal recovery information and MUST NOT make the committed turn appear to have failed. -## Runtime-Local Ports - -Native runtime interfaces own async, streaming, cancellation, and SDK-specific behavior: - -- `ModelPort` -- `ContextSource` -- `ContextTransform` -- `ContextPackingStrategy` -- `PermissionPort` -- `ToolPort` -- `DurabilityPort` (atomic semantic-event and checkpoint persistence) -- `Clock` -- `IdGenerator` -- post-commit effect ports - -Portable TypeSpec models will be promoted only after the Rust state machine and -conformance vectors establish stable semantics. Native interfaces themselves are not -generated. +## Engine Ports + +The stable permission, tool, durability, and post-commit effect boundaries are canonical +TypeSpec protocols: + +- `EnginePermissionPort.authorize(ModelToolRequest) -> EnginePermissionDecision` is async + and runtime-cancellable. +- `EngineToolPort.execute(ModelToolRequest) -> ModelToolResult` is async and + runtime-cancellable. +- `EngineDurabilityPort.append(EngineEvent) -> void` is async and non-cancellable. +- `EngineDurabilityPort.appendWithCheckpoint(EngineEvent[], EngineCheckpoint) -> void` + is async, atomic, and non-cancellable. +- `EnginePostCommitPort.afterCommit(effectId, TurnCommit) -> void` is async, + runtime-cancellable, and non-fatal after the turn is committed. + +Runtime cancellation is projected as a native language seam rather than a wire or model +field: `CancellationToken` in C#, `context.Context` in Go, optional `AbortSignal` in +TypeScript, the runtime `CancellationToken` reference in Rust, and the runtime +`CancellationToken` signal in Python. Port failures remain native runtime errors such as +`PortError`; they are not portable wire models. + +Richer runtime interfaces continue to own SDK-specific streaming, context assembly, +host policy, retry, clocks, identifiers, and provider reconciliation behavior. ## Rust-First Conformance Gate diff --git a/spec/vectors/engine/port_contracts.json b/spec/vectors/engine/port_contracts.json new file mode 100644 index 000000000..0ed4a1cc5 --- /dev/null +++ b/spec/vectors/engine/port_contracts.json @@ -0,0 +1,122 @@ +{ + "version": "1", + "legacyHarnessSha256": "c534140f1aae07034cedea13feec3d66a9acbc0df46869858753aef39e24fe69", + "nativeErrors": ["PortError"], + "protocols": { + "EnginePermissionPort": { + "methods": { + "authorize": { + "returns": "EnginePermissionDecision", + "params": { + "request": "ModelToolRequest" + }, + "optional": false, + "sync": false, + "runtimeCancellable": true, + "atomic": false, + "nonFatal": false + } + } + }, + "EngineToolPort": { + "methods": { + "execute": { + "returns": "ModelToolResult", + "params": { + "request": "ModelToolRequest" + }, + "optional": false, + "sync": false, + "runtimeCancellable": true, + "atomic": false, + "nonFatal": false + } + } + }, + "EngineDurabilityPort": { + "methods": { + "append": { + "returns": "void", + "params": { + "event": "EngineEvent" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": false, + "nonFatal": false + }, + "appendWithCheckpoint": { + "returns": "void", + "params": { + "events": "EngineEvent[]", + "checkpoint": "EngineCheckpoint" + }, + "optional": false, + "sync": false, + "runtimeCancellable": false, + "atomic": true, + "nonFatal": false + } + } + }, + "EnginePostCommitPort": { + "methods": { + "afterCommit": { + "returns": "void", + "params": { + "effectId": "string", + "commit": "TurnCommit" + }, + "optional": false, + "sync": false, + "runtimeCancellable": true, + "atomic": false, + "nonFatal": true + } + } + }, + "Executor": { + "methods": { + "execute": { + "returns": "unknown", + "params": { + "agent": "Prompty", + "messages": "Message[]" + }, + "optional": false, + "runtimeCancellable": true, + "sync": false, + "atomic": false, + "nonFatal": false + }, + "executeStream": { + "returns": "unknown", + "params": { + "agent": "Prompty", + "messages": "Message[]" + }, + "optional": true, + "runtimeCancellable": true, + "sync": false, + "atomic": false, + "nonFatal": false + }, + "formatToolMessages": { + "returns": "Message[]", + "params": { + "rawResponse": "unknown", + "toolCalls": "ToolCall[]", + "toolResults": "string[]", + "textContent": "string?" + }, + "optional": false, + "runtimeCancellable": false, + "sync": true, + "atomic": false, + "nonFatal": false + } + } + } + } +} diff --git a/spec/vectors/model/connection_roundtrip_vectors.json b/spec/vectors/model/connection_roundtrip_vectors.json new file mode 100644 index 000000000..0336c31ad --- /dev/null +++ b/spec/vectors/model/connection_roundtrip_vectors.json @@ -0,0 +1,106 @@ +{ + "version": "1", + "description": "Cross-runtime Connection load/save/reload contracts. Unknown string discriminators are forward-compatible Connection values, not CustomTool values: runtimes must preserve the exact kind and complete JSON-compatible payload without coercing to a known/default connection kind.", + "vectors": [ + { + "name": "known_reference_connection_roundtrip_unchanged", + "operation": "load-save-reload", + "input": { + "kind": "reference", + "authenticationMode": "system", + "usageDescription": "Exercise the known discriminator control", + "name": "shared-connection", + "target": "model-service" + }, + "expected": { + "kind": "reference", + "authenticationMode": "system", + "usageDescription": "Exercise the known discriminator control", + "name": "shared-connection", + "target": "model-service" + } + }, + { + "name": "unknown_connection_kind_preserves_payload", + "operation": "load-save-reload", + "input": { + "kind": "future-auth", + "authenticationMode": "system", + "usageDescription": "Exercise forward-compatible connection preservation", + "endpoint": "https://future.example.test", + "tenant": "example-tenant", + "priority": 5, + "enabled": false, + "weight": 0.1, + "regions": [ + "west", + "east" + ], + "notes": null, + "providerOptions": { + "audience": "prompty", + "features": [ + "delegation", + { + "name": "nested-option", + "enabled": true + } + ], + "retry": { + "maxAttempts": 3, + "backoffSeconds": 0.1 + }, + "nullable": null + } + }, + "expected": { + "kind": "future-auth", + "authenticationMode": "system", + "usageDescription": "Exercise forward-compatible connection preservation", + "endpoint": "https://future.example.test", + "tenant": "example-tenant", + "priority": 5, + "enabled": false, + "weight": 0.1, + "regions": [ + "west", + "east" + ], + "notes": null, + "providerOptions": { + "audience": "prompty", + "features": [ + "delegation", + { + "name": "nested-option", + "enabled": true + } + ], + "retry": { + "maxAttempts": 3, + "backoffSeconds": 0.1 + }, + "nullable": null + } + } + }, + { + "name": "unknown_connection_case_collision_preserves_payload", + "operation": "load-save-reload", + "input": { + "kind": "Reference", + "name": "case-sensitive-unknown", + "payload": { + "mode": "future" + } + }, + "expected": { + "kind": "Reference", + "name": "case-sensitive-unknown", + "payload": { + "mode": "future" + } + } + } + ] +} diff --git a/spec/vectors/model/content_part_discriminator_vectors.json b/spec/vectors/model/content_part_discriminator_vectors.json new file mode 100644 index 000000000..41c237dbd --- /dev/null +++ b/spec/vectors/model/content_part_discriminator_vectors.json @@ -0,0 +1,45 @@ +{ + "version": "1", + "description": "Cross-runtime strict ContentPart discriminator contracts. ContentPart is closed and case-sensitive: unknown kinds are rejected rather than preserved like unknown Connection values or dispatched like unknown Tool values.", + "vectors": [ + { + "name": "known_text_content_part_loads", + "operation": "load", + "input": { + "kind": "text", + "value": "hello" + }, + "expected": { + "kind": "text", + "value": "hello" + } + }, + { + "name": "unknown_content_part_kind_is_rejected", + "operation": "load-error", + "input": { + "kind": "video", + "source": "https://example.test/video.mp4", + "durationSeconds": 3 + }, + "expected": { + "error": "unknown-discriminator", + "discriminator": "kind", + "value": "video" + } + }, + { + "name": "content_part_case_collision_is_rejected", + "operation": "load-error", + "input": { + "kind": "Text", + "value": "case-sensitive" + }, + "expected": { + "error": "unknown-discriminator", + "discriminator": "kind", + "value": "Text" + } + } + ] +} diff --git a/spec/vectors/model/named_collection_vectors.json b/spec/vectors/model/named_collection_vectors.json new file mode 100644 index 000000000..9c69f9804 --- /dev/null +++ b/spec/vectors/model/named_collection_vectors.json @@ -0,0 +1,428 @@ +{ + "version": "1", + "description": "Cross-runtime named-collection load/save/reload contracts. Canonical serialization uses a name-keyed object only when every parsed name is non-empty and unique; otherwise it uses a whole-collection array fallback without omission, collision, or synthetic names. Immediate primitive Property values infer kind and default without leaking direct-coercion example semantics. Array-valued entries in name-keyed object form are rejected recursively, while arrays in declared entry fields remain valid.", + "vectors": [ + { + "name": "unique_names_use_canonical_object_form", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "unique-names", + "inputs": [ + { + "name": "alpha", + "kind": "string", + "description": "first entry" + }, + { + "name": "beta", + "kind": "boolean", + "required": true + } + ] + }, + "expected": { + "collectionFormat": "object", + "entries": [ + { + "name": "alpha", + "kind": "string", + "description": "first entry" + }, + { + "name": "beta", + "kind": "boolean", + "required": true + } + ] + } + }, + { + "name": "missing_and_empty_names_use_lossless_array_fallback", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "unnamed-inputs", + "inputs": [ + { + "name": "head", + "kind": "string", + "description": "preserve the named head" + }, + { + "kind": "integer", + "default": 7 + }, + { + "name": "", + "kind": "boolean", + "example": false + }, + { + "name": "tail", + "kind": "array", + "default": [ + 1, + null, + { + "nested": [ + "x", + 2 + ] + } + ], + "items": { + "kind": "object" + } + } + ] + }, + "expected": { + "collectionFormat": "array", + "preserveOrder": true, + "entries": [ + { + "name": "head", + "kind": "string", + "description": "preserve the named head" + }, + { + "name": "", + "kind": "integer", + "default": 7 + }, + { + "name": "", + "kind": "boolean", + "example": false + }, + { + "name": "tail", + "kind": "array", + "default": [ + 1, + null, + { + "nested": [ + "x", + 2 + ] + } + ], + "items": { + "kind": "object" + } + } + ] + } + }, + { + "name": "duplicate_names_use_lossless_array_fallback", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "duplicate-inputs", + "inputs": [ + { + "name": "same", + "kind": "string", + "description": "first duplicate" + }, + { + "name": "same", + "kind": "integer", + "default": 2 + } + ] + }, + "expected": { + "collectionFormat": "array", + "preserveOrder": true, + "entries": [ + { + "name": "same", + "kind": "string", + "description": "first duplicate" + }, + { + "name": "same", + "kind": "integer", + "default": 2 + } + ] + } + }, + { + "name": "unnamed_composite_omits_empty_name_stably", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "unnamed-composite", + "inputs": [ + { + "kind": "object", + "description": "preserve unnamed composite payload", + "properties": { + "nested": { + "kind": "string", + "default": "kept" + } + } + } + ] + }, + "expected": { + "collectionFormat": "array", + "preserveOrder": true, + "wireEntries": [ + { + "index": 0, + "absentFields": [ + "name" + ] + } + ], + "entries": [ + { + "name": "", + "kind": "object", + "description": "preserve unnamed composite payload", + "properties": { + "nested": { + "kind": "string", + "default": "kept" + } + } + } + ] + } + }, + { + "name": "empty_object_key_reloads_as_unnamed_array_entry", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "empty-object-key", + "inputs": { + "": { + "kind": "string", + "description": "preserve empty key payload" + }, + "tail": { + "kind": "boolean" + } + } + }, + "expected": { + "collectionFormat": "array", + "entries": [ + { + "name": "", + "kind": "string", + "description": "preserve empty key payload" + }, + { + "name": "tail", + "kind": "boolean" + } + ] + } + }, + { + "name": "array_in_declared_property_field_remains_valid", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "declared-array-field", + "inputs": { + "aliases": { + "kind": "array", + "default": [ + "Ada", + "Grace", + null + ], + "items": { + "kind": "string" + } + } + } + }, + "expected": { + "collectionFormat": "object", + "entries": [ + { + "name": "aliases", + "kind": "array", + "default": [ + "Ada", + "Grace", + null + ], + "items": { + "kind": "string" + } + } + ] + } + }, + { + "name": "string_scalar_in_name_keyed_inputs_infers_property", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "string-scalar-input", + "inputs": { + "city": "Seattle" + } + }, + "expected": { + "collectionFormat": "object", + "absentEntryFields": [ + "example" + ], + "entries": [ + { + "name": "city", + "kind": "string", + "default": "Seattle" + } + ] + } + }, + { + "name": "integer_scalar_in_name_keyed_inputs_infers_property", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "integer-scalar-input", + "inputs": { + "city": 3 + } + }, + "expected": { + "collectionFormat": "object", + "absentEntryFields": [ + "example" + ], + "entries": [ + { + "name": "city", + "kind": "integer", + "default": 3 + } + ] + } + }, + { + "name": "float_scalar_in_name_keyed_inputs_infers_property", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "float-scalar-input", + "inputs": { + "city": 1.5 + } + }, + "expected": { + "collectionFormat": "object", + "absentEntryFields": [ + "example" + ], + "entries": [ + { + "name": "city", + "kind": "float", + "default": 1.5 + } + ] + } + }, + { + "name": "boolean_scalar_in_name_keyed_inputs_infers_property", + "operation": "load-save-reload", + "collectionPath": "inputs", + "input": { + "name": "boolean-scalar-input", + "inputs": { + "city": true + } + }, + "expected": { + "collectionFormat": "object", + "absentEntryFields": [ + "example" + ], + "entries": [ + { + "name": "city", + "kind": "boolean", + "default": true + } + ] + } + }, + { + "name": "scalar_array_shorthand_in_name_keyed_inputs_is_rejected", + "operation": "load-error", + "input": { + "name": "invalid-scalar-array-shorthand", + "inputs": { + "arrayDefault": [ + 1, + "two", + null + ] + } + }, + "expected": { + "error": "invalid-named-collection-entry", + "path": "inputs.arrayDefault", + "valueCategory": "array" + } + }, + { + "name": "array_value_in_name_keyed_inputs_is_rejected", + "operation": "load-error", + "input": { + "name": "invalid-top-level-array", + "inputs": { + "arrayEntry": [ + { + "kind": "string" + } + ] + } + }, + "expected": { + "error": "invalid-named-collection-entry", + "path": "inputs.arrayEntry", + "valueCategory": "array" + } + }, + { + "name": "array_value_in_nested_properties_is_rejected", + "operation": "load-error", + "input": { + "name": "invalid-recursive-array", + "inputs": { + "profile": { + "kind": "object", + "properties": { + "arrayEntry": [ + { + "kind": "string" + } + ] + } + } + } + }, + "expected": { + "error": "invalid-named-collection-entry", + "path": "inputs.profile.properties.arrayEntry", + "valueCategory": "array" + } + } + ] +} diff --git a/spec/vectors/model/property_scalar_coercion_vectors.json b/spec/vectors/model/property_scalar_coercion_vectors.json new file mode 100644 index 000000000..90b622d0b --- /dev/null +++ b/spec/vectors/model/property_scalar_coercion_vectors.json @@ -0,0 +1,44 @@ +{ + "version": "1", + "description": "Atomic cross-runtime Property scalar coercion contract. Direct generated-model JSON loading infers the exact primitive kind and stores the unmodified scalar in example. All four cases are required together.", + "vectors": [ + { + "name": "all_primitive_property_scalars_coerce_atomically", + "operation": "load", + "cases": [ + { + "name": "string", + "input": "example", + "expected": { + "kind": "string", + "example": "example" + } + }, + { + "name": "integer", + "input": 4, + "expected": { + "kind": "integer", + "example": 4 + } + }, + { + "name": "float", + "input": 3.14, + "expected": { + "kind": "float", + "example": 3.14 + } + }, + { + "name": "boolean", + "input": false, + "expected": { + "kind": "boolean", + "example": false + } + } + ] + } + ] +} diff --git a/spec/vectors/model/record_unknown_nullability_vectors.json b/spec/vectors/model/record_unknown_nullability_vectors.json new file mode 100644 index 000000000..32f9bec1f --- /dev/null +++ b/spec/vectors/model/record_unknown_nullability_vectors.json @@ -0,0 +1,364 @@ +{ + "version": "1", + "description": "Cross-runtime Record nullability contracts. Optionality controls whether the record itself may be absent; present records permit explicit null values at every nesting depth. Load/save/reload must preserve null-valued keys and must not conflate present-null with absence.", + "vectors": [ + { + "name": "message_metadata_preserves_null_values", + "operation": "load-save-reload", + "model": "Message", + "fieldPath": "metadata", + "input": { + "role": "user", + "parts": [], + "metadata": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "prompty_metadata_preserves_null_values", + "operation": "load-save-reload", + "model": "Prompty", + "fieldPath": "metadata", + "input": { + "name": "nullable-prompty-metadata", + "metadata": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "model_info_additional_properties_preserve_null_values", + "operation": "load-save-reload", + "model": "ModelInfo", + "fieldPath": "additionalProperties", + "input": { + "id": "nullable-provider-model", + "additionalProperties": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "turn_model_request_inputs_preserve_null_values", + "operation": "load-save-reload", + "model": "TurnModelRequest", + "fieldPath": "inputs", + "input": { + "sessionId": "sess_nullable", + "turnId": "turn_nullable", + "iteration": 0, + "inputs": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "run_turn_request_inputs_preserve_null_values", + "operation": "load-save-reload", + "model": "RunTurnRequest", + "fieldPath": "inputs", + "input": { + "sessionId": "sess_nullable", + "turnId": "turn_nullable", + "inputs": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "turn_model_response_checkpoint_state_preserves_null_values", + "operation": "load-save-reload", + "model": "TurnModelResponse", + "fieldPath": "checkpointState", + "input": { + "checkpointState": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "host_tool_request_arguments_preserve_null_values", + "operation": "load-save-reload", + "model": "HostToolRequest", + "fieldPath": "arguments", + "input": { + "toolName": "nullable-tool", + "arguments": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "turn_event_payload_preserves_null_values", + "operation": "load-save-reload", + "model": "TurnEvent", + "fieldPath": "payload", + "input": { + "id": "evt_turn_nullable", + "type": "turn_start", + "timestamp": "2026-07-01T00:00:00Z", + "payload": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + { + "name": "session_event_payload_preserves_null_values", + "operation": "load-save-reload", + "model": "SessionEvent", + "fieldPath": "payload", + "input": { + "id": "evt_session_nullable", + "type": "session_start", + "timestamp": "2026-07-01T00:00:00Z", + "payload": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + }, + "expected": { + "direct": null, + "nested": { + "value": null + }, + "list": [ + "text", + null, + { + "deep": [ + null + ] + } + ] + } + } + ] +}