diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs index d73197cb3..11628abdb 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Text.Json; using Prompty.Core; namespace Prompty.Anthropic.Tests; @@ -112,8 +113,18 @@ public void FormatToolMessages_BatchesToolResultsIntoSingleUserMessage() new() { Id = "call_2", Name = "get_time", Arguments = """{"tz":"PST"}""" }, }; var toolResults = new List { "72°F", "3:00 PM" }; + using var rawResponse = JsonDocument.Parse( + """ + { + "content": [ + {"type":"text","text":"Let me check."}, + {"type":"tool_use","id":"call_1","name":"get_weather","input":{"city":"Seattle"}}, + {"type":"tool_use","id":"call_2","name":"get_time","input":{"tz":"PST"}} + ] + } + """); - var messages = executor.FormatToolMessages("raw", toolCalls, toolResults, "Let me check."); + var messages = executor.FormatToolMessages(rawResponse.RootElement, toolCalls, toolResults, "Let me check."); // Should be 2 messages: assistant + single user message (NOT 3) Assert.Equal(2, messages.Count); @@ -121,11 +132,11 @@ public void FormatToolMessages_BatchesToolResultsIntoSingleUserMessage() // Assistant message preserves text + tool_use content blocks Assert.Equal(Role.Assistant, messages[0].Role); Assert.Equal("Let me check.", messages[0].Text); - var content = Assert.IsType>>(messages[0].Metadata["content"]); - Assert.Equal(3, content.Count); // 1 text + 2 tool_use - Assert.Equal("text", content[0]["type"]); - Assert.Equal("tool_use", content[1]["type"]); - Assert.Equal("tool_use", content[2]["type"]); + var content = Assert.IsType(messages[0].Metadata["content"]); + Assert.Equal(3, content.GetArrayLength()); // 1 text + 2 tool_use + Assert.Equal("text", content[0].GetProperty("type").GetString()); + Assert.Equal("tool_use", content[1].GetProperty("type").GetString()); + Assert.Equal("tool_use", content[2].GetProperty("type").GetString()); // Single user message with batched tool_result blocks Assert.Equal(Role.User, messages[1].Role); @@ -148,12 +159,155 @@ public void FormatToolMessages_NoTextContent_OmitsTextBlock() new() { Id = "call_1", Name = "fn", Arguments = "{}" }, }; var toolResults = new List { "result" }; + using var rawResponse = JsonDocument.Parse( + """{"content":[{"type":"tool_use","id":"call_1","name":"fn","input":{}}]}"""); - var messages = executor.FormatToolMessages("raw", toolCalls, toolResults); + var messages = executor.FormatToolMessages(rawResponse.RootElement, toolCalls, toolResults); + var content = Assert.IsType(messages[0].Metadata["content"]); + Assert.Single(content.EnumerateArray()); // Only tool_use, no text block + Assert.Equal("tool_use", content[0].GetProperty("type").GetString()); + } + + [Fact] + public void FormatToolMessages_RoundTripsCorrelatedBlocksIntoNextRequest() + { + var executor = new Anthropic.AnthropicExecutor(); + var agent = TestHelpers.CreateAgent(provider: "anthropic"); + using var rawResponse = JsonDocument.Parse( + """ + { + "content": [ + {"type":"thinking","thinking":"I should look this up.","signature":"signed-thinking"}, + {"type":"text","text":"Checking."}, + {"type":"tool_use","id":"call_1","name":"lookup","input":{"key":"value"}} + ] + } + """); + var messages = executor.FormatToolMessages( + rawResponse.RootElement, + [new ToolCall { Id = "call_1", Name = "lookup", Arguments = """{"key":"value"}""" }], + ["result"], + "Checking."); + + var body = executor.BuildRequestBody(agent, messages, stream: false); + var wireMessages = Assert.IsType>>(body["messages"]); + + var assistantContent = Assert.IsType(wireMessages[0]["content"]); + Assert.Equal("thinking", assistantContent[0].GetProperty("type").GetString()); + Assert.Equal("signed-thinking", assistantContent[0].GetProperty("signature").GetString()); + Assert.Equal("text", assistantContent[1].GetProperty("type").GetString()); + Assert.Equal("tool_use", assistantContent[2].GetProperty("type").GetString()); + Assert.Equal("call_1", assistantContent[2].GetProperty("id").GetString()); + + var userContent = Assert.IsType>>(wireMessages[1]["content"]); + Assert.Equal("tool_result", userContent[0]["type"]); + Assert.Equal("call_1", userContent[0]["tool_use_id"]); + Assert.Equal("result", userContent[0]["content"]); + } + + [Fact] + public async Task FormatToolMessages_ReconstructsStreamingToolAndThinkingBlocks() + { + var executor = new Anthropic.AnthropicExecutor(); + var stream = new PromptyStream(StreamEvents( + """{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"I should look."}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"signed"}}""", + """{"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Checking "}}""", + """{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"now."}}""", + """{"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"call_1","name":"lookup","input":{}}}""", + """{"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"key\":\"value\"}"}}""")); + await foreach (var _ in stream) + { + } + + var messages = executor.FormatToolMessages( + stream, + [new ToolCall { Id = "call_1", Name = "lookup", Arguments = """{"key":"value"}""" }], + ["result"]); var content = Assert.IsType>>(messages[0].Metadata["content"]); - Assert.Single(content); // Only tool_use, no text block - Assert.Equal("tool_use", content[0]["type"]); + + Assert.Equal("thinking", content[0]["type"]?.ToString()); + Assert.Equal("I should look.", content[0]["thinking"]); + Assert.Equal("signed", content[0]["signature"]); + Assert.Equal("text", content[1]["type"]?.ToString()); + Assert.Equal("Checking now.", content[1]["text"]); + Assert.Equal("tool_use", content[2]["type"]?.ToString()); + Assert.Equal("call_1", content[2]["id"]?.ToString()); + Assert.Equal("""{"key":"value"}""", Assert.IsType(content[2]["input"]).GetRawText()); } -} + [Fact] + public async Task FormatToolMessages_PreservesValidInitialInputWhenStreamEndsMidJson() + { + var executor = new Anthropic.AnthropicExecutor(); + var stream = new PromptyStream(StreamEvents( + """{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call_1","name":"lookup","input":{}}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"key\":"}}""")); + await foreach (var _ in stream) + { + } + + var messages = executor.FormatToolMessages( + stream, + [new ToolCall { Id = "call_1", Name = "lookup", Arguments = """{"key":""" }], + ["invalid arguments"]); + var content = Assert.IsType>>(messages[0].Metadata["content"]); + + Assert.Equal("{}", Assert.IsType(content[0]["input"]).GetRawText()); + } + + [Fact] + public async Task FormatToolMessages_SynthesizesBlocksWhenStreamItemsAreUnavailable() + { + var executor = new Anthropic.AnthropicExecutor(); + var stream = new PromptyStream(StreamEvents()); + await foreach (var _ in stream) + { + } + + var messages = executor.FormatToolMessages( + stream, + [new ToolCall { Id = "call_1", Name = "lookup", Arguments = """{"key":"value"}""" }], + ["result"], + "Checking."); + var content = Assert.IsType>>(messages[0].Metadata["content"]); + + Assert.Equal("text", content[0]["type"]); + Assert.Equal("Checking.", content[0]["text"]); + Assert.Equal("tool_use", content[1]["type"]); + Assert.Equal("call_1", content[1]["id"]); + Assert.Equal("""{"key":"value"}""", Assert.IsType(content[1]["input"]).GetRawText()); + } + + [Fact] + public void BuildRequestBody_SingleToolResultMetadata_PreservesCorrelation() + { + var executor = new Anthropic.AnthropicExecutor(); + var agent = TestHelpers.CreateAgent(provider: "anthropic"); + var message = new Message + { + Role = Role.Tool, + Parts = [new TextPart { Value = "result" }], + Metadata = new Dictionary { ["tool_use_id"] = "call_2" }, + }; + + var body = executor.BuildRequestBody(agent, [message], stream: false); + var wireMessages = Assert.IsType>>(body["messages"]); + var content = Assert.IsType>>(wireMessages[0]["content"]); + + Assert.Equal("call_2", content[0]["tool_use_id"]); + } + + private static async IAsyncEnumerable StreamEvents(params string[] events) + { + foreach (var json in events) + { + await Task.Yield(); + using var document = JsonDocument.Parse(json); + yield return document.RootElement.Clone(); + } + } +} diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs new file mode 100644 index 000000000..51b2486ab --- /dev/null +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net; +using System.Net.Sockets; +using System.Text; +using Prompty.Core; + +namespace Prompty.Anthropic.Tests; + +/// +/// Tests Anthropic model discovery protocol and pagination behavior. +/// +public class AnthropicModelDiscoveryTests +{ + [Fact] + public void ModelLister_ImplementsGeneratedProtocol() + { + Assert.IsAssignableFrom(new AnthropicModelLister()); + } + + [Fact] + public async Task ListModelsAsync_FollowsPaginationAndPreservesProviderPayload() + { + using var socket = new TcpListener(IPAddress.Loopback, 0); + socket.Start(); + var port = ((IPEndPoint)socket.LocalEndpoint).Port; + socket.Stop(); + + using var listener = new HttpListener(); + listener.Prefixes.Add($"http://localhost:{port}/"); + listener.Start(); + var requests = new List<(string Query, string? ApiKey, string? Version)>(); + var server = Task.Run(async () => + { + for (var page = 0; page < 2; page++) + { + var context = await listener.GetContextAsync(); + requests.Add(( + context.Request.Url?.Query ?? string.Empty, + context.Request.Headers["x-api-key"], + context.Request.Headers["anthropic-version"])); + var body = page == 0 + ? """ + { + "data": [{"id":"claude-first","display_name":"Claude First","type":"model"}], + "has_more": true, + "last_id": "claude-first" + } + """ + : """ + { + "data": [{"id":"claude-second","created_at":"2025-01-01T00:00:00Z","type":"model"}], + "has_more": false, + "last_id": "claude-second" + } + """; + var bytes = Encoding.UTF8.GetBytes(body); + context.Response.ContentType = "application/json"; + context.Response.ContentLength64 = bytes.Length; + await context.Response.OutputStream.WriteAsync(bytes); + context.Response.Close(); + } + }); + + var models = await AnthropicModels.ListModelsAsync( + new ApiKeyConnection + { + Endpoint = $"http://localhost:{port}", + ApiKey = "test-anthropic-key", + }); + + await server; + Assert.Equal(2, models.Count); + Assert.Equal("Claude First", models[0].DisplayName); + Assert.Equal("2025-01-01T00:00:00Z", models[1].AdditionalProperties!["created_at"]?.ToString()); + Assert.Equal("?limit=100", requests[0].Query); + Assert.Equal("?limit=100&after_id=claude-first", requests[1].Query); + Assert.All(requests, request => + { + Assert.Equal("test-anthropic-key", request.ApiKey); + Assert.Equal("2023-06-01", request.Version); + }); + } +} diff --git a/runtime/csharp/Prompty.Anthropic.Tests/TestHelpers.cs b/runtime/csharp/Prompty.Anthropic.Tests/TestHelpers.cs index 080dba9fe..370f15abd 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/TestHelpers.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/TestHelpers.cs @@ -97,7 +97,7 @@ internal static Message CreateAssistantWithToolCalls(string text, List { Role = Role.Assistant, Parts = [new TextPart { Value = text }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = toolCalls, }, @@ -113,7 +113,7 @@ internal static Message CreateToolMessage(string toolCallId, string content) { Role = Role.Tool, Parts = [new TextPart { Value = content }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCallId, }, diff --git a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs index a64f6198d..a7f6088a1 100644 --- a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs +++ b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs @@ -151,25 +151,38 @@ private static HttpRequestMessage CreateRequest(string endpoint, string apiKey, _ => "user", }; - // Handle tool results + if (msg.Metadata is not null + && msg.Metadata.TryGetValue("tool_results", out var toolResults)) + { + return new Dictionary + { + ["role"] = role, + ["content"] = toolResults, + }; + } + + if (msg.Metadata is not null + && msg.Metadata.TryGetValue("tool_use_id", out var toolUseId)) + { + return BuildToolResultMessage(toolUseId?.ToString() ?? "", msg.Text); + } + + if (msg.Metadata is not null + && msg.Metadata.TryGetValue("content", out var rawContent)) + { + return new Dictionary + { + ["role"] = role, + ["content"] = rawContent, + }; + } + if (msg.Role == Role.Tool) { var toolCallId = msg.Metadata is not null && msg.Metadata.TryGetValue("tool_call_id", out var id) ? id?.ToString() ?? "" : ""; - return new Dictionary - { - ["role"] = "user", - ["content"] = new List> - { - new() - { - ["type"] = "tool_result", - ["tool_use_id"] = toolCallId, - ["content"] = msg.Text, - } - }, - }; + return BuildToolResultMessage(toolCallId, msg.Text); } // Build content blocks @@ -207,15 +220,26 @@ private static HttpRequestMessage CreateRequest(string endpoint, string apiKey, } } - // Simplify single text content - if (content.Count == 1 && content[0]["type"]?.ToString() == "text") - { - return new() { ["role"] = role, ["content"] = content[0]["text"] }; - } - return new() { ["role"] = role, ["content"] = content }; } + private static Dictionary BuildToolResultMessage(string toolUseId, string content) + { + return new Dictionary + { + ["role"] = "user", + ["content"] = new List> + { + new() + { + ["type"] = "tool_result", + ["tool_use_id"] = toolUseId, + ["content"] = content, + } + }, + }; + } + private static List>? ToolsToWire(Core.Prompty agent) { if (agent.Tools is null || agent.Tools.Count == 0) @@ -315,35 +339,21 @@ public List FormatToolMessages( { var messages = new List(); - // --- Assistant message with ALL content blocks (text + tool_use) --- - var rawContent = new List>(); - if (!string.IsNullOrEmpty(textContent)) - { - rawContent.Add(new() { ["type"] = "text", ["text"] = textContent }); - } - foreach (var tc in toolCalls) - { - rawContent.Add(new() - { - ["type"] = "tool_use", - ["id"] = tc.Id, - ["name"] = tc.Name, - ["input"] = JsonSerializer.Deserialize(tc.Arguments), - }); - } - messages.Add(new Message { Role = Role.Assistant, Parts = !string.IsNullOrEmpty(textContent) ? [new TextPart { Value = textContent }] : [], - Metadata = new Dictionary { ["content"] = rawContent }, + Metadata = new Dictionary + { + ["content"] = GetRawContent(rawResponse, toolCalls, textContent), + }, }); // --- Single user message with batched tool_result blocks --- var toolResultBlocks = new List>(); - for (var i = 0; i < toolCalls.Count; i++) + for (var i = 0; i < Math.Min(toolCalls.Count, toolResults.Count); i++) { toolResultBlocks.Add(new() { @@ -357,9 +367,172 @@ public List FormatToolMessages( { Role = Role.User, Parts = toolResults.Select(r => (ContentPart)new TextPart { Value = r }).ToList(), - Metadata = new Dictionary { ["tool_results"] = toolResultBlocks }, + Metadata = new Dictionary { ["tool_results"] = toolResultBlocks }, }); return messages; } + + private static object GetRawContent( + object rawResponse, + IReadOnlyList toolCalls, + string? textContent) + { + if (rawResponse is PromptyStream stream) + { + return ReconstructStreamContent(stream.Items, toolCalls, textContent); + } + + if (rawResponse is JsonElement { ValueKind: JsonValueKind.Object } element + && element.TryGetProperty("content", out var content)) + { + return content.Clone(); + } + + if (rawResponse is IReadOnlyDictionary dictionary + && dictionary.TryGetValue("content", out var value) + && value is not null) + { + return value; + } + + return Array.Empty(); + } + + private static List> ReconstructStreamContent( + IReadOnlyList items, + IReadOnlyList toolCalls, + string? textContent) + { + var blocks = new SortedDictionary>(); + var textDeltas = new Dictionary>(); + var thinkingDeltas = new Dictionary>(); + var signatureDeltas = new Dictionary>(); + var inputDeltas = new Dictionary>(); + + foreach (var item in items) + { + if (item is not JsonElement evt + || !evt.TryGetProperty("type", out var eventType)) + { + continue; + } + + if (eventType.GetString() == "content_block_start" + && evt.TryGetProperty("index", out var startIndex) + && evt.TryGetProperty("content_block", out var contentBlock)) + { + blocks[startIndex.GetInt32()] = + JsonSerializer.Deserialize>(contentBlock.GetRawText()) ?? []; + continue; + } + + if (eventType.GetString() != "content_block_delta" + || !evt.TryGetProperty("index", out var deltaIndex) + || !evt.TryGetProperty("delta", out var delta) + || !delta.TryGetProperty("type", out var deltaType)) + { + continue; + } + + var index = deltaIndex.GetInt32(); + switch (deltaType.GetString()) + { + case "text_delta": + AppendDelta(textDeltas, index, delta, "text"); + break; + case "thinking_delta": + AppendDelta(thinkingDeltas, index, delta, "thinking"); + break; + case "signature_delta": + AppendDelta(signatureDeltas, index, delta, "signature"); + break; + case "input_json_delta": + AppendDelta(inputDeltas, index, delta, "partial_json"); + break; + } + } + + foreach (var (index, block) in blocks) + { + if (textDeltas.TryGetValue(index, out var text)) + { + block["text"] = string.Concat(text); + } + if (thinkingDeltas.TryGetValue(index, out var thinking)) + { + block["thinking"] = string.Concat(thinking); + } + if (signatureDeltas.TryGetValue(index, out var signature)) + { + block["signature"] = string.Concat(signature); + } + if (inputDeltas.TryGetValue(index, out var input)) + { + var json = string.Concat(input); + try + { + block["input"] = JsonSerializer.Deserialize(json) ?? new Dictionary(); + } + catch (JsonException) + { + // Keep the valid initial input block when the provider stops mid-JSON. + } + } + } + + if (blocks.Count == 0) + { + var index = 0; + if (!string.IsNullOrEmpty(textContent)) + { + blocks[index++] = new Dictionary + { + ["type"] = "text", + ["text"] = textContent, + }; + } + foreach (var toolCall in toolCalls) + { + object input; + try + { + input = JsonSerializer.Deserialize(toolCall.Arguments) + ?? new Dictionary(); + } + catch (JsonException) + { + input = new Dictionary(); + } + blocks[index++] = new Dictionary + { + ["type"] = "tool_use", + ["id"] = toolCall.Id, + ["name"] = toolCall.Name, + ["input"] = input, + }; + } + } + + return blocks.Values.ToList(); + } + + private static void AppendDelta( + Dictionary> target, + int index, + JsonElement delta, + string property) + { + if (!delta.TryGetProperty(property, out var value) + || string.IsNullOrEmpty(value.GetString())) + { + return; + } + if (!target.TryGetValue(index, out var values)) + { + values = []; + target[index] = values; + } + values.Add(value.GetString()!); + } } diff --git a/runtime/csharp/Prompty.Anthropic/Models.cs b/runtime/csharp/Prompty.Anthropic/Models.cs new file mode 100644 index 000000000..db34dd358 --- /dev/null +++ b/runtime/csharp/Prompty.Anthropic/Models.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http.Headers; +using System.Text.Json; +using Prompty.Core; + +namespace Prompty.Anthropic; + +/// +/// Lists Anthropic models and maps provider payloads to generated model types. +/// +public sealed class AnthropicModelLister : IModelLister +{ + /// + public async Task> ListModelsAsync(object connection) + { + if (connection is not Connection typedConnection) + throw new ArgumentException("Anthropic model listing requires a generated Connection.", nameof(connection)); + + return [.. await AnthropicModels.ListModelsAsync(typedConnection)]; + } +} + +/// +/// Model discovery for the Anthropic Models API. +/// +public static class AnthropicModels +{ + private const string DefaultEndpoint = "https://api.anthropic.com"; + private const string ApiVersion = "2023-06-01"; + private static readonly HttpClient HttpClient = new(); + + /// + /// List all available Anthropic models, following cursor-based pagination. + /// + public static async Task> ListModelsAsync( + Connection connection, + CancellationToken cancellationToken = default) + { + if (connection is not ApiKeyConnection keyConnection) + throw new InvalidOperationException( + $"Connection kind '{connection.Kind}' is not supported for Anthropic model listing. Use 'key'."); + + var apiKey = string.IsNullOrWhiteSpace(keyConnection.ApiKey) + ? Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") + : keyConnection.ApiKey; + if (string.IsNullOrWhiteSpace(apiKey)) + throw new InvalidOperationException( + "Anthropic API key is required. Set connection.apiKey or ANTHROPIC_API_KEY."); + + var endpoint = string.IsNullOrWhiteSpace(keyConnection.Endpoint) + ? DefaultEndpoint + : keyConnection.Endpoint.TrimEnd('/'); + var models = new List(); + string? afterId = null; + + do + { + var query = afterId is null + ? "limit=100" + : $"limit=100&after_id={Uri.EscapeDataString(afterId)}"; + using var request = new HttpRequestMessage(HttpMethod.Get, $"{endpoint}/v1/models?{query}"); + request.Headers.Add("x-api-key", apiKey); + request.Headers.Add("anthropic-version", ApiVersion); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + using var response = await HttpClient.SendAsync(request, cancellationToken); + var body = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + throw new InvalidOperationException( + $"Anthropic list models failed: {(int)response.StatusCode} {response.ReasonPhrase} - " + + body[..Math.Min(body.Length, 300)]); + + using var document = JsonDocument.Parse(body); + if (document.RootElement.TryGetProperty("data", out var data) + && data.ValueKind == JsonValueKind.Array) + { + models.AddRange(data.EnumerateArray().Select(MapModel)); + } + + var hasMore = document.RootElement.TryGetProperty("has_more", out var hasMoreValue) + && hasMoreValue.ValueKind == JsonValueKind.True; + afterId = hasMore + && document.RootElement.TryGetProperty("last_id", out var lastId) + && lastId.ValueKind == JsonValueKind.String + ? lastId.GetString() + : null; + } + while (afterId is not null); + + return models.AsReadOnly(); + } + + /// + /// Map a raw Anthropic model payload to the generated provider-neutral contract. + /// + public static ModelInfo MapModel(JsonElement model) + { + var info = new ModelInfo + { + Id = GetString(model, "id") ?? string.Empty, + DisplayName = GetString(model, "display_name"), + OwnedBy = "anthropic", + ContextWindow = GetInt(model, "context_length"), + InputModalities = GetStringList(model, "input_modalities"), + OutputModalities = GetStringList(model, "output_modalities"), + AdditionalProperties = ModelDiscovery.PreserveRaw(model), + }; + ModelDiscovery.Enrich("anthropic", info); + return info; + } + + private static string? GetString(JsonElement element, string name) => + element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + + private static int? GetInt(JsonElement element, string name) => + element.TryGetProperty(name, out var value) + && value.ValueKind == JsonValueKind.Number + && value.TryGetInt32(out var number) + ? number + : null; + + private static IList? GetStringList(JsonElement element, string name) => + element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Array + ? value.EnumerateArray().Where(item => item.ValueKind == JsonValueKind.String) + .Select(item => item.GetString()!).ToArray() + : null; +} diff --git a/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs b/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs index ce0bcb19a..b8eadf5e4 100644 --- a/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs @@ -83,7 +83,7 @@ public List FormatToolMessages( { Role = Role.Assistant, Parts = [new TextPart { Value = textContent ?? "" }], - Metadata = new Dictionary { ["tool_calls"] = toolCalls } + Metadata = new Dictionary { ["tool_calls"] = toolCalls } }); // one tool-result message per call for (int i = 0; i < toolCalls.Count; i++) @@ -92,7 +92,7 @@ public List FormatToolMessages( { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], - Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); } return msgs; diff --git a/runtime/csharp/Prompty.Core.Tests/CompactionTests.cs b/runtime/csharp/Prompty.Core.Tests/CompactionTests.cs index 628b53cf2..e20dc9e7e 100644 --- a/runtime/csharp/Prompty.Core.Tests/CompactionTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/CompactionTests.cs @@ -14,7 +14,7 @@ private static Message ToolCallMsg(string name, string args) => { Role = Role.Assistant, Parts = [new TextPart { Value = "" }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = new List { diff --git a/runtime/csharp/Prompty.Core.Tests/HarnessAdaptersTests.cs b/runtime/csharp/Prompty.Core.Tests/HarnessAdaptersTests.cs index 1884d3f75..8721c6318 100644 --- a/runtime/csharp/Prompty.Core.Tests/HarnessAdaptersTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/HarnessAdaptersTests.cs @@ -39,6 +39,8 @@ public void JsonlEventJournalWriter_WritesRecords() Assert.Equal(new[] { "turn", "session", "summary" }, lines.Select(line => line["kind"].GetString()).ToArray()); Assert.Equal("turn-event", lines[0]["event"].GetProperty("id").GetString()); Assert.Equal("session-event", lines[1]["event"].GetProperty("id").GetString()); + Assert.Equal(JsonValueKind.Null, lines[0]["event"].GetProperty("payload").GetProperty("nullable").ValueKind); + Assert.Equal(JsonValueKind.Null, lines[1]["event"].GetProperty("payload").GetProperty("nullable").ValueKind); Assert.Equal("session-1", lines[2]["summary"].GetProperty("sessionId").GetString()); Assert.DoesNotContain("\r\n", File.ReadAllText(path)); } @@ -121,7 +123,7 @@ public async Task FunctionHostToolExecutor_ExecutesRegisteredHandlers() { RequestId = "exec-1", ToolName = "add", - Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } + Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } }); Assert.True(result.Success); @@ -144,6 +146,29 @@ public async Task FunctionHostToolExecutor_PassesEmptyArguments() Assert.Equal(0, result.Result); } + [Fact] + public async Task FunctionHostToolExecutor_PreservesExplicitNullArguments() + { + var executor = new FunctionHostToolExecutor(new Dictionary + { + ["inspect"] = (args, _) => + { + Assert.True(args.ContainsKey("nullable")); + Assert.Null(args["nullable"]); + return Task.FromResult("observed"); + } + }); + + var result = await executor.ExecuteAsync(new HostToolRequest + { + ToolName = "inspect", + Arguments = new Dictionary { ["nullable"] = null } + }); + + Assert.True(result.Success); + Assert.Equal("observed", result.Result); + } + [Fact] public async Task FunctionHostToolExecutor_ReturnsFailureResults() { @@ -168,7 +193,7 @@ public async Task FunctionHostToolExecutor_ReturnsFailureResults() Id = "turn-event", Type = TurnEventType.TurnStart, Timestamp = "2026-06-10T00:00:00Z", - Payload = new Dictionary { ["phase"] = "start" } + Payload = new Dictionary { ["phase"] = "start", ["nullable"] = null } }; private static SessionEvent SessionEvent() => new() @@ -177,6 +202,6 @@ public async Task FunctionHostToolExecutor_ReturnsFailureResults() Type = SessionEventType.SessionStart, Timestamp = "2026-06-10T00:00:00Z", SessionId = "session-1", - Payload = new Dictionary { ["phase"] = "start" } + Payload = new Dictionary { ["phase"] = "start", ["nullable"] = null } }; } diff --git a/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs b/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs new file mode 100644 index 000000000..a9dd0ee84 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs @@ -0,0 +1,347 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +[Collection("InvokerRegistry")] +public sealed class LiveTurnIntegrationTests : IDisposable +{ + private const string Provider = "live-turn-test"; + private const string Format = "live-turn-format"; + + public LiveTurnIntegrationTests() + { + InvokerRegistry.Clear(); + InvokerRegistry.RegisterRenderer(Format, new PassthroughRenderer()); + InvokerRegistry.RegisterParser(Format, new PassthroughParser()); + InvokerRegistry.RegisterProcessor(Provider, new PassthroughProcessor()); + } + + public void Dispose() => InvokerRegistry.Clear(); + + [Fact] + public async Task TurnWithEngineRequest_PersistsAtomicCheckpoints_AndResumesWithoutReinvocation() + { + var executor = new QueueExecutor("durable result"); + InvokerRegistry.RegisterExecutor(Provider, executor); + var durability = new AtomicRecordingDurability(); + durability.FailAfter = EngineEventKind.ModelInvocationCompleted; + var options = new TurnEnginePipelineOptions { Durability = durability }; + + await Assert.ThrowsAsync(() => + Pipeline.TurnWithEngineRequestAsync( + NewAgent(), + NewRequest("durable-session", "durable-turn"), + options)); + Assert.NotEmpty(durability.AtomicAppends); + Assert.All(durability.AtomicAppends, append => + { + Assert.NotEmpty(append.Events); + var semanticTail = append.Events.Last(@event => @event.Kind != EngineEventKind.CheckpointCreated); + Assert.Equal(semanticTail.Sequence, append.Checkpoint.LastSequence); + }); + + var checkpoint = durability.AtomicAppends[^1].Checkpoint; + durability.FailAfter = null; + var resume = TurnEngineRequest.ResumeFrom(checkpoint, maxIterations: 3, checkpoint.LastSequence); + var resumed = await Pipeline.TurnWithEngineRequestAsync(NewAgent(), resume, options); + + Assert.Equal("durable result", resumed); + Assert.Equal(1, executor.InvocationCount); + } + + [Fact] + public async Task TurnWithEngineRequest_CancellationPersistsTerminalCheckpoint() + { + InvokerRegistry.RegisterExecutor(Provider, new QueueExecutor("unused")); + var durability = new AtomicRecordingDurability(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => + Pipeline.TurnWithEngineRequestAsync( + NewAgent(), + NewRequest("cancel-session", "cancel-turn"), + new TurnEnginePipelineOptions { Durability = durability }, + cancellation.Token)); + + var events = durability.StandaloneEvents + .Concat(durability.AtomicAppends.SelectMany(append => append.Events)); + Assert.Contains(events, @event => @event.Kind == EngineEventKind.TurnCancelled); + if (durability.AtomicAppends.Count > 0) + { + Assert.Equal( + durability.AtomicAppends[^1].Events[^1].Sequence, + durability.AtomicAppends[^1].Checkpoint.LastSequence); + } + } + + [Fact] + public async Task TurnWithEngineRequest_RejectsNonObjectInputs() + { + var executor = new QueueExecutor("unused"); + InvokerRegistry.RegisterExecutor(Provider, executor); + var request = NewRequest("invalid-input-session", "invalid-input-turn"); + request.Inputs = "not-an-input-object"; + + var error = await Assert.ThrowsAsync(() => + Pipeline.TurnWithEngineRequestAsync(NewAgent(), request)); + + Assert.Contains("string-keyed dictionary or JSON object", error.Message, StringComparison.Ordinal); + Assert.Equal(0, executor.InvocationCount); + } + + [Fact] + public async Task TurnAsync_RejectsParallelToolsBeforeInvokingProvider() + { + var executor = new QueueExecutor("unused"); + InvokerRegistry.RegisterExecutor(Provider, executor); + var agent = NewAgent(); + agent.Tools = [new FunctionTool { Name = "lookup", Kind = "function" }]; + + var error = await Assert.ThrowsAsync(() => + Pipeline.TurnAsync(agent, parallelToolCalls: true)); + + Assert.Contains("sequentially", error.Message); + Assert.Equal(0, executor.InvocationCount); + } + + [Fact] + public async Task TurnAsync_RawFinalSkipsOutputGuardrail() + { + var rawResponse = new object(); + InvokerRegistry.RegisterExecutor(Provider, new QueueExecutor(rawResponse)); + var outputGuardrailCalled = false; + var guardrails = new Guardrails( + output: _ => + { + outputGuardrailCalled = true; + return new GuardrailResult(false, "raw output must not be inspected"); + }); + + var result = await Pipeline.TurnAsync(NewAgent(), raw: true, guardrails: guardrails); + + Assert.Same(rawResponse, result); + Assert.False(outputGuardrailCalled); + } + + [Fact] + public async Task TurnAsync_ProcessesAccumulatedStreamingItems_AndProjectsTokens() + { + var stream = new PromptyStream(StreamItems("one", "two")); + InvokerRegistry.RegisterExecutor(Provider, new QueueExecutor(stream)); + InvokerRegistry.RegisterProcessor(Provider, new AccumulatedStreamProcessor()); + var tokens = new List(); + EventCallback onEvent = (type, data) => + { + if (type == AgentEventType.Token) + { + tokens.Add(data["token"]!.ToString()!); + } + }; + + var result = await Pipeline.TurnAsync(NewAgent(), onEvent: onEvent); + + Assert.Equal("onetwo", result); + Assert.Equal(["one", "two"], tokens); + Assert.Equal(["one", "two"], stream.Items); + } + + [Fact] + public async Task TurnAsync_CancellationBetweenToolsCommitsFirstResultBeforeStopping() + { + var executor = new QueueExecutor(new ToolCallResult + { + ToolCalls = + [ + new ToolCall { Id = "call-1", Name = "first", Arguments = "{}" }, + new ToolCall { Id = "call-2", Name = "second", Arguments = "{}" }, + ], + }); + InvokerRegistry.RegisterExecutor(Provider, executor); + using var cancellation = new CancellationTokenSource(); + var executed = new List(); + var tools = new Dictionary>> + { + ["first"] = _ => + { + executed.Add("first"); + cancellation.Cancel(); + return Task.FromResult("first-result"); + }, + ["second"] = _ => + { + executed.Add("second"); + return Task.FromResult("second-result"); + }, + }; + var agent = NewAgent(); + agent.Tools = + [ + new FunctionTool { Name = "first", Kind = "function" }, + new FunctionTool { Name = "second", Kind = "function" }, + ]; + var toolResults = new List(); + EventCallback onEvent = (type, data) => + { + if (type == AgentEventType.ToolResult) + { + toolResults.Add(data["result"]!.ToString()!); + } + }; + + await Assert.ThrowsAnyAsync(() => + Pipeline.TurnAsync( + agent, + tools: tools, + onEvent: onEvent, + cancellationToken: cancellation.Token)); + + Assert.Equal(["first"], executed); + Assert.Equal(["first-result"], toolResults); + } + + [Fact] + public async Task TurnAsync_CancellationAfterFinalToolCommitsConversationBeforeStopping() + { + var executor = new QueueExecutor(new ToolCallResult + { + ToolCalls = [new ToolCall { Id = "call-1", Name = "only", Arguments = "{}" }], + }); + InvokerRegistry.RegisterExecutor(Provider, executor); + using var cancellation = new CancellationTokenSource(); + var tools = new Dictionary>> + { + ["only"] = _ => + { + cancellation.Cancel(); + return Task.FromResult("only-result"); + }, + }; + var agent = NewAgent(); + agent.Tools = [new FunctionTool { Name = "only", Kind = "function" }]; + var events = new List(); + EventCallback onEvent = (type, _) => events.Add(type); + + await Assert.ThrowsAnyAsync(() => + Pipeline.TurnAsync( + agent, + tools: tools, + onEvent: onEvent, + cancellationToken: cancellation.Token)); + + Assert.Contains(AgentEventType.ToolResult, events); + Assert.Contains(AgentEventType.MessagesUpdated, events); + Assert.Contains(AgentEventType.Cancelled, events); + } + + private static TurnEngineRequest NewRequest(string sessionId, string turnId) => + new(sessionId, turnId, []) + { + Inputs = new Dictionary(), + MaxIterations = 3, + MaxModelAttempts = 1, + }; + + private static Prompty NewAgent() => new() + { + Name = "live-turn-agent", + Instructions = "hello", + Model = new Model { Id = "test-model", Provider = Provider }, + Template = new Template + { + Format = new FormatConfig { Kind = Format }, + Parser = new ParserConfig { Kind = Format }, + }, + }; + + private sealed class QueueExecutor(params object[] responses) : IExecutor + { + private readonly Queue _responses = new(responses); + + public int InvocationCount { get; private set; } + + public Task ExecuteAsync(Prompty agent, List messages) + { + InvocationCount++; + return Task.FromResult(_responses.Dequeue()); + } + + public List FormatToolMessages( + object rawResponse, + List toolCalls, + List toolResults, + string? textContent = null) + { + var messages = new List { Message.Assistant(textContent ?? string.Empty) }; + messages.AddRange(toolCalls.Select((call, index) => Message.ToolResult(call.Id, toolResults[index]))); + return messages; + } + } + + private sealed class PassthroughRenderer : IRenderer + { + public Task RenderAsync(Prompty agent, string template, Dictionary inputs) => + Task.FromResult(template); + } + + private sealed class PassthroughParser : IParser + { + public Task> ParseAsync( + Prompty agent, + string rendered, + Dictionary? context) => + Task.FromResult(new List { Message.User(rendered) }); + } + + private sealed class PassthroughProcessor : IProcessor + { + public Task ProcessAsync(Prompty agent, object response) => Task.FromResult(response); + } + + private sealed class AccumulatedStreamProcessor : IProcessor + { + public Task ProcessAsync(Prompty agent, object response) + { + var stream = Assert.IsType(response); + return Task.FromResult(string.Concat(stream.Items.Cast())); + } + } + + private static async IAsyncEnumerable StreamItems(params object[] items) + { + foreach (var item in items) + { + await Task.Yield(); + yield return item; + } + } + + private sealed class AtomicRecordingDurability : IEngineDurabilityPort + { + public List<(IReadOnlyList Events, EngineCheckpoint Checkpoint)> AtomicAppends { get; } = []; + + public List StandaloneEvents { get; } = []; + + public EngineEventKind? FailAfter { get; set; } + + public Task AppendAsync(EngineEvent @event) + { + StandaloneEvents.Add(@event); + return Task.CompletedTask; + } + + public Task AppendWithCheckpointAsync( + IReadOnlyList events, + EngineCheckpoint checkpoint) + { + AtomicAppends.Add((events, checkpoint)); + if (FailAfter is not null && events.Any(@event => @event.Kind == FailAfter)) + { + throw PortError.Configuration("simulated durability crash"); + } + return Task.CompletedTask; + } + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs b/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs index f874eab7f..a70a2101e 100644 --- a/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs @@ -190,6 +190,44 @@ public void Load_Minimal_Works() Assert.Contains("Hello world.", agent.Instructions); } + [Fact] + public void Load_WithoutFrontmatter_TreatsEntireFileAsInstructions() + { + var root = Directory.CreateTempSubdirectory("prompty-loader-"); + try + { + var path = Path.Combine(root.FullName, "body-only.prompty"); + File.WriteAllText(path, "system:\nYou are helpful."); + + var agent = PromptyLoader.Load(path); + + Assert.Equal("system:\nYou are helpful.", agent.Instructions); + } + finally + { + root.Delete(recursive: true); + } + } + + [Fact] + public void Load_MissingClosingFrontmatterDelimiter_Throws() + { + var root = Directory.CreateTempSubdirectory("prompty-loader-"); + try + { + var path = Path.Combine(root.FullName, "invalid.prompty"); + File.WriteAllText(path, "---\nname: invalid\nsystem:\nHello"); + + var error = Assert.Throws(() => PromptyLoader.Load(path)); + + Assert.Contains("closing delimiter", error.Message); + } + finally + { + root.Delete(recursive: true); + } + } + // --- Env resolution --- [Fact] @@ -224,6 +262,31 @@ public void Load_EnvDefault_UsesDefault() Assert.Equal("https://fallback.openai.azure.com", conn.Endpoint); } + [Fact] + public void Load_EnvResolution_RecursesThroughMetadataAndArrays() + { + Environment.SetEnvironmentVariable("PROMPTY_NESTED_VALUE", "resolved"); + var root = Directory.CreateTempSubdirectory("prompty-loader-"); + try + { + var path = Path.Combine(root.FullName, "nested.prompty"); + File.WriteAllText( + path, + "---\nname: nested\nmetadata:\n nested:\n values:\n - ${env:PROMPTY_NESTED_VALUE}\n---\nHello"); + + var agent = PromptyLoader.Load(path); + var nested = Assert.IsAssignableFrom(agent.Metadata!["nested"]); + var values = Assert.IsAssignableFrom(nested["values"]); + + Assert.Equal("resolved", values[0]); + } + finally + { + Environment.SetEnvironmentVariable("PROMPTY_NESTED_VALUE", null); + root.Delete(recursive: true); + } + } + [Fact] public void Load_EnvMissing_Throws() { @@ -422,6 +485,30 @@ public void Load_SetsSourcePath() Assert.EndsWith("minimal.prompty", sourcePath); } + [Fact] + public void Load_PreservesExplicitNullMetadata() + { + var root = Directory.CreateTempSubdirectory("prompty-loader-null-"); + try + { + var path = Path.Combine(root.FullName, "nullable-metadata.prompty"); + File.WriteAllText( + path, + "---\nname: nullable-metadata\nmetadata:\n nullable: null\nmodel: gpt-4\n---\nsystem:\nHello."); + + var agent = PromptyLoader.Load(path); + + Assert.NotNull(agent.Metadata); + Assert.True(agent.Metadata.ContainsKey("nullable")); + Assert.Null(agent.Metadata["nullable"]); + Assert.True(agent.Metadata.ContainsKey("__source_path")); + } + finally + { + root.Delete(recursive: true); + } + } + // --- Error cases --- [Fact] diff --git a/runtime/csharp/Prompty.Core.Tests/ParserTests.cs b/runtime/csharp/Prompty.Core.Tests/ParserTests.cs index cc9e3d725..27f2b215e 100644 --- a/runtime/csharp/Prompty.Core.Tests/ParserTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ParserTests.cs @@ -33,19 +33,43 @@ public async Task Parse_MultipleRoles() } [Fact] - public async Task Parse_DeveloperRole() + public async Task Parse_DeveloperRole_IsPlainSystemContent() { var messages = await _parser.ParseAsync(CreateAgent(), "developer:\nInstructions here.", null); Assert.Single(messages); - Assert.Equal(Role.Developer, messages[0].Role); + Assert.Equal(Role.System, messages[0].Role); + Assert.Equal("developer:\nInstructions here.", messages[0].Text); } [Fact] - public async Task Parse_ToolRole() + public async Task Parse_ToolRole_IsPlainSystemContent() { var messages = await _parser.ParseAsync(CreateAgent(), "tool:\nTool response", null); Assert.Single(messages); - Assert.Equal(Role.Tool, messages[0].Role); + Assert.Equal(Role.System, messages[0].Role); + Assert.Equal("tool:\nTool response", messages[0].Text); + } + + [Theory] + [InlineData("SYSTEM:\nUppercase", Role.System)] + [InlineData(" user: \nIndented", Role.User)] + [InlineData("# assistant:\nHeading", Role.Assistant)] + public async Task Parse_CanonicalMarkerVariants(string input, Role expectedRole) + { + var messages = await _parser.ParseAsync(CreateAgent(), input, null); + Assert.Single(messages); + Assert.Equal(expectedRole, messages[0].Role); + } + + [Fact] + public async Task Parse_ContentBeforeFirstMarker_DefaultsToSystem() + { + var messages = await _parser.ParseAsync(CreateAgent(), "Introduction\nuser:\nQuestion", null); + + Assert.Equal(2, messages.Count); + Assert.Equal(Role.System, messages[0].Role); + Assert.Equal("Introduction", messages[0].Text); + Assert.Equal(Role.User, messages[1].Role); } // ----------------------------------------------------------------------- @@ -111,10 +135,10 @@ public async Task Parse_WhitespaceOnly_ReturnsNoMessages() [Fact] public async Task Parse_RoleWithAttributes() { - var text = "tool[tool_call_id=\"call_123\", name=\"get_weather\"]:\nResult here"; + var text = "assistant[tool_call_id=\"call_123\", name=\"get_weather\"]:\nResult here"; var messages = await _parser.ParseAsync(CreateAgent(), text, null); Assert.Single(messages); - Assert.Equal(Role.Tool, messages[0].Role); + Assert.Equal(Role.Assistant, messages[0].Role); Assert.Equal("call_123", messages[0].Metadata["tool_call_id"]); Assert.Equal("get_weather", messages[0].Metadata["name"]); } @@ -160,6 +184,101 @@ public async Task PreRender_Then_Parse_ValidatesNonce() Assert.False(messages[0].Metadata?.ContainsKey("nonce") ?? false); } + [Fact] + public void PreRender_Then_Parse_ClearsNonceForSubsequentSyncParse() + { + var parser = new PromptyChatParser(); + var (sanitized, _) = parser.PreRender("system:\nStrict"); + + var strictMessages = parser.Parse(sanitized); + var plainMessages = parser.Parse("user:\nPlain"); + + Assert.Single(strictMessages); + Assert.Single(plainMessages); + Assert.Equal("Plain", plainMessages[0].Text); + } + + [Fact] + public async Task PreRender_Then_ParseAsync_ClearsNonceForSubsequentAsyncParse() + { + var parser = new PromptyChatParser(); + var (sanitized, _) = parser.PreRender("system:\nStrict"); + + var strictMessages = await parser.ParseAsync(CreateAgent(), sanitized, null); + var plainMessages = await parser.ParseAsync(CreateAgent(), "user:\nPlain", null); + + Assert.Single(strictMessages); + Assert.Single(plainMessages); + Assert.Equal("Plain", plainMessages[0].Text); + } + + [Fact] + public async Task PreRender_Then_ParseAsync_IsolatesConcurrentNonceLifecycles() + { + var parser = new PromptyChatParser(); + using var barrier = new Barrier(2); + + Task ParseOnBranchAsync(string content) => + Task.Run(async () => + { + var (sanitized, _) = parser.PreRender($"user:\n{content}"); + barrier.SignalAndWait(); + var messages = await parser.ParseAsync(CreateAgent(), sanitized, null); + return Assert.Single(messages); + }); + + var messages = await Task.WhenAll( + ParseOnBranchAsync("First"), + ParseOnBranchAsync("Second")).WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(["First", "Second"], messages.Select(message => message.Text).Order()); + } + + [Fact] + public async Task PreRender_Then_Parse_RejectsInjectedMarkerWithoutNonce() + { + var parser = new PromptyChatParser(); + var (_, context) = parser.PreRender("system:\nHello"); + var nonce = context["nonce"]; + + InvalidOperationException? error = null; + try + { + await parser.ParseAsync( + CreateAgent(), + $"system[nonce=\"{nonce}\"]:\nHello\nuser:\nInjected", + null); + } + catch (InvalidOperationException caught) + { + error = caught; + } + + var plainMessages = await parser.ParseAsync(CreateAgent(), "user:\nRecovered", null); + + Assert.NotNull(error); + var nonNullError = error!; + Assert.Contains("nonce mismatch", nonNullError.Message); + Assert.Single(plainMessages); + Assert.Equal("Recovered", plainMessages[0].Text); + } + + [Fact] + public void PreRender_Then_Parse_WrongNonceStillRejectsAndClearsNonce() + { + var parser = new PromptyChatParser(); + var (sanitized, context) = parser.PreRender("system:\nHello"); + var nonce = context["nonce"]!.ToString()!; + var wrongNonce = sanitized.Replace(nonce, "wrong-nonce", StringComparison.Ordinal); + + var error = Assert.Throws(() => parser.Parse(wrongNonce)); + var plainMessages = parser.Parse("user:\nRecovered"); + + Assert.Contains("nonce mismatch", error.Message); + Assert.Single(plainMessages); + Assert.Equal("Recovered", plainMessages[0].Text); + } + // ----------------------------------------------------------------------- // Thread Nonce Pattern Recognition // ----------------------------------------------------------------------- diff --git a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs index 4c111edcb..c008d8cbe 100644 --- a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs @@ -537,17 +537,22 @@ public void ExpandThreadMarkers_CopiesMetadata() new() { Role = Role.System, - Parts = [new TextPart { Value = "Prefix __PROMPTY_THREAD_abcd1234_conv__" }], - Metadata = new Dictionary { ["source"] = "test" }, + Parts = [new TextPart { Value = "Prefix __PROMPTY_THREAD_abcd1234_conv__ Suffix" }], + Metadata = new Dictionary { ["source"] = "test", ["nullable"] = null }, }, }; var inputs = new Dictionary { ["conv"] = threadMessages }; var result = Pipeline.ExpandThreadMarkers(messages, inputs); - // The "before" fragment should carry the original message's metadata - Assert.Equal(2, result.Count); + // Both fragments should carry all metadata, including explicit null values. + Assert.Equal(3, result.Count); Assert.Equal("test", result[0].Metadata["source"]); + Assert.True(result[0].Metadata.ContainsKey("nullable")); + Assert.Null(result[0].Metadata["nullable"]); + Assert.Equal("test", result[2].Metadata["source"]); + Assert.True(result[2].Metadata.ContainsKey("nullable")); + Assert.Null(result[2].Metadata["nullable"]); } // ----------------------------------------------------------------------- @@ -608,10 +613,10 @@ public List FormatToolMessages(object rawResponse, List toolC { var messages = new List { - new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, + new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, }; for (var i = 0; i < toolCalls.Count; i++) - messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); + messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); return messages; } } @@ -643,10 +648,10 @@ public List FormatToolMessages(object rawResponse, List toolC { var messages = new List { - new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, + new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, }; for (var i = 0; i < toolCalls.Count; i++) - messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); + messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); return messages; } } diff --git a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs index a972bcfb3..95bbb1410 100644 --- a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs @@ -48,7 +48,7 @@ public List FormatToolMessages( { Role = Role.Assistant, Parts = [new TextPart { Value = textContent ?? "" }], - Metadata = new Dictionary { ["tool_calls"] = toolCalls } + Metadata = new Dictionary { ["tool_calls"] = toolCalls } }); for (int i = 0; i < toolCalls.Count; i++) { @@ -56,7 +56,7 @@ public List FormatToolMessages( { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], - Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); } return msgs; @@ -379,7 +379,7 @@ public async Task ToolDispatch_ParseError_ReturnsErrorStringToLlm() } [Fact] - public async Task ToolDispatch_ParallelPath_HandlerThrows_LoopContinues() + public async Task ToolDispatch_ParallelPath_IsRejectedForDeterministicDurability() { var executor = ResilienceHelper.Register(); @@ -407,9 +407,11 @@ public async Task ToolDispatch_ParallelPath_HandlerThrows_LoopContinues() ["bad_tool"] = _ => throw new InvalidOperationException("parallel boom") }; - var result = await Pipeline.TurnAsync(agent, tools: tools, parallelToolCalls: true); - Assert.Equal("parallel recovered", result); - Assert.Equal(2, executor.Calls.Count); + var error = await Assert.ThrowsAsync( + () => Pipeline.TurnAsync(agent, tools: tools, parallelToolCalls: true)); + + Assert.Contains("sequentially", error.Message); + Assert.Empty(executor.Calls); } } diff --git a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs index 91519b0a8..1d451674c 100644 --- a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs @@ -59,12 +59,6 @@ public async Task RenderVectors_AllPass() var mustacheRenderer = new MustacheRenderer(); var failures = new List(); - // Known Jinja2.NET compatibility issues - var knownSkips = new HashSet - { - "for_loop", // Jinja2.NET strips whitespace inside for loops differently - }; - foreach (var vec in vectors) { var name = vec.GetProperty("name").GetString()!; @@ -79,10 +73,6 @@ public async Task RenderVectors_AllPass() if (!expected.TryGetProperty("rendered", out var renderedEl)) continue; - // Skip known compatibility issues - if (knownSkips.Contains(name)) - continue; - var template = input.GetProperty("template").GetString()!; var engine = input.GetProperty("engine").GetString()!; var inputsEl = input.GetProperty("inputs"); @@ -295,6 +285,26 @@ public void LoadVectors_AllPass() } } + [Fact] + public void FunctionToolLoadVector_DeclaresBindingsExpectation() + { + var vector = LoadVectors("load") + .Single(vector => vector.GetProperty("name").GetString() == "tools_function_load"); + var expectedTool = vector.GetProperty("expected") + .GetProperty("tools") + .EnumerateArray() + .Single(tool => + tool.GetProperty("kind").GetString() == "function" && + tool.GetProperty("name").GetString() == "get_weather"); + + Assert.True( + expectedTool.TryGetProperty("bindings", out var bindings), + "tools_function_load must declare expected FunctionTool bindings"); + Assert.True( + bindings.ValueKind is JsonValueKind.Object or JsonValueKind.Array, + "FunctionTool bindings expectations must use the equivalent map or list wire form"); + } + // ========================================================================= // Helpers // ========================================================================= diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineFailureTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineFailureTests.cs new file mode 100644 index 000000000..92f9010bd --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineFailureTests.cs @@ -0,0 +1,309 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// Covers effect failures, recovery contracts, context composition, retry, and best-effort streaming. +public class TurnEngineFailureTests +{ + [Fact] + public async Task StructuredToolOutput_IsSerializedAsJsonForTheModel() + { + var output = new Dictionary + { + ["ok"] = true, + ["items"] = new List { 1, "two" }, + }; + var tools = new StructuredOutputToolPort(output); + var model = new CapturingModelPort( + [ + new ModelInvocationResponse + { + AssistantMessages = [Message.Assistant("Using a tool.")], + ToolRequests = [new ModelToolRequest { Id = "call-json", Name = "json", Arguments = new Dictionary() }], + }, + new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }, + ]); + var engine = CreateEngine(model, tools); + + var result = await engine.RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + var toolMessage = Assert.Single(model.Requests[1].Context.Messages, message => message.Role == Role.Tool); + var text = Assert.IsType(Assert.Single(toolMessage.Parts)); + Assert.Equal("""{"items":[1,"two"],"ok":true}""", text.Value); + } + + [Fact] + public async Task ContextPort_CanAddMessagesAndDecisions() + { + var model = new CapturingModelPort( + [ + new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }, + ]); + var effects = CreateEffects(model, new EchoToolPort(), context: new AppendingContextPort()); + var engine = new TurnEngine(effects); + + var result = await engine.RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + var snapshot = Assert.Single(result.Snapshots!); + Assert.Equal(2, snapshot.Messages.Count); + Assert.Equal("candidate-1", Assert.Single(snapshot.Decisions!).CandidateId); + Assert.Same(snapshot, Assert.Single(model.Requests).Context); + } + + [Fact] + public async Task InvalidContextSnapshot_CommitsContextErrorWithoutInvokingModel() + { + var model = new ThrowingModelPort(); + var effects = CreateEffects(model, new EchoToolPort(), context: new InvalidContextPort()); + var result = await new TurnEngine(effects).RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Failed, result.Commit.Status); + var output = Assert.IsType>(result.Commit.Output); + Assert.Equal("context_error", output["errorKind"]); + } + + [Fact] + public async Task TransientModelFailure_RetriesTheSameSnapshot() + { + var model = new TransientModelPort(); + var retry = new RecordingRetryPort(); + var effects = CreateEffects(model, new EchoToolPort(), retry: retry); + var result = await new TurnEngine(effects).RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal(2, model.InvocationCount); + Assert.Single(retry.Requests); + Assert.Same(model.Snapshots[0], model.Snapshots[1]); + } + + [Fact] + public async Task AtomicDurabilityFailure_ThrowsRecoveryRequiredWithCheckpoint() + { + var durability = new FailingCheckpointDurabilityPort(); + var effects = CreateEffects( + new CapturingModelPort( + [ + new ModelInvocationResponse { Output = "provider committed", AssistantMessages = [], ToolRequests = [] }, + ]), + new EchoToolPort(), + durability: durability); + var engine = new TurnEngine(effects); + + var error = await Assert.ThrowsAsync( + () => engine.RunAsync(NewRequest(), CancellationToken.None)); + + Assert.Equal("model response", error.Stage); + Assert.True(error.Checkpoint.FinalOutputReady); + Assert.Equal("provider committed", error.Checkpoint.PendingOutput); + Assert.Empty(error.ToolResults); + Assert.DoesNotContain(durability.Events, evt => evt.Kind == EngineEventKind.ModelInvocationCompleted); + } + + [Fact] + public async Task StreamFailure_DoesNotChangeSemanticExecution() + { + var model = new StreamingModelPort(); + var effects = CreateEffects(model, new EchoToolPort(), stream: new FailingStreamPort()); + var result = await new TurnEngine(effects).RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal("done", result.Commit.Output); + } + + [Fact] + public async Task NonPortStreamFailure_DoesNotChangeSemanticExecution() + { + var model = new StreamingModelPort(); + var effects = CreateEffects(model, new EchoToolPort(), stream: new FailingStreamPort(usePortError: false)); + var result = await new TurnEngine(effects).RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal("done", result.Commit.Output); + } + + [Fact] + public async Task PostCommitEffectId_UsesUtf8ByteLengths() + { + var postCommit = new RecordingPostCommitPort(); + var effects = new TurnEngineEffects + { + Model = new CapturingModelPort( + [ + new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }, + ]), + Tools = new EchoToolPort(), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + PostCommit = postCommit, + }; + var request = new TurnEngineRequest("séssion", "türn", [Message.User("Hello")]) { MaxIterations = 10 }; + + await new TurnEngine(effects).RunAsync(request, CancellationToken.None); + + Assert.Equal("post_commit:8:séssion:5:türn", Assert.Single(postCommit.Commits).EffectId); + } + + private static TurnEngineRequest NewRequest() => + new("session-1", "turn-1", [Message.User("Hello")]) { MaxIterations = 10 }; + + private static TurnEngine CreateEngine(IEngineModelPort model, IEngineToolPort tools) => + new(CreateEffects(model, tools)); + + private static TurnEngineEffects CreateEffects( + IEngineModelPort model, + IEngineToolPort tools, + IEngineContextPort? context = null, + IEngineRetryPolicyPort? retry = null, + IEngineDurabilityPort? durability = null, + IEngineModelStreamPort? stream = null) => + new() + { + Model = model, + Tools = tools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Context = context ?? new PassthroughEngineContextPort(), + Retry = retry ?? new NoopRetryPolicyPort(), + Durability = durability ?? new RecordingDurabilityPort(), + Stream = stream ?? new NoopModelStreamPort(), + }; + + private sealed class StructuredOutputToolPort(object output) : IEngineToolPort + { + public Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) => + Task.FromResult(new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = ModelToolOutcome.Success, + Output = output, + }); + } + + private sealed class CapturingModelPort(IEnumerable responses) : IEngineModelPort + { + private readonly Queue _responses = new(responses); + + public List Requests { get; } = []; + + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + Requests.Add(request); + return Task.FromResult(_responses.Dequeue()); + } + } + + private sealed class AppendingContextPort : IEngineContextPort + { + public Task PrepareAsync(ContextRequest request, CancellationToken cancellationToken) => + Task.FromResult(new ModelInvocationContextSnapshot + { + Id = $"context:{request.InvocationId}", + SessionId = request.SessionId, + TurnId = request.TurnId, + InvocationId = request.InvocationId, + Iteration = request.Iteration, + Messages = [.. request.Messages, Message.System("Recalled context")], + Decisions = + [ + new InvocationContextDecision + { + CandidateId = "candidate-1", + Disposition = InvocationContextDisposition.Included, + Reason = "included by test context", + }, + ], + StablePrefixMessages = request.StablePrefixMessages, + ContextState = request.ContextState, + }); + } + + private sealed class InvalidContextPort : IEngineContextPort + { + public Task PrepareAsync(ContextRequest request, CancellationToken cancellationToken) => + Task.FromResult(new ModelInvocationContextSnapshot + { + Id = "wrong", + SessionId = "wrong-session", + TurnId = request.TurnId, + InvocationId = request.InvocationId, + Iteration = request.Iteration, + Messages = request.Messages, + Decisions = [], + StablePrefixMessages = request.StablePrefixMessages, + ContextState = request.ContextState, + }); + } + + private sealed class TransientModelPort : IEngineModelPort + { + public int InvocationCount { get; private set; } + + public List Snapshots { get; } = []; + + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + InvocationCount++; + Snapshots.Add(request.Context); + return InvocationCount == 1 + ? throw new PortError("transient") + : Task.FromResult(new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }); + } + } + + private sealed class RecordingRetryPort : IEngineRetryPolicyPort + { + public List Requests { get; } = []; + + public Task BackoffAsync(RetryPolicyRequest request, CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.CompletedTask; + } + } + + private sealed class FailingCheckpointDurabilityPort : IEngineDurabilityPort + { + public List Events { get; } = []; + + public Task AppendAsync(EngineEvent @event) + { + Events.Add(@event); + return Task.CompletedTask; + } + + public Task AppendWithCheckpointAsync(IReadOnlyList events, EngineCheckpoint checkpoint) => + throw new PortError("atomic store unavailable"); + } + + private sealed class StreamingModelPort : IEngineModelPort + { + public async Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + await stream.EmitAsync(new ModelStreamChunk.Text("partial")); + return new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }; + } + } + + private sealed class FailingStreamPort(bool usePortError = true) : IEngineModelStreamPort + { + public Task EmitAsync(ModelStreamChunk chunk) => + usePortError + ? throw new PortError("stream sink unavailable") + : throw new IOException("stream sink unavailable"); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs new file mode 100644 index 000000000..33f14b5ba --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs @@ -0,0 +1,248 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// +/// Covers behavior that the shared vectors do not exercise directly: indeterminate tool-effect +/// and model-effect reconciliation (an effect whose real-world outcome is unknown must block the +/// turn until a host supplies an explicit determinate resolution, and resuming must never re-run +/// the unknown effect), non-fatal post-commit failures, mid-run cancellation, and strictly +/// sequential (never concurrent) tool execution. +/// +public class TurnEngineReconciliationTests +{ + [Fact] + public async Task IndeterminateToolEffect_BlocksTurnUntilExplicitReconciliation() + { + var toolRequest = new ModelToolRequest { Id = "call-a", Name = "echo", Arguments = new Dictionary { ["value"] = "A" } }; + var steps = new List { new() { Assistant = "Using a tool.", Tools = [toolRequest] } }; + + var durability = new RecordingDurabilityPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = new IndeterminateToolPort("call-a", new EchoToolPort()), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = durability, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Do something")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.ReconciliationRequired, result.Commit.Status); + + var checkpoint = durability.Checkpoints.Last(); + Assert.True(checkpoint.ReconciliationRequired); + Assert.Equal(ModelToolOutcome.Indeterminate, checkpoint.CompletedToolResults!.Single().Outcome); + Assert.Null(checkpoint.ModelReconciliation); + + // Resuming without an explicit resolution must not silently proceed or re-run the tool — + // it re-reports the same reconciliation-required outcome. + var unresolvedTools = new EchoToolPort(forbiddenRequestIds: ["call-a"]); + var unresolvedEngine = new TurnEngine(new TurnEngineEffects + { + Model = new ThrowingModelPort(), + Tools = unresolvedTools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }); + var unresolvedResume = TurnEngineRequest.ResumeFrom(checkpoint, maxIterations: 10, lastJournalSequence: 0); + var unresolvedResult = await unresolvedEngine.RunAsync(unresolvedResume, CancellationToken.None); + Assert.Equal(EngineTurnStatus.ReconciliationRequired, unresolvedResult.Commit.Status); + Assert.Empty(unresolvedTools.ExecutedRequestIds); + + // Attempting to resolve with another indeterminate outcome is rejected outright. + var stillIndeterminate = new ModelToolResult { RequestId = "call-a", Name = "echo", Outcome = ModelToolOutcome.Indeterminate, Output = "still unknown" }; + Assert.Throws(() => + TurnEngineRequest.ResumeAfterReconciliation(checkpoint, 10, 0, stillIndeterminate)); + + // Resolving with an explicit determinate outcome resumes and completes without re-running + // the tool at all. + var resolvedResult = new ModelToolResult { RequestId = "call-a", Name = "echo", Outcome = ModelToolOutcome.Success, Output = "A" }; + var resolvedTools = new EchoToolPort(forbiddenRequestIds: ["call-a"]); + var resolvedEngine = new TurnEngine(new TurnEngineEffects + { + Model = new ScriptedModelPort([new ScriptedModelStep { Output = "done" }]), + Tools = resolvedTools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }); + var resolvedRequest = TurnEngineRequest.ResumeAfterReconciliation(checkpoint, 10, 0, resolvedResult); + var finalResult = await resolvedEngine.RunAsync(resolvedRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, finalResult.Commit.Status); + Assert.Equal("done", finalResult.Commit.Output); + Assert.Empty(resolvedTools.ExecutedRequestIds); + Assert.Equal(ModelToolOutcome.Success, finalResult.ToolResults!.Single(t => t.RequestId == "call-a").Outcome); + } + + [Fact] + public async Task IndeterminateModelEffect_BlocksTurnUntilExplicitReconciliation() + { + var durability = new RecordingDurabilityPort(); + var failureMetadata = new Dictionary { ["nullable"] = null }; + var effects = new TurnEngineEffects + { + Model = new IndeterminateModelPort(failureMetadata), + Tools = new EchoToolPort(), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = durability, + }; + + var engine = new TurnEngine(effects); + // A single model attempt is enough to force reconciliation: an indeterminate outcome is + // never retried, regardless of MaxModelAttempts. + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Hello")]) { MaxIterations = 10, MaxModelAttempts = 3 }; + var result = await engine.RunAsync(request, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.ReconciliationRequired, result.Commit.Status); + Assert.Equal(1, ((IndeterminateModelPort)effects.Model).InvocationCount); + + var checkpoint = durability.Checkpoints.Last(); + Assert.True(checkpoint.ReconciliationRequired); + Assert.NotNull(checkpoint.ModelReconciliation); + Assert.Equal(checkpoint.ActiveInvocationId, checkpoint.ModelReconciliation!.InvocationId); + Assert.NotNull(checkpoint.ModelReconciliation.Metadata); + Assert.True(checkpoint.ModelReconciliation.Metadata.ContainsKey("nullable")); + Assert.Null(checkpoint.ModelReconciliation.Metadata["nullable"]); + var reloadedReconciliation = ModelReconciliationState.Load(checkpoint.ModelReconciliation.Save()); + Assert.NotNull(reloadedReconciliation.Metadata); + Assert.True(reloadedReconciliation.Metadata.ContainsKey("nullable")); + Assert.Null(reloadedReconciliation.Metadata["nullable"]); + + // Resolving with an explicit model response resumes without re-invoking the model port. + var resolvedResponse = new ModelInvocationResponse { Output = "resolved output", AssistantMessages = [], ToolRequests = [] }; + var resumedEngine = new TurnEngine(new TurnEngineEffects + { + Model = new ThrowingModelPort(), + Tools = new EchoToolPort(), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }); + var resumeRequest = TurnEngineRequest.ResumeAfterModelReconciliation(checkpoint, 10, 0, resolvedResponse); + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, resumedResult.Commit.Status); + Assert.Equal("resolved output", resumedResult.Commit.Output); + } + + [Fact] + public async Task PostCommitFailure_IsReportedNonFatally() + { + var postCommit = new FailingPostCommitPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort([new ScriptedModelStep { Output = "Hello back" }]), + Tools = new EchoToolPort(), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + PostCommit = postCommit, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Hello")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, CancellationToken.None); + + // The turn itself still committed successfully — a broken post-commit sink never + // uncommits the turn — but the failure is surfaced for the host to notice and retry. + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal("Hello back", result.Commit.Output); + Assert.NotNull(result.PostCommitError); + Assert.Contains(FailingPostCommitPort.FailureMessage, result.PostCommitError); + Assert.Single(postCommit.AttemptedEffectIds); + } + + [Fact] + public async Task Cancellation_BetweenSequentialTools_StopsBeforeTheSecondToolRuns() + { + var toolA = new ModelToolRequest { Id = "call-a", Name = "echo", Arguments = new Dictionary { ["value"] = "A" } }; + var toolB = new ModelToolRequest { Id = "call-b", Name = "echo", Arguments = new Dictionary { ["value"] = "B" } }; + var steps = new List + { + new() { Assistant = "Using tools.", Tools = [toolA, toolB] }, + new() { Output = "should never be reached" }, + }; + + using var cts = new CancellationTokenSource(); + var tools = new CancelAfterFirstToolPort(cts, "call-a"); + var durability = new RecordingDurabilityPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = tools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = durability, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Add two values")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, cts.Token); + + Assert.Equal(EngineTurnStatus.Cancelled, result.Commit.Status); + Assert.Equal(["call-a"], tools.ExecutedRequestIds); + Assert.Single(result.ToolResults!); + Assert.Equal(EngineEventKind.TurnCancelled, durability.Events.Last().Kind); + } + + [Fact] + public async Task Tools_ExecuteStrictlySequentially_NeverConcurrently() + { + var toolA = new ModelToolRequest { Id = "call-a", Name = "echo", Arguments = new Dictionary { ["value"] = "A" } }; + var toolB = new ModelToolRequest { Id = "call-b", Name = "echo", Arguments = new Dictionary { ["value"] = "B" } }; + var steps = new List + { + new() { Assistant = "Using tools.", Tools = [toolA, toolB] }, + new() { Output = "A then B" }, + }; + var tools = new EchoToolPort(new Dictionary { ["call-a"] = "A", ["call-b"] = "B" }); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = tools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Add two values")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal(1, tools.MaxObservedConcurrency); + Assert.Equal(["call-a", "call-b"], tools.ExecutedRequestIds); + } + + /// Tool port that cancels the shared token immediately after its first configured request completes. + private sealed class CancelAfterFirstToolPort(CancellationTokenSource cancellationSource, string cancelAfterRequestId) : IEngineToolPort + { + public List ExecutedRequestIds { get; } = []; + + public Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) + { + if (ExecutedRequestIds.Contains(cancelAfterRequestId)) + { + throw new InvalidOperationException($"tool '{request.Id}' must not execute after cancellation"); + } + + ExecutedRequestIds.Add(request.Id); + var result = new ModelToolResult { RequestId = request.Id, Name = request.Name, Outcome = ModelToolOutcome.Success, Output = request.Id }; + if (request.Id == cancelAfterRequestId) + { + cancellationSource.Cancel(); + } + + return Task.FromResult(result); + } + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineResumeTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineResumeTests.cs new file mode 100644 index 000000000..e294a9335 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineResumeTests.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// +/// Proves the canonical can resume from a durably persisted +/// without duplicating already-committed effects: a checkpoint +/// captured right after the first of two sequential tool results resumes without re-running +/// that tool, continues the event sequence after the checkpoint's last committed sequence +/// (or the journal tail, whichever is larger), a checkpoint with a ready final output commits +/// without ever re-invoking the model, and a checkpoint resumed with no iteration budget left +/// fails immediately instead of silently looping. +/// +public class TurnEngineResumeTests +{ + /// + /// Runs the two-sequential-tool scenario (mirrors the "ordered_tool_round" shared vector) + /// to completion while recording every checkpoint the engine persists along the way, so + /// individual tests can resume from any mid-run checkpoint. + /// + private static async Task<(RecordingDurabilityPort Durability, TurnEngineResult Result)> RunOrderedToolRoundAsync() + { + var steps = new List + { + new() + { + Assistant = "I will use the tools.", + Tools = + [ + new ModelToolRequest { Id = "call-a", Name = "echo", Arguments = new Dictionary { ["value"] = "A" } }, + new ModelToolRequest { Id = "call-b", Name = "echo", Arguments = new Dictionary { ["value"] = "B" } }, + ], + }, + new() { Output = "A then B" }, + }; + var toolOutputs = new Dictionary { ["call-a"] = "A", ["call-b"] = "B" }; + + var durability = new RecordingDurabilityPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = new EchoToolPort(toolOutputs), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = durability, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Add two values")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, CancellationToken.None); + return (durability, result); + } + + [Fact] + public async Task Resume_AfterFirstToolResult_DoesNotRerunThatTool() + { + var (durability, originalResult) = await RunOrderedToolRoundAsync(); + Assert.Equal(EngineTurnStatus.Success, originalResult.Commit.Status); + + // The checkpoint persisted immediately after "call-a" completes: exactly one completed + // tool result, and "call-b" still pending. + var checkpointAfterFirstTool = durability.Checkpoints.First(c => + (c.CompletedToolResults?.Count ?? 0) == 1 + && (c.PendingToolRequests?.Count ?? 0) == 1); + + Assert.Equal("call-a", checkpointAfterFirstTool.CompletedToolResults![0].RequestId); + Assert.Equal("call-b", checkpointAfterFirstTool.PendingToolRequests![0].Id); + + // Resume with a tool port that throws if "call-a" is ever asked to execute again, and a + // model port that only knows about the final output step (the tool round's own model + // response is already durably captured in the checkpoint and must not be re-requested). + var resumedDurability = new RecordingDurabilityPort(); + var resumedTools = new EchoToolPort( + new Dictionary { ["call-b"] = "B" }, + forbiddenRequestIds: ["call-a"]); + var resumedEffects = new TurnEngineEffects + { + Model = new ScriptedModelPort([new ScriptedModelStep { Output = "A then B" }]), + Tools = resumedTools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = resumedDurability, + }; + + var resumedEngine = new TurnEngine(resumedEffects); + var resumeRequest = TurnEngineRequest.ResumeFrom(checkpointAfterFirstTool, maxIterations: 10, lastJournalSequence: 0); + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, resumedResult.Commit.Status); + Assert.Equal("A then B", resumedResult.Commit.Output); + Assert.Equal(["call-b"], resumedTools.ExecutedRequestIds); + Assert.Equal(2, resumedResult.ToolResults?.Count); + Assert.Equal(["call-a", "call-b"], resumedResult.ToolResults!.Select(t => t.RequestId)); + + // The resumed run must continue the event sequence strictly after the checkpoint's last + // committed sequence — never restart from 1, which would duplicate already-durable events. + Assert.All(resumedDurability.Events, e => Assert.True(e.Sequence > checkpointAfterFirstTool.LastSequence)); + Assert.Equal(checkpointAfterFirstTool.LastSequence + 1, resumedDurability.Events[0].Sequence); + } + + [Fact] + public async Task Resume_ContinuesAfterMaxOfCheckpointAndJournalTail() + { + var (durability, _) = await RunOrderedToolRoundAsync(); + var checkpointAfterFirstTool = durability.Checkpoints.First(c => + (c.CompletedToolResults?.Count ?? 0) == 1 && (c.PendingToolRequests?.Count ?? 0) == 1); + + // Simulate a journal whose tail is further ahead than the checkpoint itself (for example, + // a plain, non-checkpointed event was appended after the checkpoint before the host + // crashed). Resuming must continue after the larger of the two, never the checkpoint alone. + var journalTail = checkpointAfterFirstTool.LastSequence + 5; + + var resumedDurability = new RecordingDurabilityPort(); + var resumedEffects = new TurnEngineEffects + { + Model = new ScriptedModelPort([new ScriptedModelStep { Output = "A then B" }]), + Tools = new EchoToolPort(new Dictionary { ["call-b"] = "B" }, forbiddenRequestIds: ["call-a"]), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = resumedDurability, + }; + + var resumedEngine = new TurnEngine(resumedEffects); + var resumeRequest = TurnEngineRequest.ResumeFrom(checkpointAfterFirstTool, maxIterations: 10, lastJournalSequence: journalTail); + Assert.Equal(journalTail, resumeRequest.InitialSequence); + + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, resumedResult.Commit.Status); + Assert.Equal(journalTail + 1, resumedDurability.Events[0].Sequence); + } + + [Fact] + public async Task Resume_WhenFinalOutputAlreadyReady_NeverReinvokesModel() + { + var (durability, _) = await RunOrderedToolRoundAsync(); + var finalCheckpoint = durability.Checkpoints.Last(c => c.FinalOutputReady); + + var resumedEffects = new TurnEngineEffects + { + Model = new ThrowingModelPort(), + Tools = new EchoToolPort(forbiddenRequestIds: ["call-a", "call-b"]), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }; + + var resumedEngine = new TurnEngine(resumedEffects); + var resumeRequest = TurnEngineRequest.ResumeFrom(finalCheckpoint, maxIterations: 10, lastJournalSequence: 0); + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, resumedResult.Commit.Status); + Assert.Equal("A then B", resumedResult.Commit.Output); + } + + [Fact] + public async Task Resume_WithNoIterationBudgetRemaining_FailsWithMaxIterations() + { + var (durability, _) = await RunOrderedToolRoundAsync(); + var checkpointAfterFirstTool = durability.Checkpoints.First(c => + (c.CompletedToolResults?.Count ?? 0) == 1 && (c.PendingToolRequests?.Count ?? 0) == 1); + + var resumedEffects = new TurnEngineEffects + { + Model = new ThrowingModelPort(), + Tools = new EchoToolPort(forbiddenRequestIds: ["call-a", "call-b"]), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }; + + var resumedEngine = new TurnEngine(resumedEffects); + // maxIterations equal to the checkpoint's own iteration means the resumed run starts + // with no budget left at all: the while loop condition is false on the first check. + var resumeRequest = TurnEngineRequest.ResumeFrom( + checkpointAfterFirstTool, + maxIterations: checkpointAfterFirstTool.Iteration, + lastJournalSequence: 0); + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Failed, resumedResult.Commit.Status); + var output = Assert.IsType>(resumedResult.Commit.Output); + Assert.Equal("max_iterations", output["errorKind"]); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs new file mode 100644 index 000000000..855a877ec --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs @@ -0,0 +1,245 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +// ----------------------------------------------------------------------- +// Deterministic test doubles shared by the canonical TurnEngine test suite +// (TurnEngineVectorTests, TurnEngineResumeTests, TurnEngineReconciliationTests). +// ----------------------------------------------------------------------- + +/// Deterministic, monotonically increasing clock for engine tests. +internal sealed class FakeEngineClock : IEngineClock +{ + private long _tick; + + public string Now() => $"2024-01-01T00:00:{_tick++:D2}Z"; +} + +/// Deterministic per-kind sequential id generator for engine tests. +internal sealed class FakeEngineIdGenerator : IEngineIdGenerator +{ + private readonly Dictionary _counters = []; + + public string NextId(string kind) + { + var next = _counters.TryGetValue(kind, out var current) ? current + 1 : 1; + _counters[kind] = next; + return $"{kind}-{next}"; + } +} + +/// One scripted model turn step: either a final output, or an assistant message plus tool requests. +internal sealed class ScriptedModelStep +{ + public string? Assistant { get; init; } + + public object? Output { get; init; } + + public IList Tools { get; init; } = []; + + public InvocationContextPortability? NextPortability { get; init; } + + public IList? DelegatedState { get; init; } +} + +/// Model port that replays a fixed script of responses, one per invocation, strictly in order. +internal sealed class ScriptedModelPort : IEngineModelPort +{ + private readonly Queue _steps; + + public ScriptedModelPort(IEnumerable steps) => _steps = new Queue(steps); + + public int InvocationCount { get; private set; } + + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + InvocationCount++; + if (_steps.Count == 0) + { + throw PortError.Configuration("scripted model script exhausted"); + } + + var step = _steps.Dequeue(); + var response = new ModelInvocationResponse + { + Output = step.Output, + AssistantMessages = step.Assistant is null ? [] : [Message.Assistant(step.Assistant)], + ToolRequests = [.. step.Tools], + }; + + if (step.NextPortability is not null) + { + response.NextContextState = new InvocationContextState + { + Portability = step.NextPortability.Value, + DelegatedState = step.DelegatedState ?? [], + }; + } + + return Task.FromResult(response); + } +} + +/// Model port that always throws — proves the model is never re-invoked after resume/reconciliation. +internal sealed class ThrowingModelPort : IEngineModelPort +{ + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) => + throw new InvalidOperationException("model must not be invoked again"); +} + +/// Model port whose single scripted attempt always fails with an indeterminate (outcome-unknown) error. +internal sealed class IndeterminateModelPort : IEngineModelPort +{ + private readonly IDictionary? _metadata; + + public IndeterminateModelPort(IDictionary? metadata = null) => _metadata = metadata; + + public int InvocationCount { get; private set; } + + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + InvocationCount++; + throw PortError.Indeterminate("model invocation outcome is unknown", _metadata); + } +} + +/// +/// Tool port that resolves each request's output from a lookup table and records execution +/// order/concurrency, so tests can prove tools run sequentially and in model-provided order. +/// +internal sealed class EchoToolPort : IEngineToolPort +{ + private readonly IReadOnlyDictionary _outputs; + private readonly HashSet _forbiddenRequestIds; + private int _inFlight; + + public EchoToolPort(IReadOnlyDictionary? outputs = null, IEnumerable? forbiddenRequestIds = null) + { + _outputs = outputs ?? new Dictionary(); + _forbiddenRequestIds = forbiddenRequestIds is null ? [] : [.. forbiddenRequestIds]; + } + + /// Request ids executed so far, in the order they were executed. + public List ExecutedRequestIds { get; } = []; + + /// The largest number of concurrently in-flight tool executions observed. + public int MaxObservedConcurrency { get; private set; } + + public async Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) + { + if (_forbiddenRequestIds.Contains(request.Id)) + { + throw new InvalidOperationException($"tool request '{request.Id}' must not execute again"); + } + + var concurrent = Interlocked.Increment(ref _inFlight); + MaxObservedConcurrency = Math.Max(MaxObservedConcurrency, concurrent); + try + { + // Yield to make any accidental parallel invocation observable. + await Task.Yield(); + ExecutedRequestIds.Add(request.Id); + var output = _outputs.TryGetValue(request.Id, out var value) ? value : request.Name; + return new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = ModelToolOutcome.Success, + Output = output, + }; + } + finally + { + Interlocked.Decrement(ref _inFlight); + } + } +} + +/// Tool port whose single configured request id fails with an indeterminate (outcome-unknown) error. +internal sealed class IndeterminateToolPort : IEngineToolPort +{ + private readonly string _requestId; + private readonly IEngineToolPort _inner; + + public IndeterminateToolPort(string requestId, IEngineToolPort inner) + { + _requestId = requestId; + _inner = inner; + } + + public Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) => + request.Id == _requestId + ? throw PortError.Indeterminate($"tool '{request.Id}' outcome is unknown") + : _inner.ExecuteAsync(request, cancellationToken); +} + +/// Permission port that denies tool requests whose name is in the deny list. +internal sealed class DenyByNamePermissionPort : IEnginePermissionPort +{ + private readonly HashSet _denied; + + public DenyByNamePermissionPort(IEnumerable deniedNames) => _denied = [.. deniedNames]; + + public Task AuthorizeAsync(ModelToolRequest request, CancellationToken cancellationToken) => + Task.FromResult(_denied.Contains(request.Name) + ? new EnginePermissionDecision { Approved = false, Reason = "Permission was denied" } + : new EnginePermissionDecision { Approved = true }); +} + +/// Durability port that records every appended event/checkpoint in memory for assertions. +internal sealed class RecordingDurabilityPort : IEngineDurabilityPort +{ + public List Events { get; } = []; + + public List Checkpoints { get; } = []; + + public Task AppendAsync(EngineEvent @event) + { + Events.Add(@event); + return Task.CompletedTask; + } + + public Task AppendWithCheckpointAsync(IReadOnlyList events, EngineCheckpoint checkpoint) + { + Events.AddRange(events); + Checkpoints.Add(checkpoint); + return Task.CompletedTask; + } +} + +/// Post-commit port that always fails, to prove post-commit failures are reported non-fatally. +internal sealed class FailingPostCommitPort : IEnginePostCommitPort +{ + public const string FailureMessage = "post-commit sink unavailable"; + + public List AttemptedEffectIds { get; } = []; + + public Task AfterCommitAsync(string effectId, TurnCommit commit, CancellationToken cancellationToken) + { + AttemptedEffectIds.Add(effectId); + throw PortError.Configuration(FailureMessage); + } +} + +/// Post-commit port that records every commit it was asked to run an effect for. +internal sealed class RecordingPostCommitPort : IEnginePostCommitPort +{ + public List<(string EffectId, TurnCommit Commit)> Commits { get; } = []; + + public Task AfterCommitAsync(string effectId, TurnCommit commit, CancellationToken cancellationToken) + { + Commits.Add((effectId, commit)); + return Task.CompletedTask; + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineVectorTests.cs new file mode 100644 index 000000000..9b8a6b837 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineVectorTests.cs @@ -0,0 +1,248 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// +/// Executes the shared canonical engine vectors in spec/vectors/engine/turn_vectors.json +/// against the C# . These vectors are the normative minimum: event +/// ordering, stable-prefix snapshot bookkeeping, sequential ordered tool execution, permission +/// denial being fed back to the model as a normal turn, delegated provider state, and +/// cancellation before context preparation must all match the Rust reference engine exactly. +/// +public class TurnEngineVectorTests +{ + private static readonly string SpecDir = FindSpecDir(); + + [Fact] + public async Task FinalOutput_MatchesVector() => await RunCaseAsync("final_output"); + + [Fact] + public async Task OrderedToolRound_MatchesVector() => await RunCaseAsync("ordered_tool_round"); + + [Fact] + public async Task PermissionDenialIsModelVisible_MatchesVector() => await RunCaseAsync("permission_denial_is_model_visible"); + + [Fact] + public async Task DelegatedProviderState_MatchesVector() => await RunCaseAsync("delegated_provider_state"); + + [Fact] + public async Task CancelBeforeContext_MatchesVector() => await RunCaseAsync("cancel_before_context"); + + private static async Task RunCaseAsync(string name) + { + var testCase = LoadCases().First(c => c.GetProperty("name").GetString() == name); + + var messages = ParseMessages(testCase.GetProperty("messages")); + var steps = ParseSteps(testCase.GetProperty("model")); + var toolOutputs = testCase.TryGetProperty("toolOutputs", out var toolOutputsEl) + ? toolOutputsEl.EnumerateObject().ToDictionary(p => p.Name, p => p.Value.GetString() ?? string.Empty) + : []; + var denyTools = testCase.TryGetProperty("denyTools", out var denyToolsEl) + ? denyToolsEl.EnumerateArray().Select(e => e.GetString()!).ToArray() + : []; + var cancelBeforeRun = testCase.TryGetProperty("cancelBeforeRun", out var cancelEl) && cancelEl.GetBoolean(); + + var durability = new RecordingDurabilityPort(); + var postCommit = new RecordingPostCommitPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = new EchoToolPort(toolOutputs), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Permission = new DenyByNamePermissionPort(denyTools), + Durability = durability, + PostCommit = postCommit, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", messages) { MaxIterations = 10 }; + + using var cts = new CancellationTokenSource(); + if (cancelBeforeRun) + { + cts.Cancel(); + } + + var result = await engine.RunAsync(request, cts.Token); + var expected = testCase.GetProperty("expected"); + + Assert.Equal(ParseStatus(expected.GetProperty("status").GetString()!), result.Commit.Status); + Assert.Equal(expected.GetProperty("iterations").GetInt32(), result.Commit.Iterations); + Assert.Equal(expected.GetProperty("snapshots").GetInt32(), result.Snapshots?.Count ?? 0); + Assert.Equal(expected.GetProperty("toolResults").GetInt32(), result.ToolResults?.Count ?? 0); + + if (expected.TryGetProperty("output", out var outputEl)) + { + Assert.Equal(outputEl.GetString(), result.Commit.Output); + } + + if (expected.TryGetProperty("snapshotStablePrefixes", out var prefixesEl)) + { + var expectedPrefixes = prefixesEl.EnumerateArray().Select(e => e.GetInt32()).ToArray(); + var actualPrefixes = (result.Snapshots ?? []).Select(s => s.StablePrefixMessages).ToArray(); + Assert.Equal(expectedPrefixes, actualPrefixes); + } + + if (expected.TryGetProperty("toolResultOrder", out var orderEl)) + { + var expectedOrder = orderEl.EnumerateArray().Select(e => e.GetString()).ToArray(); + var actualOrder = (result.ToolResults ?? []).Select(t => t.RequestId).ToArray(); + Assert.Equal(expectedOrder, actualOrder); + } + + if (expected.TryGetProperty("eventKinds", out var eventKindsEl)) + { + var expectedKinds = eventKindsEl.EnumerateArray().Select(e => e.GetString()).ToArray(); + var actualKinds = durability.Events.Select(e => EngineEventKindParser.ToValue(e.Kind)).ToArray(); + Assert.Equal(expectedKinds, actualKinds); + } + + if (expected.TryGetProperty("snapshotPortability", out var snapshotPortabilityEl)) + { + var expectedPortability = snapshotPortabilityEl.EnumerateArray().Select(e => e.GetString()).ToArray(); + var actualPortability = (result.Snapshots ?? []) + .Select(s => PortabilityToWire(s.ContextState.Portability)) + .ToArray(); + Assert.Equal(expectedPortability, actualPortability); + } + + if (expected.TryGetProperty("commitPortability", out var commitPortabilityEl)) + { + Assert.Equal(commitPortabilityEl.GetString(), PortabilityToWire(result.Commit.ContextState.Portability)); + } + + if (expected.TryGetProperty("delegatedState", out var delegatedStateEl)) + { + Assert.Equal(delegatedStateEl.GetInt32(), result.Commit.ContextState.DelegatedState?.Count ?? 0); + } + + // The final_output/ordered_tool_round/permission_denial/delegated_provider_state vectors + // all commit successfully, so the non-fatal post-commit effect must always run exactly once. + if (result.Commit.Status == EngineTurnStatus.Success) + { + Assert.Single(postCommit.Commits); + Assert.Null(result.PostCommitError); + } + } + + private static EngineTurnStatus ParseStatus(string value) => value switch + { + "success" => EngineTurnStatus.Success, + "failed" => EngineTurnStatus.Failed, + "cancelled" => EngineTurnStatus.Cancelled, + "reconciliation_required" => EngineTurnStatus.ReconciliationRequired, + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "unknown status"), + }; + + private static string PortabilityToWire(InvocationContextPortability portability) => portability switch + { + InvocationContextPortability.Portable => "portable", + InvocationContextPortability.Delegated => "delegated", + InvocationContextPortability.Opaque => "opaque", + _ => throw new ArgumentOutOfRangeException(nameof(portability), portability, "unknown portability"), + }; + + private static InvocationContextPortability ParsePortability(string value) => value switch + { + "portable" => InvocationContextPortability.Portable, + "delegated" => InvocationContextPortability.Delegated, + "opaque" => InvocationContextPortability.Opaque, + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "unknown portability"), + }; + + private static List ParseMessages(JsonElement messagesEl) => + [.. messagesEl.EnumerateArray().Select(ParseMessage)]; + + private static Message ParseMessage(JsonElement messageEl) + { + var role = messageEl.GetProperty("role").GetString(); + var content = messageEl.GetProperty("content").GetString() ?? string.Empty; + return role switch + { + "assistant" => Message.Assistant(content), + "system" => Message.System(content), + _ => Message.User(content), + }; + } + + private static List ParseSteps(JsonElement modelEl) => + [.. modelEl.EnumerateArray().Select(ParseStep)]; + + private static ScriptedModelStep ParseStep(JsonElement stepEl) + { + var tools = stepEl.TryGetProperty("tools", out var toolsEl) + ? toolsEl.EnumerateArray().Select(ParseToolRequest).ToList() + : []; + + var delegatedState = stepEl.TryGetProperty("delegatedState", out var delegatedEl) + ? delegatedEl.EnumerateArray().Select(ParseDelegatedStateReference).ToList() + : null; + + return new ScriptedModelStep + { + Assistant = stepEl.TryGetProperty("assistant", out var assistantEl) ? assistantEl.GetString() : null, + Output = stepEl.TryGetProperty("output", out var outputEl) ? outputEl.GetString() : null, + Tools = tools, + NextPortability = stepEl.TryGetProperty("nextPortability", out var portabilityEl) + ? ParsePortability(portabilityEl.GetString()!) + : null, + DelegatedState = delegatedState, + }; + } + + private static ModelToolRequest ParseToolRequest(JsonElement toolEl) => new() + { + Id = toolEl.GetProperty("id").GetString()!, + Name = toolEl.GetProperty("name").GetString()!, + Arguments = toolEl.TryGetProperty("arguments", out var argsEl) ? JsonElementToObject(argsEl) : null, + }; + + private static DelegatedStateReference ParseDelegatedStateReference(JsonElement el) => new() + { + Provider = el.GetProperty("provider").GetString()!, + Kind = el.GetProperty("kind").GetString()!, + Id = el.GetProperty("id").GetString()!, + }; + + private static object? JsonElementToObject(JsonElement el) => el.ValueKind switch + { + JsonValueKind.String => el.GetString(), + JsonValueKind.Number => el.TryGetInt64(out var l) ? l : el.GetDouble(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null => null, + JsonValueKind.Array => el.EnumerateArray().Select(JsonElementToObject).ToList(), + JsonValueKind.Object => el.EnumerateObject().ToDictionary(p => p.Name, p => JsonElementToObject(p.Value)), + _ => null, + }; + + private static JsonElement[] LoadCases() + { + var path = Path.Combine(SpecDir, "vectors", "engine", "turn_vectors.json"); + var json = File.ReadAllText(path); + using var doc = JsonDocument.Parse(json); + return [.. doc.RootElement.GetProperty("cases").EnumerateArray().Select(e => e.Clone())]; + } + + private static string FindSpecDir() + { + var dir = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine(dir, "spec"); + if (Directory.Exists(candidate)) + { + return candidate; + } + + dir = Path.GetDirectoryName(dir) ?? dir; + } + + var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..")); + return Path.Combine(projectRoot, "spec"); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs index 82d8ead42..fed8a8215 100644 --- a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs @@ -22,11 +22,16 @@ public async Task ReferenceTurnRunner_EmitsJournalsAndCheckpoints() checkpointStore, new AllowAllPermissionResolver(), new FunctionHostToolExecutor(new Dictionary()), - request => Task.FromResult(new TurnModelResponse + request => { - Output = new Dictionary { ["text"] = $"hello {request.Inputs["name"]}" }, - CheckpointState = new Dictionary { ["stable"] = true } - }), + Assert.True(request.Inputs!.ContainsKey("nullable")); + Assert.Null(request.Inputs["nullable"]); + return Task.FromResult(new TurnModelResponse + { + Output = new Dictionary { ["text"] = $"hello {request.Inputs["name"]}" }, + CheckpointState = new Dictionary { ["stable"] = true, ["nullable"] = null } + }); + }, FixedClock(), FixedIds()); @@ -34,7 +39,7 @@ public async Task ReferenceTurnRunner_EmitsJournalsAndCheckpoints() { SessionId = "session-1", TurnId = "turn-1", - Inputs = new Dictionary { ["name"] = "Ada" }, + Inputs = new Dictionary { ["name"] = "Ada", ["nullable"] = null }, Options = new TurnOptions { MaxIterations = 3 } }); @@ -49,7 +54,13 @@ public async Task ReferenceTurnRunner_EmitsJournalsAndCheckpoints() sink.SessionEvents.Select(sessionEvent => sessionEvent.Type).ToArray()); var checkpoint = await checkpointStore.LoadAsync("session-1", "turn-1-checkpoint-0"); Assert.NotNull(checkpoint); - Assert.True((bool)checkpoint.State!["stable"]); + Assert.True(Assert.IsType(checkpoint.State!["stable"])); + Assert.True(checkpoint.State.ContainsKey("nullable")); + Assert.Null(checkpoint.State["nullable"]); + var turnStart = sink.TurnEvents.Single(turnEvent => turnEvent.Type == TurnEventType.TurnStart); + var turnInputs = Assert.IsAssignableFrom>(turnStart.Payload["inputs"]); + Assert.True(turnInputs.ContainsKey("nullable")); + Assert.Null(turnInputs["nullable"]); Assert.Equal( ["session", "turn", "turn", "turn", "session", "turn", "session", "summary"], JournalKinds(journalPath)); @@ -67,14 +78,20 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() try { var sink = new CollectingEventSink(); + var permissionResolver = new CapturingPermissionResolver(); var runner = new ReferenceTurnRunner( sink, new JsonlEventJournalWriter(Path.Combine(directory.FullName, "trace.jsonl")), new InMemoryCheckpointStore(), - new AllowAllPermissionResolver(), + permissionResolver, new FunctionHostToolExecutor(new Dictionary { - ["add"] = (args, _) => Task.FromResult(Convert.ToInt32(args["a"]) + Convert.ToInt32(args["b"])) + ["add"] = (args, _) => + { + Assert.True(args.ContainsKey("nullable")); + Assert.Null(args["nullable"]); + return Task.FromResult(Convert.ToInt32(args["a"]) + Convert.ToInt32(args["b"])); + } }), request => { @@ -89,7 +106,7 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() RequestId = "exec-1", ToolCallId = "call-1", ToolName = "add", - Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } + Arguments = new Dictionary { ["a"] = 2, ["b"] = 3, ["nullable"] = null } } ] }); @@ -97,7 +114,7 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() return Task.FromResult(new TurnModelResponse { - Output = new Dictionary { ["toolResult"] = request.ToolResults[0].Result } + Output = new Dictionary { ["toolResult"] = request.ToolResults![0].Result } }); }, FixedClock(), @@ -106,7 +123,13 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() var result = await runner.RunAsync(new RunTurnRequest { SessionId = "session-1", TurnId = "turn-1" }); Assert.Equal(5, Assert.IsType>(result.Output)["toolResult"]); - Assert.True(result.ToolResults[0].Success); + Assert.True(result.ToolResults![0].Success); + Assert.NotNull(permissionResolver.Request); + Assert.NotNull(permissionResolver.Request.Details); + var permissionArguments = + Assert.IsAssignableFrom>(permissionResolver.Request.Details["arguments"]); + Assert.True(permissionArguments.ContainsKey("nullable")); + Assert.Null(permissionArguments["nullable"]); Assert.Equal( [ TurnEventType.TurnStart, @@ -158,7 +181,7 @@ public async Task ReferenceTurnRunner_DeniedPermissionSkipsExecution() return Task.FromResult(new TurnModelResponse { - Output = new Dictionary { ["denied"] = request.ToolResults[0].ErrorKind } + Output = new Dictionary { ["denied"] = request.ToolResults![0].ErrorKind } }); }, FixedClock(), @@ -168,6 +191,10 @@ public async Task ReferenceTurnRunner_DeniedPermissionSkipsExecution() Assert.Equal("permission_denied", Assert.IsType>(result.Output)["denied"]); Assert.DoesNotContain(sink.TurnEvents, turnEvent => turnEvent.Type == TurnEventType.ToolExecutionStart); + Assert.Contains( + sink.TurnEvents, + turnEvent => turnEvent.Type == TurnEventType.ToolResult + && turnEvent.Payload["errorKind"] as string == "permission_denied"); } finally { @@ -200,7 +227,7 @@ public async Task ReferenceTurnRunner_HostToolFailureIsReplayable() }); } - return Task.FromResult(new TurnModelResponse { Output = request.ToolResults[0].Save() }); + return Task.FromResult(new TurnModelResponse { Output = request.ToolResults![0].Save() }); }, FixedClock(), FixedIds()); @@ -277,7 +304,7 @@ await runner.RunAsync(new RunTurnRequest { SessionId = vectors.SessionId, TurnId = vectors.TurnId, - Inputs = scenario.Inputs ?? new Dictionary(), + Inputs = scenario.Inputs ?? new Dictionary(), Options = new TurnOptions { MaxIterations = scenario.MaxIterations } }); @@ -292,6 +319,24 @@ await runner.RunAsync(new RunTurnRequest private static Func FixedClock() => () => "2026-06-28T00:00:00Z"; + private sealed class CapturingPermissionResolver : IPermissionResolver + { + public PermissionRequest? Request { get; private set; } + + public Task RequestAsync(PermissionRequest request) + { + Request = request; + return Task.FromResult(new PermissionDecision + { + RequestId = request.RequestId, + ToolCallId = request.ToolCallId, + Permission = request.Permission, + Approved = true, + Reason = "captured" + }); + } + } + private static Func FixedIds() { var index = 0; @@ -314,8 +359,8 @@ private static Func> ModelForScenario( { return Task.FromResult(new TurnModelResponse { - Output = new Dictionary { ["text"] = $"hello {request.Inputs["name"]}" }, - CheckpointState = new Dictionary { ["stable"] = true } + Output = new Dictionary { ["text"] = $"hello {request.Inputs!["name"]}" }, + CheckpointState = new Dictionary { ["stable"] = true } }); } @@ -330,7 +375,7 @@ private static Func> ModelForScenario( RequestId = "exec-1", ToolCallId = "call-1", ToolName = name == "tool_failure" ? "fail" : "add", - Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } + Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } } ] }); @@ -340,8 +385,8 @@ private static Func> ModelForScenario( { Output = new Dictionary { - ["toolResult"] = request.ToolResults[0].Result, - ["errorKind"] = request.ToolResults[0].ErrorKind + ["toolResult"] = request.ToolResults![0].Result, + ["errorKind"] = request.ToolResults![0].ErrorKind } }); }; @@ -428,7 +473,7 @@ public static ReplayVectors Load() private sealed record ReplayScenario( string Name, - Dictionary? Inputs, + Dictionary? Inputs, int? MaxIterations, string[] Expected) { @@ -442,9 +487,9 @@ public static ReplayScenario Load(JsonElement element) } } - private static Dictionary ToDictionary(JsonElement element) + private static Dictionary ToDictionary(JsonElement element) { - return element.EnumerateObject().ToDictionary(property => property.Name, property => ToObject(property.Value)!); + return element.EnumerateObject().ToDictionary(property => property.Name, property => ToObject(property.Value)); } private static object? ToObject(JsonElement element) diff --git a/runtime/csharp/Prompty.Core/Data/model_capabilities.json b/runtime/csharp/Prompty.Core/Data/model_capabilities.json new file mode 100644 index 000000000..7c7f6db19 --- /dev/null +++ b/runtime/csharp/Prompty.Core/Data/model_capabilities.json @@ -0,0 +1,55 @@ +{ + "description": "Cross-runtime fallback capability data for provider model discovery. Some provider /models endpoints (Anthropic, Foundry) return capability fields directly; others (OpenAI) return only ids. To keep discovery results consistent across providers AND across runtimes, every Prompty runtime embeds THIS file and applies one shared rule: provider-supplied fields always win; entries here only fill fields the provider left empty (fill-only-missing). Model ids are matched by longest prefix within a provider's list. This dataset is deliberately NOT emitted from TypeSpec: it is volatile provider data (context windows, modalities, new model families) refreshed as a snapshot, whereas TypeSpec owns the structural ModelInfo contract. Fields use the canonical camelCase ModelInfo names. A missing 'contextWindow' means unknown; a present empty modality list (e.g. []) is intentional (e.g. embeddings produce no textual/image output modality).", + "match": "longest_prefix", + "providers": { + "openai": [ + { + "prefix": "gpt-4o-mini", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4o", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4-turbo", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4", + "contextWindow": 8192, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-3.5-turbo", + "contextWindow": 16385, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "text-embedding-3-small", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "text-embedding-3-large", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "dall-e-3", + "inputModalities": ["text"], + "outputModalities": ["image"] + } + ] + } +} diff --git a/runtime/csharp/Prompty.Core/FrontmatterParser.cs b/runtime/csharp/Prompty.Core/FrontmatterParser.cs index 270e8687f..c07e586d2 100644 --- a/runtime/csharp/Prompty.Core/FrontmatterParser.cs +++ b/runtime/csharp/Prompty.Core/FrontmatterParser.cs @@ -1,5 +1,4 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Text.RegularExpressions; using YamlDotNet.Serialization; namespace Prompty.Core; @@ -8,18 +7,13 @@ namespace Prompty.Core; /// Splits .prompty file content into YAML frontmatter and markdown body. /// Frontmatter is delimited by --- or +++ markers. /// -public static partial class FrontmatterParser +public static class FrontmatterParser { - // Matches --- or +++ delimited frontmatter followed by body content. - // Group 1: frontmatter YAML, Group 2: markdown body. - [GeneratedRegex(@"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", RegexOptions.Singleline)] - private static partial Regex FrontmatterRegex(); - /// /// Parse .prompty file content into a dictionary. /// If frontmatter markers are present, splits frontmatter (YAML) from body (markdown). /// The body is stored under the "instructions" key. - /// If no frontmatter markers, treats entire content as YAML. + /// If no frontmatter markers are present, treats the entire content as instructions. /// /// Raw .prompty file content. /// Dictionary with parsed frontmatter fields and optional "instructions" key. @@ -27,24 +21,41 @@ public static partial class FrontmatterParser { ArgumentNullException.ThrowIfNull(contents); - // Check for frontmatter markers var trimmed = contents.TrimStart(); - if (trimmed.StartsWith("---") || trimmed.StartsWith("+++")) + var openingLineEnd = trimmed.IndexOf('\n'); + var openingLine = (openingLineEnd >= 0 ? trimmed[..openingLineEnd] : trimmed).Trim(); + if (openingLine is not "---" and not "+++") { - var match = FrontmatterRegex().Match(contents); - if (match.Success) - { - var frontmatter = match.Groups[1].Value; - var body = match.Groups[2].Value; + return new Dictionary { ["instructions"] = contents }; + } + + if (openingLineEnd < 0) + { + return new Dictionary { ["instructions"] = string.Empty }; + } - var data = DeserializeYaml(frontmatter); - data["instructions"] = body; + var frontmatterStart = openingLineEnd + 1; + var lineStart = frontmatterStart; + while (lineStart <= trimmed.Length) + { + var lineEnd = trimmed.IndexOf('\n', lineStart); + var line = (lineEnd >= 0 ? trimmed[lineStart..lineEnd] : trimmed[lineStart..]).Trim(); + if (line is "---" or "+++") + { + var data = DeserializeYaml(trimmed[frontmatterStart..lineStart]); + data["instructions"] = lineEnd >= 0 ? trimmed[(lineEnd + 1)..] : string.Empty; return data; } + + if (lineEnd < 0) + { + break; + } + + lineStart = lineEnd + 1; } - // No frontmatter markers — treat entire content as YAML - return DeserializeYaml(contents); + throw new InvalidOperationException("Opening frontmatter delimiter does not have a closing delimiter."); } private static Dictionary DeserializeYaml(string yaml) diff --git a/runtime/csharp/Prompty.Core/HarnessAdapters.cs b/runtime/csharp/Prompty.Core/HarnessAdapters.cs index ef0348751..92a3aab4e 100644 --- a/runtime/csharp/Prompty.Core/HarnessAdapters.cs +++ b/runtime/csharp/Prompty.Core/HarnessAdapters.cs @@ -234,7 +234,7 @@ public Task RequestAsync(PermissionRequest request) } } -public delegate Task HostToolHandler(IDictionary arguments, HostToolRequest request); +public delegate Task HostToolHandler(IDictionary arguments, HostToolRequest request); /// /// Dispatches host tool requests to registered local functions. @@ -267,7 +267,7 @@ public async Task ExecuteAsync(HostToolRequest request) try { - var result = await handler(request.Arguments ?? new Dictionary(), request); + var result = await handler(request.Arguments ?? new Dictionary(), request); return new HostToolResult { RequestId = request.RequestId, diff --git a/runtime/csharp/Prompty.Core/Jinja2Renderer.cs b/runtime/csharp/Prompty.Core/Jinja2Renderer.cs index 1caabb2f8..0bd0298f8 100644 --- a/runtime/csharp/Prompty.Core/Jinja2Renderer.cs +++ b/runtime/csharp/Prompty.Core/Jinja2Renderer.cs @@ -1,13 +1,20 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Text.RegularExpressions; + namespace Prompty.Core; /// /// Renders Prompty templates using the Jinja2 template engine (via Jinja2.NET). /// Registered under key "jinja2". /// -public class Jinja2Renderer : IRenderer +public partial class Jinja2Renderer : IRenderer { + private const string ProtectedWhitespace = "__PROMPTY_JINJA_CONTROL_WHITESPACE__"; + + [GeneratedRegex(@"[ \t]+(?=\{%\s*endfor\s*%\})")] + private static partial Regex LoopBoundaryWhitespaceRegex(); + /// /// The most recently generated nonces from rendering, for thread expansion. /// @@ -19,7 +26,10 @@ public Task RenderAsync(Prompty agent, string template, Dictionary string.Concat(Enumerable.Repeat(ProtectedWhitespace, match.Length))); + var jinja = new Jinja2.NET.Template(protectedTemplate); // Convert to IDictionary for Jinja2.NET (no nulls) var context = new Dictionary(); @@ -29,7 +39,7 @@ public Task RenderAsync(Prompty agent, string template, DictionaryRuntime-only configuration for a public turn backed by the canonical . +public sealed class TurnEnginePipelineOptions +{ + public Dictionary>>? Tools { get; init; } + + public bool Raw { get; init; } + + public EventCallback? OnEvent { get; init; } + + public int? ContextBudget { get; init; } + + public Guardrails? Guardrails { get; init; } + + public Steering? Steering { get; init; } + + public CompactionStrategy? Compaction { get; init; } + + public IEngineDurabilityPort? Durability { get; init; } + + public IEnginePermissionPort? Permission { get; init; } + + public IEnginePostCommitPort? PostCommit { get; init; } +} + +/// Adapts the public pipeline registry and hooks to the canonical turn engine ports. +internal static class LiveTurn +{ + private static long _turnId; + + public static Task RunAsync( + Prompty agent, + Dictionary? inputs, + Dictionary>>? tools, + int maxIterations, + bool raw, + EventCallback? onEvent, + CancellationToken cancellationToken, + int? contextBudget, + Guardrails? guardrails, + Steering? steering, + bool parallelToolCalls, + int maxLlmRetries, + CompactionStrategy? compaction) + { + if (parallelToolCalls) + { + throw new ArgumentException( + "parallelToolCalls=true is not supported by the canonical engine; " + + "tool effects execute sequentially for deterministic durable ordering.", + nameof(parallelToolCalls)); + } + + var id = Interlocked.Increment(ref _turnId); + var agentMode = (agent.Tools?.Count ?? 0) > 0 || (tools?.Count ?? 0) > 0 + || guardrails is not null || steering is not null || contextBudget is not null; + var request = new TurnEngineRequest($"pipeline-session-{id}", $"pipeline-turn-{id}", []) + { + Inputs = inputs ?? new Dictionary(), + MaxIterations = Math.Max(maxIterations, 1), + MaxModelAttempts = agentMode ? Math.Max(maxLlmRetries, 1) : 1, + }; + var options = new TurnEnginePipelineOptions + { + Tools = tools, + Raw = raw, + OnEvent = onEvent, + ContextBudget = contextBudget, + Guardrails = guardrails, + Steering = steering, + Compaction = compaction, + }; + return RunAsync(agent, request, options, cancellationToken); + } + + public static async Task RunAsync( + Prompty agent, + TurnEngineRequest request, + TurnEnginePipelineOptions? options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(agent); + ArgumentNullException.ThrowIfNull(request); + options ??= new TurnEnginePipelineOptions(); + var inputs = NormalizeInputs(request.Inputs); + + var provider = agent.Model?.Provider ?? "openai"; + var executor = InvokerRegistry.GetExecutor(provider); + var processor = InvokerRegistry.GetProcessor(provider); + var agentMode = (agent.Tools?.Count ?? 0) > 0 || (options.Tools?.Count ?? 0) > 0; + var failures = new LiveFailureState(); + var authorization = new LivePermissionPort(options.Permission, options.Guardrails); + var durability = new LiveDurabilityPort( + options.Durability ?? new NoopDurabilityPort(), + options.OnEvent, + agent, + agentMode, + request.MaxIterations); + var engine = new TurnEngine(new TurnEngineEffects + { + Model = new LiveModelPort(agent, executor, processor, options.Raw, agentMode, failures), + Tools = new LiveToolPort( + agent, + options.Tools, + inputs, + authorization, + options.OnEvent), + Clock = new LiveClock(), + Ids = new LiveIds(), + Policy = new LivePolicyPort( + agent, + inputs, + options.ContextBudget, + options.Guardrails, + options.Steering, + options.Compaction, + failures), + Retry = new LiveRetryPort(options.OnEvent), + Conversation = new LiveConversationPort(executor), + Permission = authorization, + Durability = durability, + PostCommit = options.PostCommit ?? new NoopPostCommitPort(), + Stream = new LiveStreamPort(options.OnEvent), + }); + + TurnEngineResult result; + try + { + result = await engine.RunAsync(request, cancellationToken).ConfigureAwait(false); + } + catch + { + durability.EmitUncommittedError(); + throw; + } + + return result.Commit.Status switch + { + EngineTurnStatus.Success => result.Commit.Output!, + EngineTurnStatus.Cancelled => throw new OperationCanceledException("Operation cancelled", cancellationToken), + EngineTurnStatus.Failed or EngineTurnStatus.ReconciliationRequired => + throw MapFailure(result, request.MaxModelAttempts, agentMode, failures), + _ => throw new InvalidOperationException($"Unsupported engine status '{result.Commit.Status}'."), + }; + } + + private static Dictionary? NormalizeInputs(object? inputs) + { + return inputs switch + { + null => null, + Dictionary values => values, + IReadOnlyDictionary values => values.ToDictionary(), + JsonElement { ValueKind: JsonValueKind.Object } value => + value.Deserialize>(), + _ => throw new ArgumentException( + "Turn engine inputs must be a string-keyed dictionary or JSON object.", + nameof(inputs)), + }; + } + + private static Exception MapFailure( + TurnEngineResult result, + int maxModelAttempts, + bool agentMode, + LiveFailureState failures) + { + var output = result.Commit.Output as IReadOnlyDictionary; + var kind = output?.GetValueOrDefault("errorKind")?.ToString() ?? "engine_error"; + var message = output?.GetValueOrDefault("message")?.ToString() ?? "TurnEngine failed."; + return kind switch + { + "input_guardrail_denied" or "output_guardrail_denied" => new GuardrailError(failures.GuardrailReason ?? message), + "model_error" when !agentMode && failures.InvokerError is not null => failures.InvokerError, + "model_error" => new ExecuteError( + $"LLM call failed after {maxModelAttempts} retries: {message}", + [.. result.Commit.Messages]), + "max_iterations" => new InvalidOperationException( + $"Agent loop exceeded maximum iterations ({result.Commit.Iterations})."), + "prepare_error" when failures.InvokerError is not null => failures.InvokerError, + _ => failures.InvokerError ?? new InvalidOperationException(message), + }; + } + + private static object? MetadataValue(IDictionary? metadata, string key) + => metadata is not null && metadata.TryGetValue(key, out var value) ? value : null; + + private sealed class LiveFailureState + { + public Exception? InvokerError { get; set; } + + public string? GuardrailReason { get; set; } + + public bool SkipOutputGuardrail { get; set; } + } + + private sealed class LiveModelPort( + Prompty agent, + IExecutor executor, + IProcessor processor, + bool raw, + bool agentMode, + LiveFailureState failures) : IEngineModelPort + { + public async Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + cancellationToken.ThrowIfCancellationRequested(); + object rawResponse; + try + { + rawResponse = await executor.ExecuteAsync(agent, [.. request.Context.Messages]).ConfigureAwait(false); + if (rawResponse is PromptyStream rawStream) + { + await foreach (var chunk in rawStream.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + if (chunk is string text && text.Length > 0) + { + await stream.EmitAsync(new ModelStreamChunk.Text(text)).ConfigureAwait(false); + } + } + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception error) + { + failures.InvokerError = error; + throw new PortError(error.Message); + } + + object processed; + try + { + failures.SkipOutputGuardrail = raw && !agentMode; + processed = raw && !agentMode + ? rawResponse + : await processor.ProcessAsync(agent, rawResponse).ConfigureAwait(false); + } + catch (Exception error) + { + failures.InvokerError = error; + throw new PortError(error.Message); + } + + if (processed is ToolCallResult toolResult && toolResult.ToolCalls.Count > 0) + { + return new ModelInvocationResponse + { + AssistantMessages = [], + ToolRequests = toolResult.ToolCalls.Select(call => new ModelToolRequest + { + Id = call.Id, + Name = call.Name, + Arguments = ParseArgumentsValue(call.Arguments), + Metadata = new Dictionary { ["argumentsText"] = call.Arguments }, + }).ToList(), + Metadata = new Dictionary + { + ["rawResponse"] = rawResponse, + ["textContent"] = toolResult.Content ?? string.Empty, + }, + }; + } + + return new ModelInvocationResponse + { + Output = processed, + AssistantMessages = [], + ToolRequests = [], + NextContextState = new InvocationContextState + { + Portability = InvocationContextPortability.Portable, + DelegatedState = [], + }, + Metadata = new Dictionary { ["rawResponse"] = rawResponse }, + }; + } + + private static object ParseArgumentsValue(string arguments) + { + try + { + return JsonSerializer.Deserialize(arguments) ?? arguments; + } + catch (JsonException) + { + return arguments; + } + } + } + + private sealed class LiveConversationPort(IExecutor executor) : IEngineConversationPort + { + public IList FormatToolExchange( + ModelInvocationResponse response, + IReadOnlyList results) + { + var requests = response.ToolRequests ?? []; + var calls = requests.Select(request => new ToolCall + { + Id = request.Id, + Name = request.Name, + Arguments = MetadataValue(request.Metadata, "argumentsText")?.ToString() + ?? request.ModelArgumentsText(), + }).ToList(); + var orderedResults = requests.Select(request => + results.First(result => result.RequestId == request.Id).ModelText()).ToList(); + var rawResponse = MetadataValue(response.Metadata, "rawResponse") + ?? throw PortError.Configuration("provider response metadata is missing rawResponse"); + var content = MetadataValue(response.Metadata, "textContent")?.ToString(); + try + { + return executor.FormatToolMessages(rawResponse, calls, orderedResults, content); + } + catch (Exception error) + { + throw PortError.Configuration(error.Message); + } + } + } + + private sealed class LiveToolPort( + Prompty agent, + Dictionary>>? tools, + Dictionary? inputs, + LivePermissionPort authorization, + EventCallback? onEvent) : IEngineToolPort + { + public async Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var call = new ToolCall + { + Id = request.Id, + Name = request.Name, + Arguments = MetadataValue(request.Metadata, "argumentsText")?.ToString() + ?? request.ModelArgumentsText(), + }; + if (authorization.TakeRewrite(request.Id) is { } rewritten) + { + call.Arguments = JsonSerializer.Serialize(rewritten); + } + + string output; + try + { + output = await ToolDispatch.DispatchAsync(agent, call, tools, inputs).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception error) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.Error, new Dictionary + { + ["tool"] = request.Name, + ["error"] = error.Message, + }); + output = $"Error: Tool '{request.Name}' failed: {error.Message}"; + } + var failed = output.StartsWith("Error:", StringComparison.Ordinal); + return new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = failed ? ModelToolOutcome.Failed : ModelToolOutcome.Success, + Output = output, + ErrorKind = failed ? "tool_error" : null, + }; + } + } + + private sealed class LivePermissionPort( + IEnginePermissionPort? inner, + Guardrails? guardrails) : IEnginePermissionPort + { + private readonly Dictionary> _rewrites = []; + + public async Task AuthorizeAsync( + ModelToolRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (inner is not null) + { + var decision = await inner.AuthorizeAsync(request, cancellationToken).ConfigureAwait(false); + if (!decision.Approved) + { + return decision; + } + } + if (guardrails is null) + { + return new EnginePermissionDecision { Approved = true }; + } + + var arguments = ToolDispatch.ParseArguments(request.ModelArgumentsText()); + var result = guardrails.CheckTool(request.Name, arguments); + if (!result.Allowed) + { + return new EnginePermissionDecision + { + Approved = false, + Reason = $"Error: Tool guardrail denied: {result.Reason ?? "Tool denied"}", + }; + } + if (result.Rewrite is Dictionary rewrite) + { + _rewrites[request.Id] = rewrite; + } + return new EnginePermissionDecision { Approved = true }; + } + + public Dictionary? TakeRewrite(string requestId) + => _rewrites.Remove(requestId, out var rewrite) ? rewrite : null; + } + + private sealed class LivePolicyPort( + Prompty agent, + Dictionary? inputs, + int? contextBudget, + Guardrails? guardrails, + Steering? steering, + CompactionStrategy? compaction, + LiveFailureState failures) : IEngineHostPolicyPort + { + private bool _prepared; + + public async Task BeforeModelAsync( + HostPolicyRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var messages = request.Messages.ToList(); + var stablePrefix = Math.Min(request.StablePrefixMessages, messages.Count); + if (!_prepared) + { + try + { + messages = await Pipeline.PrepareAsync(agent, inputs).ConfigureAwait(false); + } + catch (Exception error) + { + failures.InvokerError = error; + throw new HostPolicyException("prepare_error", error.Message); + } + stablePrefix = messages.Count; + _prepared = true; + } + + var steeringMessages = steering?.Drain() ?? []; + messages.AddRange(steeringMessages); + var trimmed = 0; + if (contextBudget is not null) + { + var before = messages.ToList(); + var (droppedCount, droppedMessages) = ContextWindow.TrimToContextWindow(messages, contextBudget.Value); + trimmed = droppedCount; + if (droppedCount > 0 && compaction is not null) + { + await Pipeline.ApplyCompactionAsync(compaction, droppedMessages, messages, onEvent: null).ConfigureAwait(false); + } + stablePrefix = Math.Min(stablePrefix, CommonPrefixLength(before, messages)); + } + + if (guardrails is not null) + { + var check = guardrails.CheckInput(messages); + if (!check.Allowed) + { + failures.GuardrailReason = check.Reason; + throw new HostPolicyException("input_guardrail_denied", check.Reason ?? "Input guardrail denied"); + } + if (check.Rewrite is List rewritten) + { + messages = rewritten; + stablePrefix = Math.Min(stablePrefix, messages.Count); + } + } + + return new HostPolicyResult + { + Messages = messages, + StablePrefixMessages = stablePrefix, + Metadata = new Dictionary + { + ["steeringCount"] = steeringMessages.Count, + ["trimmedCount"] = trimmed, + ["notifyMessagesUpdated"] = steeringMessages.Count > 0 || trimmed > 0, + }, + }; + } + + public Task BeforeCommitAsync( + FinalOutputPolicyRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var output = request.Output; + if (guardrails is not null && !failures.SkipOutputGuardrail) + { + var message = output switch + { + Message existing => existing, + string text => Message.Assistant(text), + _ => Message.Assistant(output?.ToString() ?? string.Empty), + }; + var check = guardrails.CheckOutput(message); + if (!check.Allowed) + { + failures.GuardrailReason = check.Reason; + throw new HostPolicyException("output_guardrail_denied", check.Reason ?? "Output guardrail denied"); + } + if (check.Rewrite is not null) + { + output = check.Rewrite; + } + } + return Task.FromResult(new FinalOutputPolicyResult { Output = output }); + } + + private static int CommonPrefixLength(IList left, IList right) + { + var length = Math.Min(left.Count, right.Count); + var common = 0; + while (common < length && left[common].ToJson(indent: false) == right[common].ToJson(indent: false)) + { + common++; + } + return common; + } + } + + private sealed class LiveRetryPort(EventCallback? onEvent) : IEngineRetryPolicyPort + { + public async Task BackoffAsync(RetryPolicyRequest request, CancellationToken cancellationToken) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.Status, new Dictionary + { + ["message"] = $"LLM call failed, retrying (attempt {request.NextAttempt}/{request.MaxAttempts})...", + }); + AgentEvents.EmitEvent(onEvent, AgentEventType.Retry, new Dictionary + { + ["operation"] = "llm", + ["attempt"] = request.NextAttempt, + ["maxAttempts"] = request.MaxAttempts, + }); + var delay = Math.Min(Math.Pow(2, request.FailedAttempts) + Random.Shared.NextDouble(), 60); + await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken) + .ConfigureAwait(false); + } + } + + private sealed class LiveStreamPort(EventCallback? onEvent) : IEngineModelStreamPort + { + public Task EmitAsync(ModelStreamChunk chunk) + { + if (chunk is ModelStreamChunk.Text text) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.Token, new Dictionary { ["token"] = text.Value }); + } + else if (chunk is ModelStreamChunk.Thinking thinking) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.Thinking, new Dictionary { ["thinking"] = thinking.Value }); + } + return Task.CompletedTask; + } + } + + private sealed class LiveDurabilityPort( + IEngineDurabilityPort inner, + EventCallback? onEvent, + Prompty agent, + bool agentMode, + int maxIterations) : IEngineDurabilityPort + { + private IList _messages = []; + private int _iterations; + private bool _terminal; + + public async Task AppendAsync(EngineEvent @event) + { + await inner.AppendAsync(@event).ConfigureAwait(false); + Project(@event); + } + + public async Task AppendWithCheckpointAsync( + IReadOnlyList events, + EngineCheckpoint checkpoint) + { + await inner.AppendWithCheckpointAsync(events, checkpoint).ConfigureAwait(false); + _messages = checkpoint.Messages; + _iterations = checkpoint.CompletedModelIterations; + foreach (var @event in events) + { + Project(@event); + } + } + + public void EmitUncommittedError() => EmitTerminal("error", null); + + private void Project(EngineEvent @event) + { + var payload = @event.Payload as IReadOnlyDictionary; + switch (@event.Kind) + { + case EngineEventKind.TurnStarted: + AgentEvents.EmitEvent(onEvent, AgentEventType.TurnStart, new Dictionary + { + ["agent"] = agent.Name, + ["maxIterations"] = maxIterations, + }); + break; + case EngineEventKind.ModelInvocationStarted: + AgentEvents.EmitEvent(onEvent, AgentEventType.LlmStart, new Dictionary + { + ["provider"] = agent.Model?.Provider, + ["modelId"] = agent.Model?.Id, + ["messageCount"] = payload?.GetValueOrDefault("messageCount"), + ["attempt"] = payload?.GetValueOrDefault("attempt"), + ["iteration"] = @event.Iteration, + }); + break; + case EngineEventKind.ModelInvocationCompleted: + case EngineEventKind.ModelInvocationReconciled: + AgentEvents.EmitEvent(onEvent, AgentEventType.LlmComplete, new Dictionary + { + ["iteration"] = @event.Iteration, + }); + break; + case EngineEventKind.PermissionRequested: + AgentEvents.EmitEvent(onEvent, AgentEventType.PermissionRequested, new Dictionary + { + ["iteration"] = @event.Iteration, + ["request"] = payload?.GetValueOrDefault("toolRequest"), + }); + break; + case EngineEventKind.PermissionResolved: + AgentEvents.EmitEvent(onEvent, AgentEventType.PermissionCompleted, new Dictionary + { + ["iteration"] = @event.Iteration, + ["decision"] = payload?.GetValueOrDefault("decision"), + }); + break; + case EngineEventKind.ToolExecutionStarted: + if (payload?.GetValueOrDefault("toolRequest") is ModelToolRequest request) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallStart, new Dictionary + { + ["tool"] = request.Name, + ["arguments"] = request.ModelArgumentsText(), + }); + } + break; + case EngineEventKind.ToolExecutionCompleted: + if (payload?.GetValueOrDefault("toolResult") is ModelToolResult result) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, new Dictionary + { + ["tool"] = result.Name, + ["result"] = result.ModelText(), + }); + AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, new Dictionary + { + ["name"] = result.Name, + ["success"] = result.Outcome == ModelToolOutcome.Success, + ["result"] = result.ModelText(), + ["errorKind"] = result.ErrorKind, + }); + } + break; + case EngineEventKind.ConversationUpdated: + AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, new Dictionary + { + ["messages"] = _messages, + }); + break; + case EngineEventKind.TurnCommitted: + AgentEvents.EmitEvent(onEvent, AgentEventType.Done, new Dictionary + { + ["iterations"] = _iterations, + }); + EmitTerminal("success", payload?.GetValueOrDefault("output")); + break; + case EngineEventKind.TurnCancelled: + AgentEvents.EmitEvent(onEvent, AgentEventType.Cancelled, new Dictionary()); + EmitTerminal("cancelled", null); + break; + case EngineEventKind.TurnFailed: + case EngineEventKind.TurnReconciliationRequired: + EmitTerminal("error", null); + break; + } + } + + private void EmitTerminal(string status, object? response) + { + if (_terminal) + { + return; + } + _terminal = true; + AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, new Dictionary + { + ["iterations"] = agentMode ? _iterations : 0, + ["status"] = status, + ["response"] = response, + }); + } + } + + private sealed class LiveClock : IEngineClock + { + public string Now() => DateTimeOffset.UtcNow.ToString("O"); + } + + private sealed class LiveIds : IEngineIdGenerator + { + private long _id; + + public string NextId(string kind) => $"{kind}-{Interlocked.Increment(ref _id)}"; + } +} + +internal static class LiveTurnModelExtensions +{ + public static string ModelArgumentsText(this ModelToolRequest request) => request.Arguments switch + { + null => string.Empty, + string text => text, + var value => JsonSerializer.Serialize(value), + }; +} diff --git a/runtime/csharp/Prompty.Core/ModelDiscovery.cs b/runtime/csharp/Prompty.Core/ModelDiscovery.cs new file mode 100644 index 000000000..6710bc79e --- /dev/null +++ b/runtime/csharp/Prompty.Core/ModelDiscovery.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Reflection; +using System.Text.Json; + +namespace Prompty.Core; + +/// +/// Enriches generated model-discovery results from the shared capability dataset. +/// +public static class ModelDiscovery +{ + private const string ResourceName = "Prompty.Core.Data.model_capabilities.json"; + private static readonly Lazy> CapabilityTable = + new(LoadCapabilityTable); + + /// + /// Fill missing capability fields without replacing values supplied by a provider. + /// + public static void Enrich(string provider, ModelInfo info) + { + if (!CapabilityTable.Value.TryGetValue(provider, out var entries)) + return; + + var entry = entries.FirstOrDefault(candidate => PrefixMatches(info.Id, candidate.Prefix)); + if (entry is null) + return; + + info.ContextWindow ??= entry.ContextWindow; + info.InputModalities ??= entry.InputModalities?.ToArray(); + info.OutputModalities ??= entry.OutputModalities?.ToArray(); + } + + /// + /// Convert a provider JSON object to a dictionary while preserving nested raw values. + /// + public static IDictionary PreserveRaw(JsonElement value) + { + if (value.ValueKind != JsonValueKind.Object) + throw new ArgumentException("Provider model payload must be a JSON object.", nameof(value)); + + return value.EnumerateObject() + .ToDictionary(property => property.Name, property => (object?)property.Value.Clone()); + } + + private static bool PrefixMatches(string id, string prefix) + { + if (!id.StartsWith(prefix, StringComparison.Ordinal)) + return false; + if (id.Length == prefix.Length) + return true; + + return !char.IsAsciiLetterOrDigit(id[prefix.Length]); + } + + private static IReadOnlyDictionary LoadCapabilityTable() + { + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(ResourceName) + ?? throw new InvalidOperationException($"Embedded capability dataset '{ResourceName}' was not found."); + using var document = JsonDocument.Parse(stream); + var providers = document.RootElement.GetProperty("providers"); + var result = new Dictionary(); + + foreach (var provider in providers.EnumerateObject()) + { + result[provider.Name] = provider.Value.EnumerateArray() + .Select(CapabilityEntry.FromJson) + .OrderByDescending(entry => entry.Prefix.Length) + .ToArray(); + } + + return result; + } + + private sealed record CapabilityEntry( + string Prefix, + int? ContextWindow, + IList? InputModalities, + IList? OutputModalities) + { + public static CapabilityEntry FromJson(JsonElement value) => + new( + value.GetProperty("prefix").GetString()!, + value.TryGetProperty("contextWindow", out var contextWindow) ? contextWindow.GetInt32() : null, + ReadModalities(value, "inputModalities"), + ReadModalities(value, "outputModalities")); + + private static IList? ReadModalities(JsonElement value, string propertyName) => + value.TryGetProperty(propertyName, out var modalities) + ? modalities.EnumerateArray().Select(item => item.GetString()!).ToArray() + : null; + } +} diff --git a/runtime/csharp/Prompty.Core/Pipeline.cs b/runtime/csharp/Prompty.Core/Pipeline.cs index a476ca3a7..c0bba3d2d 100644 --- a/runtime/csharp/Prompty.Core/Pipeline.cs +++ b/runtime/csharp/Prompty.Core/Pipeline.cs @@ -10,20 +10,6 @@ namespace Prompty.Core; /// public static class Pipeline { - private static void EmitFailedTurnEnd(EventCallback? onEvent, Exception exception, int iterations, object? response = null) - { - var payload = new Dictionary - { - ["iterations"] = iterations, - ["status"] = exception is OperationCanceledException ? "cancelled" : "error" - }; - if (response is not null) - { - payload["response"] = response; - } - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, payload); - } - // ----------------------------------------------------------------------- // Input Validation // ----------------------------------------------------------------------- @@ -229,10 +215,14 @@ public static async Task InvokeAsync( // ----------------------------------------------------------------------- /// - /// Conversational round-trip: Prepare → [Execute → check tool_calls → execute tools → loop] → Process. - /// If no tools are provided, performs a single Prepare → Execute → Process pass. - /// Calls executor and processor directly (not via RunAsync). + /// Conversational round-trip through the canonical durable turn engine. + /// If no tools are provided, the engine performs a single Prepare → Execute → Process pass. /// + /// + /// Cancellation is cooperative at engine phase boundaries. The current generated provider and + /// tool protocols do not expose a native , so an in-flight + /// non-streaming provider or tool call may finish before cancellation is observed. + /// public static async Task TurnAsync( Prompty agent, Dictionary? inputs = null, @@ -250,428 +240,47 @@ public static async Task TurnAsync( CompactionStrategy? compaction = null) { var label = turnNumber.HasValue ? $"turn {turnNumber.Value}" : "turn"; - return await Trace.TraceAsync($"prompty.turn", async (emit) => + return await Trace.TraceAsync("prompty.turn", async (emit) => { emit("signature", "prompty.turn"); - emit("inputs", new Dictionary { ["agent"] = agent.Name, ["label"] = label, ["maxIterations"] = maxIterations }); - var messages = await PrepareAsync(agent, inputs); - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnStart, - new Dictionary - { - ["agent"] = agent.Name, - ["inputs"] = inputs ?? new Dictionary(), - ["maxIterations"] = maxIterations - }); - - // Simple path: no tools on agent, no user tools, and no agent-loop features → single execute + process - var hasAgentTools = agent.Tools is not null && agent.Tools.Count > 0; - var hasUserTools = tools is not null && tools.Count > 0; - var hasAgentFeatures = guardrails is not null || steering is not null || contextBudget is not null; - if (!hasAgentTools && !hasUserTools && !hasAgentFeatures) + emit("inputs", new Dictionary { - AgentEvents.EmitEvent(onEvent, AgentEventType.LlmStart, - new Dictionary - { - ["provider"] = agent.Model?.Provider, - ["modelId"] = agent.Model?.Id, - ["messageCount"] = messages.Count, - ["attempt"] = 0 - }); - object response; - try - { - response = await ExecuteAsync(agent, messages); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, 0); - throw; - } - AgentEvents.EmitEvent(onEvent, AgentEventType.LlmComplete, new Dictionary()); - if (raw) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, - new Dictionary { ["iterations"] = 0, ["status"] = "success", ["response"] = response }); - return response; - } - object processed; - try - { - processed = await ProcessAsync(agent, response); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, 0, response); - throw; - } - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, - new Dictionary { ["iterations"] = 0, ["status"] = "success", ["response"] = processed }); - return processed; - } - - // Agent loop: execute → check tool_calls → dispatch tools → loop - var executor = InvokerRegistry.GetExecutor(agent.Model?.Provider ?? "openai"); - object? response2 = null; - int iteration = 0; - - while (true) - { - if (iteration >= maxIterations) - { - EmitFailedTurnEnd(onEvent, new InvalidOperationException("Agent loop exceeded maximum iterations."), iteration); - throw new InvalidOperationException( - $"Agent loop exceeded maximum iterations ({maxIterations})."); - } - - // Cancellation check at loop start - try - { - cancellationToken.ThrowIfCancellationRequested(); - } - catch (OperationCanceledException) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Cancelled, - new Dictionary { ["iteration"] = iteration, ["reason"] = "cancellation_requested" }); - EmitFailedTurnEnd(onEvent, new OperationCanceledException(), iteration); - throw; - } - - // Drain steering messages - if (steering is not null) - { - var steered = steering.Drain(); - if (steered.Count > 0) - { - messages.AddRange(steered); - AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, - new Dictionary { ["source"] = "steering", ["count"] = steered.Count }); - } - } - - // Context window trimming - if (contextBudget is not null) - { - var (droppedCount, droppedMessages) = ContextWindow.TrimToContextWindow(messages, contextBudget.Value); - if (droppedCount > 0) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, - new Dictionary { ["source"] = "context_trim", ["dropped"] = droppedCount }); - - if (compaction is not null) - { - await ApplyCompactionAsync(compaction, droppedMessages, messages, onEvent); - } - } - } - - // Input guardrail - if (guardrails is not null) - { - var inputCheck = guardrails.CheckInput(messages); - if (!inputCheck.Allowed) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Error, - new Dictionary { ["guardrail"] = "input", ["reason"] = inputCheck.Reason }); - EmitFailedTurnEnd(onEvent, new GuardrailError(inputCheck.Reason ?? "Input guardrail denied"), iteration); - throw new GuardrailError(inputCheck.Reason ?? "Input guardrail denied"); - } - if (inputCheck.Rewrite is List rewrittenMessages) - { - messages = rewrittenMessages; - } - } - - // Cancellation check before LLM call - try - { - cancellationToken.ThrowIfCancellationRequested(); - } - catch (OperationCanceledException) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Cancelled, - new Dictionary { ["iteration"] = iteration, ["reason"] = "cancelled_before_llm" }); - EmitFailedTurnEnd(onEvent, new OperationCanceledException(), iteration); - throw; - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.Status, - new Dictionary { ["iteration"] = iteration, ["phase"] = "executing" }); - - AgentEvents.EmitEvent(onEvent, AgentEventType.LlmStart, - new Dictionary - { - ["provider"] = agent.Model?.Provider, - ["modelId"] = agent.Model?.Id, - ["messageCount"] = messages.Count, - ["attempt"] = 0, - ["iteration"] = iteration - }); - try - { - response2 = await InvokeWithRetryAsync(agent, messages, maxLlmRetries, onEvent, cancellationToken); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration); - throw; - } - AgentEvents.EmitEvent(onEvent, AgentEventType.LlmComplete, - new Dictionary { ["iteration"] = iteration }); - - // If response is a stream, consume it fully before processing. - if (response2 is PromptyStream stream) - { - try - { - await foreach (var chunk in stream) - { - if (chunk is string tokenText && tokenText.Length > 0) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Token, - new Dictionary { ["token"] = tokenText }); - } - } - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration, response2); - throw; - } - response2 = stream; - } - - object result; - try - { - result = raw ? response2! : await ProcessAsync(agent, response2!); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration, response2); - throw; - } + ["agent"] = agent.Name, + ["label"] = label, + ["maxIterations"] = maxIterations, + }); + return await LiveTurn.RunAsync( + agent, + inputs, + tools, + maxIterations, + raw, + onEvent, + cancellationToken, + contextBudget, + guardrails, + steering, + parallelToolCalls, + maxLlmRetries, + compaction).ConfigureAwait(false); + }).ConfigureAwait(false); - if (result is ToolCallResult toolResult && toolResult.ToolCalls.Count > 0) - { - // Dispatch tool calls (parallel or sequential) - var toolResults = new List(); - - if (parallelToolCalls && toolResult.ToolCalls.Count > 1) - { - // Parallel dispatch via Task.WhenAll - var tasks = new List>(); - for (int ti = 0; ti < toolResult.ToolCalls.Count; ti++) - { - var call = toolResult.ToolCalls[ti]; - var capturedIndex = ti; - - // Tool guardrail (with rewrite support) - if (guardrails is not null) - { - var args = ToolDispatch.ParseArguments(call.Arguments); - var toolCheck = guardrails.CheckTool(call.Name, args); - if (!toolCheck.Allowed) - { - var deniedMsg = $"Tool denied by guardrail: {toolCheck.Reason}"; - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, - new Dictionary { ["tool"] = call.Name, ["result"] = deniedMsg }); - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, - new Dictionary { ["name"] = call.Name, ["success"] = false, ["result"] = deniedMsg, ["errorKind"] = "guardrail_denied" }); - tasks.Add(Task.FromResult((capturedIndex, deniedMsg))); - continue; - } - if (toolCheck.Rewrite is Dictionary rewrittenArgs) - { - call.Arguments = System.Text.Json.JsonSerializer.Serialize(rewrittenArgs); - } - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallStart, - new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - - tasks.Add(Task.Run(async () => - { - var toolStarted = DateTimeOffset.UtcNow; - string toolResponse; - try - { - toolResponse = await Trace.TraceAsync("Prompty.Core.ToolDispatch.Execute", async (toolEmit) => - { - toolEmit("inputs", new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - return await ToolDispatch.DispatchAsync(agent, call, tools, inputs) ?? string.Empty; - }); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - toolResponse = $"Error: Tool '{call.Name}' failed: {ex.Message}"; - AgentEvents.EmitEvent(onEvent, AgentEventType.Error, - new Dictionary { ["tool"] = call.Name, ["error"] = ex.Message }); - } - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, - new Dictionary - { - ["name"] = call.Name, - ["success"] = !toolResponse.StartsWith("Error:", StringComparison.Ordinal), - ["result"] = toolResponse, - ["durationMs"] = (DateTimeOffset.UtcNow - toolStarted).TotalMilliseconds, - ["errorKind"] = toolResponse.StartsWith("Error:", StringComparison.Ordinal) ? "tool_error" : null - }); - return (capturedIndex, toolResponse); - })); - } - - (int Index, string Result)[] completed; - try - { - completed = await Task.WhenAll(tasks); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration); - throw; - } - // Maintain order - var ordered = new string[toolResult.ToolCalls.Count]; - foreach (var (index, res) in completed) - { - ordered[index] = res; - } - toolResults.AddRange(ordered); - - for (int ti = 0; ti < toolResult.ToolCalls.Count; ti++) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, - new Dictionary { ["tool"] = toolResult.ToolCalls[ti].Name, ["result"] = toolResults[ti] }); - } - } - else - { - // Sequential dispatch - foreach (var call in toolResult.ToolCalls) - { - // Tool guardrail (with rewrite support) - if (guardrails is not null) - { - var args = ToolDispatch.ParseArguments(call.Arguments); - var toolCheck = guardrails.CheckTool(call.Name, args); - if (!toolCheck.Allowed) - { - var deniedMsg = $"Tool denied by guardrail: {toolCheck.Reason}"; - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, - new Dictionary { ["tool"] = call.Name, ["result"] = deniedMsg }); - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, - new Dictionary { ["name"] = call.Name, ["success"] = false, ["result"] = deniedMsg, ["errorKind"] = "guardrail_denied" }); - toolResults.Add(deniedMsg); - continue; - } - if (toolCheck.Rewrite is Dictionary rewrittenArgs) - { - call.Arguments = System.Text.Json.JsonSerializer.Serialize(rewrittenArgs); - } - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallStart, - new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - - var toolStarted = DateTimeOffset.UtcNow; - string toolResponse; - try - { - toolResponse = await Trace.TraceAsync("Prompty.Core.ToolDispatch.Execute", async (toolEmit) => - { - toolEmit("inputs", new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - return await ToolDispatch.DispatchAsync(agent, call, tools, inputs); - }); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - toolResponse = $"Error: Tool '{call.Name}' failed: {ex.Message}"; - AgentEvents.EmitEvent(onEvent, AgentEventType.Error, - new Dictionary { ["tool"] = call.Name, ["error"] = ex.Message }); - } - catch (OperationCanceledException ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration); - throw; - } - toolResults.Add(toolResponse); - - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, - new Dictionary { ["tool"] = call.Name, ["result"] = toolResponse }); - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, - new Dictionary - { - ["name"] = call.Name, - ["success"] = !toolResponse.StartsWith("Error:", StringComparison.Ordinal), - ["result"] = toolResponse, - ["durationMs"] = (DateTimeOffset.UtcNow - toolStarted).TotalMilliseconds, - ["errorKind"] = toolResponse.StartsWith("Error:", StringComparison.Ordinal) ? "tool_error" : null - }); - } - } - - // Delegate message formatting to the executor (provider-specific) - List toolMessages; - try - { - toolMessages = executor.FormatToolMessages( - response2, toolResult.ToolCalls, toolResults, toolResult.Content); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration, response2); - throw; - } - messages.AddRange(toolMessages); - - AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, - new Dictionary { ["source"] = "tool_results", ["count"] = toolMessages.Count }); - - iteration++; - continue; - } - - // Output guardrail on final response - if (guardrails is not null) - { - var outputMsg = result switch - { - string resultText => new Message - { - Role = Role.Assistant, - Parts = [new TextPart { Value = resultText }] - }, - Message msg => msg, - _ => new Message - { - Role = Role.Assistant, - Parts = [new TextPart { Value = result?.ToString() ?? "" }] - }, - }; - var outputCheck = guardrails.CheckOutput(outputMsg); - if (!outputCheck.Allowed) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Error, - new Dictionary { ["guardrail"] = "output", ["reason"] = outputCheck.Reason }); - EmitFailedTurnEnd(onEvent, new GuardrailError(outputCheck.Reason ?? "Output guardrail denied"), iteration + 1, result); - throw new GuardrailError(outputCheck.Reason ?? "Output guardrail denied"); - } - if (outputCheck.Rewrite is not null) - { - result = outputCheck.Rewrite; - } - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.Done, - new Dictionary { ["iterations"] = iteration + 1 }); - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, - new Dictionary { ["iterations"] = iteration + 1, ["status"] = "success", ["response"] = result }); - - return result!; - } - }); } + /// + /// Executes or resumes a turn through the canonical engine using caller-owned durability and effect ports. + /// + /// + /// Runtime-local effect ports receive the native token. Adapters over generated protocols remain + /// boundary-cancellable until the schema/emitter supports language-native cancellation seams. + /// + public static Task TurnWithEngineRequestAsync( + Prompty agent, + TurnEngineRequest request, + TurnEnginePipelineOptions? options = null, + CancellationToken cancellationToken = default) + => LiveTurn.RunAsync(agent, request, options, cancellationToken); + /// /// Conversational round-trip with path-based loading. /// @@ -823,58 +432,6 @@ internal static void ReplaceSummaryMessage(List messages, string newSum } } - // ----------------------------------------------------------------------- - // LLM Retry Helper (§9.10) - // ----------------------------------------------------------------------- - - /// - /// Invoke ExecuteAsync with exponential backoff retry on transient failures. - /// - private static async Task InvokeWithRetryAsync( - Prompty agent, - List messages, - int maxRetries, - EventCallback? onEvent, - CancellationToken cancellationToken) - { - var attempts = 0; - while (true) - { - try - { - return await ExecuteAsync(agent, messages); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - attempts++; - if (attempts >= maxRetries) - { - throw new ExecuteError( - $"LLM call failed after {maxRetries} retries: {ex.Message}", - new List(messages)); - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.Status, - new Dictionary - { - ["message"] = $"LLM call failed, retrying (attempt {attempts + 1}/{maxRetries})..." - }); - AgentEvents.EmitEvent(onEvent, AgentEventType.Retry, - new Dictionary - { - ["operation"] = "llm", - ["attempt"] = attempts + 1, - ["maxAttempts"] = maxRetries, - ["reason"] = ex.Message - }); - - // Exponential backoff with jitter, capped at 60s - var backoff = Math.Min(Math.Pow(2, attempts) + Random.Shared.NextDouble(), 60); - await Task.Delay(TimeSpan.FromSeconds(backoff), cancellationToken); - } - } - } - // ----------------------------------------------------------------------- // Thread Expansion // ----------------------------------------------------------------------- @@ -923,7 +480,7 @@ internal static List ExpandThreadMarkers( { Role = msg.Role, Parts = [new TextPart { Value = before }], - Metadata = msg.Metadata is not null ? new Dictionary(msg.Metadata) : new Dictionary(), + Metadata = msg.Metadata is not null ? new Dictionary(msg.Metadata) : new Dictionary(), }); } @@ -937,7 +494,7 @@ internal static List ExpandThreadMarkers( { Role = msg.Role, Parts = [new TextPart { Value = after }], - Metadata = msg.Metadata is not null ? new Dictionary(msg.Metadata) : new Dictionary(), + Metadata = msg.Metadata is not null ? new Dictionary(msg.Metadata) : new Dictionary(), }); } } diff --git a/runtime/csharp/Prompty.Core/Prompty.Core.csproj b/runtime/csharp/Prompty.Core/Prompty.Core.csproj index 151ca0d14..a5ce22719 100644 --- a/runtime/csharp/Prompty.Core/Prompty.Core.csproj +++ b/runtime/csharp/Prompty.Core/Prompty.Core.csproj @@ -21,6 +21,7 @@ + diff --git a/runtime/csharp/Prompty.Core/PromptyChatParser.cs b/runtime/csharp/Prompty.Core/PromptyChatParser.cs index a8c5cf90e..8472f9902 100644 --- a/runtime/csharp/Prompty.Core/PromptyChatParser.cs +++ b/runtime/csharp/Prompty.Core/PromptyChatParser.cs @@ -13,10 +13,12 @@ namespace Prompty.Core; public partial class PromptyChatParser : IParser, IPreRenderable { /// - /// Regex matching role marker lines: "role:" or "role[attrs]:" + /// Regex matching canonical role marker lines with optional heading syntax and attributes. /// Captures role name and optional attributes. /// - [GeneratedRegex(@"^(system|user|assistant|developer|tool)(\[.*?\])?:\s*$", RegexOptions.Multiline)] + [GeneratedRegex( + @"^\s*#?\s*(system|user|assistant)(\[(?:\w+\s*=\s*""?[^""]*""?\s*,?\s*)+\])?\s*:\s*$", + RegexOptions.IgnoreCase | RegexOptions.Multiline)] private static partial Regex RoleMarkerRegex(); /// @@ -52,7 +54,7 @@ public partial class PromptyChatParser : IParser, IPreRenderable var sanitized = RoleMarkerRegex().Replace(template, match => { - var role = match.Groups[1].Value; + var role = match.Groups[1].Value.ToLowerInvariant(); var existingAttrs = match.Groups[2].Value; // e.g. "[key=val]" or "" if (string.IsNullOrEmpty(existingAttrs)) return $"{role}[nonce=\"{nonce}\"]:\n"; @@ -82,70 +84,91 @@ public Task> ParseAsync(Prompty agent, string rendered, Dictionary /// internal List Parse(string rendered) { - var messages = new List(); - var lines = rendered.Split('\n'); - string? currentRole = null; - var currentContent = new List(); - Dictionary? currentAttrs = null; - - foreach (var line in lines) + try { - var match = RoleMarkerRegex().Match(line); - if (match.Success) + var messages = new List(); + var lines = rendered.Split('\n'); + string? currentRole = null; + var currentContent = new List(); + Dictionary? currentAttrs = null; + + foreach (var line in lines) { - // Flush previous message - if (currentRole is not null) + var match = RoleMarkerRegex().Match(line); + if (match.Success) + { + if (currentRole is not null) + { + messages.Add(CreateMessage(currentRole, currentContent, currentAttrs, validateNonce: true)); + } + else if (currentContent.Count > 0) + { + var leadingText = string.Join("\n", currentContent).Trim('\r', '\n'); + if (!string.IsNullOrEmpty(leadingText)) + { + messages.Add(new Message + { + Role = Role.System, + Parts = [new TextPart { Value = leadingText }] + }); + } + } + + currentRole = match.Groups[1].Value; + currentAttrs = ParseAttributes(match.Groups[2].Value); + currentContent = []; + } + else { - messages.Add(CreateMessage(currentRole, currentContent, currentAttrs)); + currentContent.Add(line); } + } - currentRole = match.Groups[1].Value; - currentAttrs = ParseAttributes(match.Groups[2].Value); - currentContent = []; + // Flush last message + if (currentRole is not null) + { + messages.Add(CreateMessage(currentRole, currentContent, currentAttrs, validateNonce: true)); } - else + else if (currentContent.Count > 0) { - currentContent.Add(line); + // No role markers at all — treat as system message + var text = string.Join("\n", currentContent).Trim(); + if (!string.IsNullOrEmpty(text)) + { + messages.Add(new Message + { + Role = Role.System, + Parts = [new TextPart { Value = text }] + }); + } } - } - // Flush last message - if (currentRole is not null) - { - messages.Add(CreateMessage(currentRole, currentContent, currentAttrs)); + return messages; } - else if (currentContent.Count > 0) + finally { - // No role markers at all — treat as system message - var text = string.Join("\n", currentContent).Trim(); - if (!string.IsNullOrEmpty(text)) - { - messages.Add(new Message - { - Role = Role.System, - Parts = [new TextPart { Value = text }] - }); - } + _renderNonce.Value = null; } - - return messages; } // ----------------------------------------------------------------------- // Helpers // ----------------------------------------------------------------------- - private Message CreateMessage(string role, List contentLines, Dictionary? attrs) + private Message CreateMessage( + string role, + List contentLines, + Dictionary? attrs, + bool validateNonce) { - // Validate nonce if strict mode was used - if (_renderNonce.Value is not null && attrs is not null) + if (validateNonce && _renderNonce.Value is not null) { - if (!attrs.TryGetValue("nonce", out var foundNonce) || foundNonce != _renderNonce.Value) + if (attrs is null || !attrs.TryGetValue("nonce", out var foundNonce) || foundNonce != _renderNonce.Value) { throw new InvalidOperationException( $"Role marker injection detected: nonce mismatch for '{role}:' marker."); } - attrs.Remove("nonce"); // Don't pass nonce through to metadata + attrs.Remove("nonce"); } var text = string.Join("\n", contentLines); @@ -160,7 +183,7 @@ private Message CreateMessage(string role, List contentLines, Dictionary if (attrs is not null && attrs.Count > 0) { - message.Metadata ??= new Dictionary(); + message.Metadata ??= new Dictionary(); foreach (var kvp in attrs) message.Metadata[kvp.Key] = kvp.Value; } diff --git a/runtime/csharp/Prompty.Core/PromptyLoader.cs b/runtime/csharp/Prompty.Core/PromptyLoader.cs index d3e1891cf..b1c5754a2 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoader.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoader.cs @@ -65,16 +65,12 @@ private static Prompty Build(string contents, string fullPath, PromptyLoadOption // 1. Split frontmatter + body var data = FrontmatterParser.Parse(contents); - // 2. Load via typed model with ${env:}/${file:} resolution - var ctx = new LoadContext - { - PreProcess = ReferenceResolver.CreatePreProcess(fullPath, options?.AllowedFileRoots), - }; - - var agent = Prompty.Load(data, ctx); + // 2. Resolve the complete untyped tree before generated model loading. + data = ReferenceResolver.CreatePreProcess(fullPath, options?.AllowedFileRoots)(data); + var agent = Prompty.Load(data, new LoadContext()); // 3. Attach source path in metadata - agent.Metadata ??= new Dictionary(); + agent.Metadata ??= new Dictionary(); agent.Metadata["__source_path"] = fullPath; return agent; diff --git a/runtime/csharp/Prompty.Core/ReferenceResolver.cs b/runtime/csharp/Prompty.Core/ReferenceResolver.cs index 3e765f42d..3510efee7 100644 --- a/runtime/csharp/Prompty.Core/ReferenceResolver.cs +++ b/runtime/csharp/Prompty.Core/ReferenceResolver.cs @@ -34,44 +34,67 @@ public static class ReferenceResolver } /// - /// Walks a dictionary and resolves any string values matching ${protocol:value} patterns. - /// Only processes top-level string values in the given dictionary (recursive walking is - /// handled by LoadContext calling PreProcess on each nested dict). + /// Recursively walks a dictionary and resolves string values matching ${protocol:value} patterns. /// internal static Dictionary ResolveReferences( Dictionary data, string parentDir, IReadOnlyCollection allowedRoots) { - foreach (var key in data.Keys.ToList()) - { - var value = data[key]; - if (value is not string str) - continue; - if (!str.StartsWith(RefPrefix) || !str.EndsWith(RefSuffix)) - continue; + ResolveValue(data, "", parentDir, allowedRoots); + return data; + } - var inner = str[RefPrefix.Length..^RefSuffix.Length]; - var colonIndex = inner.IndexOf(':'); - if (colonIndex < 0) - continue; + private static object? ResolveValue( + object? value, + string key, + string parentDir, + IReadOnlyCollection allowedRoots) + { + if (value is System.Collections.IDictionary dictionary) + { + foreach (var childKey in dictionary.Keys.Cast().ToList()) + { + if (childKey is string childName) + { + dictionary[childKey] = ResolveValue(dictionary[childKey], childName, parentDir, allowedRoots); + } + } - var protocol = inner[..colonIndex].ToLowerInvariant(); - var remainder = inner[(colonIndex + 1)..]; + return value; + } - switch (protocol) + if (value is System.Collections.IList list) + { + for (var index = 0; index < list.Count; index++) { - case "env": - data[key] = ResolveEnvVar(remainder, key); - break; - case "file": - data[key] = ResolveFileRef(remainder, parentDir, allowedRoots, key); - break; - // Unknown protocol: leave unchanged per spec §4 + list[index] = ResolveValue(list[index], $"{key}[{index}]", parentDir, allowedRoots); } + + return value; } - return data; + if (value is not string str || !str.StartsWith(RefPrefix) || !str.EndsWith(RefSuffix)) + { + return value; + } + + var inner = str[RefPrefix.Length..^RefSuffix.Length]; + var colonIndex = inner.IndexOf(':'); + if (colonIndex < 0) + { + return value; + } + + var protocol = inner[..colonIndex].ToLowerInvariant(); + var remainder = inner[(colonIndex + 1)..]; + var resolved = protocol switch + { + "env" => ResolveEnvVar(remainder, key), + "file" => ResolveFileRef(remainder, parentDir, allowedRoots, key), + _ => value + }; + return resolved; } /// diff --git a/runtime/csharp/Prompty.Core/TurnEngine.cs b/runtime/csharp/Prompty.Core/TurnEngine.cs new file mode 100644 index 000000000..f4edd6b2f --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEngine.cs @@ -0,0 +1,1258 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Prompty.Core; + +/// +/// The bundle of runtime-local ports the uses to invoke the model, +/// enforce host policy, resolve permissions, execute tools, persist durable state, and run +/// post-commit effects. Only , , , and +/// are required; every other port has a conservative no-op default. +/// +public sealed class TurnEngineEffects +{ + /// The model invocation port. Required. + public required IEngineModelPort Model { get; init; } + + /// The tool execution port. Required. + public required IEngineToolPort Tools { get; init; } + + /// The deterministic clock used to timestamp every emitted event. Required. + public required IEngineClock Clock { get; init; } + + /// The deterministic id generator used for run/invocation/checkpoint/event ids. Required. + public required IEngineIdGenerator Ids { get; init; } + + /// The streaming sink handed to the model port. Defaults to a no-op sink. + public IEngineModelStreamPort Stream { get; init; } = new NoopModelStreamPort(); + + /// The context assembly port. Defaults to preserving the canonical messages unchanged. + public IEngineContextPort Context { get; init; } = new PassthroughEngineContextPort(); + + /// The host policy port. Defaults to a pass-through policy. + public IEngineHostPolicyPort Policy { get; init; } = new NoopHostPolicyPort(); + + /// The retry/backoff policy port. Defaults to an immediate no-op backoff. + public IEngineRetryPolicyPort Retry { get; init; } = new NoopRetryPolicyPort(); + + /// The conversation formatting port. Defaults to a single synthetic tool-result message per result. + public IEngineConversationPort Conversation { get; init; } = new DefaultConversationPort(); + + /// The permission resolution port. Defaults to approving every tool request. + public IEnginePermissionPort Permission { get; init; } = new AllowAllPermissionsPort(); + + /// The durability port used to append events and checkpoints. Defaults to an in-memory no-op. + public IEngineDurabilityPort Durability { get; init; } = new NoopDurabilityPort(); + + /// The post-commit effect port. Defaults to a no-op that always succeeds. + public IEnginePostCommitPort PostCommit { get; init; } = new NoopPostCommitPort(); +} + +/// +/// The canonical turn engine: a deterministic, resumable state machine that drives a single +/// conversational turn to completion by orchestrating host policy, context preparation, model +/// invocation, permissioned tool execution, and durable checkpointing. +/// +/// +/// This is a line-for-line behavioral port of the Rust reference implementation at +/// runtime/rust/prompty/src/engine/turn.rs. Every emitted , +/// every field, and every commit/cancellation/reconciliation path +/// mirrors the Rust engine so that the two runtimes agree on wire-visible behavior for the +/// shared vectors in spec/vectors/engine/turn_vectors.json. +/// +public sealed class TurnEngine +{ + private readonly TurnEngineEffects _effects; + + /// Creates a new engine bound to the supplied port bundle. + public TurnEngine(TurnEngineEffects effects) + { + _effects = effects ?? throw new ArgumentNullException(nameof(effects)); + } + + /// Resumes a turn from a durable checkpoint. + public Task ResumeAsync(ResumeContext resume, CancellationToken cancellationToken) => + RunAsync(TurnEngineRequest.FromResume(resume), cancellationToken); + + /// Runs a turn to completion (success, cancellation, failure, or reconciliation-required). + public async Task RunAsync(TurnEngineRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ValidateRequest(request); + + if (string.IsNullOrEmpty(request.RunId)) + { + request.RunId = _effects.Ids.NextId("run"); + } + + var state = new TurnState(request); + + await EmitAsync(state, EngineEventKind.TurnStarted, invocationId: null, iteration: null, new Dictionary + { + ["maxIterations"] = state.MaxIterations, + ["startIteration"] = state.Iteration, + ["inputs"] = state.Inputs, + }).ConfigureAwait(false); + + if (state.ModelReconciliationResolution is not null) + { + var response = state.ModelReconciliationResolution; + state.ModelReconciliationResolution = null; + var reconciliation = state.ModelReconciliation + ?? throw new TurnEngineInvalidRequestException("model reconciliation response is missing durable reconciliation state"); + + state.ReconciliationRequired = false; + state.ModelReconciliation = null; + + var applyError = state.ApplyModelResponse(reconciliation.InvocationId, response); + if (applyError is not null) + { + return await CommitFailedAsync(state, "provider_state_error", applyError, cancellationToken).ConfigureAwait(false); + } + + await PersistModelReconciliationAsync(state, reconciliation.InvocationId, reconciliation, response).ConfigureAwait(false); + } + + if (state.ReconciliationResolution is not null) + { + var resolution = state.ReconciliationResolution; + state.ReconciliationResolution = null; + await PersistReconciliationAsync(state, resolution).ConfigureAwait(false); + } + + if (state.ReconciliationRequired) + { + return await CommitReconciliationAsync( + state, + "effect_outcome_unknown", + "Checkpoint requires explicit effect reconciliation", + cancellationToken).ConfigureAwait(false); + } + + if (state.FinalOutputReady) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + state.Output = state.PendingOutput; + return await ApplyFinalPolicyAsync(state, cancellationToken).ConfigureAwait(false); + } + + while (state.Iteration < state.MaxIterations) + { + if (state.PendingToolRequests.Count == 0 && state.PendingModelResponse is not null) + { + var invocationId = state.ActiveInvocationId ?? _effects.Ids.NextId("invocation"); + + List results; + try + { + results = FinalizeToolExchange(state); + } + catch (PortError error) + { + return await CommitFailedAsync(state, "conversation_format_error", error.Message, cancellationToken).ConfigureAwait(false); + } + + await PersistToolExchangeAsync(state, invocationId, results).ConfigureAwait(false); + state.ActiveInvocationId = null; + state.Iteration += 1; + continue; + } + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + if (state.PendingToolRequests.Count > 0) + { + var invocationId = state.ActiveInvocationId ?? _effects.Ids.NextId("invocation"); + var toolRequest = state.PendingToolRequests[0]; + state.PendingToolRequests.RemoveAt(0); + + ModelToolResult toolResult; + try + { + toolResult = await ExecuteToolAsync(state, invocationId, toolRequest, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + catch (ToolPermissionFailure error) + { + return await CommitFailedAsync(state, "permission_error", error.Error.Message, cancellationToken).ConfigureAwait(false); + } + catch (ToolConfigurationFailure error) + { + return await CommitFailedAsync(state, "tool_configuration_error", error.Error.Message, cancellationToken).ConfigureAwait(false); + } + + var outcomeUnknown = toolResult.Outcome == ModelToolOutcome.Indeterminate; + state.ToolResults.Add(toolResult); + if (state.PendingModelResponse is null) + { + state.Messages.Add(Message.ToolResult(toolRequest.Id, toolResult.ModelText())); + } + + await PersistToolResultAsync(state, invocationId, toolRequest).ConfigureAwait(false); + + if (outcomeUnknown) + { + return await CommitReconciliationAsync( + state, + "effect_outcome_unknown", + "Tool effect outcome is unknown and requires reconciliation", + cancellationToken).ConfigureAwait(false); + } + + if (state.PendingToolRequests.Count == 0 && state.PendingModelResponse is null) + { + state.ActiveInvocationId = null; + state.Iteration += 1; + } + + continue; + } + + var freshInvocationId = _effects.Ids.NextId("invocation"); + + if (state.PolicyAppliedForIteration) + { + state.PolicyAppliedForIteration = false; + } + else + { + var policyRequest = new HostPolicyRequest + { + SessionId = state.SessionId, + TurnId = state.TurnId, + Iteration = state.Iteration, + Messages = [.. state.Messages], + StablePrefixMessages = state.StablePrefixMessages, + Inputs = state.Inputs, + }; + + HostPolicyResult policyResult; + try + { + policyResult = await _effects.Policy.BeforeModelAsync(policyRequest, cancellationToken).ConfigureAwait(false); + } + catch (HostPolicyException error) + { + return await CommitFailedAsync(state, error.ErrorKind, error.Message, cancellationToken).ConfigureAwait(false); + } + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + if (policyResult.StablePrefixMessages < 0 || policyResult.StablePrefixMessages > policyResult.Messages.Count) + { + return await CommitFailedAsync( + state, + "policy_error", + "host policy stable prefix exceeds rewritten message count", + cancellationToken).ConfigureAwait(false); + } + + var policyChanged = !MessagesEqual(state.Messages, policyResult.Messages) + || state.StablePrefixMessages != policyResult.StablePrefixMessages; + if (policyChanged) + { + state.Messages = policyResult.Messages; + state.StablePrefixMessages = policyResult.StablePrefixMessages; + await PersistPolicyUpdateAsync(state, freshInvocationId, policyResult.Metadata).ConfigureAwait(false); + state.PolicyAppliedForIteration = false; + } + } + + ModelInvocationContextSnapshot snapshot; + try + { + snapshot = await _effects.Context.PrepareAsync( + new ContextRequest + { + SessionId = state.SessionId, + TurnId = state.TurnId, + InvocationId = freshInvocationId, + Iteration = state.Iteration, + Messages = [.. state.Messages], + StablePrefixMessages = Math.Min(state.StablePrefixMessages, state.Messages.Count), + ContextState = new InvocationContextState + { + Portability = state.Portability, + DelegatedState = [.. state.DelegatedState], + }, + Inputs = state.Inputs, + }, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + catch (PortError error) + { + return await CommitFailedAsync(state, "context_error", error.Message, cancellationToken).ConfigureAwait(false); + } + + var contextError = ValidateContextSnapshot(snapshot, state, freshInvocationId); + if (contextError is not null) + { + return await CommitFailedAsync(state, "context_error", contextError, cancellationToken).ConfigureAwait(false); + } + + await EmitAsync(state, EngineEventKind.ContextPrepared, freshInvocationId, state.Iteration, snapshot).ConfigureAwait(false); + state.Snapshots.Add(snapshot); + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + var modelRequest = new ModelInvocationRequest { Context = snapshot }; + state.ActiveInvocationId = freshInvocationId; + + var attempt = 0; + ModelInvocationResponse? modelResponse = null; + while (modelResponse is null) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + await EmitAsync(state, EngineEventKind.ModelInvocationStarted, freshInvocationId, state.Iteration, new Dictionary + { + ["snapshotId"] = snapshot.Id, + ["attempt"] = attempt, + ["messageCount"] = snapshot.Messages.Count, + }).ConfigureAwait(false); + + try + { + modelResponse = await _effects.Model.InvokeAsync( + modelRequest, + cancellationToken, + new BestEffortModelStreamPort(_effects.Stream)).ConfigureAwait(false); + } + catch (PortError failure) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + attempt += 1; + var outcomeUnknown = failure.OutcomeUnknown; + var exhausted = outcomeUnknown || attempt >= state.MaxModelAttempts; + var reason = failure.Message; + + await EmitAsync(state, EngineEventKind.ModelInvocationFailed, freshInvocationId, state.Iteration, new Dictionary + { + ["attempt"] = attempt - 1, + ["exhausted"] = exhausted, + ["outcomeUnknown"] = outcomeUnknown, + ["message"] = reason, + }).ConfigureAwait(false); + + if (outcomeUnknown) + { + state.ReconciliationRequired = true; + state.ModelReconciliation = new ModelReconciliationState + { + InvocationId = freshInvocationId, + Request = modelRequest, + FailedAttempt = attempt - 1, + Message = reason, + Metadata = failure.Metadata, + }; + + await PersistModelReconciliationRequiredAsync(state, freshInvocationId).ConfigureAwait(false); + return await CommitReconciliationAsync(state, "model_outcome_unknown", reason, cancellationToken).ConfigureAwait(false); + } + + if (exhausted) + { + return await CommitFailedAsync(state, "model_error", reason, cancellationToken).ConfigureAwait(false); + } + + try + { + await _effects.Retry.BackoffAsync( + new RetryPolicyRequest + { + FailedAttempts = attempt, + NextAttempt = attempt + 1, + MaxAttempts = state.MaxModelAttempts, + Reason = reason, + }, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + catch (PortError source) + { + return await CommitFailedAsync(state, "retry_policy_error", source.Message, cancellationToken).ConfigureAwait(false); + } + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + } + } + + state.ModelReconciliation = null; + state.ReconciliationRequired = false; + + var applyModelError = state.ApplyModelResponse(freshInvocationId, modelResponse); + if (applyModelError is not null) + { + return await CommitFailedAsync(state, "provider_state_error", applyModelError, cancellationToken).ConfigureAwait(false); + } + + await PersistModelResponseAsync(state, freshInvocationId, modelResponse).ConfigureAwait(false); + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + if (state.FinalOutputReady) + { + state.Output = state.PendingOutput; + return await ApplyFinalPolicyAsync(state, cancellationToken).ConfigureAwait(false); + } + } + + return await CommitFailedAsync(state, "max_iterations", "Maximum model iterations reached", cancellationToken).ConfigureAwait(false); + } + + private static void ValidateRequest(TurnEngineRequest request) + { + if (string.IsNullOrEmpty(request.SessionId)) + { + throw new TurnEngineInvalidRequestException("session_id is required"); + } + + if (string.IsNullOrEmpty(request.TurnId)) + { + throw new TurnEngineInvalidRequestException("turn_id is required"); + } + + if (request.MaxModelAttempts <= 0) + { + throw new TurnEngineInvalidRequestException("max_model_attempts must be greater than zero"); + } + + if (request.StartIteration > request.MaxIterations) + { + throw new TurnEngineInvalidRequestException("start_iteration must not exceed max_iterations"); + } + + if (request.StablePrefixMessages > request.Messages.Count) + { + throw new TurnEngineInvalidRequestException("stable_prefix_messages exceeds initial message count"); + } + + if (request.Portability == InvocationContextPortability.Portable && request.DelegatedState.Count > 0) + { + throw new TurnEngineInvalidRequestException("portable turns cannot begin with delegated provider state"); + } + } + + private List FinalizeToolExchange(TurnState state) + { + var response = state.PendingModelResponse; + if (response is null) + { + return []; + } + + state.PendingModelResponse = null; + + if (response.ToolRequests is null || response.ToolRequests.Count == 0) + { + return []; + } + + var results = new List(); + foreach (var request in response.ToolRequests) + { + var match = state.ToolResults.FirstOrDefault(result => result.RequestId == request.Id); + if (match is not null) + { + results.Add(match); + } + } + + if (results.Count != response.ToolRequests.Count) + { + state.PendingModelResponse = response; + throw PortError.Configuration("tool exchange is incomplete and cannot be formatted"); + } + + IList messages; + try + { + messages = _effects.Conversation.FormatToolExchange(response, results); + } + catch (PortError) + { + state.PendingModelResponse = response; + throw; + } + + foreach (var message in messages) + { + state.Messages.Add(message); + } + + return results; + } + + private async Task ExecuteToolAsync( + TurnState state, + string invocationId, + ModelToolRequest request, + CancellationToken cancellationToken) + { + await EmitAsync(state, EngineEventKind.PermissionRequested, invocationId, state.Iteration, new Dictionary + { + ["toolRequest"] = request, + }).ConfigureAwait(false); + + EnginePermissionDecision decision; + try + { + decision = await _effects.Permission.AuthorizeAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (PortError source) + { + throw new ToolPermissionFailure(source); + } + + await EmitPermissionResolvedAsync(state, invocationId, request, decision).ConfigureAwait(false); + + if (!decision.Approved) + { + var errorKind = decision.Metadata is not null + && decision.Metadata.TryGetValue("errorKind", out var kindValue) + && kindValue is string kind + ? kind + : "permission_denied"; + + return new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = ModelToolOutcome.Failed, + Output = decision.Reason ?? "Permission denied", + ErrorKind = errorKind, + Metadata = decision.Metadata, + }; + } + + cancellationToken.ThrowIfCancellationRequested(); + + await EmitAsync(state, EngineEventKind.ToolExecutionStarted, invocationId, state.Iteration, new Dictionary + { + ["toolRequest"] = request, + }).ConfigureAwait(false); + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + return await _effects.Tools.ExecuteAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (PortError error) when (error.ConfigurationError) + { + throw new ToolConfigurationFailure(error); + } + catch (PortError error) + { + return new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = error.OutcomeUnknown ? ModelToolOutcome.Indeterminate : ModelToolOutcome.Failed, + Output = error.OutcomeUnknown + ? $"Tool '{request.Name}' outcome is unknown and requires reconciliation: {error.Message}" + : $"Tool '{request.Name}' failed: {error.Message}", + ErrorKind = error.OutcomeUnknown ? "effect_outcome_unknown" : "tool_error", + }; + } + } + + private Task EmitPermissionResolvedAsync( + TurnState state, + string invocationId, + ModelToolRequest request, + EnginePermissionDecision decision) => + EmitAsync(state, EngineEventKind.PermissionResolved, invocationId, state.Iteration, new Dictionary + { + ["toolRequestId"] = request.Id, + ["decision"] = decision, + }); + + private static string? ValidateContextSnapshot( + ModelInvocationContextSnapshot snapshot, + TurnState state, + string invocationId) + { + if (snapshot.SessionId != state.SessionId + || snapshot.TurnId != state.TurnId + || snapshot.InvocationId != invocationId + || snapshot.Iteration != state.Iteration) + { + return $"snapshot identity ({snapshot.SessionId}/{snapshot.TurnId}/{snapshot.InvocationId}/{snapshot.Iteration}) " + + $"does not match request ({state.SessionId}/{state.TurnId}/{invocationId}/{state.Iteration})"; + } + + if (snapshot.StablePrefixMessages < 0 || snapshot.StablePrefixMessages > snapshot.Messages.Count) + { + return $"stable prefix contains {snapshot.StablePrefixMessages} messages but snapshot contains {snapshot.Messages.Count}"; + } + + if (snapshot.ContextState is null) + { + return "snapshot context state is required"; + } + + if (snapshot.ContextState.Portability == InvocationContextPortability.Portable + && snapshot.ContextState.DelegatedState?.Count > 0) + { + return "portable snapshots cannot contain delegated provider state"; + } + + if (snapshot.ContextState.Portability == InvocationContextPortability.Delegated + && (snapshot.ContextState.DelegatedState is null || snapshot.ContextState.DelegatedState.Count == 0)) + { + return "delegated snapshots must identify provider-held state"; + } + + return null; + } + + private static bool MessagesEqual(IList left, IList right) + { + if (ReferenceEquals(left, right)) + { + return true; + } + + if (left.Count != right.Count) + { + return false; + } + + for (var i = 0; i < left.Count; i++) + { + if (left[i].ToJson(indent: false) != right[i].ToJson(indent: false)) + { + return false; + } + } + + return true; + } + + private async Task PersistCheckpointAsync( + string stage, + string requestId, + TurnState state, + EngineCheckpoint checkpoint, + IReadOnlyList events) + { + try + { + await _effects.Durability.AppendWithCheckpointAsync(events, checkpoint).ConfigureAwait(false); + } + catch (PortError source) + { + throw new TurnEngineRecoveryRequiredException(stage, requestId, checkpoint, [.. state.ToolResults], source); + } + } + + private async Task PersistPolicyUpdateAsync( + TurnState state, + string invocationId, + IDictionary? metadata) + { + var sequence = state.Sequence + 1; + state.PolicyAppliedForIteration = true; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: true); + var evt = BuildEvent(state, sequence, EngineEventKind.PolicyApplied, invocationId, state.Iteration, new Dictionary + { + ["messages"] = state.Messages, + ["stablePrefixMessages"] = state.StablePrefixMessages, + ["metadata"] = metadata, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("host policy", invocationId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistToolExchangeAsync( + TurnState state, + string invocationId, + IReadOnlyList results) + { + var sequence = state.Sequence; + var events = new List(results.Count + 2); + foreach (var result in results) + { + sequence += 1; + events.Add(BuildEvent(state, sequence, EngineEventKind.ToolResultCommitted, invocationId, state.Iteration, new Dictionary + { + ["toolResult"] = result, + })); + } + + sequence += 1; + events.Add(BuildEvent(state, sequence, EngineEventKind.ConversationUpdated, invocationId, state.Iteration, new Dictionary + { + ["messageCount"] = state.Messages.Count, + })); + + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + events.Add(BuildCheckpointEvent(state, checkpoint, invocationId)); + + await PersistCheckpointAsync("tool exchange", invocationId, state, checkpoint, events).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistModelReconciliationRequiredAsync(TurnState state, string invocationId) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var reconciliation = state.ModelReconciliation + ?? throw new InvalidOperationException("model reconciliation state must exist before persistence"); + var evt = BuildEvent(state, sequence, EngineEventKind.ModelReconciliationRequired, invocationId, state.Iteration, reconciliation); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("model reconciliation", invocationId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistModelReconciliationAsync( + TurnState state, + string invocationId, + ModelReconciliationState reconciliation, + ModelInvocationResponse response) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var evt = BuildEvent(state, sequence, EngineEventKind.ModelInvocationReconciled, invocationId, state.Iteration, new Dictionary + { + ["reconciliation"] = reconciliation, + ["hasOutput"] = response.Output is not null, + ["toolRequests"] = response.ToolRequests?.Count ?? 0, + ["metadata"] = response.Metadata, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("model reconciliation resolution", invocationId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistModelResponseAsync( + TurnState state, + string invocationId, + ModelInvocationResponse response) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var evt = BuildEvent(state, sequence, EngineEventKind.ModelInvocationCompleted, invocationId, state.Iteration, new Dictionary + { + ["hasOutput"] = response.Output is not null, + ["toolRequests"] = response.ToolRequests?.Count ?? 0, + ["nextPortability"] = response.NextContextState?.Portability, + ["delegatedState"] = response.NextContextState?.DelegatedState, + ["metadata"] = response.Metadata, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("model response", invocationId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistToolResultAsync(TurnState state, string invocationId, ModelToolRequest request) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var result = state.ToolResults.Count > 0 + ? state.ToolResults[^1] + : throw new InvalidOperationException("tool result must be recorded before persistence"); + var evt = BuildEvent(state, sequence, EngineEventKind.ToolExecutionCompleted, invocationId, state.Iteration, new Dictionary + { + ["toolResult"] = result, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("tool result", request.Id, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistReconciliationAsync(TurnState state, ModelToolResult result) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var invocationId = state.ActiveInvocationId ?? "reconciliation"; + var evt = BuildEvent(state, sequence, EngineEventKind.ToolResultReconciled, invocationId, state.Iteration, new Dictionary + { + ["toolResult"] = result, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("tool reconciliation", result.RequestId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private EngineEvent BuildCheckpointEvent(TurnState state, EngineCheckpoint checkpoint, string invocationId) => + BuildEvent(state, checkpoint.LastSequence + 1, EngineEventKind.CheckpointCreated, invocationId, checkpoint.Iteration, new Dictionary + { + ["checkpointId"] = checkpoint.Id, + ["includedThroughSequence"] = checkpoint.LastSequence, + }); + + private EngineCheckpoint BuildCheckpoint(TurnState state, long lastSequence, bool resumeSameIteration) + { + var lastToolResult = state.ToolResults.Count > 0 ? state.ToolResults[^1] : null; + return new EngineCheckpoint + { + Id = _effects.Ids.NextId("checkpoint"), + SessionId = state.SessionId, + TurnId = state.TurnId, + RunId = state.RunId, + ParentRunId = state.ParentRunId, + DelegationDepth = state.DelegationDepth, + Iteration = state.Iteration, + LastSequence = lastSequence, + Messages = [.. state.Messages], + StablePrefixMessages = state.StablePrefixMessages, + Inputs = state.Inputs, + ActiveInvocationId = state.ActiveInvocationId, + PendingToolRequests = [.. state.PendingToolRequests], + CompletedToolResults = [.. state.ToolResults], + CompletedModelIterations = state.CompletedModelIterations, + ReconciliationRequired = state.ReconciliationRequired || lastToolResult?.Outcome == ModelToolOutcome.Indeterminate, + ModelReconciliation = state.ModelReconciliation, + PendingOutput = state.PendingOutput, + FinalOutputReady = state.FinalOutputReady, + PendingModelResponse = state.PendingModelResponse, + ResumeSameIteration = resumeSameIteration, + PolicyAppliedForIteration = state.PolicyAppliedForIteration, + ContextState = new InvocationContextState + { + Portability = state.Portability, + DelegatedState = state.DelegatedState, + }, + }; + } + + private EngineEvent BuildEvent( + TurnState state, + long sequence, + EngineEventKind kind, + string? invocationId, + int? iteration, + object? payload) => + new() + { + Sequence = sequence, + Id = _effects.Ids.NextId("event"), + Timestamp = _effects.Clock.Now(), + SessionId = state.SessionId, + TurnId = state.TurnId, + RunId = state.RunId, + ParentRunId = state.ParentRunId, + DelegationDepth = state.DelegationDepth, + InvocationId = invocationId, + Iteration = iteration, + Kind = kind, + Payload = payload, + }; + + private async Task EmitAsync(TurnState state, EngineEventKind kind, string? invocationId, int? iteration, object? payload) + { + var sequence = state.Sequence + 1; + var evt = BuildEvent(state, sequence, kind, invocationId, iteration, payload); + try + { + await _effects.Durability.AppendAsync(evt).ConfigureAwait(false); + } + catch (PortError source) + { + throw new TurnEnginePortException("event journal", source); + } + + state.Sequence = sequence; + } + + private Task CommitSuccessAsync(TurnState state, CancellationToken cancellationToken) => + CommitAsync(state, EngineTurnStatus.Success, EngineEventKind.TurnCommitted, cancellationToken); + + private async Task ApplyFinalPolicyAsync(TurnState state, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + var request = new FinalOutputPolicyRequest + { + SessionId = state.SessionId, + TurnId = state.TurnId, + Iteration = state.Iteration, + Messages = [.. state.Messages], + Output = state.Output, + Inputs = state.Inputs, + }; + + FinalOutputPolicyResult result; + try + { + result = await _effects.Policy.BeforeCommitAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HostPolicyException error) + { + return cancellationToken.IsCancellationRequested + ? await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false) + : await CommitFailedAsync(state, error.ErrorKind, error.Message, cancellationToken).ConfigureAwait(false); + } + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + state.Output = result.Output; + return await CommitSuccessAsync(state, cancellationToken).ConfigureAwait(false); + } + + private Task CommitCancelledAsync(TurnState state, CancellationToken cancellationToken) => + CommitAsync(state, EngineTurnStatus.Cancelled, EngineEventKind.TurnCancelled, cancellationToken); + + private Task CommitFailedAsync(TurnState state, string errorKind, string message, CancellationToken cancellationToken) + { + state.Output = new Dictionary { ["errorKind"] = errorKind, ["message"] = message }; + return CommitAsync(state, EngineTurnStatus.Failed, EngineEventKind.TurnFailed, cancellationToken); + } + + private Task CommitReconciliationAsync( + TurnState state, + string errorKind, + string message, + CancellationToken cancellationToken) + { + state.Output = new Dictionary { ["errorKind"] = errorKind, ["message"] = message }; + return CommitAsync(state, EngineTurnStatus.ReconciliationRequired, EngineEventKind.TurnReconciliationRequired, cancellationToken); + } + + private async Task CommitAsync( + TurnState state, + EngineTurnStatus status, + EngineEventKind kind, + CancellationToken cancellationToken) + { + var iteration = state.Iteration; + var terminalPayload = new Dictionary { ["status"] = status, ["output"] = state.Output }; + await EmitAsync(state, kind, invocationId: null, iteration, terminalPayload).ConfigureAwait(false); + + var commit = new TurnCommit + { + SessionId = state.SessionId, + TurnId = state.TurnId, + Status = status, + Output = state.Output, + Messages = [.. state.Messages], + Iterations = state.CompletedModelIterations, + LastSequence = state.Sequence, + ContextState = new InvocationContextState + { + Portability = state.Portability, + DelegatedState = state.DelegatedState, + }, + ModelReconciliation = state.ModelReconciliation, + }; + + string? postCommitError = null; + if (status == EngineTurnStatus.Success) + { + var sessionIdLength = System.Text.Encoding.UTF8.GetByteCount(commit.SessionId); + var turnIdLength = System.Text.Encoding.UTF8.GetByteCount(commit.TurnId); + var effectId = $"post_commit:{sessionIdLength}:{commit.SessionId}:{turnIdLength}:{commit.TurnId}"; + + TurnEnginePortException? startError = null; + try + { + await EmitAsync(state, EngineEventKind.PostCommitStarted, invocationId: null, iteration, new Dictionary + { + ["effectId"] = effectId, + }).ConfigureAwait(false); + } + catch (TurnEnginePortException error) + { + startError = error; + } + + if (startError is not null) + { + postCommitError = + $"post-commit effect '{effectId}' was not started because its start event could not be persisted: {startError.Message}"; + } + else + { + try + { + await _effects.PostCommit.AfterCommitAsync(effectId, commit, cancellationToken).ConfigureAwait(false); + + try + { + await EmitAsync(state, EngineEventKind.PostCommitCompleted, invocationId: null, iteration, new Dictionary + { + ["effectId"] = effectId, + }).ConfigureAwait(false); + } + catch (TurnEnginePortException completionError) + { + postCommitError = + $"post-commit effect '{effectId}' completed, but its completion event could not be persisted: {completionError.Message}"; + } + } + catch (PortError source) + { + var message = source.Message; + string? eventError = null; + try + { + await EmitAsync(state, EngineEventKind.PostCommitFailed, invocationId: null, iteration, new Dictionary + { + ["effectId"] = effectId, + ["message"] = message, + }).ConfigureAwait(false); + } + catch (TurnEnginePortException failureEventError) + { + eventError = failureEventError.Message; + } + + postCommitError = eventError is not null + ? $"{message}; failure event for post-commit effect '{effectId}' could not be persisted: {eventError}" + : message; + } + } + } + + commit.LastSequence = state.Sequence; + + return new TurnEngineResult + { + Commit = commit, + Snapshots = state.Snapshots, + ToolResults = state.ToolResults, + PostCommitError = postCommitError, + }; + } + + /// Signals that itself failed. + private sealed class ToolPermissionFailure(PortError error) : Exception(error.Message) + { + public PortError Error { get; } = error; + } + + /// Signals that failed with a configuration error. + private sealed class ToolConfigurationFailure(PortError error) : Exception(error.Message) + { + public PortError Error { get; } = error; + } + + /// Prevents observational stream failures from changing semantic model execution. + private sealed class BestEffortModelStreamPort(IEngineModelStreamPort inner) : IEngineModelStreamPort + { + public async Task EmitAsync(ModelStreamChunk chunk) + { + try + { + await inner.EmitAsync(chunk).ConfigureAwait(false); + } + catch (Exception error) when (error is not OperationCanceledException) + { + // Stream sinks own delivery diagnostics; the canonical turn outcome remains semantic-only. + } + } + } + + /// + /// The mutable in-flight state of a single turn. This is a direct transliteration of the Rust + /// engine's TurnState struct and is never exposed outside . + /// + private sealed class TurnState + { + public TurnState(TurnEngineRequest request) + { + SessionId = request.SessionId; + TurnId = request.TurnId; + RunId = request.RunId; + ParentRunId = request.ParentRunId; + DelegationDepth = request.DelegationDepth; + Messages = [.. request.Messages]; + Inputs = request.Inputs; + MaxIterations = request.MaxIterations; + MaxModelAttempts = request.MaxModelAttempts; + StablePrefixMessages = request.StablePrefixMessages; + Portability = request.Portability; + DelegatedState = [.. request.DelegatedState]; + ActiveInvocationId = request.ActiveInvocationId; + PendingToolRequests = [.. request.PendingToolRequests]; + ReconciliationRequired = request.ReconciliationRequired; + ModelReconciliation = request.ModelReconciliation; + CompletedModelIterations = request.CompletedModelIterations; + PendingOutput = request.PendingOutput; + FinalOutputReady = request.FinalOutputReady; + PendingModelResponse = request.PendingModelResponse; + PolicyAppliedForIteration = request.PolicyAppliedForIteration; + ReconciliationResolution = request.ReconciliationResolution; + ModelReconciliationResolution = request.ModelReconciliationResolution; + + Iteration = request.StartIteration; + Sequence = request.InitialSequence; + Output = null; + Snapshots = []; + ToolResults = [.. request.CompletedToolResults]; + } + + public string SessionId { get; } + + public string TurnId { get; } + + public string RunId { get; set; } + + public string? ParentRunId { get; } + + public int DelegationDepth { get; } + + public IList Messages { get; set; } + + public object? Inputs { get; } + + public int MaxIterations { get; } + + public int MaxModelAttempts { get; } + + public int StablePrefixMessages { get; set; } + + public InvocationContextPortability Portability { get; set; } + + public IList DelegatedState { get; set; } + + public string? ActiveInvocationId { get; set; } + + public IList PendingToolRequests { get; set; } + + public bool ReconciliationRequired { get; set; } + + public ModelReconciliationState? ModelReconciliation { get; set; } + + public int CompletedModelIterations { get; set; } + + public object? PendingOutput { get; set; } + + public bool FinalOutputReady { get; set; } + + public ModelInvocationResponse? PendingModelResponse { get; set; } + + public bool PolicyAppliedForIteration { get; set; } + + public ModelToolResult? ReconciliationResolution { get; set; } + + public ModelInvocationResponse? ModelReconciliationResolution { get; set; } + + public int Iteration { get; set; } + + public long Sequence { get; set; } + + public object? Output { get; set; } + + public IList Snapshots { get; set; } + + public IList ToolResults { get; set; } + + /// Applies a model response to the state. Returns an error message on failure, else null. + public string? ApplyModelResponse(string invocationId, ModelInvocationResponse response) + { + CompletedModelIterations += 1; + + if (response.ToolRequests is null || response.ToolRequests.Count == 0) + { + foreach (var message in response.AssistantMessages ?? []) + { + Messages.Add(message); + } + + PendingModelResponse = null; + } + else + { + PendingModelResponse = response; + } + + var error = ApplyProviderState(response); + if (error is not null) + { + return error; + } + + ActiveInvocationId = invocationId; + PendingToolRequests = response.ToolRequests is null ? [] : [.. response.ToolRequests]; + PendingOutput = response.Output; + FinalOutputReady = PendingToolRequests.Count == 0; + return null; + } + + /// Adopts provider-supplied context state. Returns an error message on invariant violation. + public string? ApplyProviderState(ModelInvocationResponse response) + { + if (response.NextContextState is not null) + { + Portability = response.NextContextState.Portability; + DelegatedState = response.NextContextState.DelegatedState is null + ? [] + : [.. response.NextContextState.DelegatedState]; + } + else if (Portability == InvocationContextPortability.Portable) + { + DelegatedState = []; + } + + if (Portability == InvocationContextPortability.Portable && DelegatedState.Count > 0) + { + return "portable provider state cannot retain delegated references"; + } + + if (Portability == InvocationContextPortability.Delegated && DelegatedState.Count == 0) + { + return "delegated provider state requires at least one reference"; + } + + return null; + } + } +} diff --git a/runtime/csharp/Prompty.Core/TurnEngineExceptions.cs b/runtime/csharp/Prompty.Core/TurnEngineExceptions.cs new file mode 100644 index 000000000..cedae7a99 --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEngineExceptions.cs @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Prompty.Core; + +/// +/// Error raised by a runtime-local engine port (model, tool, permission, host policy, +/// retry policy, post-commit, or durability). Mirrors the Rust reference's +/// PortError: a plain message plus two orthogonal signal flags that change how +/// the canonical state machine reacts to the failure. +/// +public class PortError : Exception +{ + /// + /// True when the effect's real-world outcome cannot be determined (for example, a + /// network timeout after a tool call may have already been dispatched). The engine + /// treats this as a signal to stop and require explicit reconciliation rather than + /// retrying or failing outright, since retrying could duplicate a side effect. + /// + public bool OutcomeUnknown { get; } + + /// + /// True when the failure reflects a host misconfiguration (for example, an unknown + /// tool name) rather than a transient runtime failure. Configuration errors are not + /// retried and are surfaced as a distinct commit failure kind. + /// + public bool ConfigurationError { get; } + + /// Optional structured metadata describing the failure. + public IDictionary? Metadata { get; } + + public PortError( + string message, + bool outcomeUnknown = false, + bool configurationError = false, + IDictionary? metadata = null) + : base(message) + { + OutcomeUnknown = outcomeUnknown; + ConfigurationError = configurationError; + Metadata = metadata; + } + + /// Create a port error whose effect outcome is unknown and requires reconciliation. + public static PortError Indeterminate(string message, IDictionary? metadata = null) => + new(message, outcomeUnknown: true, metadata: metadata); + + /// Create a port error that reflects a host misconfiguration. + public static PortError Configuration(string message) => new(message, configurationError: true); +} + +/// +/// Error raised by . Unlike , +/// host policy failures always carry an explicit that the engine +/// forwards verbatim into the committed failure output. +/// +public class HostPolicyException : Exception +{ + /// Machine-readable failure category surfaced on the committed turn output. + public string ErrorKind { get; } + + public HostPolicyException(string errorKind, string message) + : base(message) + { + ErrorKind = errorKind; + } +} + +/// +/// Base type for errors that prevent the canonical from producing +/// a committed at all. These are distinct from ordinary +/// turn outcomes (success, cancelled, failed, reconciliation-required), which are all +/// returned as a normal rather than thrown. +/// +public abstract class TurnEngineException : Exception +{ + protected TurnEngineException(string message, Exception? innerException = null) + : base(message, innerException) + { + } +} + +/// Thrown when a or resume record fails validation. +public sealed class TurnEngineInvalidRequestException : TurnEngineException +{ + public TurnEngineInvalidRequestException(string message) + : base($"invalid turn request: {message}") + { + } +} + +/// +/// Thrown when appending a single, non-checkpointed event to the durability port fails +/// (for example, the plain TurnStarted or ContextPrepared events). This is +/// unrecoverable for the current run because no checkpoint was persisted to resume from. +/// +public sealed class TurnEnginePortException : TurnEngineException +{ + /// The engine stage that failed (for example, "event journal"). + public string Stage { get; } + + public TurnEnginePortException(string stage, PortError source) + : base($"{stage} failed: {source.Message}", source) + { + Stage = stage; + } +} + +/// +/// Thrown when an atomic append-with-checkpoint durability write fails while persisting a +/// semantic effect (policy update, model response, tool result, tool exchange, or +/// reconciliation). The caller can use and +/// to recover: the in-memory effect already happened, but it was never durably recorded, +/// so a host must decide how to reconcile before resuming. +/// +public sealed class TurnEngineRecoveryRequiredException : TurnEngineException +{ + /// The persistence stage that failed (for example, "tool result"). + public string Stage { get; } + + /// Identifier of the effect that could not be durably recorded. + public string RequestId { get; } + + /// The checkpoint that was built but never durably appended. + public EngineCheckpoint Checkpoint { get; } + + /// Tool results completed so far in this run, for host-side recovery bookkeeping. + public IReadOnlyList ToolResults { get; } + + public TurnEngineRecoveryRequiredException( + string stage, + string requestId, + EngineCheckpoint checkpoint, + IReadOnlyList toolResults, + PortError source) + : base($"{stage} durability failed after effect '{requestId}': {source.Message}", source) + { + Stage = stage; + RequestId = requestId; + Checkpoint = checkpoint; + ToolResults = toolResults; + } +} diff --git a/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs b/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs new file mode 100644 index 000000000..6b69a2f63 --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft. All rights reserved. + +// --- Runtime helpers (manually maintained) --- +// These extend generated Model/pipeline types with small convenience members the +// canonical turn engine needs. They mirror the `impl` blocks the Rust reference adds +// directly on its generated types (see runtime/rust/prompty/src/engine/ports.rs and +// model_ext.rs). New files only — no generated file under Model/ is modified. + +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; + +namespace Prompty.Core; + +public partial class Message +{ + /// + /// Build the message a host appends to the conversation for one tool result: a + /// message whose text is the result's model-visible output, + /// tagged with the originating tool request id so it can be located again (for + /// example, when resuming after an indeterminate tool-effect reconciliation). + /// + public static Message ToolResult(string requestId, string text) => new() + { + Role = Role.Tool, + Parts = [new TextPart { Value = text }], + Metadata = new Dictionary { ["tool_call_id"] = requestId }, + }; +} + +public partial class ModelToolResult +{ + /// + /// Render this tool result's output as model-visible text, tolerating an absent output. + /// A plain string output is used as-is; any other JSON-ish value is stringified. + /// + public string ModelText() => Output switch + { + null => string.Empty, + string text => text, + var value => SerializeCanonicalJson(value), + }; + + private static string SerializeCanonicalJson(object value) + { + var element = JsonSerializer.SerializeToElement(value); + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter( + stream, + new JsonWriterOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping })) + { + WriteCanonicalJson(writer, element); + } + + return Encoding.UTF8.GetString(stream.ToArray()); + } + + private static void WriteCanonicalJson(Utf8JsonWriter writer, JsonElement element) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + foreach (var property in element.EnumerateObject().OrderBy(property => property.Name, StringComparer.Ordinal)) + { + writer.WritePropertyName(property.Name); + WriteCanonicalJson(writer, property.Value); + } + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (var item in element.EnumerateArray()) + { + WriteCanonicalJson(writer, item); + } + writer.WriteEndArray(); + break; + default: + element.WriteTo(writer); + break; + } + } +} + +public partial class ResumeContext +{ + /// + /// The journal sequence a resumed run must continue after: the larger of the + /// recorded journal tail and the checkpoint's own last committed sequence. + /// + public long ResumeSequence() => Math.Max(LastJournalSequence, Checkpoint.LastSequence); +} diff --git a/runtime/csharp/Prompty.Core/TurnEnginePorts.cs b/runtime/csharp/Prompty.Core/TurnEnginePorts.cs new file mode 100644 index 000000000..fe100d906 --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEnginePorts.cs @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Prompty.Core; + +// ----------------------------------------------------------------------- +// Runtime-local effect ports used by the canonical turn engine. +// +// The generated Model/pipeline interfaces (IPermissionResolver, IHostToolExecutor, +// IEventJournalWriter, ICheckpointStore, etc.) predate the canonical EngineEvent / +// EngineCheckpoint / TurnEngineResult contract and use different request/response +// shapes with no cancellation support. Rather than repurposing or editing those +// generated interfaces, this file defines new, distinctly named runtime-local ports +// that consume the generated domain types (ModelInvocationRequest/Response, +// ModelToolRequest/Result, EnginePermissionDecision, EngineEvent, EngineCheckpoint, +// TurnCommit, ...) directly. This mirrors the Rust reference engine's runtime-local +// traits in runtime/rust/prompty/src/engine/ports.rs. +// +// Cancellation is intentionally a runtime seam, not serialized model data. The +// canonical engine checks it at semantic boundaries and passes the native token to +// these runtime-local ports. Generated protocols such as IExecutor do not yet accept +// native cancellation, so adapters over them can only observe cancellation before or +// after an in-flight call (and while draining a stream). Durability writes remain +// deliberately non-cancellable once persistence starts. +// ----------------------------------------------------------------------- + +/// An ephemeral, non-durable chunk streamed while a model invocation is in flight. +public abstract record ModelStreamChunk +{ + private ModelStreamChunk() + { + } + + /// A partial text delta. + public sealed record Text(string Value) : ModelStreamChunk; + + /// A partial "thinking"/reasoning delta. + public sealed record Thinking(string Value) : ModelStreamChunk; + + /// An opaque, provider-specific streaming payload. + public sealed record Provider(object? Value) : ModelStreamChunk; +} + +/// Delivers ephemeral model stream chunks. Delivery failure must not alter semantic execution. +public interface IEngineModelStreamPort +{ + Task EmitAsync(ModelStreamChunk chunk); +} + +/// Builds the immutable context snapshot used for one model invocation. +public interface IEngineContextPort +{ + /// Context assembly or validation failed. + Task PrepareAsync(ContextRequest request, CancellationToken cancellationToken); +} + +/// Invokes the model for one turn iteration. +public interface IEngineModelPort +{ + /// The invocation failed. Set when the + /// effect may have already happened on the provider side and requires reconciliation instead of a retry. + Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream); +} + +/// Host policy applied before each model call and before the final commit. +public interface IEngineHostPolicyPort +{ + /// The policy deterministically rejected the turn. + Task BeforeModelAsync(HostPolicyRequest request, CancellationToken cancellationToken); + + /// The policy deterministically rejected the turn. + Task BeforeCommitAsync(FinalOutputPolicyRequest request, CancellationToken cancellationToken); +} + +/// Backoff policy applied between failed model invocation attempts. +public interface IEngineRetryPolicyPort +{ + /// + /// Wait (or otherwise apply backoff) before the next model attempt. + /// + /// Backoff was cancelled. + /// The retry policy itself failed. + Task BackoffAsync(RetryPolicyRequest request, CancellationToken cancellationToken); +} + +/// Converts one completed model/tool batch into provider-valid conversation messages. +public interface IEngineConversationPort +{ + /// The batch could not be formatted (for example, results are incomplete). + IList FormatToolExchange(ModelInvocationResponse response, IReadOnlyList results); +} + +/// Authorizes a single tool request before it executes. +public interface IEnginePermissionPort +{ + /// The permission service itself failed (not the same as a denial, which is a + /// normal with Approved == false). + Task AuthorizeAsync(ModelToolRequest request, CancellationToken cancellationToken); +} + +/// Executes a single approved tool request. +public interface IEngineToolPort +{ + /// The tool failed. Set for a host + /// misconfiguration (unknown tool, invalid binding) and when the + /// effect may have already happened and requires reconciliation. + Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken); +} + +/// +/// Durable event journal and checkpoint store. Unlike the other engine ports, durability +/// writes are not cancellable: once the engine decides to persist an effect it must either +/// succeed or fail explicitly so the in-memory state and the durable record never diverge. +/// +public interface IEngineDurabilityPort +{ + /// The event could not be appended. + Task AppendAsync(EngineEvent @event); + + /// + /// Atomically append one or more events and persist the checkpoint that reflects them, so + /// resuming from the checkpoint can never duplicate the effects the events describe. + /// + /// The events/checkpoint could not be durably persisted. + Task AppendWithCheckpointAsync(IReadOnlyList events, EngineCheckpoint checkpoint); +} + +/// Runs a host side effect after a turn commits successfully (for example, updating usage counters). +public interface IEnginePostCommitPort +{ + /// The post-commit effect failed. This is reported non-fatally on + /// — it never uncommits the turn. + Task AfterCommitAsync(string effectId, TurnCommit commit, CancellationToken cancellationToken); +} + +/// Supplies deterministic or live timestamps for engine events and checkpoints. +public interface IEngineClock +{ + /// Returns the current timestamp, formatted however the host's durable log expects. + string Now(); +} + +/// Supplies deterministic or live identifiers for engine events, checkpoints, runs, and invocations. +public interface IEngineIdGenerator +{ + /// Returns a new identifier for the given identifier kind (for example, "event", "checkpoint", "run"). + string NextId(string kind); +} + +// ----------------------------------------------------------------------- +// Default / no-op port implementations, mirroring ports.rs. +// ----------------------------------------------------------------------- + +/// Permission port that approves every tool request. For hosts that explicitly allow all tools. +public sealed class AllowAllPermissionsPort : IEnginePermissionPort +{ + public Task AuthorizeAsync(ModelToolRequest request, CancellationToken cancellationToken) => + Task.FromResult(new EnginePermissionDecision { Approved = true, Reason = "allow_all" }); +} + +/// Durability port for explicitly non-durable execution profiles. Persists nothing. +public sealed class NoopDurabilityPort : IEngineDurabilityPort +{ + public Task AppendAsync(EngineEvent @event) => Task.CompletedTask; + + public Task AppendWithCheckpointAsync(IReadOnlyList events, EngineCheckpoint checkpoint) => Task.CompletedTask; +} + +/// Post-commit port that performs no side effect. +public sealed class NoopPostCommitPort : IEnginePostCommitPort +{ + public Task AfterCommitAsync(string effectId, TurnCommit commit, CancellationToken cancellationToken) => Task.CompletedTask; +} + +/// Model stream port that drops every chunk. +public sealed class NoopModelStreamPort : IEngineModelStreamPort +{ + public Task EmitAsync(ModelStreamChunk chunk) => Task.CompletedTask; +} + +/// Builds the canonical snapshot without adding external context candidates. +public sealed class PassthroughEngineContextPort : IEngineContextPort +{ + public Task PrepareAsync(ContextRequest request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(new ModelInvocationContextSnapshot + { + Id = $"context:{request.InvocationId}", + SessionId = request.SessionId, + TurnId = request.TurnId, + InvocationId = request.InvocationId, + Iteration = request.Iteration, + Messages = [.. request.Messages], + Decisions = [], + StablePrefixMessages = request.StablePrefixMessages, + ContextState = request.ContextState, + }); + } +} + +/// Host policy port that leaves messages, stable prefix, and final output unchanged. +public sealed class NoopHostPolicyPort : IEngineHostPolicyPort +{ + public Task BeforeModelAsync(HostPolicyRequest request, CancellationToken cancellationToken) => + Task.FromResult(new HostPolicyResult + { + Messages = request.Messages, + StablePrefixMessages = request.StablePrefixMessages, + }); + + public Task BeforeCommitAsync(FinalOutputPolicyRequest request, CancellationToken cancellationToken) => + Task.FromResult(new FinalOutputPolicyResult { Output = request.Output }); +} + +/// Retry policy with no delay and no side effects. +public sealed class NoopRetryPolicyPort : IEngineRetryPolicyPort +{ + public Task BackoffAsync(RetryPolicyRequest request, CancellationToken cancellationToken) => Task.CompletedTask; +} + +/// Provider-neutral fallback that preserves assistant messages and appends ordered tool result messages. +public sealed class DefaultConversationPort : IEngineConversationPort +{ + public IList FormatToolExchange(ModelInvocationResponse response, IReadOnlyList results) + { + var messages = new List(response.AssistantMessages ?? []); + foreach (var request in response.ToolRequests ?? []) + { + var result = results.FirstOrDefault(r => r.RequestId == request.Id); + if (result is not null) + { + messages.Add(Message.ToolResult(request.Id, result.ModelText())); + } + } + + return messages; + } +} diff --git a/runtime/csharp/Prompty.Core/TurnEngineRequest.cs b/runtime/csharp/Prompty.Core/TurnEngineRequest.cs new file mode 100644 index 000000000..27161e880 --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEngineRequest.cs @@ -0,0 +1,374 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Prompty.Core; + +/// +/// Request accepted by the canonical . Mirrors the Rust reference's +/// TurnEngineRequest (runtime/rust/prompty/src/engine/turn.rs) field for field. A fresh +/// turn is built with ; an +/// interrupted turn is resumed with / and the +/// reconciliation-specific resume factories. +/// +public sealed class TurnEngineRequest +{ + public TurnEngineRequest(string sessionId, string turnId, IList messages) + { + SessionId = sessionId; + TurnId = turnId; + Messages = messages; + StablePrefixMessages = messages.Count; + } + + /// Session this turn belongs to. + public string SessionId { get; set; } + + /// Turn identifier, unique within the session. + public string TurnId { get; set; } + + /// Stable identifier of this engine run. Empty means the engine assigns one at run start. + public string RunId { get; set; } = string.Empty; + + /// Run identifier of the parent run when this run was delegated. + public string? ParentRunId { get; set; } + + /// Zero-based delegation nesting depth; 0 for a top-level run. + public int DelegationDepth { get; set; } + + /// Conversation messages seen so far. + public IList Messages { get; set; } + + /// Opaque turn inputs, echoed into context snapshots and host policy requests. + public object? Inputs { get; set; } + + /// Maximum number of model/tool iterations permitted for this run. + public int MaxIterations { get; set; } = 10; + + /// Maximum model invocation attempts per iteration before failing the turn. + public int MaxModelAttempts { get; set; } = 3; + + /// Iteration to execute first. Non-zero values are used when resuming a checkpoint. + public int StartIteration { get; set; } + + /// Last committed event sequence before this run. + public long InitialSequence { get; set; } + + /// Number of leading messages the engine and host policy must not rewrite. + public int StablePrefixMessages { get; set; } + + /// Whether the provider-side context can be reconstructed from alone. + public InvocationContextPortability Portability { get; set; } = InvocationContextPortability.Portable; + + /// Opaque references to delegated provider-side state, required when is not portable. + public IList DelegatedState { get; set; } = []; + + /// Invocation id in progress when this request was built, if any. + public string? ActiveInvocationId { get; set; } + + /// Tool requests from the most recent model response not yet executed. + public IList PendingToolRequests { get; set; } = []; + + /// Tool results already committed for the in-flight model/tool exchange. + public IList CompletedToolResults { get; set; } = []; + + /// Number of model iterations completed so far. + public int CompletedModelIterations { get; set; } + + /// Whether the turn is blocked pending an explicit reconciliation resolution. + public bool ReconciliationRequired { get; set; } + + /// Durable reconciliation state for an indeterminate model invocation, if any. + public ModelReconciliationState? ModelReconciliation { get; set; } + + /// Output pending final host policy application. + public object? PendingOutput { get; set; } + + /// Whether a model response with no further tool requests is ready to commit. + public bool FinalOutputReady { get; set; } + + /// Model response awaiting completion of its tool exchange. + public ModelInvocationResponse? PendingModelResponse { get; set; } + + /// Whether host policy has already been applied for the current iteration (skip re-applying once). + public bool PolicyAppliedForIteration { get; set; } + + /// Resolved tool result supplied when resuming after an indeterminate tool effect. + public ModelToolResult? ReconciliationResolution { get; set; } + + /// Resolved model response supplied when resuming after an indeterminate model invocation. + public ModelInvocationResponse? ModelReconciliationResolution { get; set; } + + /// Set the stable run identifier for this run. An empty value lets the engine assign one at run start. + public TurnEngineRequest WithRunId(string runId) + { + RunId = runId; + return this; + } + + /// + /// Mark this run as delegated from a parent run, carrying the parent run identifier and + /// nesting one level deeper than the parent. + /// + public TurnEngineRequest DelegatedUnder(string parentRunId, int parentDelegationDepth) + { + ParentRunId = parentRunId; + DelegationDepth = parentDelegationDepth + 1; + return this; + } + + /// Resume while continuing a journal whose tail may follow the checkpoint. + public static TurnEngineRequest ResumeFrom(EngineCheckpoint checkpoint, int maxIterations, long lastJournalSequence) + { + var noPendingWork = (checkpoint.PendingToolRequests is null || checkpoint.PendingToolRequests.Count == 0) + && checkpoint.PendingModelResponse is null + && !checkpoint.FinalOutputReady + && !checkpoint.ReconciliationRequired; + var startIteration = checkpoint.ResumeSameIteration + ? checkpoint.Iteration + : noPendingWork ? checkpoint.Iteration + 1 : checkpoint.Iteration; + + return new TurnEngineRequest(checkpoint.SessionId, checkpoint.TurnId, [.. checkpoint.Messages]) + { + RunId = checkpoint.RunId, + ParentRunId = checkpoint.ParentRunId, + DelegationDepth = checkpoint.DelegationDepth, + StablePrefixMessages = checkpoint.StablePrefixMessages, + Inputs = checkpoint.Inputs, + MaxIterations = maxIterations, + MaxModelAttempts = 3, + StartIteration = startIteration, + InitialSequence = Math.Max(lastJournalSequence, checkpoint.LastSequence), + Portability = checkpoint.ContextState.Portability, + DelegatedState = checkpoint.ContextState.DelegatedState is null ? [] : [.. checkpoint.ContextState.DelegatedState], + ActiveInvocationId = checkpoint.ActiveInvocationId, + PendingToolRequests = checkpoint.PendingToolRequests is null ? [] : [.. checkpoint.PendingToolRequests], + CompletedToolResults = checkpoint.CompletedToolResults is null ? [] : [.. checkpoint.CompletedToolResults], + CompletedModelIterations = checkpoint.CompletedModelIterations, + ReconciliationRequired = checkpoint.ReconciliationRequired, + ModelReconciliation = checkpoint.ModelReconciliation, + PendingOutput = checkpoint.PendingOutput, + FinalOutputReady = checkpoint.FinalOutputReady, + PendingModelResponse = checkpoint.PendingModelResponse, + PolicyAppliedForIteration = checkpoint.PolicyAppliedForIteration, + ReconciliationResolution = null, + ModelReconciliationResolution = null, + }; + } + + /// + /// Resume after the host resolves an indeterminate tool effect. Patches the checkpoint's + /// completed tool result and, if the conversation batch was already flushed, the matching + /// tool-result message, then resumes from the patched checkpoint. + /// + /// + /// The checkpoint does not require tool reconciliation, the resolved result is still + /// indeterminate, or the checkpoint has no matching indeterminate tool request. + /// + public static TurnEngineRequest ResumeAfterReconciliation( + EngineCheckpoint checkpoint, + int maxIterations, + long lastJournalSequence, + ModelToolResult resolvedResult) + { + if (!checkpoint.ReconciliationRequired) + { + throw new TurnEngineInvalidRequestException("checkpoint does not require reconciliation"); + } + + if (checkpoint.ModelReconciliation is not null) + { + throw new TurnEngineInvalidRequestException( + "checkpoint requires model reconciliation, not tool reconciliation"); + } + + if (resolvedResult.Outcome == ModelToolOutcome.Indeterminate) + { + throw new TurnEngineInvalidRequestException("resolved tool result must have a determinate outcome"); + } + + var resolved = CloneForPatch(checkpoint); + var results = resolved.CompletedToolResults ??= []; + var index = IndexOfToolResult(results, resolvedResult.RequestId); + if (index < 0) + { + throw new TurnEngineInvalidRequestException( + $"checkpoint does not contain indeterminate tool request '{resolvedResult.RequestId}'"); + } + + if (results[index].Outcome != ModelToolOutcome.Indeterminate) + { + throw new TurnEngineInvalidRequestException( + $"tool request '{resolvedResult.RequestId}' is already determinate"); + } + + results[index] = resolvedResult; + + if (resolved.PendingModelResponse is null) + { + var messageIndex = IndexOfToolResultMessage(resolved.Messages, resolvedResult.RequestId); + if (messageIndex < 0) + { + throw new TurnEngineInvalidRequestException( + $"checkpoint is missing the tool result message for '{resolvedResult.RequestId}'"); + } + + resolved.Messages[messageIndex] = Message.ToolResult(resolvedResult.RequestId, resolvedResult.ModelText()); + } + + resolved.ReconciliationRequired = false; + + var request = ResumeFrom(resolved, maxIterations, lastJournalSequence); + request.ReconciliationResolution = resolvedResult; + return request; + } + + /// + /// Resume after the host resolves an indeterminate model invocation, replaying the same + /// iteration with the resolved response instead of re-invoking the model. + /// + /// + /// The checkpoint does not require model reconciliation, or the reconciliation's invocation + /// id no longer matches the checkpoint's active invocation. + /// + public static TurnEngineRequest ResumeAfterModelReconciliation( + EngineCheckpoint checkpoint, + int maxIterations, + long lastJournalSequence, + ModelInvocationResponse resolvedResponse) + { + if (!checkpoint.ReconciliationRequired) + { + throw new TurnEngineInvalidRequestException("checkpoint does not require reconciliation"); + } + + var reconciliation = checkpoint.ModelReconciliation + ?? throw new TurnEngineInvalidRequestException( + "checkpoint requires tool reconciliation, not model reconciliation"); + + if (checkpoint.ActiveInvocationId != reconciliation.InvocationId) + { + throw new TurnEngineInvalidRequestException( + "model reconciliation identity does not match the active invocation"); + } + + var request = ResumeFrom(checkpoint, maxIterations, lastJournalSequence); + request.StartIteration = checkpoint.Iteration; + request.ReconciliationRequired = false; + request.ModelReconciliationResolution = resolvedResponse; + return request; + } + + /// + /// Build a resume request from the durable generated . Threads + /// from the durable record rather than + /// defaulting it. Use the reconciliation-specific overloads when the checkpoint is blocked + /// pending a resolved model or tool outcome. + /// + public static TurnEngineRequest FromResume(ResumeContext resume) + { + var request = ResumeFrom( + resume.Checkpoint, + Math.Max(resume.MaxIterations, 0), + Math.Max(resume.ResumeSequence(), 0)); + request.ApplyResumeAttempts(resume); + return request; + } + + /// + /// Build a resume request from a after the host resolves an + /// indeterminate tool effect recorded in the checkpoint. + /// + public static TurnEngineRequest FromResumeAfterReconciliation(ResumeContext resume, ModelToolResult resolvedResult) + { + var request = ResumeAfterReconciliation( + resume.Checkpoint, + Math.Max(resume.MaxIterations, 0), + Math.Max(resume.ResumeSequence(), 0), + resolvedResult); + request.ApplyResumeAttempts(resume); + return request; + } + + /// + /// Build a resume request from a after the host resolves an + /// indeterminate model invocation recorded in the checkpoint. + /// + public static TurnEngineRequest FromResumeAfterModelReconciliation( + ResumeContext resume, + ModelInvocationResponse resolvedResponse) + { + var request = ResumeAfterModelReconciliation( + resume.Checkpoint, + Math.Max(resume.MaxIterations, 0), + Math.Max(resume.ResumeSequence(), 0), + resolvedResponse); + request.ApplyResumeAttempts(resume); + return request; + } + + private void ApplyResumeAttempts(ResumeContext resume) + { + if (resume.MaxModelAttempts > 0) + { + MaxModelAttempts = resume.MaxModelAttempts; + } + } + + private static int IndexOfToolResult(IList results, string requestId) + { + for (var i = 0; i < results.Count; i++) + { + if (results[i].RequestId == requestId) + { + return i; + } + } + + return -1; + } + + private static int IndexOfToolResultMessage(IList messages, string requestId) + { + for (var i = 0; i < messages.Count; i++) + { + if (messages[i].Metadata.TryGetValue("tool_call_id", out var value) && + value is string toolCallId && toolCallId == requestId) + { + return i; + } + } + + return -1; + } + + /// + /// Shallow-clone a checkpoint's mutable collections so reconciliation patching never + /// mutates the caller's original checkpoint instance. + /// + private static EngineCheckpoint CloneForPatch(EngineCheckpoint checkpoint) => new() + { + Id = checkpoint.Id, + SessionId = checkpoint.SessionId, + TurnId = checkpoint.TurnId, + RunId = checkpoint.RunId, + ParentRunId = checkpoint.ParentRunId, + DelegationDepth = checkpoint.DelegationDepth, + Iteration = checkpoint.Iteration, + LastSequence = checkpoint.LastSequence, + Messages = [.. checkpoint.Messages], + StablePrefixMessages = checkpoint.StablePrefixMessages, + Inputs = checkpoint.Inputs, + ActiveInvocationId = checkpoint.ActiveInvocationId, + PendingToolRequests = checkpoint.PendingToolRequests is null ? null : [.. checkpoint.PendingToolRequests], + CompletedToolResults = checkpoint.CompletedToolResults is null ? null : [.. checkpoint.CompletedToolResults], + CompletedModelIterations = checkpoint.CompletedModelIterations, + ReconciliationRequired = checkpoint.ReconciliationRequired, + ModelReconciliation = checkpoint.ModelReconciliation, + PendingOutput = checkpoint.PendingOutput, + FinalOutputReady = checkpoint.FinalOutputReady, + PendingModelResponse = checkpoint.PendingModelResponse, + ResumeSameIteration = checkpoint.ResumeSameIteration, + PolicyAppliedForIteration = checkpoint.PolicyAppliedForIteration, + ContextState = checkpoint.ContextState, + Metadata = checkpoint.Metadata, + }; +} diff --git a/runtime/csharp/Prompty.Core/TurnRunner.cs b/runtime/csharp/Prompty.Core/TurnRunner.cs index 928caaac5..bf0423f3a 100644 --- a/runtime/csharp/Prompty.Core/TurnRunner.cs +++ b/runtime/csharp/Prompty.Core/TurnRunner.cs @@ -37,7 +37,7 @@ public ReferenceTurnRunner( public async Task RunAsync(RunTurnRequest request) { var options = request.Options ?? new TurnOptions(); - var inputs = request.Inputs ?? new Dictionary(); + var inputs = request.Inputs ?? new Dictionary(); var maxIterations = options.MaxIterations ?? 10; var checkpoints = new List(); var allToolResults = new List(); @@ -47,12 +47,12 @@ public async Task RunAsync(RunTurnRequest request) var status = RunTurnStatus.Success; var iterations = 0; - RecordSession(SessionEventType.SessionStart, request.SessionId, request.TurnId, new Dictionary + RecordSession(SessionEventType.SessionStart, request.SessionId, request.TurnId, new Dictionary { ["sessionId"] = request.SessionId, ["schemaVersion"] = "1" }); - RecordTurn(TurnEventType.TurnStart, request.TurnId, 0, new Dictionary + RecordTurn(TurnEventType.TurnStart, request.TurnId, 0, new Dictionary { ["inputs"] = inputs, ["maxIterations"] = maxIterations @@ -61,7 +61,7 @@ public async Task RunAsync(RunTurnRequest request) for (var iteration = 0; iteration < maxIterations; iteration++) { iterations = iteration + 1; - RecordTurn(TurnEventType.LlmStart, request.TurnId, iteration, new Dictionary { ["attempt"] = 0 }); + RecordTurn(TurnEventType.LlmStart, request.TurnId, iteration, new Dictionary { ["attempt"] = 0 }); var modelResponse = await _invokeModel(new TurnModelRequest { SessionId = request.SessionId, @@ -71,7 +71,7 @@ public async Task RunAsync(RunTurnRequest request) Options = options, ToolResults = pendingToolResults }); - RecordTurn(TurnEventType.LlmComplete, request.TurnId, iteration, new Dictionary()); + RecordTurn(TurnEventType.LlmComplete, request.TurnId, iteration, new Dictionary()); var checkpoint = await SaveCheckpointAsync(request.SessionId, request.TurnId, iteration, modelResponse); checkpoints.Add(checkpoint); @@ -92,7 +92,7 @@ public async Task RunAsync(RunTurnRequest request) allToolResults.Add(toolResult); } - RecordTurn(TurnEventType.MessagesUpdated, request.TurnId, iteration, new Dictionary + RecordTurn(TurnEventType.MessagesUpdated, request.TurnId, iteration, new Dictionary { ["toolResults"] = pendingToolResults.Select(result => result.Save()).ToList() }); @@ -102,7 +102,7 @@ public async Task RunAsync(RunTurnRequest request) { status = RunTurnStatus.Error; output = new Dictionary { ["message"] = "Maximum turn iterations reached" }; - RecordTurn(TurnEventType.Error, request.TurnId, iterations, new Dictionary + RecordTurn(TurnEventType.Error, request.TurnId, iterations, new Dictionary { ["errorKind"] = "max_iterations", ["message"] = "Maximum turn iterations reached" @@ -115,7 +115,7 @@ public async Task RunAsync(RunTurnRequest request) ["status"] = status == RunTurnStatus.Success ? "success" : "error", ["response"] = output }!); - RecordSession(SessionEventType.SessionEnd, request.SessionId, request.TurnId, new Dictionary + RecordSession(SessionEventType.SessionEnd, request.SessionId, request.TurnId, new Dictionary { ["sessionId"] = request.SessionId, ["status"] = status == RunTurnStatus.Success ? "success" : "error", @@ -149,7 +149,7 @@ private async Task SaveCheckpointAsync(string sessionId, string turn ["output"] = response.Output, ["toolRequests"] = (response.ToolRequests ?? []).Select(request => request.Save()).ToList() }; - foreach (var (key, value) in response.CheckpointState ?? new Dictionary()) + foreach (var (key, value) in response.CheckpointState ?? new Dictionary()) { state[key] = value; } @@ -189,7 +189,7 @@ private async Task ResolveAndExecuteToolAsync(string turnId, int if (!decision.Approved) { - return new HostToolResult + var deniedResult = new HostToolResult { RequestId = toolRequest.RequestId, ToolCallId = toolRequest.ToolCallId, @@ -198,6 +198,8 @@ private async Task ResolveAndExecuteToolAsync(string turnId, int ErrorKind = "permission_denied", Result = new Dictionary { ["message"] = decision.Reason ?? "Permission denied" } }; + RecordTurn(TurnEventType.ToolResult, turnId, iteration, deniedResult.Save()); + return deniedResult; } RecordTurn(TurnEventType.ToolExecutionStart, turnId, iteration, toolRequest.Save()); @@ -207,7 +209,11 @@ private async Task ResolveAndExecuteToolAsync(string turnId, int return result; } - private void RecordTurn(TurnEventType type, string turnId, int iteration, IDictionary payload) + private void RecordTurn( + TurnEventType type, + string turnId, + int iteration, + IDictionary payload) { var turnEvent = new TurnEvent { @@ -222,7 +228,7 @@ private void RecordTurn(TurnEventType type, string turnId, int iteration, IDicti _journal.AppendTurn(turnEvent); } - private void RecordSession(SessionEventType type, string sessionId, string turnId, IDictionary payload) + private void RecordSession(SessionEventType type, string sessionId, string turnId, IDictionary payload) { var sessionEvent = new SessionEvent { diff --git a/runtime/csharp/Prompty.Foundry.Tests/FoundryModelDiscoveryTests.cs b/runtime/csharp/Prompty.Foundry.Tests/FoundryModelDiscoveryTests.cs index 2c3b59ccb..364e65613 100644 --- a/runtime/csharp/Prompty.Foundry.Tests/FoundryModelDiscoveryTests.cs +++ b/runtime/csharp/Prompty.Foundry.Tests/FoundryModelDiscoveryTests.cs @@ -87,4 +87,21 @@ public async Task ListModelsAsync_ReferenceDeploymentClient_ReturnsDeploymentsWi Assert.Equal(new[] { "text", "json" }, model.OutputModalities); Assert.NotNull(model.AdditionalProperties); } + + [Fact] + public void MapCatalogModel_PreservesExplicitJsonNull() + { + using var document = System.Text.Json.JsonDocument.Parse( + """{"id":"gpt-test","owned_by":"contoso","nullable":null}"""); + + var model = FoundryModels.MapCatalogModel(document.RootElement); + + Assert.NotNull(model.AdditionalProperties); + var nullable = Assert.IsType(model.AdditionalProperties["nullable"]); + Assert.Equal(System.Text.Json.JsonValueKind.Null, nullable.ValueKind); + using var saved = System.Text.Json.JsonDocument.Parse(model.ToJson(indent: false)); + Assert.Equal( + System.Text.Json.JsonValueKind.Null, + saved.RootElement.GetProperty("additionalProperties").GetProperty("nullable").ValueKind); + } } diff --git a/runtime/csharp/Prompty.Foundry.Tests/TestHelpers.cs b/runtime/csharp/Prompty.Foundry.Tests/TestHelpers.cs index 51ef13ab5..59690f74c 100644 --- a/runtime/csharp/Prompty.Foundry.Tests/TestHelpers.cs +++ b/runtime/csharp/Prompty.Foundry.Tests/TestHelpers.cs @@ -97,7 +97,7 @@ internal static Message CreateAssistantWithToolCalls(string text, List { Role = Role.Assistant, Parts = [new TextPart { Value = text }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = toolCalls, }, @@ -113,7 +113,7 @@ internal static Message CreateToolMessage(string toolCallId, string content) { Role = Role.Tool, Parts = [new TextPart { Value = content }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCallId, }, diff --git a/runtime/csharp/Prompty.Foundry/Models.cs b/runtime/csharp/Prompty.Foundry/Models.cs index f61c271c1..e9405b4a6 100644 --- a/runtime/csharp/Prompty.Foundry/Models.cs +++ b/runtime/csharp/Prompty.Foundry/Models.cs @@ -11,6 +11,21 @@ namespace Prompty.Foundry; +/// +/// Foundry implementation of the generated model-listing protocol. +/// +public sealed class FoundryModelLister : IModelLister +{ + /// + public async Task> ListModelsAsync(object connection) + { + if (connection is not Connection typedConnection) + throw new ArgumentException("Foundry model listing requires a generated Connection.", nameof(connection)); + + return [.. await FoundryModels.ListModelsAsync(typedConnection)]; + } +} + /// /// Model discovery for Azure OpenAI / Microsoft Foundry endpoints. /// Creates the appropriate client (API key or Entra ID) and delegates @@ -85,29 +100,56 @@ private static async Task> ListDeploymentsAsync( return models.AsReadOnly(); } - private static ModelInfo MapDeployment(JsonElement deployment) + /// + /// Map a raw Foundry deployment payload to the generated provider-neutral contract. + /// + public static ModelInfo MapDeployment(JsonElement deployment) { var properties = TryGetObject(deployment, "properties"); var model = properties is not null ? TryGetObject(properties.Value, "model") : null; var capabilities = properties is not null ? TryGetObject(properties.Value, "capabilities") : null; capabilities ??= model is not null ? TryGetObject(model.Value, "capabilities") : null; + capabilities ??= TryGetObject(deployment, "capabilities"); - return new ModelInfo + var info = new ModelInfo { Id = GetString(deployment, "name") ?? string.Empty, - DisplayName = model is not null ? GetString(model.Value, "name") : null, - OwnedBy = model is not null ? GetString(model.Value, "publisher") ?? "azure" : "azure", + DisplayName = GetString(deployment, "modelName") + ?? (model is not null ? GetString(model.Value, "name") : null), + OwnedBy = GetString(deployment, "modelPublisher") + ?? (model is not null ? GetString(model.Value, "publisher") : null) + ?? "azure", ContextWindow = capabilities is not null ? GetInt(capabilities.Value, "maxContextLength", "contextWindow", "context_length") - : model is not null ? GetInt(model.Value, "maxContextLength") : null, + : null, InputModalities = capabilities is not null ? GetStringList(capabilities.Value, "inputModalities", "input_modalities", "supportedInputModalities") : null, OutputModalities = capabilities is not null ? GetStringList(capabilities.Value, "outputModalities", "output_modalities", "supportedOutputModalities") : null, - AdditionalProperties = new Dictionary { ["deployment"] = deployment.Clone() }, + AdditionalProperties = ModelDiscovery.PreserveRaw(deployment), + }; + info.ContextWindow ??= model is not null ? GetInt(model.Value, "maxContextLength") : null; + info.ContextWindow ??= GetInt(deployment, "maxContextLength"); + ModelDiscovery.Enrich("foundry", info); + return info; + } + + /// + /// Map a raw Azure OpenAI catalog payload to the generated provider-neutral contract. + /// + public static ModelInfo MapCatalogModel(JsonElement model) + { + var info = new ModelInfo + { + Id = GetString(model, "id") ?? string.Empty, + OwnedBy = GetString(model, "owned_by"), + ContextWindow = GetInt(model, "maxContextLength"), + AdditionalProperties = ModelDiscovery.PreserveRaw(model), }; + ModelDiscovery.Enrich("foundry", info); + return info; } private static JsonElement? TryGetObject(JsonElement element, string name) => diff --git a/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs index 563759e14..133ff23c6 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs @@ -327,10 +327,10 @@ private static List DefaultFormatToolMessages(List toolCalls, { var messages = new List { - new() { Role = Role.Assistant, Parts = string.IsNullOrEmpty(textContent) ? [] : [new TextPart { Value = textContent }], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, + new() { Role = Role.Assistant, Parts = string.IsNullOrEmpty(textContent) ? [] : [new TextPart { Value = textContent }], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, }; for (var i = 0; i < toolCalls.Count; i++) - messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name } }); + messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name } }); return messages; } } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/Integration/IntegrationTestBase.cs b/runtime/csharp/Prompty.OpenAI.Tests/Integration/IntegrationTestBase.cs index e0d32acfd..fad91d480 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/Integration/IntegrationTestBase.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/Integration/IntegrationTestBase.cs @@ -77,7 +77,7 @@ protected static Core.Prompty MakeOpenAIAgent( ModelOptions? options = null, IList? tools = null, IList? outputs = null, - IDictionary? metadata = null) + IDictionary? metadata = null) { var apiKey = GetEnvOrSkip("OPENAI_API_KEY"); model ??= OpenAIModel; @@ -129,7 +129,7 @@ protected static Core.Prompty MakeFoundryAgent( ModelOptions? options = null, IList? tools = null, IList? outputs = null, - IDictionary? metadata = null) + IDictionary? metadata = null) { var apiKey = GetEnvOrSkip("AZURE_OPENAI_API_KEY"); var endpoint = GetEnvOrSkip("AZURE_OPENAI_ENDPOINT"); @@ -180,7 +180,7 @@ protected static Core.Prompty MakeAnthropicAgent( ModelOptions? options = null, IList? tools = null, IList? outputs = null, - IDictionary? metadata = null) + IDictionary? metadata = null) { var apiKey = GetEnvOrSkip("ANTHROPIC_API_KEY"); diff --git a/runtime/csharp/Prompty.OpenAI.Tests/Integration/ResponsesApiTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/Integration/ResponsesApiTests.cs index 115c9838b..8e02b9c09 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/Integration/ResponsesApiTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/Integration/ResponsesApiTests.cs @@ -64,7 +64,7 @@ public async Task OpenAI_ResponsesApi_Streaming() var agent = MakeOpenAIAgent( apiType: "responses", options: new ModelOptions { Temperature = 0.5f, MaxOutputTokens = 200 }, - metadata: new Dictionary { ["stream"] = true }); + metadata: new Dictionary { ["stream"] = true }); var messages = new List { diff --git a/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingAgentTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingAgentTests.cs index f05430da5..3a4030312 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingAgentTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingAgentTests.cs @@ -37,7 +37,7 @@ private static void RegisterProviders() private static Dictionary>> WeatherToolFunctions() => new() { ["get_weather"] = GetWeatherAsync }; - private static Dictionary StreamingMetadata() => + private static Dictionary StreamingMetadata() => new() { ["stream"] = true }; // ----------------------------------------------------------------------- diff --git a/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingTests.cs index ad10a94f5..3678582e3 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingTests.cs @@ -15,7 +15,7 @@ public class StreamingTests : IntegrationTestBase { private static void EnableStreaming(Core.Prompty agent) { - agent.Metadata ??= new Dictionary(); + agent.Metadata ??= new Dictionary(); agent.Metadata["stream"] = true; } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/ResponsesApiTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/ResponsesApiTests.cs index d84620320..fdac335ab 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/ResponsesApiTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/ResponsesApiTests.cs @@ -167,7 +167,7 @@ public void MessageToResponsesInput_FunctionCallPassthrough_ReturnsStoredItem() { Role = Role.Assistant, Parts = [new TextPart { Value = "" }], - Metadata = new Dictionary + Metadata = new Dictionary { ["responses_function_call"] = storedItem, }, diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs index a4a0af53d..93c987758 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs @@ -369,7 +369,16 @@ public async Task AgentLoop_ExtensionVectors(string _name, JsonElement input, Js } // Execute and verify - if (expected.TryGetProperty("error", out var errorProp)) + if (expected.TryGetProperty("rust_expected_error", out var rustError) + && rustError.GetString()?.Contains("parallel_tool_calls=true", StringComparison.Ordinal) == true) + { + var error = await Assert.ThrowsAsync(() => + Pipeline.TurnAsync(agent, tools: toolFunctions, onEvent: onEvent, + cancellationToken: cts?.Token ?? default, contextBudget: contextBudget, + guardrails: guardrails, steering: steering, parallelToolCalls: parallelToolCalls)); + Assert.Contains("sequentially", error.Message); + } + else if (expected.TryGetProperty("error", out var errorProp)) { var errorMsg = errorProp.GetString() ?? ""; if (errorMsg == "CancelledError" || errorMsg.Contains("cancelled", StringComparison.OrdinalIgnoreCase)) @@ -722,10 +731,10 @@ public List FormatToolMessages(object rawResponse, List toolC { var messages = new List { - new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, + new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, }; for (var i = 0; i < toolCalls.Count; i++) - messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name } }); + messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name } }); return messages; } } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorDiscoveryTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorDiscoveryTests.cs new file mode 100644 index 000000000..27bd14e18 --- /dev/null +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorDiscoveryTests.cs @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Anthropic; +using Prompty.Core; +using Prompty.Foundry; + +namespace Prompty.OpenAI.Tests; + +/// +/// Executes the shared provider discovery and capability-enrichment contracts. +/// +public class SpecVectorDiscoveryTests +{ + private static readonly string SpecDir = FindSpecDir(); + + [Theory] + [MemberData(nameof(DiscoveryVectors))] + public void Discovery_Mapping_MatchesSharedVector( + string name, + string provider, + string shape, + JsonElement input, + JsonElement expected) + { + var actual = (provider, shape) switch + { + ("openai", "model") => OpenAIModels.MapModel(input), + ("anthropic", "model") => AnthropicModels.MapModel(input), + ("foundry", "deployment") => FoundryModels.MapDeployment(input), + ("foundry", "catalog") => FoundryModels.MapCatalogModel(input), + _ => throw new InvalidOperationException($"Unsupported discovery vector {provider}/{shape}."), + }; + + AssertJsonEqual(expected, JsonSerializer.SerializeToElement(actual.Save()), name); + } + + [Theory] + [MemberData(nameof(EnrichmentVectors))] + public void Enrichment_MatchesSharedVector( + string name, + string provider, + JsonElement input, + JsonElement expected) + { + var model = ModelInfo.Load((Dictionary)ConvertValue(input)!); + + ModelDiscovery.Enrich(provider, model); + + AssertJsonEqual(expected, JsonSerializer.SerializeToElement(model.Save()), name); + } + + [Fact] + public void EmbeddedCapabilityDataset_MatchesCanonicalSpec() + { + var canonical = JsonNode.Parse(File.ReadAllText(Path.Combine(SpecDir, "data", "model_capabilities.json"))); + var embeddedPath = Path.Combine( + FindRepositoryRoot(), + "runtime", + "csharp", + "Prompty.Core", + "Data", + "model_capabilities.json"); + var embedded = JsonNode.Parse(File.ReadAllText(embeddedPath)); + + Assert.True(JsonNode.DeepEquals(canonical, embedded), "The C# capability dataset copy has drifted from spec/data."); + } + + [Fact] + public void Enrichment_IsCaseSensitiveAndReturnsIndependentLists() + { + var upperCase = new ModelInfo { Id = "GPT-4O" }; + ModelDiscovery.Enrich("openai", upperCase); + Assert.Null(upperCase.ContextWindow); + + var first = new ModelInfo { Id = "gpt-4o" }; + ModelDiscovery.Enrich("openai", first); + first.InputModalities![0] = "mutated"; + + var second = new ModelInfo { Id = "gpt-4o" }; + ModelDiscovery.Enrich("openai", second); + Assert.Equal("text", second.InputModalities![0]); + } + + public static IEnumerable DiscoveryVectors() => + LoadVectors("discovery_vectors.json") + .Select(vector => new object[] + { + vector.GetProperty("name").GetString()!, + vector.GetProperty("provider").GetString()!, + vector.GetProperty("shape").GetString()!, + vector.GetProperty("input").Clone(), + vector.GetProperty("expected").Clone(), + }); + + public static IEnumerable EnrichmentVectors() => + LoadVectors("enrichment_vectors.json") + .Select(vector => new object[] + { + vector.GetProperty("name").GetString()!, + vector.GetProperty("provider").GetString()!, + vector.GetProperty("input").Clone(), + vector.GetProperty("expected").Clone(), + }); + + private static JsonElement[] LoadVectors(string fileName) + { + using var document = JsonDocument.Parse( + File.ReadAllText(Path.Combine(SpecDir, "vectors", "discovery", fileName))); + return document.RootElement.GetProperty("vectors").EnumerateArray().Select(item => item.Clone()).ToArray(); + } + + private static void AssertJsonEqual(JsonElement expected, JsonElement actual, string name) + { + var expectedNode = JsonNode.Parse(expected.GetRawText()); + var actualNode = JsonNode.Parse(actual.GetRawText()); + Assert.True( + JsonNode.DeepEquals(expectedNode, actualNode), + $"[{name}] expected {expected.GetRawText()}, actual {actual.GetRawText()}"); + } + + private static object? ConvertValue(JsonElement value) => + value.ValueKind switch + { + JsonValueKind.Object => value.EnumerateObject() + .ToDictionary(property => property.Name, property => ConvertValue(property.Value)), + JsonValueKind.Array => value.EnumerateArray().Select(ConvertValue).ToList(), + JsonValueKind.String => value.GetString(), + JsonValueKind.Number when value.TryGetInt32(out var intValue) => intValue, + JsonValueKind.Number => value.GetDouble(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null => null, + _ => value.Clone(), + }; + + private static string FindSpecDir() => Path.Combine(FindRepositoryRoot(), "spec"); + + private static string FindRepositoryRoot() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + if (Directory.Exists(Path.Combine(directory, "spec"))) + return directory; + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new DirectoryNotFoundException("Could not locate repository root containing spec/."); + } +} diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs index aa12b8637..ab91d96cf 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs @@ -123,6 +123,17 @@ public void OpenAI_Chat_WireFormat(string name, JsonElement input, JsonElement e // 4. Check options (temperature, max_completion_tokens, etc.) AssertOptionFields(optionsJson, expectedBody, name); + if (agent.Model?.Options?.AdditionalProperties is not null) + { + foreach (var optionName in agent.Model.Options.AdditionalProperties.Keys) + { + Assert.True(expectedBody.TryGetProperty(optionName, out var expectedOption), + $"[{name}] Expected vector body to contain passthrough option '{optionName}'"); + Assert.True(optionsJson.TryGetProperty(optionName, out var actualOption), + $"[{name}] Serialized options omitted passthrough option '{optionName}'"); + AssertJsonSubset(expectedOption, actualOption, $"[{name}] {optionName}"); + } + } // 5. Check response_format (structured output) if (expectedBody.TryGetProperty("response_format", out var expectedRF)) @@ -143,16 +154,11 @@ public static IEnumerable OpenAIChatVectors() if (provider != "openai") continue; if (apiType != "chat") continue; - var name = vec.GetProperty("name").GetString()!; - - // Skip vectors that require features not yet implemented in C# - if (name is "chat_audio_part" or "chat_audio_mp3") - continue; // AudioPart not handled in WireFormat.BuildContentParts - - if (name == "options_additional_properties") - continue; // AdditionalProperties passthrough not implemented in BuildOptions - - yield return [name, input, vec.GetProperty("expected").GetProperty("request_body")]; + yield return [ + vec.GetProperty("name").GetString()!, + input, + vec.GetProperty("expected").GetProperty("request_body"), + ]; } } @@ -375,12 +381,10 @@ public void Anthropic_Chat_WireFormat(string name, JsonElement input, JsonElemen if (prop.Name == "messages") { - // Messages need special handling for content simplification: - // The runtime may simplify single-text content to a string while vectors use array form. Assert.Equal(prop.Value.GetArrayLength(), actualProp.GetArrayLength()); for (int i = 0; i < prop.Value.GetArrayLength(); i++) { - AssertAnthropicMessageMatches(actualProp[i], prop.Value[i], $"[{name}] messages[{i}]"); + AssertJsonSubset(prop.Value[i], actualProp[i], $"[{name}] messages[{i}]"); } } else @@ -649,63 +653,6 @@ private static void AssertMessageMatches(JsonElement actual, JsonElement expecte } } - /// - /// Compares an Anthropic message where the runtime may simplify single-text content - /// to a string (e.g., "content": "Hello") while vectors use the array form - /// (e.g., "content": [{"type": "text", "text": "Hello"}]). - /// - private static void AssertAnthropicMessageMatches(JsonElement actual, JsonElement expected, string context) - { - // Check role - if (expected.TryGetProperty("role", out var expectedRole)) - { - Assert.True(actual.TryGetProperty("role", out var actualRole), - $"{context}: missing 'role' in actual. Actual: {actual.GetRawText()}"); - Assert.Equal(expectedRole.GetString(), actualRole.GetString()); - } - - // Check content with simplification handling - if (expected.TryGetProperty("content", out var expectedContent)) - { - Assert.True(actual.TryGetProperty("content", out var actualContent), - $"{context}: missing 'content' in actual. Actual: {actual.GetRawText()}"); - - if (expectedContent.ValueKind == JsonValueKind.Array && actualContent.ValueKind == JsonValueKind.String) - { - // Runtime simplified: expected array with single text, actual is string - if (expectedContent.GetArrayLength() == 1) - { - var part = expectedContent[0]; - if (part.TryGetProperty("text", out var textProp)) - { - Assert.Equal(textProp.GetString(), actualContent.GetString()); - return; - } - } - Assert.Fail($"{context}: expected array content but got simplified string '{actualContent.GetString()}'"); - } - else if (expectedContent.ValueKind == JsonValueKind.String && actualContent.ValueKind == JsonValueKind.Array) - { - // Opposite: expected string, actual array - if (actualContent.GetArrayLength() == 1) - { - var part = actualContent[0]; - if (part.TryGetProperty("text", out var textProp)) - { - Assert.Equal(expectedContent.GetString(), textProp.GetString()); - return; - } - } - Assert.Fail($"{context}: expected string content but got array: {actualContent.GetRawText()}"); - } - else - { - // Same kind — use standard comparison - AssertJsonSubset(expectedContent, actualContent, $"{context}.content"); - } - } - } - /// /// Compares a Responses API input item. The SDK serializes items with extra fields /// (type: "message", content: [{type: "input_text", text: "..."}]) while vectors diff --git a/runtime/csharp/Prompty.OpenAI.Tests/StreamingTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/StreamingTests.cs index 8a12b745c..8be5571de 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/StreamingTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/StreamingTests.cs @@ -108,7 +108,7 @@ public async Task ProcessedStream_ExtractsDeltaText() // The processor should wrap it in a ProcessedStream var agent = TestHelpers.CreateAgent(); - agent.Metadata = new Dictionary { ["stream"] = true }; + agent.Metadata = new Dictionary { ["stream"] = true }; var result = await processor.ProcessAsync(agent, rawStream); Assert.IsType(result); @@ -195,7 +195,7 @@ public async Task AnthropicProcessor_HandlesStreamInput() public void StreamMetadata_EnablesStreaming() { var agent = TestHelpers.CreateAgent(); - agent.Metadata = new Dictionary { ["stream"] = true }; + agent.Metadata = new Dictionary { ["stream"] = true }; var streamVal = agent.Metadata.TryGetValue("stream", out var val) && val is true; Assert.True(streamVal); diff --git a/runtime/csharp/Prompty.OpenAI.Tests/TestHelpers.cs b/runtime/csharp/Prompty.OpenAI.Tests/TestHelpers.cs index 64961ef40..798150417 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/TestHelpers.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/TestHelpers.cs @@ -97,7 +97,7 @@ internal static Message CreateAssistantWithToolCalls(string text, List { Role = Role.Assistant, Parts = [new TextPart { Value = text }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = toolCalls, }, @@ -113,7 +113,7 @@ internal static Message CreateToolMessage(string toolCallId, string content) { Role = Role.Tool, Parts = [new TextPart { Value = content }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCallId, }, diff --git a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs index e14ead199..80b454a5a 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System.Text.Json; +using System.ClientModel.Primitives; using OpenAI.Chat; using Prompty.Core; using Prompty.OpenAI; @@ -73,6 +74,56 @@ public void MessageToWire_UserMultimodalMessage_ReturnsUserChatMessage() Assert.IsType(result); } + [Fact] + public void MessageToWire_AudioDataUri_PreservesSource() + { + var msg = new Message + { + Role = Role.User, + Parts = + [ + new AudioPart + { + Source = "data:audio/wav;base64,YXVkaW8=", + MediaType = "audio/wav", + }, + ], + }; + + var result = WireFormat.MessageToWire(msg); + var data = ModelReaderWriter.Write(result, ModelReaderWriterOptions.Json); + using var json = JsonDocument.Parse(data.ToStream()); + var inputAudio = json.RootElement.GetProperty("content")[0].GetProperty("input_audio"); + + Assert.Equal("data:audio/wav;base64,YXVkaW8=", inputAudio.GetProperty("data").GetString()); + Assert.Equal("wav", inputAudio.GetProperty("format").GetString()); + } + + [Fact] + public void MessageToWire_AudioUrl_PreservesSource() + { + var msg = new Message + { + Role = Role.User, + Parts = + [ + new AudioPart + { + Source = "https://example.com/audio.wav", + MediaType = "audio/wav", + }, + ], + }; + + var result = WireFormat.MessageToWire(msg); + var data = ModelReaderWriter.Write(result, ModelReaderWriterOptions.Json); + using var json = JsonDocument.Parse(data.ToStream()); + var inputAudio = json.RootElement.GetProperty("content")[0].GetProperty("input_audio"); + + Assert.Equal("https://example.com/audio.wav", inputAudio.GetProperty("data").GetString()); + Assert.Equal("wav", inputAudio.GetProperty("format").GetString()); + } + [Fact] public void MessageToWire_AssistantMessage_ReturnsAssistantChatMessage() { @@ -94,7 +145,7 @@ public void MessageToWire_AssistantWithToolCalls_PreservesToolCalls() { Role = Role.Assistant, Parts = [new TextPart { Value = "" }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = new List { @@ -116,7 +167,7 @@ public void MessageToWire_ToolMessage_ReturnsToolChatMessage() { Role = Role.Tool, Parts = [new TextPart { Value = "72°F and sunny" }], - Metadata = new Dictionary { ["tool_call_id"] = "call_1" }, + Metadata = new Dictionary { ["tool_call_id"] = "call_1" }, }; var result = WireFormat.MessageToWire(msg); @@ -430,5 +481,53 @@ public void BuildOptions_AllOptions_SetsEverything() Assert.Equal(123L, result.Seed); Assert.Single(result.StopSequences); } -} + [Fact] + public void BuildOptions_AdditionalProperties_DoNotOverrideCanonicalOptions() + { + var agent = new Core.Prompty + { + Model = new Model + { + Options = new ModelOptions + { + Temperature = 0.2f, + AdditionalProperties = new Dictionary + { + ["temperature"] = 0.9, + ["logprobs"] = true, + }, + }, + }, + }; + + var result = WireFormat.BuildOptions(agent); + var json = JsonDocument.Parse( + ModelReaderWriter.Write(result, ModelReaderWriterOptions.Json).ToStream()).RootElement; + + Assert.Equal(0.2, json.GetProperty("temperature").GetDouble(), precision: 6); + Assert.True(json.GetProperty("logprobs").GetBoolean()); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("nested.option")] + public void BuildOptions_UnsafeAdditionalPropertyName_Throws(string optionName) + { + var agent = new Core.Prompty + { + Model = new Model + { + Options = new ModelOptions + { + AdditionalProperties = new Dictionary { [optionName] = true }, + }, + }, + }; + + var error = Assert.Throws(() => WireFormat.BuildOptions(agent)); + + Assert.Contains("cannot be represented safely", error.Message); + } +} diff --git a/runtime/csharp/Prompty.OpenAI/Models.cs b/runtime/csharp/Prompty.OpenAI/Models.cs index bd5f99d81..c2b85bef7 100644 --- a/runtime/csharp/Prompty.OpenAI/Models.cs +++ b/runtime/csharp/Prompty.OpenAI/Models.cs @@ -1,35 +1,34 @@ // Copyright (c) Microsoft. All rights reserved. using System.ClientModel; +using System.Text.Json; using OpenAI; using OpenAI.Models; using Prompty.Core; namespace Prompty.OpenAI; +/// +/// OpenAI implementation of the generated model-listing protocol. +/// +public sealed class OpenAIModelLister : IModelLister +{ + /// + public async Task> ListModelsAsync(object connection) + { + if (connection is not Connection typedConnection) + throw new ArgumentException("OpenAI model listing requires a generated Connection.", nameof(connection)); + + return [.. await OpenAIModels.ListModelsAsync(typedConnection)]; + } +} + /// /// Model discovery for OpenAI — lists available models and enriches /// sparse API responses with known context window and modality metadata. /// public static class OpenAIModels { - /// - /// Known model metadata used to enrich the sparse data returned by GET /v1/models. - /// Keys are sorted by descending length so prefix matching finds the most specific - /// match first (e.g. "gpt-4o-mini" before "gpt-4o" before "gpt-4"). - /// - private static readonly (string Key, int? ContextWindow, string[] Inputs, string[] Outputs)[] KnownModels = - [ - ("text-embedding-3-large", 8191, ["text"], []), - ("text-embedding-3-small", 8191, ["text"], []), - ("gpt-3.5-turbo", 16385, ["text"], ["text"]), - ("gpt-4o-mini", 128000, ["text", "image"], ["text"]), - ("gpt-4-turbo", 128000, ["text", "image"], ["text"]), - ("dall-e-3", null, ["text"], ["image"]), - ("gpt-4o", 128000, ["text", "image"], ["text"]), - ("gpt-4", 8192, ["text"], ["text"]), - ]; - /// /// List models available from an OpenAI endpoint using connection credentials. /// @@ -58,8 +57,15 @@ public static async Task> ListModelsAsync( { Id = m.Id, OwnedBy = m.OwnedBy, + AdditionalProperties = new Dictionary + { + ["id"] = m.Id, + ["object"] = "model", + ["created"] = m.CreatedAt.ToUnixTimeSeconds(), + ["owned_by"] = m.OwnedBy, + }, }; - Enrich(info); + ModelDiscovery.Enrich("openai", info); models.Add(info); } @@ -73,17 +79,22 @@ public static async Task> ListModelsAsync( /// internal static void Enrich(ModelInfo info) { - foreach (var (key, contextWindow, inputs, outputs) in KnownModels) + ModelDiscovery.Enrich("openai", info); + } + + /// + /// Map a raw OpenAI model payload to the generated provider-neutral contract. + /// + public static ModelInfo MapModel(JsonElement model) + { + var info = new ModelInfo { - if (string.Equals(info.Id, key, StringComparison.OrdinalIgnoreCase) - || info.Id.StartsWith(key + "-", StringComparison.OrdinalIgnoreCase)) - { - info.ContextWindow = contextWindow; - info.InputModalities = inputs; - info.OutputModalities = outputs; - return; - } - } + Id = model.TryGetProperty("id", out var id) ? id.GetString() ?? string.Empty : string.Empty, + OwnedBy = model.TryGetProperty("owned_by", out var ownedBy) ? ownedBy.GetString() : null, + AdditionalProperties = ModelDiscovery.PreserveRaw(model), + }; + ModelDiscovery.Enrich("openai", info); + return info; } private static OpenAIClient CreateClient(Connection connection) diff --git a/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs b/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs index 335b635bd..6c3dc640b 100644 --- a/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs +++ b/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs @@ -193,7 +193,7 @@ public virtual List FormatToolMessages( { Role = Role.Assistant, Parts = [], - Metadata = new Dictionary { ["responses_function_call"] = (ResponseItem)fc }, + Metadata = new Dictionary { ["responses_function_call"] = (ResponseItem)fc }, }); } @@ -204,7 +204,7 @@ public virtual List FormatToolMessages( { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name, @@ -224,7 +224,7 @@ public virtual List FormatToolMessages( { Role = Role.Assistant, Parts = assistantParts, - Metadata = new Dictionary { ["tool_calls"] = toolCalls.ToList() }, + Metadata = new Dictionary { ["tool_calls"] = toolCalls.ToList() }, }); // --- One tool message per result --- @@ -234,7 +234,7 @@ public virtual List FormatToolMessages( { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name, diff --git a/runtime/csharp/Prompty.OpenAI/Prompty.OpenAI.csproj b/runtime/csharp/Prompty.OpenAI/Prompty.OpenAI.csproj index 6c8c5d436..a99192c29 100644 --- a/runtime/csharp/Prompty.OpenAI/Prompty.OpenAI.csproj +++ b/runtime/csharp/Prompty.OpenAI/Prompty.OpenAI.csproj @@ -4,7 +4,7 @@ net9.0 enable enable - OPENAI001;CS1591 + OPENAI001;SCME0001;CS1591 true 2.0.0-beta.4 Prompty.OpenAI diff --git a/runtime/csharp/Prompty.OpenAI/WireFormat.cs b/runtime/csharp/Prompty.OpenAI/WireFormat.cs index 2da3e4909..9ac1bbf1f 100644 --- a/runtime/csharp/Prompty.OpenAI/WireFormat.cs +++ b/runtime/csharp/Prompty.OpenAI/WireFormat.cs @@ -2,6 +2,8 @@ #pragma warning disable OPENAI001 // Responses API is in preview +using System.ClientModel.Primitives; +using System.Text; using System.Text.Json; using OpenAI.Chat; using OpenAI.Responses; @@ -62,10 +64,57 @@ private static IEnumerable BuildContentParts(IList ChatImageDetailLevel.Auto, }); break; + case AudioPart audio: + yield return BuildAudioContentPart(audio); + break; + case FilePart file: + var filePart = ChatMessageContentPart.CreateFilePart(file.Source); + filePart.Patch.Set( + "$.file"u8, + BinaryData.FromObjectAsJson(new Dictionary { ["url"] = file.Source }).ToMemory().Span); + yield return filePart; + break; } } } + private static ChatMessageContentPart BuildAudioContentPart(AudioPart audio) + { + byte[] bytes; + try + { + bytes = Convert.FromBase64String(audio.Source); + } + catch (FormatException) + { + bytes = Encoding.UTF8.GetBytes(audio.Source); + } + + var format = AudioFormat(audio.MediaType); + var part = ChatMessageContentPart.CreateInputAudioPart( + BinaryData.FromBytes(bytes), + format == "mp3" ? ChatInputAudioFormat.Mp3 : ChatInputAudioFormat.Wav); + part.Patch.Set("$.input_audio.data"u8, BinaryData.FromObjectAsJson(audio.Source).ToMemory().Span); + part.Patch.Set("$.input_audio.format"u8, BinaryData.FromObjectAsJson(format).ToMemory().Span); + return part; + } + + private static string AudioFormat(string? mediaType) + { + return mediaType?.ToLowerInvariant() switch + { + "audio/wav" or "audio/x-wav" => "wav", + "audio/mpeg" or "audio/mp3" => "mp3", + "audio/mp4" => "mp4", + "audio/ogg" => "ogg", + "audio/flac" => "flac", + "audio/webm" => "webm", + "audio/pcm" => "pcm", + { } value when value.StartsWith("audio/", StringComparison.Ordinal) => value["audio/".Length..], + _ => "wav" + }; + } + private static AssistantChatMessage BuildAssistantMessage(Message msg) { var assistant = new AssistantChatMessage(msg.Text); @@ -156,11 +205,30 @@ public static ChatCompletionOptions BuildOptions(Core.Prompty agent) options.PresencePenalty = (float)opts.PresencePenalty; if (opts.Seed is not null) options.Seed = (long)opts.Seed; + if (opts.AllowMultipleToolCalls is not null) + options.AllowParallelToolCalls = opts.AllowMultipleToolCalls; if (opts.StopSequences is not null) { foreach (var s in opts.StopSequences) options.StopSequences.Add(s); } + if (opts.AdditionalProperties is not null) + { + foreach (var (name, value) in opts.AdditionalProperties) + { + if (CanonicalOptionIsSet(name, agent, opts)) + continue; + if (string.IsNullOrWhiteSpace(name) || + name.Any(character => !char.IsAsciiLetterOrDigit(character) && character != '_')) + throw new ArgumentException( + $"OpenAI additional option name '{name}' cannot be represented safely by the SDK.", + nameof(agent)); + + var path = Encoding.UTF8.GetBytes($"$.{name}"); + var json = BinaryData.FromObjectAsJson(value).ToMemory().Span; + options.Patch.Set(path, json); + } + } // Tools var tools = ToolsToWire(agent); @@ -178,6 +246,25 @@ public static ChatCompletionOptions BuildOptions(Core.Prompty agent) return options; } + private static bool CanonicalOptionIsSet(string name, Core.Prompty agent, ModelOptions options) + { + return name switch + { + "model" or "messages" or "stream" => true, + "temperature" => options.Temperature is not null, + "max_completion_tokens" => options.MaxOutputTokens is not null, + "top_p" => options.TopP is not null, + "frequency_penalty" => options.FrequencyPenalty is not null, + "presence_penalty" => options.PresencePenalty is not null, + "seed" => options.Seed is not null, + "stop" => options.StopSequences is not null, + "parallel_tool_calls" => options.AllowMultipleToolCalls is not null, + "tools" => agent.Tools is { Count: > 0 }, + "response_format" => agent.Outputs is { Count: > 0 }, + _ => false + }; + } + // ----------------------------------------------------------------------- // Responses API wire format // -----------------------------------------------------------------------