From 996768e18a394c1f817f0287933ca1553741d73f Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Mon, 3 Aug 2026 23:01:57 -0700 Subject: [PATCH 01/19] fix(csharp): journal denied tool results Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs | 4 ++++ runtime/csharp/Prompty.Core/TurnRunner.cs | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs index 82d8ead42..ba5517400 100644 --- a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs @@ -168,6 +168,10 @@ public async Task ReferenceTurnRunner_DeniedPermissionSkipsExecution() Assert.Equal("permission_denied", Assert.IsType>(result.Output)["denied"]); Assert.DoesNotContain(sink.TurnEvents, turnEvent => turnEvent.Type == TurnEventType.ToolExecutionStart); + Assert.Contains( + sink.TurnEvents, + turnEvent => turnEvent.Type == TurnEventType.ToolResult + && turnEvent.Payload["errorKind"] as string == "permission_denied"); } finally { diff --git a/runtime/csharp/Prompty.Core/TurnRunner.cs b/runtime/csharp/Prompty.Core/TurnRunner.cs index 928caaac5..c00460ef5 100644 --- a/runtime/csharp/Prompty.Core/TurnRunner.cs +++ b/runtime/csharp/Prompty.Core/TurnRunner.cs @@ -189,7 +189,7 @@ private async Task ResolveAndExecuteToolAsync(string turnId, int if (!decision.Approved) { - return new HostToolResult + var deniedResult = new HostToolResult { RequestId = toolRequest.RequestId, ToolCallId = toolRequest.ToolCallId, @@ -198,6 +198,8 @@ private async Task ResolveAndExecuteToolAsync(string turnId, int ErrorKind = "permission_denied", Result = new Dictionary { ["message"] = decision.Reason ?? "Permission denied" } }; + RecordTurn(TurnEventType.ToolResult, turnId, iteration, deniedResult.Save()); + return deniedResult; } RecordTurn(TurnEventType.ToolExecutionStart, turnId, iteration, toolRequest.Save()); From 0aadf1c62c9ef067bac37b50b2f1446e2c6004c2 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Mon, 3 Aug 2026 23:44:28 -0700 Subject: [PATCH 02/19] fix(csharp): align core format contracts Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../csharp/Prompty.Core.Tests/LoaderTests.cs | 63 ++++++++++++++++ .../csharp/Prompty.Core.Tests/ParserTests.cs | 52 +++++++++++-- .../Prompty.Core.Tests/SpecVectorTests.cs | 10 --- .../csharp/Prompty.Core/FrontmatterParser.cs | 49 ++++++++----- runtime/csharp/Prompty.Core/Jinja2Renderer.cs | 16 +++- .../csharp/Prompty.Core/PromptyChatParser.cs | 38 +++++++--- runtime/csharp/Prompty.Core/PromptyLoader.cs | 10 +-- .../csharp/Prompty.Core/ReferenceResolver.cs | 73 ++++++++++++------- 8 files changed, 230 insertions(+), 81 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs b/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs index f874eab7f..29317693b 100644 --- a/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs @@ -190,6 +190,44 @@ public void Load_Minimal_Works() Assert.Contains("Hello world.", agent.Instructions); } + [Fact] + public void Load_WithoutFrontmatter_TreatsEntireFileAsInstructions() + { + var root = Directory.CreateTempSubdirectory("prompty-loader-"); + try + { + var path = Path.Combine(root.FullName, "body-only.prompty"); + File.WriteAllText(path, "system:\nYou are helpful."); + + var agent = PromptyLoader.Load(path); + + Assert.Equal("system:\nYou are helpful.", agent.Instructions); + } + finally + { + root.Delete(recursive: true); + } + } + + [Fact] + public void Load_MissingClosingFrontmatterDelimiter_Throws() + { + var root = Directory.CreateTempSubdirectory("prompty-loader-"); + try + { + var path = Path.Combine(root.FullName, "invalid.prompty"); + File.WriteAllText(path, "---\nname: invalid\nsystem:\nHello"); + + var error = Assert.Throws(() => PromptyLoader.Load(path)); + + Assert.Contains("closing delimiter", error.Message); + } + finally + { + root.Delete(recursive: true); + } + } + // --- Env resolution --- [Fact] @@ -224,6 +262,31 @@ public void Load_EnvDefault_UsesDefault() Assert.Equal("https://fallback.openai.azure.com", conn.Endpoint); } + [Fact] + public void Load_EnvResolution_RecursesThroughMetadataAndArrays() + { + Environment.SetEnvironmentVariable("PROMPTY_NESTED_VALUE", "resolved"); + var root = Directory.CreateTempSubdirectory("prompty-loader-"); + try + { + var path = Path.Combine(root.FullName, "nested.prompty"); + File.WriteAllText( + path, + "---\nname: nested\nmetadata:\n nested:\n values:\n - ${env:PROMPTY_NESTED_VALUE}\n---\nHello"); + + var agent = PromptyLoader.Load(path); + var nested = Assert.IsAssignableFrom(agent.Metadata!["nested"]); + var values = Assert.IsAssignableFrom(nested["values"]); + + Assert.Equal("resolved", values[0]); + } + finally + { + Environment.SetEnvironmentVariable("PROMPTY_NESTED_VALUE", null); + root.Delete(recursive: true); + } + } + [Fact] public void Load_EnvMissing_Throws() { diff --git a/runtime/csharp/Prompty.Core.Tests/ParserTests.cs b/runtime/csharp/Prompty.Core.Tests/ParserTests.cs index cc9e3d725..4e6bf4bb1 100644 --- a/runtime/csharp/Prompty.Core.Tests/ParserTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ParserTests.cs @@ -33,19 +33,43 @@ public async Task Parse_MultipleRoles() } [Fact] - public async Task Parse_DeveloperRole() + public async Task Parse_DeveloperRole_IsPlainSystemContent() { var messages = await _parser.ParseAsync(CreateAgent(), "developer:\nInstructions here.", null); Assert.Single(messages); - Assert.Equal(Role.Developer, messages[0].Role); + Assert.Equal(Role.System, messages[0].Role); + Assert.Equal("developer:\nInstructions here.", messages[0].Text); } [Fact] - public async Task Parse_ToolRole() + public async Task Parse_ToolRole_IsPlainSystemContent() { var messages = await _parser.ParseAsync(CreateAgent(), "tool:\nTool response", null); Assert.Single(messages); - Assert.Equal(Role.Tool, messages[0].Role); + Assert.Equal(Role.System, messages[0].Role); + Assert.Equal("tool:\nTool response", messages[0].Text); + } + + [Theory] + [InlineData("SYSTEM:\nUppercase", Role.System)] + [InlineData(" user: \nIndented", Role.User)] + [InlineData("# assistant:\nHeading", Role.Assistant)] + public async Task Parse_CanonicalMarkerVariants(string input, Role expectedRole) + { + var messages = await _parser.ParseAsync(CreateAgent(), input, null); + Assert.Single(messages); + Assert.Equal(expectedRole, messages[0].Role); + } + + [Fact] + public async Task Parse_ContentBeforeFirstMarker_DefaultsToSystem() + { + var messages = await _parser.ParseAsync(CreateAgent(), "Introduction\nuser:\nQuestion", null); + + Assert.Equal(2, messages.Count); + Assert.Equal(Role.System, messages[0].Role); + Assert.Equal("Introduction", messages[0].Text); + Assert.Equal(Role.User, messages[1].Role); } // ----------------------------------------------------------------------- @@ -111,10 +135,10 @@ public async Task Parse_WhitespaceOnly_ReturnsNoMessages() [Fact] public async Task Parse_RoleWithAttributes() { - var text = "tool[tool_call_id=\"call_123\", name=\"get_weather\"]:\nResult here"; + var text = "assistant[tool_call_id=\"call_123\", name=\"get_weather\"]:\nResult here"; var messages = await _parser.ParseAsync(CreateAgent(), text, null); Assert.Single(messages); - Assert.Equal(Role.Tool, messages[0].Role); + Assert.Equal(Role.Assistant, messages[0].Role); Assert.Equal("call_123", messages[0].Metadata["tool_call_id"]); Assert.Equal("get_weather", messages[0].Metadata["name"]); } @@ -160,6 +184,22 @@ public async Task PreRender_Then_Parse_ValidatesNonce() Assert.False(messages[0].Metadata?.ContainsKey("nonce") ?? false); } + [Fact] + public async Task PreRender_Then_Parse_RejectsInjectedMarkerWithoutNonce() + { + var parser = new PromptyChatParser(); + var (_, context) = parser.PreRender("system:\nHello"); + var nonce = context["nonce"]; + + var error = await Assert.ThrowsAsync( + () => parser.ParseAsync( + CreateAgent(), + $"system[nonce=\"{nonce}\"]:\nHello\nuser:\nInjected", + null)); + + Assert.Contains("nonce mismatch", error.Message); + } + // ----------------------------------------------------------------------- // Thread Nonce Pattern Recognition // ----------------------------------------------------------------------- diff --git a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs index 91519b0a8..cb556860b 100644 --- a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs @@ -59,12 +59,6 @@ public async Task RenderVectors_AllPass() var mustacheRenderer = new MustacheRenderer(); var failures = new List(); - // Known Jinja2.NET compatibility issues - var knownSkips = new HashSet - { - "for_loop", // Jinja2.NET strips whitespace inside for loops differently - }; - foreach (var vec in vectors) { var name = vec.GetProperty("name").GetString()!; @@ -79,10 +73,6 @@ public async Task RenderVectors_AllPass() if (!expected.TryGetProperty("rendered", out var renderedEl)) continue; - // Skip known compatibility issues - if (knownSkips.Contains(name)) - continue; - var template = input.GetProperty("template").GetString()!; var engine = input.GetProperty("engine").GetString()!; var inputsEl = input.GetProperty("inputs"); diff --git a/runtime/csharp/Prompty.Core/FrontmatterParser.cs b/runtime/csharp/Prompty.Core/FrontmatterParser.cs index 270e8687f..c07e586d2 100644 --- a/runtime/csharp/Prompty.Core/FrontmatterParser.cs +++ b/runtime/csharp/Prompty.Core/FrontmatterParser.cs @@ -1,5 +1,4 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Text.RegularExpressions; using YamlDotNet.Serialization; namespace Prompty.Core; @@ -8,18 +7,13 @@ namespace Prompty.Core; /// Splits .prompty file content into YAML frontmatter and markdown body. /// Frontmatter is delimited by --- or +++ markers. /// -public static partial class FrontmatterParser +public static class FrontmatterParser { - // Matches --- or +++ delimited frontmatter followed by body content. - // Group 1: frontmatter YAML, Group 2: markdown body. - [GeneratedRegex(@"^\s*(?:---|\+\+\+)(.*?)(?:---|\+\+\+)\s*(.+)$", RegexOptions.Singleline)] - private static partial Regex FrontmatterRegex(); - /// /// Parse .prompty file content into a dictionary. /// If frontmatter markers are present, splits frontmatter (YAML) from body (markdown). /// The body is stored under the "instructions" key. - /// If no frontmatter markers, treats entire content as YAML. + /// If no frontmatter markers are present, treats the entire content as instructions. /// /// Raw .prompty file content. /// Dictionary with parsed frontmatter fields and optional "instructions" key. @@ -27,24 +21,41 @@ public static partial class FrontmatterParser { ArgumentNullException.ThrowIfNull(contents); - // Check for frontmatter markers var trimmed = contents.TrimStart(); - if (trimmed.StartsWith("---") || trimmed.StartsWith("+++")) + var openingLineEnd = trimmed.IndexOf('\n'); + var openingLine = (openingLineEnd >= 0 ? trimmed[..openingLineEnd] : trimmed).Trim(); + if (openingLine is not "---" and not "+++") { - var match = FrontmatterRegex().Match(contents); - if (match.Success) - { - var frontmatter = match.Groups[1].Value; - var body = match.Groups[2].Value; + return new Dictionary { ["instructions"] = contents }; + } + + if (openingLineEnd < 0) + { + return new Dictionary { ["instructions"] = string.Empty }; + } - var data = DeserializeYaml(frontmatter); - data["instructions"] = body; + var frontmatterStart = openingLineEnd + 1; + var lineStart = frontmatterStart; + while (lineStart <= trimmed.Length) + { + var lineEnd = trimmed.IndexOf('\n', lineStart); + var line = (lineEnd >= 0 ? trimmed[lineStart..lineEnd] : trimmed[lineStart..]).Trim(); + if (line is "---" or "+++") + { + var data = DeserializeYaml(trimmed[frontmatterStart..lineStart]); + data["instructions"] = lineEnd >= 0 ? trimmed[(lineEnd + 1)..] : string.Empty; return data; } + + if (lineEnd < 0) + { + break; + } + + lineStart = lineEnd + 1; } - // No frontmatter markers — treat entire content as YAML - return DeserializeYaml(contents); + throw new InvalidOperationException("Opening frontmatter delimiter does not have a closing delimiter."); } private static Dictionary DeserializeYaml(string yaml) diff --git a/runtime/csharp/Prompty.Core/Jinja2Renderer.cs b/runtime/csharp/Prompty.Core/Jinja2Renderer.cs index 1caabb2f8..0bd0298f8 100644 --- a/runtime/csharp/Prompty.Core/Jinja2Renderer.cs +++ b/runtime/csharp/Prompty.Core/Jinja2Renderer.cs @@ -1,13 +1,20 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Text.RegularExpressions; + namespace Prompty.Core; /// /// Renders Prompty templates using the Jinja2 template engine (via Jinja2.NET). /// Registered under key "jinja2". /// -public class Jinja2Renderer : IRenderer +public partial class Jinja2Renderer : IRenderer { + private const string ProtectedWhitespace = "__PROMPTY_JINJA_CONTROL_WHITESPACE__"; + + [GeneratedRegex(@"[ \t]+(?=\{%\s*endfor\s*%\})")] + private static partial Regex LoopBoundaryWhitespaceRegex(); + /// /// The most recently generated nonces from rendering, for thread expansion. /// @@ -19,7 +26,10 @@ public Task RenderAsync(Prompty agent, string template, Dictionary string.Concat(Enumerable.Repeat(ProtectedWhitespace, match.Length))); + var jinja = new Jinja2.NET.Template(protectedTemplate); // Convert to IDictionary for Jinja2.NET (no nulls) var context = new Dictionary(); @@ -29,7 +39,7 @@ public Task RenderAsync(Prompty agent, string template, Dictionary - /// Regex matching role marker lines: "role:" or "role[attrs]:" + /// Regex matching canonical role marker lines with optional heading syntax and attributes. /// Captures role name and optional attributes. /// - [GeneratedRegex(@"^(system|user|assistant|developer|tool)(\[.*?\])?:\s*$", RegexOptions.Multiline)] + [GeneratedRegex( + @"^\s*#?\s*(system|user|assistant)(\[(?:\w+\s*=\s*""?[^""]*""?\s*,?\s*)+\])?\s*:\s*$", + RegexOptions.IgnoreCase | RegexOptions.Multiline)] private static partial Regex RoleMarkerRegex(); /// @@ -52,7 +54,7 @@ public partial class PromptyChatParser : IParser, IPreRenderable var sanitized = RoleMarkerRegex().Replace(template, match => { - var role = match.Groups[1].Value; + var role = match.Groups[1].Value.ToLowerInvariant(); var existingAttrs = match.Groups[2].Value; // e.g. "[key=val]" or "" if (string.IsNullOrEmpty(existingAttrs)) return $"{role}[nonce=\"{nonce}\"]:\n"; @@ -93,10 +95,21 @@ internal List Parse(string rendered) var match = RoleMarkerRegex().Match(line); if (match.Success) { - // Flush previous message if (currentRole is not null) { - messages.Add(CreateMessage(currentRole, currentContent, currentAttrs)); + messages.Add(CreateMessage(currentRole, currentContent, currentAttrs, validateNonce: true)); + } + else if (currentContent.Count > 0) + { + var leadingText = string.Join("\n", currentContent).Trim('\r', '\n'); + if (!string.IsNullOrEmpty(leadingText)) + { + messages.Add(new Message + { + Role = Role.System, + Parts = [new TextPart { Value = leadingText }] + }); + } } currentRole = match.Groups[1].Value; @@ -112,7 +125,7 @@ internal List Parse(string rendered) // Flush last message if (currentRole is not null) { - messages.Add(CreateMessage(currentRole, currentContent, currentAttrs)); + messages.Add(CreateMessage(currentRole, currentContent, currentAttrs, validateNonce: true)); } else if (currentContent.Count > 0) { @@ -135,17 +148,20 @@ internal List Parse(string rendered) // Helpers // ----------------------------------------------------------------------- - private Message CreateMessage(string role, List contentLines, Dictionary? attrs) + private Message CreateMessage( + string role, + List contentLines, + Dictionary? attrs, + bool validateNonce) { - // Validate nonce if strict mode was used - if (_renderNonce.Value is not null && attrs is not null) + if (validateNonce && _renderNonce.Value is not null) { - if (!attrs.TryGetValue("nonce", out var foundNonce) || foundNonce != _renderNonce.Value) + if (attrs is null || !attrs.TryGetValue("nonce", out var foundNonce) || foundNonce != _renderNonce.Value) { throw new InvalidOperationException( $"Role marker injection detected: nonce mismatch for '{role}:' marker."); } - attrs.Remove("nonce"); // Don't pass nonce through to metadata + attrs.Remove("nonce"); } var text = string.Join("\n", contentLines); diff --git a/runtime/csharp/Prompty.Core/PromptyLoader.cs b/runtime/csharp/Prompty.Core/PromptyLoader.cs index d3e1891cf..186e66d8a 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoader.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoader.cs @@ -65,13 +65,9 @@ private static Prompty Build(string contents, string fullPath, PromptyLoadOption // 1. Split frontmatter + body var data = FrontmatterParser.Parse(contents); - // 2. Load via typed model with ${env:}/${file:} resolution - var ctx = new LoadContext - { - PreProcess = ReferenceResolver.CreatePreProcess(fullPath, options?.AllowedFileRoots), - }; - - var agent = Prompty.Load(data, ctx); + // 2. Resolve the complete untyped tree before generated model loading. + data = ReferenceResolver.CreatePreProcess(fullPath, options?.AllowedFileRoots)(data); + var agent = Prompty.Load(data, new LoadContext()); // 3. Attach source path in metadata agent.Metadata ??= new Dictionary(); diff --git a/runtime/csharp/Prompty.Core/ReferenceResolver.cs b/runtime/csharp/Prompty.Core/ReferenceResolver.cs index 3e765f42d..3510efee7 100644 --- a/runtime/csharp/Prompty.Core/ReferenceResolver.cs +++ b/runtime/csharp/Prompty.Core/ReferenceResolver.cs @@ -34,44 +34,67 @@ public static class ReferenceResolver } /// - /// Walks a dictionary and resolves any string values matching ${protocol:value} patterns. - /// Only processes top-level string values in the given dictionary (recursive walking is - /// handled by LoadContext calling PreProcess on each nested dict). + /// Recursively walks a dictionary and resolves string values matching ${protocol:value} patterns. /// internal static Dictionary ResolveReferences( Dictionary data, string parentDir, IReadOnlyCollection allowedRoots) { - foreach (var key in data.Keys.ToList()) - { - var value = data[key]; - if (value is not string str) - continue; - if (!str.StartsWith(RefPrefix) || !str.EndsWith(RefSuffix)) - continue; + ResolveValue(data, "", parentDir, allowedRoots); + return data; + } - var inner = str[RefPrefix.Length..^RefSuffix.Length]; - var colonIndex = inner.IndexOf(':'); - if (colonIndex < 0) - continue; + private static object? ResolveValue( + object? value, + string key, + string parentDir, + IReadOnlyCollection allowedRoots) + { + if (value is System.Collections.IDictionary dictionary) + { + foreach (var childKey in dictionary.Keys.Cast().ToList()) + { + if (childKey is string childName) + { + dictionary[childKey] = ResolveValue(dictionary[childKey], childName, parentDir, allowedRoots); + } + } - var protocol = inner[..colonIndex].ToLowerInvariant(); - var remainder = inner[(colonIndex + 1)..]; + return value; + } - switch (protocol) + if (value is System.Collections.IList list) + { + for (var index = 0; index < list.Count; index++) { - case "env": - data[key] = ResolveEnvVar(remainder, key); - break; - case "file": - data[key] = ResolveFileRef(remainder, parentDir, allowedRoots, key); - break; - // Unknown protocol: leave unchanged per spec §4 + list[index] = ResolveValue(list[index], $"{key}[{index}]", parentDir, allowedRoots); } + + return value; } - return data; + if (value is not string str || !str.StartsWith(RefPrefix) || !str.EndsWith(RefSuffix)) + { + return value; + } + + var inner = str[RefPrefix.Length..^RefSuffix.Length]; + var colonIndex = inner.IndexOf(':'); + if (colonIndex < 0) + { + return value; + } + + var protocol = inner[..colonIndex].ToLowerInvariant(); + var remainder = inner[(colonIndex + 1)..]; + var resolved = protocol switch + { + "env" => ResolveEnvVar(remainder, key), + "file" => ResolveFileRef(remainder, parentDir, allowedRoots, key), + _ => value + }; + return resolved; } /// From 7bf3abbb602530db7189379c9a89728c72ba4fb7 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Mon, 3 Aug 2026 23:58:19 -0700 Subject: [PATCH 03/19] feat(csharp): complete OpenAI wire vectors Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SpecVectorWireTests.cs | 26 +++--- .../Prompty.OpenAI.Tests/WireFormatTests.cs | 48 ++++++++++- runtime/csharp/Prompty.OpenAI/WireFormat.cs | 86 +++++++++++++++++++ 3 files changed, 149 insertions(+), 11 deletions(-) diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs index aa12b8637..4c48ce6f9 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs @@ -123,6 +123,17 @@ public void OpenAI_Chat_WireFormat(string name, JsonElement input, JsonElement e // 4. Check options (temperature, max_completion_tokens, etc.) AssertOptionFields(optionsJson, expectedBody, name); + if (agent.Model?.Options?.AdditionalProperties is not null) + { + foreach (var optionName in agent.Model.Options.AdditionalProperties.Keys) + { + Assert.True(expectedBody.TryGetProperty(optionName, out var expectedOption), + $"[{name}] Expected vector body to contain passthrough option '{optionName}'"); + Assert.True(optionsJson.TryGetProperty(optionName, out var actualOption), + $"[{name}] Serialized options omitted passthrough option '{optionName}'"); + AssertJsonSubset(expectedOption, actualOption, $"[{name}] {optionName}"); + } + } // 5. Check response_format (structured output) if (expectedBody.TryGetProperty("response_format", out var expectedRF)) @@ -143,16 +154,11 @@ public static IEnumerable OpenAIChatVectors() if (provider != "openai") continue; if (apiType != "chat") continue; - var name = vec.GetProperty("name").GetString()!; - - // Skip vectors that require features not yet implemented in C# - if (name is "chat_audio_part" or "chat_audio_mp3") - continue; // AudioPart not handled in WireFormat.BuildContentParts - - if (name == "options_additional_properties") - continue; // AdditionalProperties passthrough not implemented in BuildOptions - - yield return [name, input, vec.GetProperty("expected").GetProperty("request_body")]; + yield return [ + vec.GetProperty("name").GetString()!, + input, + vec.GetProperty("expected").GetProperty("request_body"), + ]; } } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs index e14ead199..8b412c0eb 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System.Text.Json; +using System.ClientModel.Primitives; using OpenAI.Chat; using Prompty.Core; using Prompty.OpenAI; @@ -430,5 +431,50 @@ public void BuildOptions_AllOptions_SetsEverything() Assert.Equal(123L, result.Seed); Assert.Single(result.StopSequences); } -} + [Fact] + public void BuildOptions_AdditionalProperties_DoNotOverrideCanonicalOptions() + { + var agent = new Core.Prompty + { + Model = new Model + { + Options = new ModelOptions + { + Temperature = 0.2f, + AdditionalProperties = new Dictionary + { + ["temperature"] = 0.9, + ["logprobs"] = true, + }, + }, + }, + }; + + var result = WireFormat.BuildOptions(agent); + var json = JsonDocument.Parse( + ModelReaderWriter.Write(result, ModelReaderWriterOptions.Json).ToStream()).RootElement; + + Assert.Equal(0.2, json.GetProperty("temperature").GetDouble(), precision: 6); + Assert.True(json.GetProperty("logprobs").GetBoolean()); + } + + [Fact] + public void BuildOptions_UnsafeAdditionalPropertyName_Throws() + { + var agent = new Core.Prompty + { + Model = new Model + { + Options = new ModelOptions + { + AdditionalProperties = new Dictionary { ["nested.option"] = true }, + }, + }, + }; + + var error = Assert.Throws(() => WireFormat.BuildOptions(agent)); + + Assert.Contains("cannot be represented safely", error.Message); + } +} diff --git a/runtime/csharp/Prompty.OpenAI/WireFormat.cs b/runtime/csharp/Prompty.OpenAI/WireFormat.cs index 2da3e4909..0788da29a 100644 --- a/runtime/csharp/Prompty.OpenAI/WireFormat.cs +++ b/runtime/csharp/Prompty.OpenAI/WireFormat.cs @@ -2,6 +2,8 @@ #pragma warning disable OPENAI001 // Responses API is in preview +using System.ClientModel.Primitives; +using System.Text; using System.Text.Json; using OpenAI.Chat; using OpenAI.Responses; @@ -62,10 +64,57 @@ private static IEnumerable BuildContentParts(IList ChatImageDetailLevel.Auto, }); break; + case AudioPart audio: + yield return BuildAudioContentPart(audio); + break; + case FilePart file: + var filePart = ChatMessageContentPart.CreateFilePart(file.Source); + filePart.Patch.Set( + "$.file"u8, + BinaryData.FromObjectAsJson(new Dictionary { ["url"] = file.Source }).ToMemory().Span); + yield return filePart; + break; } } } + private static ChatMessageContentPart BuildAudioContentPart(AudioPart audio) + { + byte[] bytes; + try + { + bytes = Convert.FromBase64String(audio.Source); + } + catch (FormatException) + { + bytes = Encoding.UTF8.GetBytes(audio.Source); + } + + var format = AudioFormat(audio.MediaType); + var part = ChatMessageContentPart.CreateInputAudioPart( + BinaryData.FromBytes(bytes), + format == "mp3" ? ChatInputAudioFormat.Mp3 : ChatInputAudioFormat.Wav); + part.Patch.Set("$.input_audio.data"u8, audio.Source); + part.Patch.Set("$.input_audio.format"u8, format); + return part; + } + + private static string AudioFormat(string? mediaType) + { + return mediaType?.ToLowerInvariant() switch + { + "audio/wav" or "audio/x-wav" => "wav", + "audio/mpeg" or "audio/mp3" => "mp3", + "audio/mp4" => "mp4", + "audio/ogg" => "ogg", + "audio/flac" => "flac", + "audio/webm" => "webm", + "audio/pcm" => "pcm", + { } value when value.StartsWith("audio/", StringComparison.Ordinal) => value["audio/".Length..], + _ => "wav" + }; + } + private static AssistantChatMessage BuildAssistantMessage(Message msg) { var assistant = new AssistantChatMessage(msg.Text); @@ -156,11 +205,29 @@ public static ChatCompletionOptions BuildOptions(Core.Prompty agent) options.PresencePenalty = (float)opts.PresencePenalty; if (opts.Seed is not null) options.Seed = (long)opts.Seed; + if (opts.AllowMultipleToolCalls is not null) + options.AllowParallelToolCalls = opts.AllowMultipleToolCalls; if (opts.StopSequences is not null) { foreach (var s in opts.StopSequences) options.StopSequences.Add(s); } + if (opts.AdditionalProperties is not null) + { + foreach (var (name, value) in opts.AdditionalProperties) + { + if (CanonicalOptionIsSet(name, agent, opts)) + continue; + if (name.Any(character => !char.IsAsciiLetterOrDigit(character) && character != '_')) + throw new ArgumentException( + $"OpenAI additional option name '{name}' cannot be represented safely by the SDK.", + nameof(agent)); + + var path = Encoding.UTF8.GetBytes($"$.{name}"); + var json = BinaryData.FromObjectAsJson(value).ToMemory().Span; + options.Patch.Set(path, json); + } + } // Tools var tools = ToolsToWire(agent); @@ -178,6 +245,25 @@ public static ChatCompletionOptions BuildOptions(Core.Prompty agent) return options; } + private static bool CanonicalOptionIsSet(string name, Core.Prompty agent, ModelOptions options) + { + return name switch + { + "model" or "messages" or "stream" => true, + "temperature" => options.Temperature is not null, + "max_completion_tokens" => options.MaxOutputTokens is not null, + "top_p" => options.TopP is not null, + "frequency_penalty" => options.FrequencyPenalty is not null, + "presence_penalty" => options.PresencePenalty is not null, + "seed" => options.Seed is not null, + "stop" => options.StopSequences is not null, + "parallel_tool_calls" => options.AllowMultipleToolCalls is not null, + "tools" => agent.Tools is { Count: > 0 }, + "response_format" => agent.Outputs is { Count: > 0 }, + _ => false + }; + } + // ----------------------------------------------------------------------- // Responses API wire format // ----------------------------------------------------------------------- From 5f56064d95688fcd17074f79317f8840ea1180a6 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 00:06:42 -0700 Subject: [PATCH 04/19] feat(csharp): preserve Anthropic tool blocks Keep canonical Anthropic content arrays and replay raw assistant blocks so tool-result turns retain correlation and signed thinking metadata. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AnthropicExecutorTests.cs | 90 +++++++++++++-- .../Prompty.Anthropic/AnthropicExecutor.cs | 103 +++++++++++------- .../SpecVectorWireTests.cs | 61 +---------- 3 files changed, 144 insertions(+), 110 deletions(-) diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs index d73197cb3..7c6c52926 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Text.Json; using Prompty.Core; namespace Prompty.Anthropic.Tests; @@ -112,8 +113,18 @@ public void FormatToolMessages_BatchesToolResultsIntoSingleUserMessage() new() { Id = "call_2", Name = "get_time", Arguments = """{"tz":"PST"}""" }, }; var toolResults = new List { "72°F", "3:00 PM" }; + using var rawResponse = JsonDocument.Parse( + """ + { + "content": [ + {"type":"text","text":"Let me check."}, + {"type":"tool_use","id":"call_1","name":"get_weather","input":{"city":"Seattle"}}, + {"type":"tool_use","id":"call_2","name":"get_time","input":{"tz":"PST"}} + ] + } + """); - var messages = executor.FormatToolMessages("raw", toolCalls, toolResults, "Let me check."); + var messages = executor.FormatToolMessages(rawResponse.RootElement, toolCalls, toolResults, "Let me check."); // Should be 2 messages: assistant + single user message (NOT 3) Assert.Equal(2, messages.Count); @@ -121,11 +132,11 @@ public void FormatToolMessages_BatchesToolResultsIntoSingleUserMessage() // Assistant message preserves text + tool_use content blocks Assert.Equal(Role.Assistant, messages[0].Role); Assert.Equal("Let me check.", messages[0].Text); - var content = Assert.IsType>>(messages[0].Metadata["content"]); - Assert.Equal(3, content.Count); // 1 text + 2 tool_use - Assert.Equal("text", content[0]["type"]); - Assert.Equal("tool_use", content[1]["type"]); - Assert.Equal("tool_use", content[2]["type"]); + var content = Assert.IsType(messages[0].Metadata["content"]); + Assert.Equal(3, content.GetArrayLength()); // 1 text + 2 tool_use + Assert.Equal("text", content[0].GetProperty("type").GetString()); + Assert.Equal("tool_use", content[1].GetProperty("type").GetString()); + Assert.Equal("tool_use", content[2].GetProperty("type").GetString()); // Single user message with batched tool_result blocks Assert.Equal(Role.User, messages[1].Role); @@ -148,12 +159,69 @@ public void FormatToolMessages_NoTextContent_OmitsTextBlock() new() { Id = "call_1", Name = "fn", Arguments = "{}" }, }; var toolResults = new List { "result" }; + using var rawResponse = JsonDocument.Parse( + """{"content":[{"type":"tool_use","id":"call_1","name":"fn","input":{}}]}"""); - var messages = executor.FormatToolMessages("raw", toolCalls, toolResults); + var messages = executor.FormatToolMessages(rawResponse.RootElement, toolCalls, toolResults); - var content = Assert.IsType>>(messages[0].Metadata["content"]); - Assert.Single(content); // Only tool_use, no text block - Assert.Equal("tool_use", content[0]["type"]); + var content = Assert.IsType(messages[0].Metadata["content"]); + Assert.Single(content.EnumerateArray()); // Only tool_use, no text block + Assert.Equal("tool_use", content[0].GetProperty("type").GetString()); + } + + [Fact] + public void FormatToolMessages_RoundTripsCorrelatedBlocksIntoNextRequest() + { + var executor = new Anthropic.AnthropicExecutor(); + var agent = TestHelpers.CreateAgent(provider: "anthropic"); + using var rawResponse = JsonDocument.Parse( + """ + { + "content": [ + {"type":"thinking","thinking":"I should look this up.","signature":"signed-thinking"}, + {"type":"text","text":"Checking."}, + {"type":"tool_use","id":"call_1","name":"lookup","input":{"key":"value"}} + ] + } + """); + var messages = executor.FormatToolMessages( + rawResponse.RootElement, + [new ToolCall { Id = "call_1", Name = "lookup", Arguments = """{"key":"value"}""" }], + ["result"], + "Checking."); + + var body = executor.BuildRequestBody(agent, messages, stream: false); + var wireMessages = Assert.IsType>>(body["messages"]); + + var assistantContent = Assert.IsType(wireMessages[0]["content"]); + Assert.Equal("thinking", assistantContent[0].GetProperty("type").GetString()); + Assert.Equal("signed-thinking", assistantContent[0].GetProperty("signature").GetString()); + Assert.Equal("text", assistantContent[1].GetProperty("type").GetString()); + Assert.Equal("tool_use", assistantContent[2].GetProperty("type").GetString()); + Assert.Equal("call_1", assistantContent[2].GetProperty("id").GetString()); + + var userContent = Assert.IsType>>(wireMessages[1]["content"]); + Assert.Equal("tool_result", userContent[0]["type"]); + Assert.Equal("call_1", userContent[0]["tool_use_id"]); + Assert.Equal("result", userContent[0]["content"]); } -} + [Fact] + public void BuildRequestBody_SingleToolResultMetadata_PreservesCorrelation() + { + var executor = new Anthropic.AnthropicExecutor(); + var agent = TestHelpers.CreateAgent(provider: "anthropic"); + var message = new Message + { + Role = Role.Tool, + Parts = [new TextPart { Value = "result" }], + Metadata = new Dictionary { ["tool_use_id"] = "call_2" }, + }; + + var body = executor.BuildRequestBody(agent, [message], stream: false); + var wireMessages = Assert.IsType>>(body["messages"]); + var content = Assert.IsType>>(wireMessages[0]["content"]); + + Assert.Equal("call_2", content[0]["tool_use_id"]); + } +} diff --git a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs index a64f6198d..2074ee61e 100644 --- a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs +++ b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs @@ -151,25 +151,38 @@ private static HttpRequestMessage CreateRequest(string endpoint, string apiKey, _ => "user", }; - // Handle tool results + if (msg.Metadata is not null + && msg.Metadata.TryGetValue("tool_results", out var toolResults)) + { + return new Dictionary + { + ["role"] = role, + ["content"] = toolResults, + }; + } + + if (msg.Metadata is not null + && msg.Metadata.TryGetValue("tool_use_id", out var toolUseId)) + { + return BuildToolResultMessage(toolUseId?.ToString() ?? "", msg.Text); + } + + if (msg.Metadata is not null + && msg.Metadata.TryGetValue("content", out var rawContent)) + { + return new Dictionary + { + ["role"] = role, + ["content"] = rawContent, + }; + } + if (msg.Role == Role.Tool) { var toolCallId = msg.Metadata is not null && msg.Metadata.TryGetValue("tool_call_id", out var id) ? id?.ToString() ?? "" : ""; - return new Dictionary - { - ["role"] = "user", - ["content"] = new List> - { - new() - { - ["type"] = "tool_result", - ["tool_use_id"] = toolCallId, - ["content"] = msg.Text, - } - }, - }; + return BuildToolResultMessage(toolCallId, msg.Text); } // Build content blocks @@ -207,15 +220,26 @@ private static HttpRequestMessage CreateRequest(string endpoint, string apiKey, } } - // Simplify single text content - if (content.Count == 1 && content[0]["type"]?.ToString() == "text") - { - return new() { ["role"] = role, ["content"] = content[0]["text"] }; - } - return new() { ["role"] = role, ["content"] = content }; } + private static Dictionary BuildToolResultMessage(string toolUseId, string content) + { + return new Dictionary + { + ["role"] = "user", + ["content"] = new List> + { + new() + { + ["type"] = "tool_result", + ["tool_use_id"] = toolUseId, + ["content"] = content, + } + }, + }; + } + private static List>? ToolsToWire(Core.Prompty agent) { if (agent.Tools is null || agent.Tools.Count == 0) @@ -315,35 +339,18 @@ public List FormatToolMessages( { var messages = new List(); - // --- Assistant message with ALL content blocks (text + tool_use) --- - var rawContent = new List>(); - if (!string.IsNullOrEmpty(textContent)) - { - rawContent.Add(new() { ["type"] = "text", ["text"] = textContent }); - } - foreach (var tc in toolCalls) - { - rawContent.Add(new() - { - ["type"] = "tool_use", - ["id"] = tc.Id, - ["name"] = tc.Name, - ["input"] = JsonSerializer.Deserialize(tc.Arguments), - }); - } - messages.Add(new Message { Role = Role.Assistant, Parts = !string.IsNullOrEmpty(textContent) ? [new TextPart { Value = textContent }] : [], - Metadata = new Dictionary { ["content"] = rawContent }, + Metadata = new Dictionary { ["content"] = GetRawContent(rawResponse) }, }); // --- Single user message with batched tool_result blocks --- var toolResultBlocks = new List>(); - for (var i = 0; i < toolCalls.Count; i++) + for (var i = 0; i < Math.Min(toolCalls.Count, toolResults.Count); i++) { toolResultBlocks.Add(new() { @@ -362,4 +369,22 @@ public List FormatToolMessages( return messages; } + + private static object GetRawContent(object rawResponse) + { + if (rawResponse is JsonElement { ValueKind: JsonValueKind.Object } element + && element.TryGetProperty("content", out var content)) + { + return content.Clone(); + } + + if (rawResponse is IReadOnlyDictionary dictionary + && dictionary.TryGetValue("content", out var value) + && value is not null) + { + return value; + } + + return Array.Empty(); + } } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs index 4c48ce6f9..ab91d96cf 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorWireTests.cs @@ -381,12 +381,10 @@ public void Anthropic_Chat_WireFormat(string name, JsonElement input, JsonElemen if (prop.Name == "messages") { - // Messages need special handling for content simplification: - // The runtime may simplify single-text content to a string while vectors use array form. Assert.Equal(prop.Value.GetArrayLength(), actualProp.GetArrayLength()); for (int i = 0; i < prop.Value.GetArrayLength(); i++) { - AssertAnthropicMessageMatches(actualProp[i], prop.Value[i], $"[{name}] messages[{i}]"); + AssertJsonSubset(prop.Value[i], actualProp[i], $"[{name}] messages[{i}]"); } } else @@ -655,63 +653,6 @@ private static void AssertMessageMatches(JsonElement actual, JsonElement expecte } } - /// - /// Compares an Anthropic message where the runtime may simplify single-text content - /// to a string (e.g., "content": "Hello") while vectors use the array form - /// (e.g., "content": [{"type": "text", "text": "Hello"}]). - /// - private static void AssertAnthropicMessageMatches(JsonElement actual, JsonElement expected, string context) - { - // Check role - if (expected.TryGetProperty("role", out var expectedRole)) - { - Assert.True(actual.TryGetProperty("role", out var actualRole), - $"{context}: missing 'role' in actual. Actual: {actual.GetRawText()}"); - Assert.Equal(expectedRole.GetString(), actualRole.GetString()); - } - - // Check content with simplification handling - if (expected.TryGetProperty("content", out var expectedContent)) - { - Assert.True(actual.TryGetProperty("content", out var actualContent), - $"{context}: missing 'content' in actual. Actual: {actual.GetRawText()}"); - - if (expectedContent.ValueKind == JsonValueKind.Array && actualContent.ValueKind == JsonValueKind.String) - { - // Runtime simplified: expected array with single text, actual is string - if (expectedContent.GetArrayLength() == 1) - { - var part = expectedContent[0]; - if (part.TryGetProperty("text", out var textProp)) - { - Assert.Equal(textProp.GetString(), actualContent.GetString()); - return; - } - } - Assert.Fail($"{context}: expected array content but got simplified string '{actualContent.GetString()}'"); - } - else if (expectedContent.ValueKind == JsonValueKind.String && actualContent.ValueKind == JsonValueKind.Array) - { - // Opposite: expected string, actual array - if (actualContent.GetArrayLength() == 1) - { - var part = actualContent[0]; - if (part.TryGetProperty("text", out var textProp)) - { - Assert.Equal(expectedContent.GetString(), textProp.GetString()); - return; - } - } - Assert.Fail($"{context}: expected string content but got array: {actualContent.GetRawText()}"); - } - else - { - // Same kind — use standard comparison - AssertJsonSubset(expectedContent, actualContent, $"{context}.content"); - } - } - } - /// /// Compares a Responses API input item. The SDK serializes items with extra fields /// (type: "message", content: [{type: "input_text", text: "..."}]) while vectors From 14068cc6d681a830be1a66da0fbcad5f6bedc2f6 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 00:19:52 -0700 Subject: [PATCH 05/19] feat(csharp): complete model discovery parity Use generated model discovery contracts across providers, embed shared fill-only capability data, preserve raw provider payloads, map Foundry shapes, and paginate Anthropic models. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AnthropicModelDiscoveryTests.cs | 84 ++++++++++ runtime/csharp/Prompty.Anthropic/Models.cs | 130 +++++++++++++++ .../Prompty.Core/Data/model_capabilities.json | 55 +++++++ runtime/csharp/Prompty.Core/ModelDiscovery.cs | 93 +++++++++++ .../csharp/Prompty.Core/Prompty.Core.csproj | 1 + runtime/csharp/Prompty.Foundry/Models.cs | 54 ++++++- .../SpecVectorDiscoveryTests.cs | 152 ++++++++++++++++++ runtime/csharp/Prompty.OpenAI/Models.cs | 67 ++++---- .../Prompty.OpenAI/Prompty.OpenAI.csproj | 2 +- 9 files changed, 603 insertions(+), 35 deletions(-) create mode 100644 runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs create mode 100644 runtime/csharp/Prompty.Anthropic/Models.cs create mode 100644 runtime/csharp/Prompty.Core/Data/model_capabilities.json create mode 100644 runtime/csharp/Prompty.Core/ModelDiscovery.cs create mode 100644 runtime/csharp/Prompty.OpenAI.Tests/SpecVectorDiscoveryTests.cs diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs new file mode 100644 index 000000000..c56a90feb --- /dev/null +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net; +using System.Net.Sockets; +using System.Text; +using Prompty.Core; + +namespace Prompty.Anthropic.Tests; + +/// +/// Tests Anthropic model discovery protocol and pagination behavior. +/// +public class AnthropicModelDiscoveryTests +{ + [Fact] + public void ModelLister_ImplementsGeneratedProtocol() + { + Assert.IsAssignableFrom(new AnthropicModelLister()); + } + + [Fact] + public async Task ListModelsAsync_FollowsPaginationAndPreservesProviderPayload() + { + using var socket = new TcpListener(IPAddress.Loopback, 0); + socket.Start(); + var port = ((IPEndPoint)socket.LocalEndpoint).Port; + socket.Stop(); + + using var listener = new HttpListener(); + listener.Prefixes.Add($"http://localhost:{port}/"); + listener.Start(); + var requests = new List<(string Query, string? ApiKey, string? Version)>(); + var server = Task.Run(async () => + { + for (var page = 0; page < 2; page++) + { + var context = await listener.GetContextAsync(); + requests.Add(( + context.Request.Url?.Query ?? string.Empty, + context.Request.Headers["x-api-key"], + context.Request.Headers["anthropic-version"])); + var body = page == 0 + ? """ + { + "data": [{"id":"claude-first","display_name":"Claude First","type":"model"}], + "has_more": true, + "last_id": "claude-first" + } + """ + : """ + { + "data": [{"id":"claude-second","created_at":"2025-01-01T00:00:00Z","type":"model"}], + "has_more": false, + "last_id": "claude-second" + } + """; + var bytes = Encoding.UTF8.GetBytes(body); + context.Response.ContentType = "application/json"; + context.Response.ContentLength64 = bytes.Length; + await context.Response.OutputStream.WriteAsync(bytes); + context.Response.Close(); + } + }); + + var models = await AnthropicModels.ListModelsAsync( + new ApiKeyConnection + { + Endpoint = $"http://localhost:{port}", + ApiKey = "test-anthropic-key", + }); + + await server; + Assert.Equal(2, models.Count); + Assert.Equal("Claude First", models[0].DisplayName); + Assert.Equal("2025-01-01T00:00:00Z", models[1].AdditionalProperties!["created_at"].ToString()); + Assert.Equal("?limit=100", requests[0].Query); + Assert.Equal("?limit=100&after_id=claude-first", requests[1].Query); + Assert.All(requests, request => + { + Assert.Equal("test-anthropic-key", request.ApiKey); + Assert.Equal("2023-06-01", request.Version); + }); + } +} diff --git a/runtime/csharp/Prompty.Anthropic/Models.cs b/runtime/csharp/Prompty.Anthropic/Models.cs new file mode 100644 index 000000000..db34dd358 --- /dev/null +++ b/runtime/csharp/Prompty.Anthropic/Models.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Net.Http.Headers; +using System.Text.Json; +using Prompty.Core; + +namespace Prompty.Anthropic; + +/// +/// Lists Anthropic models and maps provider payloads to generated model types. +/// +public sealed class AnthropicModelLister : IModelLister +{ + /// + public async Task> ListModelsAsync(object connection) + { + if (connection is not Connection typedConnection) + throw new ArgumentException("Anthropic model listing requires a generated Connection.", nameof(connection)); + + return [.. await AnthropicModels.ListModelsAsync(typedConnection)]; + } +} + +/// +/// Model discovery for the Anthropic Models API. +/// +public static class AnthropicModels +{ + private const string DefaultEndpoint = "https://api.anthropic.com"; + private const string ApiVersion = "2023-06-01"; + private static readonly HttpClient HttpClient = new(); + + /// + /// List all available Anthropic models, following cursor-based pagination. + /// + public static async Task> ListModelsAsync( + Connection connection, + CancellationToken cancellationToken = default) + { + if (connection is not ApiKeyConnection keyConnection) + throw new InvalidOperationException( + $"Connection kind '{connection.Kind}' is not supported for Anthropic model listing. Use 'key'."); + + var apiKey = string.IsNullOrWhiteSpace(keyConnection.ApiKey) + ? Environment.GetEnvironmentVariable("ANTHROPIC_API_KEY") + : keyConnection.ApiKey; + if (string.IsNullOrWhiteSpace(apiKey)) + throw new InvalidOperationException( + "Anthropic API key is required. Set connection.apiKey or ANTHROPIC_API_KEY."); + + var endpoint = string.IsNullOrWhiteSpace(keyConnection.Endpoint) + ? DefaultEndpoint + : keyConnection.Endpoint.TrimEnd('/'); + var models = new List(); + string? afterId = null; + + do + { + var query = afterId is null + ? "limit=100" + : $"limit=100&after_id={Uri.EscapeDataString(afterId)}"; + using var request = new HttpRequestMessage(HttpMethod.Get, $"{endpoint}/v1/models?{query}"); + request.Headers.Add("x-api-key", apiKey); + request.Headers.Add("anthropic-version", ApiVersion); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + using var response = await HttpClient.SendAsync(request, cancellationToken); + var body = await response.Content.ReadAsStringAsync(cancellationToken); + if (!response.IsSuccessStatusCode) + throw new InvalidOperationException( + $"Anthropic list models failed: {(int)response.StatusCode} {response.ReasonPhrase} - " + + body[..Math.Min(body.Length, 300)]); + + using var document = JsonDocument.Parse(body); + if (document.RootElement.TryGetProperty("data", out var data) + && data.ValueKind == JsonValueKind.Array) + { + models.AddRange(data.EnumerateArray().Select(MapModel)); + } + + var hasMore = document.RootElement.TryGetProperty("has_more", out var hasMoreValue) + && hasMoreValue.ValueKind == JsonValueKind.True; + afterId = hasMore + && document.RootElement.TryGetProperty("last_id", out var lastId) + && lastId.ValueKind == JsonValueKind.String + ? lastId.GetString() + : null; + } + while (afterId is not null); + + return models.AsReadOnly(); + } + + /// + /// Map a raw Anthropic model payload to the generated provider-neutral contract. + /// + public static ModelInfo MapModel(JsonElement model) + { + var info = new ModelInfo + { + Id = GetString(model, "id") ?? string.Empty, + DisplayName = GetString(model, "display_name"), + OwnedBy = "anthropic", + ContextWindow = GetInt(model, "context_length"), + InputModalities = GetStringList(model, "input_modalities"), + OutputModalities = GetStringList(model, "output_modalities"), + AdditionalProperties = ModelDiscovery.PreserveRaw(model), + }; + ModelDiscovery.Enrich("anthropic", info); + return info; + } + + private static string? GetString(JsonElement element, string name) => + element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + + private static int? GetInt(JsonElement element, string name) => + element.TryGetProperty(name, out var value) + && value.ValueKind == JsonValueKind.Number + && value.TryGetInt32(out var number) + ? number + : null; + + private static IList? GetStringList(JsonElement element, string name) => + element.TryGetProperty(name, out var value) && value.ValueKind == JsonValueKind.Array + ? value.EnumerateArray().Where(item => item.ValueKind == JsonValueKind.String) + .Select(item => item.GetString()!).ToArray() + : null; +} diff --git a/runtime/csharp/Prompty.Core/Data/model_capabilities.json b/runtime/csharp/Prompty.Core/Data/model_capabilities.json new file mode 100644 index 000000000..7c7f6db19 --- /dev/null +++ b/runtime/csharp/Prompty.Core/Data/model_capabilities.json @@ -0,0 +1,55 @@ +{ + "description": "Cross-runtime fallback capability data for provider model discovery. Some provider /models endpoints (Anthropic, Foundry) return capability fields directly; others (OpenAI) return only ids. To keep discovery results consistent across providers AND across runtimes, every Prompty runtime embeds THIS file and applies one shared rule: provider-supplied fields always win; entries here only fill fields the provider left empty (fill-only-missing). Model ids are matched by longest prefix within a provider's list. This dataset is deliberately NOT emitted from TypeSpec: it is volatile provider data (context windows, modalities, new model families) refreshed as a snapshot, whereas TypeSpec owns the structural ModelInfo contract. Fields use the canonical camelCase ModelInfo names. A missing 'contextWindow' means unknown; a present empty modality list (e.g. []) is intentional (e.g. embeddings produce no textual/image output modality).", + "match": "longest_prefix", + "providers": { + "openai": [ + { + "prefix": "gpt-4o-mini", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4o", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4-turbo", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4", + "contextWindow": 8192, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-3.5-turbo", + "contextWindow": 16385, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "text-embedding-3-small", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "text-embedding-3-large", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "dall-e-3", + "inputModalities": ["text"], + "outputModalities": ["image"] + } + ] + } +} diff --git a/runtime/csharp/Prompty.Core/ModelDiscovery.cs b/runtime/csharp/Prompty.Core/ModelDiscovery.cs new file mode 100644 index 000000000..d51ace6b4 --- /dev/null +++ b/runtime/csharp/Prompty.Core/ModelDiscovery.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Reflection; +using System.Text.Json; + +namespace Prompty.Core; + +/// +/// Enriches generated model-discovery results from the shared capability dataset. +/// +public static class ModelDiscovery +{ + private const string ResourceName = "Prompty.Core.Data.model_capabilities.json"; + private static readonly Lazy> CapabilityTable = + new(LoadCapabilityTable); + + /// + /// Fill missing capability fields without replacing values supplied by a provider. + /// + public static void Enrich(string provider, ModelInfo info) + { + if (!CapabilityTable.Value.TryGetValue(provider, out var entries)) + return; + + var entry = entries.FirstOrDefault(candidate => PrefixMatches(info.Id, candidate.Prefix)); + if (entry is null) + return; + + info.ContextWindow ??= entry.ContextWindow; + info.InputModalities ??= entry.InputModalities?.ToArray(); + info.OutputModalities ??= entry.OutputModalities?.ToArray(); + } + + /// + /// Convert a provider JSON object to a dictionary while preserving nested raw values. + /// + public static IDictionary PreserveRaw(JsonElement value) + { + if (value.ValueKind != JsonValueKind.Object) + throw new ArgumentException("Provider model payload must be a JSON object.", nameof(value)); + + return value.EnumerateObject() + .ToDictionary(property => property.Name, property => (object)property.Value.Clone()); + } + + private static bool PrefixMatches(string id, string prefix) + { + if (!id.StartsWith(prefix, StringComparison.Ordinal)) + return false; + if (id.Length == prefix.Length) + return true; + + return !char.IsAsciiLetterOrDigit(id[prefix.Length]); + } + + private static IReadOnlyDictionary LoadCapabilityTable() + { + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(ResourceName) + ?? throw new InvalidOperationException($"Embedded capability dataset '{ResourceName}' was not found."); + using var document = JsonDocument.Parse(stream); + var providers = document.RootElement.GetProperty("providers"); + var result = new Dictionary(); + + foreach (var provider in providers.EnumerateObject()) + { + result[provider.Name] = provider.Value.EnumerateArray() + .Select(CapabilityEntry.FromJson) + .OrderByDescending(entry => entry.Prefix.Length) + .ToArray(); + } + + return result; + } + + private sealed record CapabilityEntry( + string Prefix, + int? ContextWindow, + IList? InputModalities, + IList? OutputModalities) + { + public static CapabilityEntry FromJson(JsonElement value) => + new( + value.GetProperty("prefix").GetString()!, + value.TryGetProperty("contextWindow", out var contextWindow) ? contextWindow.GetInt32() : null, + ReadModalities(value, "inputModalities"), + ReadModalities(value, "outputModalities")); + + private static IList? ReadModalities(JsonElement value, string propertyName) => + value.TryGetProperty(propertyName, out var modalities) + ? modalities.EnumerateArray().Select(item => item.GetString()!).ToArray() + : null; + } +} diff --git a/runtime/csharp/Prompty.Core/Prompty.Core.csproj b/runtime/csharp/Prompty.Core/Prompty.Core.csproj index 151ca0d14..a5ce22719 100644 --- a/runtime/csharp/Prompty.Core/Prompty.Core.csproj +++ b/runtime/csharp/Prompty.Core/Prompty.Core.csproj @@ -21,6 +21,7 @@ + diff --git a/runtime/csharp/Prompty.Foundry/Models.cs b/runtime/csharp/Prompty.Foundry/Models.cs index f61c271c1..e9405b4a6 100644 --- a/runtime/csharp/Prompty.Foundry/Models.cs +++ b/runtime/csharp/Prompty.Foundry/Models.cs @@ -11,6 +11,21 @@ namespace Prompty.Foundry; +/// +/// Foundry implementation of the generated model-listing protocol. +/// +public sealed class FoundryModelLister : IModelLister +{ + /// + public async Task> ListModelsAsync(object connection) + { + if (connection is not Connection typedConnection) + throw new ArgumentException("Foundry model listing requires a generated Connection.", nameof(connection)); + + return [.. await FoundryModels.ListModelsAsync(typedConnection)]; + } +} + /// /// Model discovery for Azure OpenAI / Microsoft Foundry endpoints. /// Creates the appropriate client (API key or Entra ID) and delegates @@ -85,29 +100,56 @@ private static async Task> ListDeploymentsAsync( return models.AsReadOnly(); } - private static ModelInfo MapDeployment(JsonElement deployment) + /// + /// Map a raw Foundry deployment payload to the generated provider-neutral contract. + /// + public static ModelInfo MapDeployment(JsonElement deployment) { var properties = TryGetObject(deployment, "properties"); var model = properties is not null ? TryGetObject(properties.Value, "model") : null; var capabilities = properties is not null ? TryGetObject(properties.Value, "capabilities") : null; capabilities ??= model is not null ? TryGetObject(model.Value, "capabilities") : null; + capabilities ??= TryGetObject(deployment, "capabilities"); - return new ModelInfo + var info = new ModelInfo { Id = GetString(deployment, "name") ?? string.Empty, - DisplayName = model is not null ? GetString(model.Value, "name") : null, - OwnedBy = model is not null ? GetString(model.Value, "publisher") ?? "azure" : "azure", + DisplayName = GetString(deployment, "modelName") + ?? (model is not null ? GetString(model.Value, "name") : null), + OwnedBy = GetString(deployment, "modelPublisher") + ?? (model is not null ? GetString(model.Value, "publisher") : null) + ?? "azure", ContextWindow = capabilities is not null ? GetInt(capabilities.Value, "maxContextLength", "contextWindow", "context_length") - : model is not null ? GetInt(model.Value, "maxContextLength") : null, + : null, InputModalities = capabilities is not null ? GetStringList(capabilities.Value, "inputModalities", "input_modalities", "supportedInputModalities") : null, OutputModalities = capabilities is not null ? GetStringList(capabilities.Value, "outputModalities", "output_modalities", "supportedOutputModalities") : null, - AdditionalProperties = new Dictionary { ["deployment"] = deployment.Clone() }, + AdditionalProperties = ModelDiscovery.PreserveRaw(deployment), + }; + info.ContextWindow ??= model is not null ? GetInt(model.Value, "maxContextLength") : null; + info.ContextWindow ??= GetInt(deployment, "maxContextLength"); + ModelDiscovery.Enrich("foundry", info); + return info; + } + + /// + /// Map a raw Azure OpenAI catalog payload to the generated provider-neutral contract. + /// + public static ModelInfo MapCatalogModel(JsonElement model) + { + var info = new ModelInfo + { + Id = GetString(model, "id") ?? string.Empty, + OwnedBy = GetString(model, "owned_by"), + ContextWindow = GetInt(model, "maxContextLength"), + AdditionalProperties = ModelDiscovery.PreserveRaw(model), }; + ModelDiscovery.Enrich("foundry", info); + return info; } private static JsonElement? TryGetObject(JsonElement element, string name) => diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorDiscoveryTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorDiscoveryTests.cs new file mode 100644 index 000000000..27bd14e18 --- /dev/null +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorDiscoveryTests.cs @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Prompty.Anthropic; +using Prompty.Core; +using Prompty.Foundry; + +namespace Prompty.OpenAI.Tests; + +/// +/// Executes the shared provider discovery and capability-enrichment contracts. +/// +public class SpecVectorDiscoveryTests +{ + private static readonly string SpecDir = FindSpecDir(); + + [Theory] + [MemberData(nameof(DiscoveryVectors))] + public void Discovery_Mapping_MatchesSharedVector( + string name, + string provider, + string shape, + JsonElement input, + JsonElement expected) + { + var actual = (provider, shape) switch + { + ("openai", "model") => OpenAIModels.MapModel(input), + ("anthropic", "model") => AnthropicModels.MapModel(input), + ("foundry", "deployment") => FoundryModels.MapDeployment(input), + ("foundry", "catalog") => FoundryModels.MapCatalogModel(input), + _ => throw new InvalidOperationException($"Unsupported discovery vector {provider}/{shape}."), + }; + + AssertJsonEqual(expected, JsonSerializer.SerializeToElement(actual.Save()), name); + } + + [Theory] + [MemberData(nameof(EnrichmentVectors))] + public void Enrichment_MatchesSharedVector( + string name, + string provider, + JsonElement input, + JsonElement expected) + { + var model = ModelInfo.Load((Dictionary)ConvertValue(input)!); + + ModelDiscovery.Enrich(provider, model); + + AssertJsonEqual(expected, JsonSerializer.SerializeToElement(model.Save()), name); + } + + [Fact] + public void EmbeddedCapabilityDataset_MatchesCanonicalSpec() + { + var canonical = JsonNode.Parse(File.ReadAllText(Path.Combine(SpecDir, "data", "model_capabilities.json"))); + var embeddedPath = Path.Combine( + FindRepositoryRoot(), + "runtime", + "csharp", + "Prompty.Core", + "Data", + "model_capabilities.json"); + var embedded = JsonNode.Parse(File.ReadAllText(embeddedPath)); + + Assert.True(JsonNode.DeepEquals(canonical, embedded), "The C# capability dataset copy has drifted from spec/data."); + } + + [Fact] + public void Enrichment_IsCaseSensitiveAndReturnsIndependentLists() + { + var upperCase = new ModelInfo { Id = "GPT-4O" }; + ModelDiscovery.Enrich("openai", upperCase); + Assert.Null(upperCase.ContextWindow); + + var first = new ModelInfo { Id = "gpt-4o" }; + ModelDiscovery.Enrich("openai", first); + first.InputModalities![0] = "mutated"; + + var second = new ModelInfo { Id = "gpt-4o" }; + ModelDiscovery.Enrich("openai", second); + Assert.Equal("text", second.InputModalities![0]); + } + + public static IEnumerable DiscoveryVectors() => + LoadVectors("discovery_vectors.json") + .Select(vector => new object[] + { + vector.GetProperty("name").GetString()!, + vector.GetProperty("provider").GetString()!, + vector.GetProperty("shape").GetString()!, + vector.GetProperty("input").Clone(), + vector.GetProperty("expected").Clone(), + }); + + public static IEnumerable EnrichmentVectors() => + LoadVectors("enrichment_vectors.json") + .Select(vector => new object[] + { + vector.GetProperty("name").GetString()!, + vector.GetProperty("provider").GetString()!, + vector.GetProperty("input").Clone(), + vector.GetProperty("expected").Clone(), + }); + + private static JsonElement[] LoadVectors(string fileName) + { + using var document = JsonDocument.Parse( + File.ReadAllText(Path.Combine(SpecDir, "vectors", "discovery", fileName))); + return document.RootElement.GetProperty("vectors").EnumerateArray().Select(item => item.Clone()).ToArray(); + } + + private static void AssertJsonEqual(JsonElement expected, JsonElement actual, string name) + { + var expectedNode = JsonNode.Parse(expected.GetRawText()); + var actualNode = JsonNode.Parse(actual.GetRawText()); + Assert.True( + JsonNode.DeepEquals(expectedNode, actualNode), + $"[{name}] expected {expected.GetRawText()}, actual {actual.GetRawText()}"); + } + + private static object? ConvertValue(JsonElement value) => + value.ValueKind switch + { + JsonValueKind.Object => value.EnumerateObject() + .ToDictionary(property => property.Name, property => ConvertValue(property.Value)), + JsonValueKind.Array => value.EnumerateArray().Select(ConvertValue).ToList(), + JsonValueKind.String => value.GetString(), + JsonValueKind.Number when value.TryGetInt32(out var intValue) => intValue, + JsonValueKind.Number => value.GetDouble(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null => null, + _ => value.Clone(), + }; + + private static string FindSpecDir() => Path.Combine(FindRepositoryRoot(), "spec"); + + private static string FindRepositoryRoot() + { + var directory = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + if (Directory.Exists(Path.Combine(directory, "spec"))) + return directory; + directory = Path.GetDirectoryName(directory) ?? directory; + } + + throw new DirectoryNotFoundException("Could not locate repository root containing spec/."); + } +} diff --git a/runtime/csharp/Prompty.OpenAI/Models.cs b/runtime/csharp/Prompty.OpenAI/Models.cs index bd5f99d81..5e3f725d4 100644 --- a/runtime/csharp/Prompty.OpenAI/Models.cs +++ b/runtime/csharp/Prompty.OpenAI/Models.cs @@ -1,35 +1,34 @@ // Copyright (c) Microsoft. All rights reserved. using System.ClientModel; +using System.Text.Json; using OpenAI; using OpenAI.Models; using Prompty.Core; namespace Prompty.OpenAI; +/// +/// OpenAI implementation of the generated model-listing protocol. +/// +public sealed class OpenAIModelLister : IModelLister +{ + /// + public async Task> ListModelsAsync(object connection) + { + if (connection is not Connection typedConnection) + throw new ArgumentException("OpenAI model listing requires a generated Connection.", nameof(connection)); + + return [.. await OpenAIModels.ListModelsAsync(typedConnection)]; + } +} + /// /// Model discovery for OpenAI — lists available models and enriches /// sparse API responses with known context window and modality metadata. /// public static class OpenAIModels { - /// - /// Known model metadata used to enrich the sparse data returned by GET /v1/models. - /// Keys are sorted by descending length so prefix matching finds the most specific - /// match first (e.g. "gpt-4o-mini" before "gpt-4o" before "gpt-4"). - /// - private static readonly (string Key, int? ContextWindow, string[] Inputs, string[] Outputs)[] KnownModels = - [ - ("text-embedding-3-large", 8191, ["text"], []), - ("text-embedding-3-small", 8191, ["text"], []), - ("gpt-3.5-turbo", 16385, ["text"], ["text"]), - ("gpt-4o-mini", 128000, ["text", "image"], ["text"]), - ("gpt-4-turbo", 128000, ["text", "image"], ["text"]), - ("dall-e-3", null, ["text"], ["image"]), - ("gpt-4o", 128000, ["text", "image"], ["text"]), - ("gpt-4", 8192, ["text"], ["text"]), - ]; - /// /// List models available from an OpenAI endpoint using connection credentials. /// @@ -58,8 +57,15 @@ public static async Task> ListModelsAsync( { Id = m.Id, OwnedBy = m.OwnedBy, + AdditionalProperties = new Dictionary + { + ["id"] = m.Id, + ["object"] = "model", + ["created"] = m.CreatedAt.ToUnixTimeSeconds(), + ["owned_by"] = m.OwnedBy, + }, }; - Enrich(info); + ModelDiscovery.Enrich("openai", info); models.Add(info); } @@ -73,17 +79,22 @@ public static async Task> ListModelsAsync( /// internal static void Enrich(ModelInfo info) { - foreach (var (key, contextWindow, inputs, outputs) in KnownModels) + ModelDiscovery.Enrich("openai", info); + } + + /// + /// Map a raw OpenAI model payload to the generated provider-neutral contract. + /// + public static ModelInfo MapModel(JsonElement model) + { + var info = new ModelInfo { - if (string.Equals(info.Id, key, StringComparison.OrdinalIgnoreCase) - || info.Id.StartsWith(key + "-", StringComparison.OrdinalIgnoreCase)) - { - info.ContextWindow = contextWindow; - info.InputModalities = inputs; - info.OutputModalities = outputs; - return; - } - } + Id = model.TryGetProperty("id", out var id) ? id.GetString() ?? string.Empty : string.Empty, + OwnedBy = model.TryGetProperty("owned_by", out var ownedBy) ? ownedBy.GetString() : null, + AdditionalProperties = ModelDiscovery.PreserveRaw(model), + }; + ModelDiscovery.Enrich("openai", info); + return info; } private static OpenAIClient CreateClient(Connection connection) diff --git a/runtime/csharp/Prompty.OpenAI/Prompty.OpenAI.csproj b/runtime/csharp/Prompty.OpenAI/Prompty.OpenAI.csproj index 6c8c5d436..a99192c29 100644 --- a/runtime/csharp/Prompty.OpenAI/Prompty.OpenAI.csproj +++ b/runtime/csharp/Prompty.OpenAI/Prompty.OpenAI.csproj @@ -4,7 +4,7 @@ net9.0 enable enable - OPENAI001;CS1591 + OPENAI001;SCME0001;CS1591 true 2.0.0-beta.4 Prompty.OpenAI From bc16f363896afc660815af8b238f0c2ad893b57f Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 01:19:51 -0700 Subject: [PATCH 06/19] feat(csharp): add canonical turn engine Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../TurnEngineFailureTests.cs | 309 ++++ .../TurnEngineReconciliationTests.cs | 240 ++++ .../TurnEngineResumeTests.cs | 187 +++ .../TurnEngineTestDoubles.cs | 241 ++++ .../TurnEngineVectorTests.cs | 248 ++++ runtime/csharp/Prompty.Core/TurnEngine.cs | 1258 +++++++++++++++++ .../Prompty.Core/TurnEngineExceptions.cs | 142 ++ .../Prompty.Core/TurnEngineModelExtensions.cs | 93 ++ .../csharp/Prompty.Core/TurnEnginePorts.cs | 234 +++ .../csharp/Prompty.Core/TurnEngineRequest.cs | 374 +++++ 10 files changed, 3326 insertions(+) create mode 100644 runtime/csharp/Prompty.Core.Tests/TurnEngineFailureTests.cs create mode 100644 runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs create mode 100644 runtime/csharp/Prompty.Core.Tests/TurnEngineResumeTests.cs create mode 100644 runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs create mode 100644 runtime/csharp/Prompty.Core.Tests/TurnEngineVectorTests.cs create mode 100644 runtime/csharp/Prompty.Core/TurnEngine.cs create mode 100644 runtime/csharp/Prompty.Core/TurnEngineExceptions.cs create mode 100644 runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs create mode 100644 runtime/csharp/Prompty.Core/TurnEnginePorts.cs create mode 100644 runtime/csharp/Prompty.Core/TurnEngineRequest.cs diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineFailureTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineFailureTests.cs new file mode 100644 index 000000000..92f9010bd --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineFailureTests.cs @@ -0,0 +1,309 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// Covers effect failures, recovery contracts, context composition, retry, and best-effort streaming. +public class TurnEngineFailureTests +{ + [Fact] + public async Task StructuredToolOutput_IsSerializedAsJsonForTheModel() + { + var output = new Dictionary + { + ["ok"] = true, + ["items"] = new List { 1, "two" }, + }; + var tools = new StructuredOutputToolPort(output); + var model = new CapturingModelPort( + [ + new ModelInvocationResponse + { + AssistantMessages = [Message.Assistant("Using a tool.")], + ToolRequests = [new ModelToolRequest { Id = "call-json", Name = "json", Arguments = new Dictionary() }], + }, + new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }, + ]); + var engine = CreateEngine(model, tools); + + var result = await engine.RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + var toolMessage = Assert.Single(model.Requests[1].Context.Messages, message => message.Role == Role.Tool); + var text = Assert.IsType(Assert.Single(toolMessage.Parts)); + Assert.Equal("""{"items":[1,"two"],"ok":true}""", text.Value); + } + + [Fact] + public async Task ContextPort_CanAddMessagesAndDecisions() + { + var model = new CapturingModelPort( + [ + new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }, + ]); + var effects = CreateEffects(model, new EchoToolPort(), context: new AppendingContextPort()); + var engine = new TurnEngine(effects); + + var result = await engine.RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + var snapshot = Assert.Single(result.Snapshots!); + Assert.Equal(2, snapshot.Messages.Count); + Assert.Equal("candidate-1", Assert.Single(snapshot.Decisions!).CandidateId); + Assert.Same(snapshot, Assert.Single(model.Requests).Context); + } + + [Fact] + public async Task InvalidContextSnapshot_CommitsContextErrorWithoutInvokingModel() + { + var model = new ThrowingModelPort(); + var effects = CreateEffects(model, new EchoToolPort(), context: new InvalidContextPort()); + var result = await new TurnEngine(effects).RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Failed, result.Commit.Status); + var output = Assert.IsType>(result.Commit.Output); + Assert.Equal("context_error", output["errorKind"]); + } + + [Fact] + public async Task TransientModelFailure_RetriesTheSameSnapshot() + { + var model = new TransientModelPort(); + var retry = new RecordingRetryPort(); + var effects = CreateEffects(model, new EchoToolPort(), retry: retry); + var result = await new TurnEngine(effects).RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal(2, model.InvocationCount); + Assert.Single(retry.Requests); + Assert.Same(model.Snapshots[0], model.Snapshots[1]); + } + + [Fact] + public async Task AtomicDurabilityFailure_ThrowsRecoveryRequiredWithCheckpoint() + { + var durability = new FailingCheckpointDurabilityPort(); + var effects = CreateEffects( + new CapturingModelPort( + [ + new ModelInvocationResponse { Output = "provider committed", AssistantMessages = [], ToolRequests = [] }, + ]), + new EchoToolPort(), + durability: durability); + var engine = new TurnEngine(effects); + + var error = await Assert.ThrowsAsync( + () => engine.RunAsync(NewRequest(), CancellationToken.None)); + + Assert.Equal("model response", error.Stage); + Assert.True(error.Checkpoint.FinalOutputReady); + Assert.Equal("provider committed", error.Checkpoint.PendingOutput); + Assert.Empty(error.ToolResults); + Assert.DoesNotContain(durability.Events, evt => evt.Kind == EngineEventKind.ModelInvocationCompleted); + } + + [Fact] + public async Task StreamFailure_DoesNotChangeSemanticExecution() + { + var model = new StreamingModelPort(); + var effects = CreateEffects(model, new EchoToolPort(), stream: new FailingStreamPort()); + var result = await new TurnEngine(effects).RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal("done", result.Commit.Output); + } + + [Fact] + public async Task NonPortStreamFailure_DoesNotChangeSemanticExecution() + { + var model = new StreamingModelPort(); + var effects = CreateEffects(model, new EchoToolPort(), stream: new FailingStreamPort(usePortError: false)); + var result = await new TurnEngine(effects).RunAsync(NewRequest(), CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal("done", result.Commit.Output); + } + + [Fact] + public async Task PostCommitEffectId_UsesUtf8ByteLengths() + { + var postCommit = new RecordingPostCommitPort(); + var effects = new TurnEngineEffects + { + Model = new CapturingModelPort( + [ + new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }, + ]), + Tools = new EchoToolPort(), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + PostCommit = postCommit, + }; + var request = new TurnEngineRequest("séssion", "türn", [Message.User("Hello")]) { MaxIterations = 10 }; + + await new TurnEngine(effects).RunAsync(request, CancellationToken.None); + + Assert.Equal("post_commit:8:séssion:5:türn", Assert.Single(postCommit.Commits).EffectId); + } + + private static TurnEngineRequest NewRequest() => + new("session-1", "turn-1", [Message.User("Hello")]) { MaxIterations = 10 }; + + private static TurnEngine CreateEngine(IEngineModelPort model, IEngineToolPort tools) => + new(CreateEffects(model, tools)); + + private static TurnEngineEffects CreateEffects( + IEngineModelPort model, + IEngineToolPort tools, + IEngineContextPort? context = null, + IEngineRetryPolicyPort? retry = null, + IEngineDurabilityPort? durability = null, + IEngineModelStreamPort? stream = null) => + new() + { + Model = model, + Tools = tools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Context = context ?? new PassthroughEngineContextPort(), + Retry = retry ?? new NoopRetryPolicyPort(), + Durability = durability ?? new RecordingDurabilityPort(), + Stream = stream ?? new NoopModelStreamPort(), + }; + + private sealed class StructuredOutputToolPort(object output) : IEngineToolPort + { + public Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) => + Task.FromResult(new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = ModelToolOutcome.Success, + Output = output, + }); + } + + private sealed class CapturingModelPort(IEnumerable responses) : IEngineModelPort + { + private readonly Queue _responses = new(responses); + + public List Requests { get; } = []; + + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + Requests.Add(request); + return Task.FromResult(_responses.Dequeue()); + } + } + + private sealed class AppendingContextPort : IEngineContextPort + { + public Task PrepareAsync(ContextRequest request, CancellationToken cancellationToken) => + Task.FromResult(new ModelInvocationContextSnapshot + { + Id = $"context:{request.InvocationId}", + SessionId = request.SessionId, + TurnId = request.TurnId, + InvocationId = request.InvocationId, + Iteration = request.Iteration, + Messages = [.. request.Messages, Message.System("Recalled context")], + Decisions = + [ + new InvocationContextDecision + { + CandidateId = "candidate-1", + Disposition = InvocationContextDisposition.Included, + Reason = "included by test context", + }, + ], + StablePrefixMessages = request.StablePrefixMessages, + ContextState = request.ContextState, + }); + } + + private sealed class InvalidContextPort : IEngineContextPort + { + public Task PrepareAsync(ContextRequest request, CancellationToken cancellationToken) => + Task.FromResult(new ModelInvocationContextSnapshot + { + Id = "wrong", + SessionId = "wrong-session", + TurnId = request.TurnId, + InvocationId = request.InvocationId, + Iteration = request.Iteration, + Messages = request.Messages, + Decisions = [], + StablePrefixMessages = request.StablePrefixMessages, + ContextState = request.ContextState, + }); + } + + private sealed class TransientModelPort : IEngineModelPort + { + public int InvocationCount { get; private set; } + + public List Snapshots { get; } = []; + + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + InvocationCount++; + Snapshots.Add(request.Context); + return InvocationCount == 1 + ? throw new PortError("transient") + : Task.FromResult(new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }); + } + } + + private sealed class RecordingRetryPort : IEngineRetryPolicyPort + { + public List Requests { get; } = []; + + public Task BackoffAsync(RetryPolicyRequest request, CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.CompletedTask; + } + } + + private sealed class FailingCheckpointDurabilityPort : IEngineDurabilityPort + { + public List Events { get; } = []; + + public Task AppendAsync(EngineEvent @event) + { + Events.Add(@event); + return Task.CompletedTask; + } + + public Task AppendWithCheckpointAsync(IReadOnlyList events, EngineCheckpoint checkpoint) => + throw new PortError("atomic store unavailable"); + } + + private sealed class StreamingModelPort : IEngineModelPort + { + public async Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + await stream.EmitAsync(new ModelStreamChunk.Text("partial")); + return new ModelInvocationResponse { Output = "done", AssistantMessages = [], ToolRequests = [] }; + } + } + + private sealed class FailingStreamPort(bool usePortError = true) : IEngineModelStreamPort + { + public Task EmitAsync(ModelStreamChunk chunk) => + usePortError + ? throw new PortError("stream sink unavailable") + : throw new IOException("stream sink unavailable"); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs new file mode 100644 index 000000000..0b2fd3226 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs @@ -0,0 +1,240 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// +/// Covers behavior that the shared vectors do not exercise directly: indeterminate tool-effect +/// and model-effect reconciliation (an effect whose real-world outcome is unknown must block the +/// turn until a host supplies an explicit determinate resolution, and resuming must never re-run +/// the unknown effect), non-fatal post-commit failures, mid-run cancellation, and strictly +/// sequential (never concurrent) tool execution. +/// +public class TurnEngineReconciliationTests +{ + [Fact] + public async Task IndeterminateToolEffect_BlocksTurnUntilExplicitReconciliation() + { + var toolRequest = new ModelToolRequest { Id = "call-a", Name = "echo", Arguments = new Dictionary { ["value"] = "A" } }; + var steps = new List { new() { Assistant = "Using a tool.", Tools = [toolRequest] } }; + + var durability = new RecordingDurabilityPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = new IndeterminateToolPort("call-a", new EchoToolPort()), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = durability, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Do something")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.ReconciliationRequired, result.Commit.Status); + + var checkpoint = durability.Checkpoints.Last(); + Assert.True(checkpoint.ReconciliationRequired); + Assert.Equal(ModelToolOutcome.Indeterminate, checkpoint.CompletedToolResults!.Single().Outcome); + Assert.Null(checkpoint.ModelReconciliation); + + // Resuming without an explicit resolution must not silently proceed or re-run the tool — + // it re-reports the same reconciliation-required outcome. + var unresolvedTools = new EchoToolPort(forbiddenRequestIds: ["call-a"]); + var unresolvedEngine = new TurnEngine(new TurnEngineEffects + { + Model = new ThrowingModelPort(), + Tools = unresolvedTools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }); + var unresolvedResume = TurnEngineRequest.ResumeFrom(checkpoint, maxIterations: 10, lastJournalSequence: 0); + var unresolvedResult = await unresolvedEngine.RunAsync(unresolvedResume, CancellationToken.None); + Assert.Equal(EngineTurnStatus.ReconciliationRequired, unresolvedResult.Commit.Status); + Assert.Empty(unresolvedTools.ExecutedRequestIds); + + // Attempting to resolve with another indeterminate outcome is rejected outright. + var stillIndeterminate = new ModelToolResult { RequestId = "call-a", Name = "echo", Outcome = ModelToolOutcome.Indeterminate, Output = "still unknown" }; + Assert.Throws(() => + TurnEngineRequest.ResumeAfterReconciliation(checkpoint, 10, 0, stillIndeterminate)); + + // Resolving with an explicit determinate outcome resumes and completes without re-running + // the tool at all. + var resolvedResult = new ModelToolResult { RequestId = "call-a", Name = "echo", Outcome = ModelToolOutcome.Success, Output = "A" }; + var resolvedTools = new EchoToolPort(forbiddenRequestIds: ["call-a"]); + var resolvedEngine = new TurnEngine(new TurnEngineEffects + { + Model = new ScriptedModelPort([new ScriptedModelStep { Output = "done" }]), + Tools = resolvedTools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }); + var resolvedRequest = TurnEngineRequest.ResumeAfterReconciliation(checkpoint, 10, 0, resolvedResult); + var finalResult = await resolvedEngine.RunAsync(resolvedRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, finalResult.Commit.Status); + Assert.Equal("done", finalResult.Commit.Output); + Assert.Empty(resolvedTools.ExecutedRequestIds); + Assert.Equal(ModelToolOutcome.Success, finalResult.ToolResults!.Single(t => t.RequestId == "call-a").Outcome); + } + + [Fact] + public async Task IndeterminateModelEffect_BlocksTurnUntilExplicitReconciliation() + { + var durability = new RecordingDurabilityPort(); + var effects = new TurnEngineEffects + { + Model = new IndeterminateModelPort(), + Tools = new EchoToolPort(), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = durability, + }; + + var engine = new TurnEngine(effects); + // A single model attempt is enough to force reconciliation: an indeterminate outcome is + // never retried, regardless of MaxModelAttempts. + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Hello")]) { MaxIterations = 10, MaxModelAttempts = 3 }; + var result = await engine.RunAsync(request, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.ReconciliationRequired, result.Commit.Status); + Assert.Equal(1, ((IndeterminateModelPort)effects.Model).InvocationCount); + + var checkpoint = durability.Checkpoints.Last(); + Assert.True(checkpoint.ReconciliationRequired); + Assert.NotNull(checkpoint.ModelReconciliation); + Assert.Equal(checkpoint.ActiveInvocationId, checkpoint.ModelReconciliation!.InvocationId); + + // Resolving with an explicit model response resumes without re-invoking the model port. + var resolvedResponse = new ModelInvocationResponse { Output = "resolved output", AssistantMessages = [], ToolRequests = [] }; + var resumedEngine = new TurnEngine(new TurnEngineEffects + { + Model = new ThrowingModelPort(), + Tools = new EchoToolPort(), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }); + var resumeRequest = TurnEngineRequest.ResumeAfterModelReconciliation(checkpoint, 10, 0, resolvedResponse); + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, resumedResult.Commit.Status); + Assert.Equal("resolved output", resumedResult.Commit.Output); + } + + [Fact] + public async Task PostCommitFailure_IsReportedNonFatally() + { + var postCommit = new FailingPostCommitPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort([new ScriptedModelStep { Output = "Hello back" }]), + Tools = new EchoToolPort(), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + PostCommit = postCommit, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Hello")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, CancellationToken.None); + + // The turn itself still committed successfully — a broken post-commit sink never + // uncommits the turn — but the failure is surfaced for the host to notice and retry. + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal("Hello back", result.Commit.Output); + Assert.NotNull(result.PostCommitError); + Assert.Contains(FailingPostCommitPort.FailureMessage, result.PostCommitError); + Assert.Single(postCommit.AttemptedEffectIds); + } + + [Fact] + public async Task Cancellation_BetweenSequentialTools_StopsBeforeTheSecondToolRuns() + { + var toolA = new ModelToolRequest { Id = "call-a", Name = "echo", Arguments = new Dictionary { ["value"] = "A" } }; + var toolB = new ModelToolRequest { Id = "call-b", Name = "echo", Arguments = new Dictionary { ["value"] = "B" } }; + var steps = new List + { + new() { Assistant = "Using tools.", Tools = [toolA, toolB] }, + new() { Output = "should never be reached" }, + }; + + using var cts = new CancellationTokenSource(); + var tools = new CancelAfterFirstToolPort(cts, "call-a"); + var durability = new RecordingDurabilityPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = tools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = durability, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Add two values")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, cts.Token); + + Assert.Equal(EngineTurnStatus.Cancelled, result.Commit.Status); + Assert.Equal(["call-a"], tools.ExecutedRequestIds); + Assert.Single(result.ToolResults!); + Assert.Equal(EngineEventKind.TurnCancelled, durability.Events.Last().Kind); + } + + [Fact] + public async Task Tools_ExecuteStrictlySequentially_NeverConcurrently() + { + var toolA = new ModelToolRequest { Id = "call-a", Name = "echo", Arguments = new Dictionary { ["value"] = "A" } }; + var toolB = new ModelToolRequest { Id = "call-b", Name = "echo", Arguments = new Dictionary { ["value"] = "B" } }; + var steps = new List + { + new() { Assistant = "Using tools.", Tools = [toolA, toolB] }, + new() { Output = "A then B" }, + }; + var tools = new EchoToolPort(new Dictionary { ["call-a"] = "A", ["call-b"] = "B" }); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = tools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Add two values")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, result.Commit.Status); + Assert.Equal(1, tools.MaxObservedConcurrency); + Assert.Equal(["call-a", "call-b"], tools.ExecutedRequestIds); + } + + /// Tool port that cancels the shared token immediately after its first configured request completes. + private sealed class CancelAfterFirstToolPort(CancellationTokenSource cancellationSource, string cancelAfterRequestId) : IEngineToolPort + { + public List ExecutedRequestIds { get; } = []; + + public Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) + { + if (ExecutedRequestIds.Contains(cancelAfterRequestId)) + { + throw new InvalidOperationException($"tool '{request.Id}' must not execute after cancellation"); + } + + ExecutedRequestIds.Add(request.Id); + var result = new ModelToolResult { RequestId = request.Id, Name = request.Name, Outcome = ModelToolOutcome.Success, Output = request.Id }; + if (request.Id == cancelAfterRequestId) + { + cancellationSource.Cancel(); + } + + return Task.FromResult(result); + } + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineResumeTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineResumeTests.cs new file mode 100644 index 000000000..e294a9335 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineResumeTests.cs @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// +/// Proves the canonical can resume from a durably persisted +/// without duplicating already-committed effects: a checkpoint +/// captured right after the first of two sequential tool results resumes without re-running +/// that tool, continues the event sequence after the checkpoint's last committed sequence +/// (or the journal tail, whichever is larger), a checkpoint with a ready final output commits +/// without ever re-invoking the model, and a checkpoint resumed with no iteration budget left +/// fails immediately instead of silently looping. +/// +public class TurnEngineResumeTests +{ + /// + /// Runs the two-sequential-tool scenario (mirrors the "ordered_tool_round" shared vector) + /// to completion while recording every checkpoint the engine persists along the way, so + /// individual tests can resume from any mid-run checkpoint. + /// + private static async Task<(RecordingDurabilityPort Durability, TurnEngineResult Result)> RunOrderedToolRoundAsync() + { + var steps = new List + { + new() + { + Assistant = "I will use the tools.", + Tools = + [ + new ModelToolRequest { Id = "call-a", Name = "echo", Arguments = new Dictionary { ["value"] = "A" } }, + new ModelToolRequest { Id = "call-b", Name = "echo", Arguments = new Dictionary { ["value"] = "B" } }, + ], + }, + new() { Output = "A then B" }, + }; + var toolOutputs = new Dictionary { ["call-a"] = "A", ["call-b"] = "B" }; + + var durability = new RecordingDurabilityPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = new EchoToolPort(toolOutputs), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = durability, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", [Message.User("Add two values")]) { MaxIterations = 10 }; + var result = await engine.RunAsync(request, CancellationToken.None); + return (durability, result); + } + + [Fact] + public async Task Resume_AfterFirstToolResult_DoesNotRerunThatTool() + { + var (durability, originalResult) = await RunOrderedToolRoundAsync(); + Assert.Equal(EngineTurnStatus.Success, originalResult.Commit.Status); + + // The checkpoint persisted immediately after "call-a" completes: exactly one completed + // tool result, and "call-b" still pending. + var checkpointAfterFirstTool = durability.Checkpoints.First(c => + (c.CompletedToolResults?.Count ?? 0) == 1 + && (c.PendingToolRequests?.Count ?? 0) == 1); + + Assert.Equal("call-a", checkpointAfterFirstTool.CompletedToolResults![0].RequestId); + Assert.Equal("call-b", checkpointAfterFirstTool.PendingToolRequests![0].Id); + + // Resume with a tool port that throws if "call-a" is ever asked to execute again, and a + // model port that only knows about the final output step (the tool round's own model + // response is already durably captured in the checkpoint and must not be re-requested). + var resumedDurability = new RecordingDurabilityPort(); + var resumedTools = new EchoToolPort( + new Dictionary { ["call-b"] = "B" }, + forbiddenRequestIds: ["call-a"]); + var resumedEffects = new TurnEngineEffects + { + Model = new ScriptedModelPort([new ScriptedModelStep { Output = "A then B" }]), + Tools = resumedTools, + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = resumedDurability, + }; + + var resumedEngine = new TurnEngine(resumedEffects); + var resumeRequest = TurnEngineRequest.ResumeFrom(checkpointAfterFirstTool, maxIterations: 10, lastJournalSequence: 0); + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, resumedResult.Commit.Status); + Assert.Equal("A then B", resumedResult.Commit.Output); + Assert.Equal(["call-b"], resumedTools.ExecutedRequestIds); + Assert.Equal(2, resumedResult.ToolResults?.Count); + Assert.Equal(["call-a", "call-b"], resumedResult.ToolResults!.Select(t => t.RequestId)); + + // The resumed run must continue the event sequence strictly after the checkpoint's last + // committed sequence — never restart from 1, which would duplicate already-durable events. + Assert.All(resumedDurability.Events, e => Assert.True(e.Sequence > checkpointAfterFirstTool.LastSequence)); + Assert.Equal(checkpointAfterFirstTool.LastSequence + 1, resumedDurability.Events[0].Sequence); + } + + [Fact] + public async Task Resume_ContinuesAfterMaxOfCheckpointAndJournalTail() + { + var (durability, _) = await RunOrderedToolRoundAsync(); + var checkpointAfterFirstTool = durability.Checkpoints.First(c => + (c.CompletedToolResults?.Count ?? 0) == 1 && (c.PendingToolRequests?.Count ?? 0) == 1); + + // Simulate a journal whose tail is further ahead than the checkpoint itself (for example, + // a plain, non-checkpointed event was appended after the checkpoint before the host + // crashed). Resuming must continue after the larger of the two, never the checkpoint alone. + var journalTail = checkpointAfterFirstTool.LastSequence + 5; + + var resumedDurability = new RecordingDurabilityPort(); + var resumedEffects = new TurnEngineEffects + { + Model = new ScriptedModelPort([new ScriptedModelStep { Output = "A then B" }]), + Tools = new EchoToolPort(new Dictionary { ["call-b"] = "B" }, forbiddenRequestIds: ["call-a"]), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = resumedDurability, + }; + + var resumedEngine = new TurnEngine(resumedEffects); + var resumeRequest = TurnEngineRequest.ResumeFrom(checkpointAfterFirstTool, maxIterations: 10, lastJournalSequence: journalTail); + Assert.Equal(journalTail, resumeRequest.InitialSequence); + + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, resumedResult.Commit.Status); + Assert.Equal(journalTail + 1, resumedDurability.Events[0].Sequence); + } + + [Fact] + public async Task Resume_WhenFinalOutputAlreadyReady_NeverReinvokesModel() + { + var (durability, _) = await RunOrderedToolRoundAsync(); + var finalCheckpoint = durability.Checkpoints.Last(c => c.FinalOutputReady); + + var resumedEffects = new TurnEngineEffects + { + Model = new ThrowingModelPort(), + Tools = new EchoToolPort(forbiddenRequestIds: ["call-a", "call-b"]), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }; + + var resumedEngine = new TurnEngine(resumedEffects); + var resumeRequest = TurnEngineRequest.ResumeFrom(finalCheckpoint, maxIterations: 10, lastJournalSequence: 0); + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Success, resumedResult.Commit.Status); + Assert.Equal("A then B", resumedResult.Commit.Output); + } + + [Fact] + public async Task Resume_WithNoIterationBudgetRemaining_FailsWithMaxIterations() + { + var (durability, _) = await RunOrderedToolRoundAsync(); + var checkpointAfterFirstTool = durability.Checkpoints.First(c => + (c.CompletedToolResults?.Count ?? 0) == 1 && (c.PendingToolRequests?.Count ?? 0) == 1); + + var resumedEffects = new TurnEngineEffects + { + Model = new ThrowingModelPort(), + Tools = new EchoToolPort(forbiddenRequestIds: ["call-a", "call-b"]), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Durability = new RecordingDurabilityPort(), + }; + + var resumedEngine = new TurnEngine(resumedEffects); + // maxIterations equal to the checkpoint's own iteration means the resumed run starts + // with no budget left at all: the while loop condition is false on the first check. + var resumeRequest = TurnEngineRequest.ResumeFrom( + checkpointAfterFirstTool, + maxIterations: checkpointAfterFirstTool.Iteration, + lastJournalSequence: 0); + var resumedResult = await resumedEngine.RunAsync(resumeRequest, CancellationToken.None); + + Assert.Equal(EngineTurnStatus.Failed, resumedResult.Commit.Status); + var output = Assert.IsType>(resumedResult.Commit.Output); + Assert.Equal("max_iterations", output["errorKind"]); + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs new file mode 100644 index 000000000..2cf0784f0 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +// ----------------------------------------------------------------------- +// Deterministic test doubles shared by the canonical TurnEngine test suite +// (TurnEngineVectorTests, TurnEngineResumeTests, TurnEngineReconciliationTests). +// ----------------------------------------------------------------------- + +/// Deterministic, monotonically increasing clock for engine tests. +internal sealed class FakeEngineClock : IEngineClock +{ + private long _tick; + + public string Now() => $"2024-01-01T00:00:{_tick++:D2}Z"; +} + +/// Deterministic per-kind sequential id generator for engine tests. +internal sealed class FakeEngineIdGenerator : IEngineIdGenerator +{ + private readonly Dictionary _counters = []; + + public string NextId(string kind) + { + var next = _counters.TryGetValue(kind, out var current) ? current + 1 : 1; + _counters[kind] = next; + return $"{kind}-{next}"; + } +} + +/// One scripted model turn step: either a final output, or an assistant message plus tool requests. +internal sealed class ScriptedModelStep +{ + public string? Assistant { get; init; } + + public object? Output { get; init; } + + public IList Tools { get; init; } = []; + + public InvocationContextPortability? NextPortability { get; init; } + + public IList? DelegatedState { get; init; } +} + +/// Model port that replays a fixed script of responses, one per invocation, strictly in order. +internal sealed class ScriptedModelPort : IEngineModelPort +{ + private readonly Queue _steps; + + public ScriptedModelPort(IEnumerable steps) => _steps = new Queue(steps); + + public int InvocationCount { get; private set; } + + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + InvocationCount++; + if (_steps.Count == 0) + { + throw PortError.Configuration("scripted model script exhausted"); + } + + var step = _steps.Dequeue(); + var response = new ModelInvocationResponse + { + Output = step.Output, + AssistantMessages = step.Assistant is null ? [] : [Message.Assistant(step.Assistant)], + ToolRequests = [.. step.Tools], + }; + + if (step.NextPortability is not null) + { + response.NextContextState = new InvocationContextState + { + Portability = step.NextPortability.Value, + DelegatedState = step.DelegatedState ?? [], + }; + } + + return Task.FromResult(response); + } +} + +/// Model port that always throws — proves the model is never re-invoked after resume/reconciliation. +internal sealed class ThrowingModelPort : IEngineModelPort +{ + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) => + throw new InvalidOperationException("model must not be invoked again"); +} + +/// Model port whose single scripted attempt always fails with an indeterminate (outcome-unknown) error. +internal sealed class IndeterminateModelPort : IEngineModelPort +{ + public int InvocationCount { get; private set; } + + public Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + InvocationCount++; + throw PortError.Indeterminate("model invocation outcome is unknown"); + } +} + +/// +/// Tool port that resolves each request's output from a lookup table and records execution +/// order/concurrency, so tests can prove tools run sequentially and in model-provided order. +/// +internal sealed class EchoToolPort : IEngineToolPort +{ + private readonly IReadOnlyDictionary _outputs; + private readonly HashSet _forbiddenRequestIds; + private int _inFlight; + + public EchoToolPort(IReadOnlyDictionary? outputs = null, IEnumerable? forbiddenRequestIds = null) + { + _outputs = outputs ?? new Dictionary(); + _forbiddenRequestIds = forbiddenRequestIds is null ? [] : [.. forbiddenRequestIds]; + } + + /// Request ids executed so far, in the order they were executed. + public List ExecutedRequestIds { get; } = []; + + /// The largest number of concurrently in-flight tool executions observed. + public int MaxObservedConcurrency { get; private set; } + + public async Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) + { + if (_forbiddenRequestIds.Contains(request.Id)) + { + throw new InvalidOperationException($"tool request '{request.Id}' must not execute again"); + } + + var concurrent = Interlocked.Increment(ref _inFlight); + MaxObservedConcurrency = Math.Max(MaxObservedConcurrency, concurrent); + try + { + // Yield to make any accidental parallel invocation observable. + await Task.Yield(); + ExecutedRequestIds.Add(request.Id); + var output = _outputs.TryGetValue(request.Id, out var value) ? value : request.Name; + return new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = ModelToolOutcome.Success, + Output = output, + }; + } + finally + { + Interlocked.Decrement(ref _inFlight); + } + } +} + +/// Tool port whose single configured request id fails with an indeterminate (outcome-unknown) error. +internal sealed class IndeterminateToolPort : IEngineToolPort +{ + private readonly string _requestId; + private readonly IEngineToolPort _inner; + + public IndeterminateToolPort(string requestId, IEngineToolPort inner) + { + _requestId = requestId; + _inner = inner; + } + + public Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) => + request.Id == _requestId + ? throw PortError.Indeterminate($"tool '{request.Id}' outcome is unknown") + : _inner.ExecuteAsync(request, cancellationToken); +} + +/// Permission port that denies tool requests whose name is in the deny list. +internal sealed class DenyByNamePermissionPort : IEnginePermissionPort +{ + private readonly HashSet _denied; + + public DenyByNamePermissionPort(IEnumerable deniedNames) => _denied = [.. deniedNames]; + + public Task AuthorizeAsync(ModelToolRequest request, CancellationToken cancellationToken) => + Task.FromResult(_denied.Contains(request.Name) + ? new EnginePermissionDecision { Approved = false, Reason = "Permission was denied" } + : new EnginePermissionDecision { Approved = true }); +} + +/// Durability port that records every appended event/checkpoint in memory for assertions. +internal sealed class RecordingDurabilityPort : IEngineDurabilityPort +{ + public List Events { get; } = []; + + public List Checkpoints { get; } = []; + + public Task AppendAsync(EngineEvent @event) + { + Events.Add(@event); + return Task.CompletedTask; + } + + public Task AppendWithCheckpointAsync(IReadOnlyList events, EngineCheckpoint checkpoint) + { + Events.AddRange(events); + Checkpoints.Add(checkpoint); + return Task.CompletedTask; + } +} + +/// Post-commit port that always fails, to prove post-commit failures are reported non-fatally. +internal sealed class FailingPostCommitPort : IEnginePostCommitPort +{ + public const string FailureMessage = "post-commit sink unavailable"; + + public List AttemptedEffectIds { get; } = []; + + public Task AfterCommitAsync(string effectId, TurnCommit commit, CancellationToken cancellationToken) + { + AttemptedEffectIds.Add(effectId); + throw PortError.Configuration(FailureMessage); + } +} + +/// Post-commit port that records every commit it was asked to run an effect for. +internal sealed class RecordingPostCommitPort : IEnginePostCommitPort +{ + public List<(string EffectId, TurnCommit Commit)> Commits { get; } = []; + + public Task AfterCommitAsync(string effectId, TurnCommit commit, CancellationToken cancellationToken) + { + Commits.Add((effectId, commit)); + return Task.CompletedTask; + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineVectorTests.cs new file mode 100644 index 000000000..9b8a6b837 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineVectorTests.cs @@ -0,0 +1,248 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; +using Prompty.Core; + +namespace Prompty.Core.Tests; + +/// +/// Executes the shared canonical engine vectors in spec/vectors/engine/turn_vectors.json +/// against the C# . These vectors are the normative minimum: event +/// ordering, stable-prefix snapshot bookkeeping, sequential ordered tool execution, permission +/// denial being fed back to the model as a normal turn, delegated provider state, and +/// cancellation before context preparation must all match the Rust reference engine exactly. +/// +public class TurnEngineVectorTests +{ + private static readonly string SpecDir = FindSpecDir(); + + [Fact] + public async Task FinalOutput_MatchesVector() => await RunCaseAsync("final_output"); + + [Fact] + public async Task OrderedToolRound_MatchesVector() => await RunCaseAsync("ordered_tool_round"); + + [Fact] + public async Task PermissionDenialIsModelVisible_MatchesVector() => await RunCaseAsync("permission_denial_is_model_visible"); + + [Fact] + public async Task DelegatedProviderState_MatchesVector() => await RunCaseAsync("delegated_provider_state"); + + [Fact] + public async Task CancelBeforeContext_MatchesVector() => await RunCaseAsync("cancel_before_context"); + + private static async Task RunCaseAsync(string name) + { + var testCase = LoadCases().First(c => c.GetProperty("name").GetString() == name); + + var messages = ParseMessages(testCase.GetProperty("messages")); + var steps = ParseSteps(testCase.GetProperty("model")); + var toolOutputs = testCase.TryGetProperty("toolOutputs", out var toolOutputsEl) + ? toolOutputsEl.EnumerateObject().ToDictionary(p => p.Name, p => p.Value.GetString() ?? string.Empty) + : []; + var denyTools = testCase.TryGetProperty("denyTools", out var denyToolsEl) + ? denyToolsEl.EnumerateArray().Select(e => e.GetString()!).ToArray() + : []; + var cancelBeforeRun = testCase.TryGetProperty("cancelBeforeRun", out var cancelEl) && cancelEl.GetBoolean(); + + var durability = new RecordingDurabilityPort(); + var postCommit = new RecordingPostCommitPort(); + var effects = new TurnEngineEffects + { + Model = new ScriptedModelPort(steps), + Tools = new EchoToolPort(toolOutputs), + Clock = new FakeEngineClock(), + Ids = new FakeEngineIdGenerator(), + Permission = new DenyByNamePermissionPort(denyTools), + Durability = durability, + PostCommit = postCommit, + }; + + var engine = new TurnEngine(effects); + var request = new TurnEngineRequest("session-1", "turn-1", messages) { MaxIterations = 10 }; + + using var cts = new CancellationTokenSource(); + if (cancelBeforeRun) + { + cts.Cancel(); + } + + var result = await engine.RunAsync(request, cts.Token); + var expected = testCase.GetProperty("expected"); + + Assert.Equal(ParseStatus(expected.GetProperty("status").GetString()!), result.Commit.Status); + Assert.Equal(expected.GetProperty("iterations").GetInt32(), result.Commit.Iterations); + Assert.Equal(expected.GetProperty("snapshots").GetInt32(), result.Snapshots?.Count ?? 0); + Assert.Equal(expected.GetProperty("toolResults").GetInt32(), result.ToolResults?.Count ?? 0); + + if (expected.TryGetProperty("output", out var outputEl)) + { + Assert.Equal(outputEl.GetString(), result.Commit.Output); + } + + if (expected.TryGetProperty("snapshotStablePrefixes", out var prefixesEl)) + { + var expectedPrefixes = prefixesEl.EnumerateArray().Select(e => e.GetInt32()).ToArray(); + var actualPrefixes = (result.Snapshots ?? []).Select(s => s.StablePrefixMessages).ToArray(); + Assert.Equal(expectedPrefixes, actualPrefixes); + } + + if (expected.TryGetProperty("toolResultOrder", out var orderEl)) + { + var expectedOrder = orderEl.EnumerateArray().Select(e => e.GetString()).ToArray(); + var actualOrder = (result.ToolResults ?? []).Select(t => t.RequestId).ToArray(); + Assert.Equal(expectedOrder, actualOrder); + } + + if (expected.TryGetProperty("eventKinds", out var eventKindsEl)) + { + var expectedKinds = eventKindsEl.EnumerateArray().Select(e => e.GetString()).ToArray(); + var actualKinds = durability.Events.Select(e => EngineEventKindParser.ToValue(e.Kind)).ToArray(); + Assert.Equal(expectedKinds, actualKinds); + } + + if (expected.TryGetProperty("snapshotPortability", out var snapshotPortabilityEl)) + { + var expectedPortability = snapshotPortabilityEl.EnumerateArray().Select(e => e.GetString()).ToArray(); + var actualPortability = (result.Snapshots ?? []) + .Select(s => PortabilityToWire(s.ContextState.Portability)) + .ToArray(); + Assert.Equal(expectedPortability, actualPortability); + } + + if (expected.TryGetProperty("commitPortability", out var commitPortabilityEl)) + { + Assert.Equal(commitPortabilityEl.GetString(), PortabilityToWire(result.Commit.ContextState.Portability)); + } + + if (expected.TryGetProperty("delegatedState", out var delegatedStateEl)) + { + Assert.Equal(delegatedStateEl.GetInt32(), result.Commit.ContextState.DelegatedState?.Count ?? 0); + } + + // The final_output/ordered_tool_round/permission_denial/delegated_provider_state vectors + // all commit successfully, so the non-fatal post-commit effect must always run exactly once. + if (result.Commit.Status == EngineTurnStatus.Success) + { + Assert.Single(postCommit.Commits); + Assert.Null(result.PostCommitError); + } + } + + private static EngineTurnStatus ParseStatus(string value) => value switch + { + "success" => EngineTurnStatus.Success, + "failed" => EngineTurnStatus.Failed, + "cancelled" => EngineTurnStatus.Cancelled, + "reconciliation_required" => EngineTurnStatus.ReconciliationRequired, + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "unknown status"), + }; + + private static string PortabilityToWire(InvocationContextPortability portability) => portability switch + { + InvocationContextPortability.Portable => "portable", + InvocationContextPortability.Delegated => "delegated", + InvocationContextPortability.Opaque => "opaque", + _ => throw new ArgumentOutOfRangeException(nameof(portability), portability, "unknown portability"), + }; + + private static InvocationContextPortability ParsePortability(string value) => value switch + { + "portable" => InvocationContextPortability.Portable, + "delegated" => InvocationContextPortability.Delegated, + "opaque" => InvocationContextPortability.Opaque, + _ => throw new ArgumentOutOfRangeException(nameof(value), value, "unknown portability"), + }; + + private static List ParseMessages(JsonElement messagesEl) => + [.. messagesEl.EnumerateArray().Select(ParseMessage)]; + + private static Message ParseMessage(JsonElement messageEl) + { + var role = messageEl.GetProperty("role").GetString(); + var content = messageEl.GetProperty("content").GetString() ?? string.Empty; + return role switch + { + "assistant" => Message.Assistant(content), + "system" => Message.System(content), + _ => Message.User(content), + }; + } + + private static List ParseSteps(JsonElement modelEl) => + [.. modelEl.EnumerateArray().Select(ParseStep)]; + + private static ScriptedModelStep ParseStep(JsonElement stepEl) + { + var tools = stepEl.TryGetProperty("tools", out var toolsEl) + ? toolsEl.EnumerateArray().Select(ParseToolRequest).ToList() + : []; + + var delegatedState = stepEl.TryGetProperty("delegatedState", out var delegatedEl) + ? delegatedEl.EnumerateArray().Select(ParseDelegatedStateReference).ToList() + : null; + + return new ScriptedModelStep + { + Assistant = stepEl.TryGetProperty("assistant", out var assistantEl) ? assistantEl.GetString() : null, + Output = stepEl.TryGetProperty("output", out var outputEl) ? outputEl.GetString() : null, + Tools = tools, + NextPortability = stepEl.TryGetProperty("nextPortability", out var portabilityEl) + ? ParsePortability(portabilityEl.GetString()!) + : null, + DelegatedState = delegatedState, + }; + } + + private static ModelToolRequest ParseToolRequest(JsonElement toolEl) => new() + { + Id = toolEl.GetProperty("id").GetString()!, + Name = toolEl.GetProperty("name").GetString()!, + Arguments = toolEl.TryGetProperty("arguments", out var argsEl) ? JsonElementToObject(argsEl) : null, + }; + + private static DelegatedStateReference ParseDelegatedStateReference(JsonElement el) => new() + { + Provider = el.GetProperty("provider").GetString()!, + Kind = el.GetProperty("kind").GetString()!, + Id = el.GetProperty("id").GetString()!, + }; + + private static object? JsonElementToObject(JsonElement el) => el.ValueKind switch + { + JsonValueKind.String => el.GetString(), + JsonValueKind.Number => el.TryGetInt64(out var l) ? l : el.GetDouble(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null => null, + JsonValueKind.Array => el.EnumerateArray().Select(JsonElementToObject).ToList(), + JsonValueKind.Object => el.EnumerateObject().ToDictionary(p => p.Name, p => JsonElementToObject(p.Value)), + _ => null, + }; + + private static JsonElement[] LoadCases() + { + var path = Path.Combine(SpecDir, "vectors", "engine", "turn_vectors.json"); + var json = File.ReadAllText(path); + using var doc = JsonDocument.Parse(json); + return [.. doc.RootElement.GetProperty("cases").EnumerateArray().Select(e => e.Clone())]; + } + + private static string FindSpecDir() + { + var dir = AppContext.BaseDirectory; + for (var i = 0; i < 10; i++) + { + var candidate = Path.Combine(dir, "spec"); + if (Directory.Exists(candidate)) + { + return candidate; + } + + dir = Path.GetDirectoryName(dir) ?? dir; + } + + var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..")); + return Path.Combine(projectRoot, "spec"); + } +} diff --git a/runtime/csharp/Prompty.Core/TurnEngine.cs b/runtime/csharp/Prompty.Core/TurnEngine.cs new file mode 100644 index 000000000..3cb75b56e --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEngine.cs @@ -0,0 +1,1258 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Prompty.Core; + +/// +/// The bundle of runtime-local ports the uses to invoke the model, +/// enforce host policy, resolve permissions, execute tools, persist durable state, and run +/// post-commit effects. Only , , , and +/// are required; every other port has a conservative no-op default. +/// +public sealed class TurnEngineEffects +{ + /// The model invocation port. Required. + public required IEngineModelPort Model { get; init; } + + /// The tool execution port. Required. + public required IEngineToolPort Tools { get; init; } + + /// The deterministic clock used to timestamp every emitted event. Required. + public required IEngineClock Clock { get; init; } + + /// The deterministic id generator used for run/invocation/checkpoint/event ids. Required. + public required IEngineIdGenerator Ids { get; init; } + + /// The streaming sink handed to the model port. Defaults to a no-op sink. + public IEngineModelStreamPort Stream { get; init; } = new NoopModelStreamPort(); + + /// The context assembly port. Defaults to preserving the canonical messages unchanged. + public IEngineContextPort Context { get; init; } = new PassthroughEngineContextPort(); + + /// The host policy port. Defaults to a pass-through policy. + public IEngineHostPolicyPort Policy { get; init; } = new NoopHostPolicyPort(); + + /// The retry/backoff policy port. Defaults to an immediate no-op backoff. + public IEngineRetryPolicyPort Retry { get; init; } = new NoopRetryPolicyPort(); + + /// The conversation formatting port. Defaults to a single synthetic tool-result message per result. + public IEngineConversationPort Conversation { get; init; } = new DefaultConversationPort(); + + /// The permission resolution port. Defaults to approving every tool request. + public IEnginePermissionPort Permission { get; init; } = new AllowAllPermissionsPort(); + + /// The durability port used to append events and checkpoints. Defaults to an in-memory no-op. + public IEngineDurabilityPort Durability { get; init; } = new NoopDurabilityPort(); + + /// The post-commit effect port. Defaults to a no-op that always succeeds. + public IEnginePostCommitPort PostCommit { get; init; } = new NoopPostCommitPort(); +} + +/// +/// The canonical turn engine: a deterministic, resumable state machine that drives a single +/// conversational turn to completion by orchestrating host policy, context preparation, model +/// invocation, permissioned tool execution, and durable checkpointing. +/// +/// +/// This is a line-for-line behavioral port of the Rust reference implementation at +/// runtime/rust/prompty/src/engine/turn.rs. Every emitted , +/// every field, and every commit/cancellation/reconciliation path +/// mirrors the Rust engine so that the two runtimes agree on wire-visible behavior for the +/// shared vectors in spec/vectors/engine/turn_vectors.json. +/// +public sealed class TurnEngine +{ + private readonly TurnEngineEffects _effects; + + /// Creates a new engine bound to the supplied port bundle. + public TurnEngine(TurnEngineEffects effects) + { + _effects = effects ?? throw new ArgumentNullException(nameof(effects)); + } + + /// Resumes a turn from a durable checkpoint. + public Task ResumeAsync(ResumeContext resume, CancellationToken cancellationToken) => + RunAsync(TurnEngineRequest.FromResume(resume), cancellationToken); + + /// Runs a turn to completion (success, cancellation, failure, or reconciliation-required). + public async Task RunAsync(TurnEngineRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ValidateRequest(request); + + if (string.IsNullOrEmpty(request.RunId)) + { + request.RunId = _effects.Ids.NextId("run"); + } + + var state = new TurnState(request); + + await EmitAsync(state, EngineEventKind.TurnStarted, invocationId: null, iteration: null, new Dictionary + { + ["maxIterations"] = state.MaxIterations, + ["startIteration"] = state.Iteration, + ["inputs"] = state.Inputs, + }).ConfigureAwait(false); + + if (state.ModelReconciliationResolution is not null) + { + var response = state.ModelReconciliationResolution; + state.ModelReconciliationResolution = null; + var reconciliation = state.ModelReconciliation + ?? throw new TurnEngineInvalidRequestException("model reconciliation response is missing durable reconciliation state"); + + state.ReconciliationRequired = false; + state.ModelReconciliation = null; + + var applyError = state.ApplyModelResponse(reconciliation.InvocationId, response); + if (applyError is not null) + { + return await CommitFailedAsync(state, "provider_state_error", applyError, cancellationToken).ConfigureAwait(false); + } + + await PersistModelReconciliationAsync(state, reconciliation.InvocationId, reconciliation, response).ConfigureAwait(false); + } + + if (state.ReconciliationResolution is not null) + { + var resolution = state.ReconciliationResolution; + state.ReconciliationResolution = null; + await PersistReconciliationAsync(state, resolution).ConfigureAwait(false); + } + + if (state.ReconciliationRequired) + { + return await CommitReconciliationAsync( + state, + "effect_outcome_unknown", + "Checkpoint requires explicit effect reconciliation", + cancellationToken).ConfigureAwait(false); + } + + if (state.FinalOutputReady) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + state.Output = state.PendingOutput; + return await ApplyFinalPolicyAsync(state, cancellationToken).ConfigureAwait(false); + } + + while (state.Iteration < state.MaxIterations) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + if (state.PendingToolRequests.Count == 0 && state.PendingModelResponse is not null) + { + var invocationId = state.ActiveInvocationId ?? _effects.Ids.NextId("invocation"); + + List results; + try + { + results = FinalizeToolExchange(state); + } + catch (PortError error) + { + return await CommitFailedAsync(state, "conversation_format_error", error.Message, cancellationToken).ConfigureAwait(false); + } + + await PersistToolExchangeAsync(state, invocationId, results).ConfigureAwait(false); + state.ActiveInvocationId = null; + state.Iteration += 1; + continue; + } + + if (state.PendingToolRequests.Count > 0) + { + var invocationId = state.ActiveInvocationId ?? _effects.Ids.NextId("invocation"); + var toolRequest = state.PendingToolRequests[0]; + state.PendingToolRequests.RemoveAt(0); + + ModelToolResult toolResult; + try + { + toolResult = await ExecuteToolAsync(state, invocationId, toolRequest, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + catch (ToolPermissionFailure error) + { + return await CommitFailedAsync(state, "permission_error", error.Source.Message, cancellationToken).ConfigureAwait(false); + } + catch (ToolConfigurationFailure error) + { + return await CommitFailedAsync(state, "tool_configuration_error", error.Source.Message, cancellationToken).ConfigureAwait(false); + } + + var outcomeUnknown = toolResult.Outcome == ModelToolOutcome.Indeterminate; + state.ToolResults.Add(toolResult); + if (state.PendingModelResponse is null) + { + state.Messages.Add(Message.ToolResult(toolRequest.Id, toolResult.ModelText())); + } + + await PersistToolResultAsync(state, invocationId, toolRequest).ConfigureAwait(false); + + if (outcomeUnknown) + { + return await CommitReconciliationAsync( + state, + "effect_outcome_unknown", + "Tool effect outcome is unknown and requires reconciliation", + cancellationToken).ConfigureAwait(false); + } + + if (state.PendingToolRequests.Count == 0 && state.PendingModelResponse is null) + { + state.ActiveInvocationId = null; + state.Iteration += 1; + } + + continue; + } + + var freshInvocationId = _effects.Ids.NextId("invocation"); + + if (state.PolicyAppliedForIteration) + { + state.PolicyAppliedForIteration = false; + } + else + { + var policyRequest = new HostPolicyRequest + { + SessionId = state.SessionId, + TurnId = state.TurnId, + Iteration = state.Iteration, + Messages = [.. state.Messages], + StablePrefixMessages = state.StablePrefixMessages, + Inputs = state.Inputs, + }; + + HostPolicyResult policyResult; + try + { + policyResult = await _effects.Policy.BeforeModelAsync(policyRequest, cancellationToken).ConfigureAwait(false); + } + catch (HostPolicyException error) + { + return await CommitFailedAsync(state, error.ErrorKind, error.Message, cancellationToken).ConfigureAwait(false); + } + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + if (policyResult.StablePrefixMessages < 0 || policyResult.StablePrefixMessages > policyResult.Messages.Count) + { + return await CommitFailedAsync( + state, + "policy_error", + "host policy stable prefix exceeds rewritten message count", + cancellationToken).ConfigureAwait(false); + } + + var policyChanged = !MessagesEqual(state.Messages, policyResult.Messages) + || state.StablePrefixMessages != policyResult.StablePrefixMessages; + if (policyChanged) + { + state.Messages = policyResult.Messages; + state.StablePrefixMessages = policyResult.StablePrefixMessages; + await PersistPolicyUpdateAsync(state, freshInvocationId, policyResult.Metadata).ConfigureAwait(false); + state.PolicyAppliedForIteration = false; + } + } + + ModelInvocationContextSnapshot snapshot; + try + { + snapshot = await _effects.Context.PrepareAsync( + new ContextRequest + { + SessionId = state.SessionId, + TurnId = state.TurnId, + InvocationId = freshInvocationId, + Iteration = state.Iteration, + Messages = [.. state.Messages], + StablePrefixMessages = Math.Min(state.StablePrefixMessages, state.Messages.Count), + ContextState = new InvocationContextState + { + Portability = state.Portability, + DelegatedState = [.. state.DelegatedState], + }, + Inputs = state.Inputs, + }, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + catch (PortError error) + { + return await CommitFailedAsync(state, "context_error", error.Message, cancellationToken).ConfigureAwait(false); + } + + var contextError = ValidateContextSnapshot(snapshot, state, freshInvocationId); + if (contextError is not null) + { + return await CommitFailedAsync(state, "context_error", contextError, cancellationToken).ConfigureAwait(false); + } + + await EmitAsync(state, EngineEventKind.ContextPrepared, freshInvocationId, state.Iteration, snapshot).ConfigureAwait(false); + state.Snapshots.Add(snapshot); + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + var modelRequest = new ModelInvocationRequest { Context = snapshot }; + state.ActiveInvocationId = freshInvocationId; + + var attempt = 0; + ModelInvocationResponse? modelResponse = null; + while (modelResponse is null) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + await EmitAsync(state, EngineEventKind.ModelInvocationStarted, freshInvocationId, state.Iteration, new Dictionary + { + ["snapshotId"] = snapshot.Id, + ["attempt"] = attempt, + ["messageCount"] = snapshot.Messages.Count, + }).ConfigureAwait(false); + + try + { + modelResponse = await _effects.Model.InvokeAsync( + modelRequest, + cancellationToken, + new BestEffortModelStreamPort(_effects.Stream)).ConfigureAwait(false); + } + catch (PortError failure) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + attempt += 1; + var outcomeUnknown = failure.OutcomeUnknown; + var exhausted = outcomeUnknown || attempt >= state.MaxModelAttempts; + var reason = failure.Message; + + await EmitAsync(state, EngineEventKind.ModelInvocationFailed, freshInvocationId, state.Iteration, new Dictionary + { + ["attempt"] = attempt - 1, + ["exhausted"] = exhausted, + ["outcomeUnknown"] = outcomeUnknown, + ["message"] = reason, + }).ConfigureAwait(false); + + if (outcomeUnknown) + { + state.ReconciliationRequired = true; + state.ModelReconciliation = new ModelReconciliationState + { + InvocationId = freshInvocationId, + Request = modelRequest, + FailedAttempt = attempt - 1, + Message = reason, + Metadata = failure.Metadata, + }; + + await PersistModelReconciliationRequiredAsync(state, freshInvocationId).ConfigureAwait(false); + return await CommitReconciliationAsync(state, "model_outcome_unknown", reason, cancellationToken).ConfigureAwait(false); + } + + if (exhausted) + { + return await CommitFailedAsync(state, "model_error", reason, cancellationToken).ConfigureAwait(false); + } + + try + { + await _effects.Retry.BackoffAsync( + new RetryPolicyRequest + { + FailedAttempts = attempt, + NextAttempt = attempt + 1, + MaxAttempts = state.MaxModelAttempts, + Reason = reason, + }, + cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + catch (PortError source) + { + return await CommitFailedAsync(state, "retry_policy_error", source.Message, cancellationToken).ConfigureAwait(false); + } + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + } + } + + state.ModelReconciliation = null; + state.ReconciliationRequired = false; + + var applyModelError = state.ApplyModelResponse(freshInvocationId, modelResponse); + if (applyModelError is not null) + { + return await CommitFailedAsync(state, "provider_state_error", applyModelError, cancellationToken).ConfigureAwait(false); + } + + await PersistModelResponseAsync(state, freshInvocationId, modelResponse).ConfigureAwait(false); + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + if (state.FinalOutputReady) + { + state.Output = state.PendingOutput; + return await ApplyFinalPolicyAsync(state, cancellationToken).ConfigureAwait(false); + } + } + + return await CommitFailedAsync(state, "max_iterations", "Maximum model iterations reached", cancellationToken).ConfigureAwait(false); + } + + private static void ValidateRequest(TurnEngineRequest request) + { + if (string.IsNullOrEmpty(request.SessionId)) + { + throw new TurnEngineInvalidRequestException("session_id is required"); + } + + if (string.IsNullOrEmpty(request.TurnId)) + { + throw new TurnEngineInvalidRequestException("turn_id is required"); + } + + if (request.MaxModelAttempts <= 0) + { + throw new TurnEngineInvalidRequestException("max_model_attempts must be greater than zero"); + } + + if (request.StartIteration > request.MaxIterations) + { + throw new TurnEngineInvalidRequestException("start_iteration must not exceed max_iterations"); + } + + if (request.StablePrefixMessages > request.Messages.Count) + { + throw new TurnEngineInvalidRequestException("stable_prefix_messages exceeds initial message count"); + } + + if (request.Portability == InvocationContextPortability.Portable && request.DelegatedState.Count > 0) + { + throw new TurnEngineInvalidRequestException("portable turns cannot begin with delegated provider state"); + } + } + + private List FinalizeToolExchange(TurnState state) + { + var response = state.PendingModelResponse; + if (response is null) + { + return []; + } + + state.PendingModelResponse = null; + + if (response.ToolRequests is null || response.ToolRequests.Count == 0) + { + return []; + } + + var results = new List(); + foreach (var request in response.ToolRequests) + { + var match = state.ToolResults.FirstOrDefault(result => result.RequestId == request.Id); + if (match is not null) + { + results.Add(match); + } + } + + if (results.Count != response.ToolRequests.Count) + { + state.PendingModelResponse = response; + throw PortError.Configuration("tool exchange is incomplete and cannot be formatted"); + } + + IList messages; + try + { + messages = _effects.Conversation.FormatToolExchange(response, results); + } + catch (PortError) + { + state.PendingModelResponse = response; + throw; + } + + foreach (var message in messages) + { + state.Messages.Add(message); + } + + return results; + } + + private async Task ExecuteToolAsync( + TurnState state, + string invocationId, + ModelToolRequest request, + CancellationToken cancellationToken) + { + await EmitAsync(state, EngineEventKind.PermissionRequested, invocationId, state.Iteration, new Dictionary + { + ["toolRequest"] = request, + }).ConfigureAwait(false); + + EnginePermissionDecision decision; + try + { + decision = await _effects.Permission.AuthorizeAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (PortError source) + { + throw new ToolPermissionFailure(source); + } + + await EmitPermissionResolvedAsync(state, invocationId, request, decision).ConfigureAwait(false); + + if (!decision.Approved) + { + var errorKind = decision.Metadata is not null + && decision.Metadata.TryGetValue("errorKind", out var kindValue) + && kindValue is string kind + ? kind + : "permission_denied"; + + return new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = ModelToolOutcome.Failed, + Output = decision.Reason ?? "Permission denied", + ErrorKind = errorKind, + Metadata = decision.Metadata, + }; + } + + cancellationToken.ThrowIfCancellationRequested(); + + await EmitAsync(state, EngineEventKind.ToolExecutionStarted, invocationId, state.Iteration, new Dictionary + { + ["toolRequest"] = request, + }).ConfigureAwait(false); + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + return await _effects.Tools.ExecuteAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (PortError error) when (error.ConfigurationError) + { + throw new ToolConfigurationFailure(error); + } + catch (PortError error) + { + return new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = error.OutcomeUnknown ? ModelToolOutcome.Indeterminate : ModelToolOutcome.Failed, + Output = error.OutcomeUnknown + ? $"Tool '{request.Name}' outcome is unknown and requires reconciliation: {error.Message}" + : $"Tool '{request.Name}' failed: {error.Message}", + ErrorKind = error.OutcomeUnknown ? "effect_outcome_unknown" : "tool_error", + }; + } + } + + private Task EmitPermissionResolvedAsync( + TurnState state, + string invocationId, + ModelToolRequest request, + EnginePermissionDecision decision) => + EmitAsync(state, EngineEventKind.PermissionResolved, invocationId, state.Iteration, new Dictionary + { + ["toolRequestId"] = request.Id, + ["decision"] = decision, + }); + + private static string? ValidateContextSnapshot( + ModelInvocationContextSnapshot snapshot, + TurnState state, + string invocationId) + { + if (snapshot.SessionId != state.SessionId + || snapshot.TurnId != state.TurnId + || snapshot.InvocationId != invocationId + || snapshot.Iteration != state.Iteration) + { + return $"snapshot identity ({snapshot.SessionId}/{snapshot.TurnId}/{snapshot.InvocationId}/{snapshot.Iteration}) " + + $"does not match request ({state.SessionId}/{state.TurnId}/{invocationId}/{state.Iteration})"; + } + + if (snapshot.StablePrefixMessages < 0 || snapshot.StablePrefixMessages > snapshot.Messages.Count) + { + return $"stable prefix contains {snapshot.StablePrefixMessages} messages but snapshot contains {snapshot.Messages.Count}"; + } + + if (snapshot.ContextState is null) + { + return "snapshot context state is required"; + } + + if (snapshot.ContextState.Portability == InvocationContextPortability.Portable + && snapshot.ContextState.DelegatedState?.Count > 0) + { + return "portable snapshots cannot contain delegated provider state"; + } + + if (snapshot.ContextState.Portability == InvocationContextPortability.Delegated + && (snapshot.ContextState.DelegatedState is null || snapshot.ContextState.DelegatedState.Count == 0)) + { + return "delegated snapshots must identify provider-held state"; + } + + return null; + } + + private static bool MessagesEqual(IList left, IList right) + { + if (ReferenceEquals(left, right)) + { + return true; + } + + if (left.Count != right.Count) + { + return false; + } + + for (var i = 0; i < left.Count; i++) + { + if (left[i].ToJson(indent: false) != right[i].ToJson(indent: false)) + { + return false; + } + } + + return true; + } + + private async Task PersistCheckpointAsync( + string stage, + string requestId, + TurnState state, + EngineCheckpoint checkpoint, + IReadOnlyList events) + { + try + { + await _effects.Durability.AppendWithCheckpointAsync(events, checkpoint).ConfigureAwait(false); + } + catch (PortError source) + { + throw new TurnEngineRecoveryRequiredException(stage, requestId, checkpoint, [.. state.ToolResults], source); + } + } + + private async Task PersistPolicyUpdateAsync( + TurnState state, + string invocationId, + IDictionary? metadata) + { + var sequence = state.Sequence + 1; + state.PolicyAppliedForIteration = true; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: true); + var evt = BuildEvent(state, sequence, EngineEventKind.PolicyApplied, invocationId, state.Iteration, new Dictionary + { + ["messages"] = state.Messages, + ["stablePrefixMessages"] = state.StablePrefixMessages, + ["metadata"] = metadata, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("host policy", invocationId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistToolExchangeAsync( + TurnState state, + string invocationId, + IReadOnlyList results) + { + var sequence = state.Sequence; + var events = new List(results.Count + 2); + foreach (var result in results) + { + sequence += 1; + events.Add(BuildEvent(state, sequence, EngineEventKind.ToolResultCommitted, invocationId, state.Iteration, new Dictionary + { + ["toolResult"] = result, + })); + } + + sequence += 1; + events.Add(BuildEvent(state, sequence, EngineEventKind.ConversationUpdated, invocationId, state.Iteration, new Dictionary + { + ["messageCount"] = state.Messages.Count, + })); + + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + events.Add(BuildCheckpointEvent(state, checkpoint, invocationId)); + + await PersistCheckpointAsync("tool exchange", invocationId, state, checkpoint, events).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistModelReconciliationRequiredAsync(TurnState state, string invocationId) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var reconciliation = state.ModelReconciliation + ?? throw new InvalidOperationException("model reconciliation state must exist before persistence"); + var evt = BuildEvent(state, sequence, EngineEventKind.ModelReconciliationRequired, invocationId, state.Iteration, reconciliation); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("model reconciliation", invocationId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistModelReconciliationAsync( + TurnState state, + string invocationId, + ModelReconciliationState reconciliation, + ModelInvocationResponse response) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var evt = BuildEvent(state, sequence, EngineEventKind.ModelInvocationReconciled, invocationId, state.Iteration, new Dictionary + { + ["reconciliation"] = reconciliation, + ["hasOutput"] = response.Output is not null, + ["toolRequests"] = response.ToolRequests?.Count ?? 0, + ["metadata"] = response.Metadata, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("model reconciliation resolution", invocationId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistModelResponseAsync( + TurnState state, + string invocationId, + ModelInvocationResponse response) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var evt = BuildEvent(state, sequence, EngineEventKind.ModelInvocationCompleted, invocationId, state.Iteration, new Dictionary + { + ["hasOutput"] = response.Output is not null, + ["toolRequests"] = response.ToolRequests?.Count ?? 0, + ["nextPortability"] = response.NextContextState?.Portability, + ["delegatedState"] = response.NextContextState?.DelegatedState, + ["metadata"] = response.Metadata, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("model response", invocationId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistToolResultAsync(TurnState state, string invocationId, ModelToolRequest request) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var result = state.ToolResults.Count > 0 + ? state.ToolResults[^1] + : throw new InvalidOperationException("tool result must be recorded before persistence"); + var evt = BuildEvent(state, sequence, EngineEventKind.ToolExecutionCompleted, invocationId, state.Iteration, new Dictionary + { + ["toolResult"] = result, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("tool result", request.Id, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private async Task PersistReconciliationAsync(TurnState state, ModelToolResult result) + { + var sequence = state.Sequence + 1; + var checkpoint = BuildCheckpoint(state, sequence, resumeSameIteration: false); + var invocationId = state.ActiveInvocationId ?? "reconciliation"; + var evt = BuildEvent(state, sequence, EngineEventKind.ToolResultReconciled, invocationId, state.Iteration, new Dictionary + { + ["toolResult"] = result, + }); + var checkpointEvent = BuildCheckpointEvent(state, checkpoint, invocationId); + + await PersistCheckpointAsync("tool reconciliation", result.RequestId, state, checkpoint, [evt, checkpointEvent]).ConfigureAwait(false); + state.Sequence = checkpoint.LastSequence + 1; + return checkpoint; + } + + private EngineEvent BuildCheckpointEvent(TurnState state, EngineCheckpoint checkpoint, string invocationId) => + BuildEvent(state, checkpoint.LastSequence + 1, EngineEventKind.CheckpointCreated, invocationId, checkpoint.Iteration, new Dictionary + { + ["checkpointId"] = checkpoint.Id, + ["includedThroughSequence"] = checkpoint.LastSequence, + }); + + private EngineCheckpoint BuildCheckpoint(TurnState state, long lastSequence, bool resumeSameIteration) + { + var lastToolResult = state.ToolResults.Count > 0 ? state.ToolResults[^1] : null; + return new EngineCheckpoint + { + Id = _effects.Ids.NextId("checkpoint"), + SessionId = state.SessionId, + TurnId = state.TurnId, + RunId = state.RunId, + ParentRunId = state.ParentRunId, + DelegationDepth = state.DelegationDepth, + Iteration = state.Iteration, + LastSequence = lastSequence, + Messages = [.. state.Messages], + StablePrefixMessages = state.StablePrefixMessages, + Inputs = state.Inputs, + ActiveInvocationId = state.ActiveInvocationId, + PendingToolRequests = [.. state.PendingToolRequests], + CompletedToolResults = [.. state.ToolResults], + CompletedModelIterations = state.CompletedModelIterations, + ReconciliationRequired = state.ReconciliationRequired || lastToolResult?.Outcome == ModelToolOutcome.Indeterminate, + ModelReconciliation = state.ModelReconciliation, + PendingOutput = state.PendingOutput, + FinalOutputReady = state.FinalOutputReady, + PendingModelResponse = state.PendingModelResponse, + ResumeSameIteration = resumeSameIteration, + PolicyAppliedForIteration = state.PolicyAppliedForIteration, + ContextState = new InvocationContextState + { + Portability = state.Portability, + DelegatedState = state.DelegatedState, + }, + }; + } + + private EngineEvent BuildEvent( + TurnState state, + long sequence, + EngineEventKind kind, + string? invocationId, + int? iteration, + object? payload) => + new() + { + Sequence = sequence, + Id = _effects.Ids.NextId("event"), + Timestamp = _effects.Clock.Now(), + SessionId = state.SessionId, + TurnId = state.TurnId, + RunId = state.RunId, + ParentRunId = state.ParentRunId, + DelegationDepth = state.DelegationDepth, + InvocationId = invocationId, + Iteration = iteration, + Kind = kind, + Payload = payload, + }; + + private async Task EmitAsync(TurnState state, EngineEventKind kind, string? invocationId, int? iteration, object? payload) + { + var sequence = state.Sequence + 1; + var evt = BuildEvent(state, sequence, kind, invocationId, iteration, payload); + try + { + await _effects.Durability.AppendAsync(evt).ConfigureAwait(false); + } + catch (PortError source) + { + throw new TurnEnginePortException("event journal", source); + } + + state.Sequence = sequence; + } + + private Task CommitSuccessAsync(TurnState state, CancellationToken cancellationToken) => + CommitAsync(state, EngineTurnStatus.Success, EngineEventKind.TurnCommitted, cancellationToken); + + private async Task ApplyFinalPolicyAsync(TurnState state, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + var request = new FinalOutputPolicyRequest + { + SessionId = state.SessionId, + TurnId = state.TurnId, + Iteration = state.Iteration, + Messages = [.. state.Messages], + Output = state.Output, + Inputs = state.Inputs, + }; + + FinalOutputPolicyResult result; + try + { + result = await _effects.Policy.BeforeCommitAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HostPolicyException error) + { + return cancellationToken.IsCancellationRequested + ? await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false) + : await CommitFailedAsync(state, error.ErrorKind, error.Message, cancellationToken).ConfigureAwait(false); + } + + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + + state.Output = result.Output; + return await CommitSuccessAsync(state, cancellationToken).ConfigureAwait(false); + } + + private Task CommitCancelledAsync(TurnState state, CancellationToken cancellationToken) => + CommitAsync(state, EngineTurnStatus.Cancelled, EngineEventKind.TurnCancelled, cancellationToken); + + private Task CommitFailedAsync(TurnState state, string errorKind, string message, CancellationToken cancellationToken) + { + state.Output = new Dictionary { ["errorKind"] = errorKind, ["message"] = message }; + return CommitAsync(state, EngineTurnStatus.Failed, EngineEventKind.TurnFailed, cancellationToken); + } + + private Task CommitReconciliationAsync( + TurnState state, + string errorKind, + string message, + CancellationToken cancellationToken) + { + state.Output = new Dictionary { ["errorKind"] = errorKind, ["message"] = message }; + return CommitAsync(state, EngineTurnStatus.ReconciliationRequired, EngineEventKind.TurnReconciliationRequired, cancellationToken); + } + + private async Task CommitAsync( + TurnState state, + EngineTurnStatus status, + EngineEventKind kind, + CancellationToken cancellationToken) + { + var iteration = state.Iteration; + var terminalPayload = new Dictionary { ["status"] = status, ["output"] = state.Output }; + await EmitAsync(state, kind, invocationId: null, iteration, terminalPayload).ConfigureAwait(false); + + var commit = new TurnCommit + { + SessionId = state.SessionId, + TurnId = state.TurnId, + Status = status, + Output = state.Output, + Messages = [.. state.Messages], + Iterations = state.CompletedModelIterations, + LastSequence = state.Sequence, + ContextState = new InvocationContextState + { + Portability = state.Portability, + DelegatedState = state.DelegatedState, + }, + ModelReconciliation = state.ModelReconciliation, + }; + + string? postCommitError = null; + if (status == EngineTurnStatus.Success) + { + var sessionIdLength = System.Text.Encoding.UTF8.GetByteCount(commit.SessionId); + var turnIdLength = System.Text.Encoding.UTF8.GetByteCount(commit.TurnId); + var effectId = $"post_commit:{sessionIdLength}:{commit.SessionId}:{turnIdLength}:{commit.TurnId}"; + + TurnEnginePortException? startError = null; + try + { + await EmitAsync(state, EngineEventKind.PostCommitStarted, invocationId: null, iteration, new Dictionary + { + ["effectId"] = effectId, + }).ConfigureAwait(false); + } + catch (TurnEnginePortException error) + { + startError = error; + } + + if (startError is not null) + { + postCommitError = + $"post-commit effect '{effectId}' was not started because its start event could not be persisted: {startError.Message}"; + } + else + { + try + { + await _effects.PostCommit.AfterCommitAsync(effectId, commit, cancellationToken).ConfigureAwait(false); + + try + { + await EmitAsync(state, EngineEventKind.PostCommitCompleted, invocationId: null, iteration, new Dictionary + { + ["effectId"] = effectId, + }).ConfigureAwait(false); + } + catch (TurnEnginePortException completionError) + { + postCommitError = + $"post-commit effect '{effectId}' completed, but its completion event could not be persisted: {completionError.Message}"; + } + } + catch (PortError source) + { + var message = source.Message; + string? eventError = null; + try + { + await EmitAsync(state, EngineEventKind.PostCommitFailed, invocationId: null, iteration, new Dictionary + { + ["effectId"] = effectId, + ["message"] = message, + }).ConfigureAwait(false); + } + catch (TurnEnginePortException failureEventError) + { + eventError = failureEventError.Message; + } + + postCommitError = eventError is not null + ? $"{message}; failure event for post-commit effect '{effectId}' could not be persisted: {eventError}" + : message; + } + } + } + + commit.LastSequence = state.Sequence; + + return new TurnEngineResult + { + Commit = commit, + Snapshots = state.Snapshots, + ToolResults = state.ToolResults, + PostCommitError = postCommitError, + }; + } + + /// Signals that itself failed. + private sealed class ToolPermissionFailure(PortError source) : Exception(source.Message) + { + public PortError Source { get; } = source; + } + + /// Signals that failed with a configuration error. + private sealed class ToolConfigurationFailure(PortError source) : Exception(source.Message) + { + public PortError Source { get; } = source; + } + + /// Prevents observational stream failures from changing semantic model execution. + private sealed class BestEffortModelStreamPort(IEngineModelStreamPort inner) : IEngineModelStreamPort + { + public async Task EmitAsync(ModelStreamChunk chunk) + { + try + { + await inner.EmitAsync(chunk).ConfigureAwait(false); + } + catch (Exception error) when (error is not OperationCanceledException) + { + // Stream sinks own delivery diagnostics; the canonical turn outcome remains semantic-only. + } + } + } + + /// + /// The mutable in-flight state of a single turn. This is a direct transliteration of the Rust + /// engine's TurnState struct and is never exposed outside . + /// + private sealed class TurnState + { + public TurnState(TurnEngineRequest request) + { + SessionId = request.SessionId; + TurnId = request.TurnId; + RunId = request.RunId; + ParentRunId = request.ParentRunId; + DelegationDepth = request.DelegationDepth; + Messages = [.. request.Messages]; + Inputs = request.Inputs; + MaxIterations = request.MaxIterations; + MaxModelAttempts = request.MaxModelAttempts; + StablePrefixMessages = request.StablePrefixMessages; + Portability = request.Portability; + DelegatedState = [.. request.DelegatedState]; + ActiveInvocationId = request.ActiveInvocationId; + PendingToolRequests = [.. request.PendingToolRequests]; + ReconciliationRequired = request.ReconciliationRequired; + ModelReconciliation = request.ModelReconciliation; + CompletedModelIterations = request.CompletedModelIterations; + PendingOutput = request.PendingOutput; + FinalOutputReady = request.FinalOutputReady; + PendingModelResponse = request.PendingModelResponse; + PolicyAppliedForIteration = request.PolicyAppliedForIteration; + ReconciliationResolution = request.ReconciliationResolution; + ModelReconciliationResolution = request.ModelReconciliationResolution; + + Iteration = request.StartIteration; + Sequence = request.InitialSequence; + Output = null; + Snapshots = []; + ToolResults = [.. request.CompletedToolResults]; + } + + public string SessionId { get; } + + public string TurnId { get; } + + public string RunId { get; set; } + + public string? ParentRunId { get; } + + public int DelegationDepth { get; } + + public IList Messages { get; set; } + + public object? Inputs { get; } + + public int MaxIterations { get; } + + public int MaxModelAttempts { get; } + + public int StablePrefixMessages { get; set; } + + public InvocationContextPortability Portability { get; set; } + + public IList DelegatedState { get; set; } + + public string? ActiveInvocationId { get; set; } + + public IList PendingToolRequests { get; set; } + + public bool ReconciliationRequired { get; set; } + + public ModelReconciliationState? ModelReconciliation { get; set; } + + public int CompletedModelIterations { get; set; } + + public object? PendingOutput { get; set; } + + public bool FinalOutputReady { get; set; } + + public ModelInvocationResponse? PendingModelResponse { get; set; } + + public bool PolicyAppliedForIteration { get; set; } + + public ModelToolResult? ReconciliationResolution { get; set; } + + public ModelInvocationResponse? ModelReconciliationResolution { get; set; } + + public int Iteration { get; set; } + + public long Sequence { get; set; } + + public object? Output { get; set; } + + public IList Snapshots { get; set; } + + public IList ToolResults { get; set; } + + /// Applies a model response to the state. Returns an error message on failure, else null. + public string? ApplyModelResponse(string invocationId, ModelInvocationResponse response) + { + CompletedModelIterations += 1; + + if (response.ToolRequests is null || response.ToolRequests.Count == 0) + { + foreach (var message in response.AssistantMessages ?? []) + { + Messages.Add(message); + } + + PendingModelResponse = null; + } + else + { + PendingModelResponse = response; + } + + var error = ApplyProviderState(response); + if (error is not null) + { + return error; + } + + ActiveInvocationId = invocationId; + PendingToolRequests = response.ToolRequests is null ? [] : [.. response.ToolRequests]; + PendingOutput = response.Output; + FinalOutputReady = PendingToolRequests.Count == 0; + return null; + } + + /// Adopts provider-supplied context state. Returns an error message on invariant violation. + public string? ApplyProviderState(ModelInvocationResponse response) + { + if (response.NextContextState is not null) + { + Portability = response.NextContextState.Portability; + DelegatedState = response.NextContextState.DelegatedState is null + ? [] + : [.. response.NextContextState.DelegatedState]; + } + else if (Portability == InvocationContextPortability.Portable) + { + DelegatedState = []; + } + + if (Portability == InvocationContextPortability.Portable && DelegatedState.Count > 0) + { + return "portable provider state cannot retain delegated references"; + } + + if (Portability == InvocationContextPortability.Delegated && DelegatedState.Count == 0) + { + return "delegated provider state requires at least one reference"; + } + + return null; + } + } +} diff --git a/runtime/csharp/Prompty.Core/TurnEngineExceptions.cs b/runtime/csharp/Prompty.Core/TurnEngineExceptions.cs new file mode 100644 index 000000000..cedae7a99 --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEngineExceptions.cs @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Prompty.Core; + +/// +/// Error raised by a runtime-local engine port (model, tool, permission, host policy, +/// retry policy, post-commit, or durability). Mirrors the Rust reference's +/// PortError: a plain message plus two orthogonal signal flags that change how +/// the canonical state machine reacts to the failure. +/// +public class PortError : Exception +{ + /// + /// True when the effect's real-world outcome cannot be determined (for example, a + /// network timeout after a tool call may have already been dispatched). The engine + /// treats this as a signal to stop and require explicit reconciliation rather than + /// retrying or failing outright, since retrying could duplicate a side effect. + /// + public bool OutcomeUnknown { get; } + + /// + /// True when the failure reflects a host misconfiguration (for example, an unknown + /// tool name) rather than a transient runtime failure. Configuration errors are not + /// retried and are surfaced as a distinct commit failure kind. + /// + public bool ConfigurationError { get; } + + /// Optional structured metadata describing the failure. + public IDictionary? Metadata { get; } + + public PortError( + string message, + bool outcomeUnknown = false, + bool configurationError = false, + IDictionary? metadata = null) + : base(message) + { + OutcomeUnknown = outcomeUnknown; + ConfigurationError = configurationError; + Metadata = metadata; + } + + /// Create a port error whose effect outcome is unknown and requires reconciliation. + public static PortError Indeterminate(string message, IDictionary? metadata = null) => + new(message, outcomeUnknown: true, metadata: metadata); + + /// Create a port error that reflects a host misconfiguration. + public static PortError Configuration(string message) => new(message, configurationError: true); +} + +/// +/// Error raised by . Unlike , +/// host policy failures always carry an explicit that the engine +/// forwards verbatim into the committed failure output. +/// +public class HostPolicyException : Exception +{ + /// Machine-readable failure category surfaced on the committed turn output. + public string ErrorKind { get; } + + public HostPolicyException(string errorKind, string message) + : base(message) + { + ErrorKind = errorKind; + } +} + +/// +/// Base type for errors that prevent the canonical from producing +/// a committed at all. These are distinct from ordinary +/// turn outcomes (success, cancelled, failed, reconciliation-required), which are all +/// returned as a normal rather than thrown. +/// +public abstract class TurnEngineException : Exception +{ + protected TurnEngineException(string message, Exception? innerException = null) + : base(message, innerException) + { + } +} + +/// Thrown when a or resume record fails validation. +public sealed class TurnEngineInvalidRequestException : TurnEngineException +{ + public TurnEngineInvalidRequestException(string message) + : base($"invalid turn request: {message}") + { + } +} + +/// +/// Thrown when appending a single, non-checkpointed event to the durability port fails +/// (for example, the plain TurnStarted or ContextPrepared events). This is +/// unrecoverable for the current run because no checkpoint was persisted to resume from. +/// +public sealed class TurnEnginePortException : TurnEngineException +{ + /// The engine stage that failed (for example, "event journal"). + public string Stage { get; } + + public TurnEnginePortException(string stage, PortError source) + : base($"{stage} failed: {source.Message}", source) + { + Stage = stage; + } +} + +/// +/// Thrown when an atomic append-with-checkpoint durability write fails while persisting a +/// semantic effect (policy update, model response, tool result, tool exchange, or +/// reconciliation). The caller can use and +/// to recover: the in-memory effect already happened, but it was never durably recorded, +/// so a host must decide how to reconcile before resuming. +/// +public sealed class TurnEngineRecoveryRequiredException : TurnEngineException +{ + /// The persistence stage that failed (for example, "tool result"). + public string Stage { get; } + + /// Identifier of the effect that could not be durably recorded. + public string RequestId { get; } + + /// The checkpoint that was built but never durably appended. + public EngineCheckpoint Checkpoint { get; } + + /// Tool results completed so far in this run, for host-side recovery bookkeeping. + public IReadOnlyList ToolResults { get; } + + public TurnEngineRecoveryRequiredException( + string stage, + string requestId, + EngineCheckpoint checkpoint, + IReadOnlyList toolResults, + PortError source) + : base($"{stage} durability failed after effect '{requestId}': {source.Message}", source) + { + Stage = stage; + RequestId = requestId; + Checkpoint = checkpoint; + ToolResults = toolResults; + } +} diff --git a/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs b/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs new file mode 100644 index 000000000..c63872e07 --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft. All rights reserved. + +// --- Runtime helpers (manually maintained) --- +// These extend generated Model/pipeline types with small convenience members the +// canonical turn engine needs. They mirror the `impl` blocks the Rust reference adds +// directly on its generated types (see runtime/rust/prompty/src/engine/ports.rs and +// model_ext.rs). New files only — no generated file under Model/ is modified. + +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; + +namespace Prompty.Core; + +public partial class Message +{ + /// + /// Build the message a host appends to the conversation for one tool result: a + /// message whose text is the result's model-visible output, + /// tagged with the originating tool request id so it can be located again (for + /// example, when resuming after an indeterminate tool-effect reconciliation). + /// + public static Message ToolResult(string requestId, string text) => new() + { + Role = Role.Tool, + Parts = [new TextPart { Value = text }], + Metadata = new Dictionary { ["tool_call_id"] = requestId }, + }; +} + +public partial class ModelToolResult +{ + /// + /// Render this tool result's output as model-visible text, tolerating an absent output. + /// A plain string output is used as-is; any other JSON-ish value is stringified. + /// + public string ModelText() => Output switch + { + null => string.Empty, + string text => text, + var value => SerializeCanonicalJson(value), + }; + + private static string SerializeCanonicalJson(object value) + { + var element = JsonSerializer.SerializeToElement(value); + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter( + stream, + new JsonWriterOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping })) + { + WriteCanonicalJson(writer, element); + } + + return Encoding.UTF8.GetString(stream.ToArray()); + } + + private static void WriteCanonicalJson(Utf8JsonWriter writer, JsonElement element) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + foreach (var property in element.EnumerateObject().OrderBy(property => property.Name, StringComparer.Ordinal)) + { + writer.WritePropertyName(property.Name); + WriteCanonicalJson(writer, property.Value); + } + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (var item in element.EnumerateArray()) + { + WriteCanonicalJson(writer, item); + } + writer.WriteEndArray(); + break; + default: + element.WriteTo(writer); + break; + } + } +} + +public partial class ResumeContext +{ + /// + /// The journal sequence a resumed run must continue after: the larger of the + /// recorded journal tail and the checkpoint's own last committed sequence. + /// + public long ResumeSequence() => Math.Max(LastJournalSequence, Checkpoint.LastSequence); +} diff --git a/runtime/csharp/Prompty.Core/TurnEnginePorts.cs b/runtime/csharp/Prompty.Core/TurnEnginePorts.cs new file mode 100644 index 000000000..1f194fc63 --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEnginePorts.cs @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Prompty.Core; + +// ----------------------------------------------------------------------- +// Runtime-local effect ports used by the canonical turn engine. +// +// The generated Model/pipeline interfaces (IPermissionResolver, IHostToolExecutor, +// IEventJournalWriter, ICheckpointStore, etc.) predate the canonical EngineEvent / +// EngineCheckpoint / TurnEngineResult contract and use different request/response +// shapes with no cancellation support. Rather than repurposing or editing those +// generated interfaces, this file defines new, distinctly named runtime-local ports +// that consume the generated domain types (ModelInvocationRequest/Response, +// ModelToolRequest/Result, EnginePermissionDecision, EngineEvent, EngineCheckpoint, +// TurnCommit, ...) directly. This mirrors the Rust reference engine's runtime-local +// traits in runtime/rust/prompty/src/engine/ports.rs. +// ----------------------------------------------------------------------- + +/// An ephemeral, non-durable chunk streamed while a model invocation is in flight. +public abstract record ModelStreamChunk +{ + private ModelStreamChunk() + { + } + + /// A partial text delta. + public sealed record Text(string Value) : ModelStreamChunk; + + /// A partial "thinking"/reasoning delta. + public sealed record Thinking(string Value) : ModelStreamChunk; + + /// An opaque, provider-specific streaming payload. + public sealed record Provider(object? Value) : ModelStreamChunk; +} + +/// Delivers ephemeral model stream chunks. Delivery failure must not alter semantic execution. +public interface IEngineModelStreamPort +{ + Task EmitAsync(ModelStreamChunk chunk); +} + +/// Builds the immutable context snapshot used for one model invocation. +public interface IEngineContextPort +{ + /// Context assembly or validation failed. + Task PrepareAsync(ContextRequest request, CancellationToken cancellationToken); +} + +/// Invokes the model for one turn iteration. +public interface IEngineModelPort +{ + /// The invocation failed. Set when the + /// effect may have already happened on the provider side and requires reconciliation instead of a retry. + Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream); +} + +/// Host policy applied before each model call and before the final commit. +public interface IEngineHostPolicyPort +{ + /// The policy deterministically rejected the turn. + Task BeforeModelAsync(HostPolicyRequest request, CancellationToken cancellationToken); + + /// The policy deterministically rejected the turn. + Task BeforeCommitAsync(FinalOutputPolicyRequest request, CancellationToken cancellationToken); +} + +/// Backoff policy applied between failed model invocation attempts. +public interface IEngineRetryPolicyPort +{ + /// + /// Wait (or otherwise apply backoff) before the next model attempt. + /// + /// Backoff was cancelled. + /// The retry policy itself failed. + Task BackoffAsync(RetryPolicyRequest request, CancellationToken cancellationToken); +} + +/// Converts one completed model/tool batch into provider-valid conversation messages. +public interface IEngineConversationPort +{ + /// The batch could not be formatted (for example, results are incomplete). + IList FormatToolExchange(ModelInvocationResponse response, IReadOnlyList results); +} + +/// Authorizes a single tool request before it executes. +public interface IEnginePermissionPort +{ + /// The permission service itself failed (not the same as a denial, which is a + /// normal with Approved == false). + Task AuthorizeAsync(ModelToolRequest request, CancellationToken cancellationToken); +} + +/// Executes a single approved tool request. +public interface IEngineToolPort +{ + /// The tool failed. Set for a host + /// misconfiguration (unknown tool, invalid binding) and when the + /// effect may have already happened and requires reconciliation. + Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken); +} + +/// +/// Durable event journal and checkpoint store. Unlike the other engine ports, durability +/// writes are not cancellable: once the engine decides to persist an effect it must either +/// succeed or fail explicitly so the in-memory state and the durable record never diverge. +/// +public interface IEngineDurabilityPort +{ + /// The event could not be appended. + Task AppendAsync(EngineEvent @event); + + /// + /// Atomically append one or more events and persist the checkpoint that reflects them, so + /// resuming from the checkpoint can never duplicate the effects the events describe. + /// + /// The events/checkpoint could not be durably persisted. + Task AppendWithCheckpointAsync(IReadOnlyList events, EngineCheckpoint checkpoint); +} + +/// Runs a host side effect after a turn commits successfully (for example, updating usage counters). +public interface IEnginePostCommitPort +{ + /// The post-commit effect failed. This is reported non-fatally on + /// — it never uncommits the turn. + Task AfterCommitAsync(string effectId, TurnCommit commit, CancellationToken cancellationToken); +} + +/// Supplies deterministic or live timestamps for engine events and checkpoints. +public interface IEngineClock +{ + /// Returns the current timestamp, formatted however the host's durable log expects. + string Now(); +} + +/// Supplies deterministic or live identifiers for engine events, checkpoints, runs, and invocations. +public interface IEngineIdGenerator +{ + /// Returns a new identifier for the given identifier kind (for example, "event", "checkpoint", "run"). + string NextId(string kind); +} + +// ----------------------------------------------------------------------- +// Default / no-op port implementations, mirroring ports.rs. +// ----------------------------------------------------------------------- + +/// Permission port that approves every tool request. For hosts that explicitly allow all tools. +public sealed class AllowAllPermissionsPort : IEnginePermissionPort +{ + public Task AuthorizeAsync(ModelToolRequest request, CancellationToken cancellationToken) => + Task.FromResult(new EnginePermissionDecision { Approved = true, Reason = "allow_all" }); +} + +/// Durability port for explicitly non-durable execution profiles. Persists nothing. +public sealed class NoopDurabilityPort : IEngineDurabilityPort +{ + public Task AppendAsync(EngineEvent @event) => Task.CompletedTask; + + public Task AppendWithCheckpointAsync(IReadOnlyList events, EngineCheckpoint checkpoint) => Task.CompletedTask; +} + +/// Post-commit port that performs no side effect. +public sealed class NoopPostCommitPort : IEnginePostCommitPort +{ + public Task AfterCommitAsync(string effectId, TurnCommit commit, CancellationToken cancellationToken) => Task.CompletedTask; +} + +/// Model stream port that drops every chunk. +public sealed class NoopModelStreamPort : IEngineModelStreamPort +{ + public Task EmitAsync(ModelStreamChunk chunk) => Task.CompletedTask; +} + +/// Builds the canonical snapshot without adding external context candidates. +public sealed class PassthroughEngineContextPort : IEngineContextPort +{ + public Task PrepareAsync(ContextRequest request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(new ModelInvocationContextSnapshot + { + Id = $"context:{request.InvocationId}", + SessionId = request.SessionId, + TurnId = request.TurnId, + InvocationId = request.InvocationId, + Iteration = request.Iteration, + Messages = [.. request.Messages], + Decisions = [], + StablePrefixMessages = request.StablePrefixMessages, + ContextState = request.ContextState, + }); + } +} + +/// Host policy port that leaves messages, stable prefix, and final output unchanged. +public sealed class NoopHostPolicyPort : IEngineHostPolicyPort +{ + public Task BeforeModelAsync(HostPolicyRequest request, CancellationToken cancellationToken) => + Task.FromResult(new HostPolicyResult + { + Messages = request.Messages, + StablePrefixMessages = request.StablePrefixMessages, + }); + + public Task BeforeCommitAsync(FinalOutputPolicyRequest request, CancellationToken cancellationToken) => + Task.FromResult(new FinalOutputPolicyResult { Output = request.Output }); +} + +/// Retry policy with no delay and no side effects. +public sealed class NoopRetryPolicyPort : IEngineRetryPolicyPort +{ + public Task BackoffAsync(RetryPolicyRequest request, CancellationToken cancellationToken) => Task.CompletedTask; +} + +/// Provider-neutral fallback that preserves assistant messages and appends ordered tool result messages. +public sealed class DefaultConversationPort : IEngineConversationPort +{ + public IList FormatToolExchange(ModelInvocationResponse response, IReadOnlyList results) + { + var messages = new List(response.AssistantMessages ?? []); + foreach (var request in response.ToolRequests ?? []) + { + var result = results.FirstOrDefault(r => r.RequestId == request.Id); + if (result is not null) + { + messages.Add(Message.ToolResult(request.Id, result.ModelText())); + } + } + + return messages; + } +} diff --git a/runtime/csharp/Prompty.Core/TurnEngineRequest.cs b/runtime/csharp/Prompty.Core/TurnEngineRequest.cs new file mode 100644 index 000000000..27161e880 --- /dev/null +++ b/runtime/csharp/Prompty.Core/TurnEngineRequest.cs @@ -0,0 +1,374 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Prompty.Core; + +/// +/// Request accepted by the canonical . Mirrors the Rust reference's +/// TurnEngineRequest (runtime/rust/prompty/src/engine/turn.rs) field for field. A fresh +/// turn is built with ; an +/// interrupted turn is resumed with / and the +/// reconciliation-specific resume factories. +/// +public sealed class TurnEngineRequest +{ + public TurnEngineRequest(string sessionId, string turnId, IList messages) + { + SessionId = sessionId; + TurnId = turnId; + Messages = messages; + StablePrefixMessages = messages.Count; + } + + /// Session this turn belongs to. + public string SessionId { get; set; } + + /// Turn identifier, unique within the session. + public string TurnId { get; set; } + + /// Stable identifier of this engine run. Empty means the engine assigns one at run start. + public string RunId { get; set; } = string.Empty; + + /// Run identifier of the parent run when this run was delegated. + public string? ParentRunId { get; set; } + + /// Zero-based delegation nesting depth; 0 for a top-level run. + public int DelegationDepth { get; set; } + + /// Conversation messages seen so far. + public IList Messages { get; set; } + + /// Opaque turn inputs, echoed into context snapshots and host policy requests. + public object? Inputs { get; set; } + + /// Maximum number of model/tool iterations permitted for this run. + public int MaxIterations { get; set; } = 10; + + /// Maximum model invocation attempts per iteration before failing the turn. + public int MaxModelAttempts { get; set; } = 3; + + /// Iteration to execute first. Non-zero values are used when resuming a checkpoint. + public int StartIteration { get; set; } + + /// Last committed event sequence before this run. + public long InitialSequence { get; set; } + + /// Number of leading messages the engine and host policy must not rewrite. + public int StablePrefixMessages { get; set; } + + /// Whether the provider-side context can be reconstructed from alone. + public InvocationContextPortability Portability { get; set; } = InvocationContextPortability.Portable; + + /// Opaque references to delegated provider-side state, required when is not portable. + public IList DelegatedState { get; set; } = []; + + /// Invocation id in progress when this request was built, if any. + public string? ActiveInvocationId { get; set; } + + /// Tool requests from the most recent model response not yet executed. + public IList PendingToolRequests { get; set; } = []; + + /// Tool results already committed for the in-flight model/tool exchange. + public IList CompletedToolResults { get; set; } = []; + + /// Number of model iterations completed so far. + public int CompletedModelIterations { get; set; } + + /// Whether the turn is blocked pending an explicit reconciliation resolution. + public bool ReconciliationRequired { get; set; } + + /// Durable reconciliation state for an indeterminate model invocation, if any. + public ModelReconciliationState? ModelReconciliation { get; set; } + + /// Output pending final host policy application. + public object? PendingOutput { get; set; } + + /// Whether a model response with no further tool requests is ready to commit. + public bool FinalOutputReady { get; set; } + + /// Model response awaiting completion of its tool exchange. + public ModelInvocationResponse? PendingModelResponse { get; set; } + + /// Whether host policy has already been applied for the current iteration (skip re-applying once). + public bool PolicyAppliedForIteration { get; set; } + + /// Resolved tool result supplied when resuming after an indeterminate tool effect. + public ModelToolResult? ReconciliationResolution { get; set; } + + /// Resolved model response supplied when resuming after an indeterminate model invocation. + public ModelInvocationResponse? ModelReconciliationResolution { get; set; } + + /// Set the stable run identifier for this run. An empty value lets the engine assign one at run start. + public TurnEngineRequest WithRunId(string runId) + { + RunId = runId; + return this; + } + + /// + /// Mark this run as delegated from a parent run, carrying the parent run identifier and + /// nesting one level deeper than the parent. + /// + public TurnEngineRequest DelegatedUnder(string parentRunId, int parentDelegationDepth) + { + ParentRunId = parentRunId; + DelegationDepth = parentDelegationDepth + 1; + return this; + } + + /// Resume while continuing a journal whose tail may follow the checkpoint. + public static TurnEngineRequest ResumeFrom(EngineCheckpoint checkpoint, int maxIterations, long lastJournalSequence) + { + var noPendingWork = (checkpoint.PendingToolRequests is null || checkpoint.PendingToolRequests.Count == 0) + && checkpoint.PendingModelResponse is null + && !checkpoint.FinalOutputReady + && !checkpoint.ReconciliationRequired; + var startIteration = checkpoint.ResumeSameIteration + ? checkpoint.Iteration + : noPendingWork ? checkpoint.Iteration + 1 : checkpoint.Iteration; + + return new TurnEngineRequest(checkpoint.SessionId, checkpoint.TurnId, [.. checkpoint.Messages]) + { + RunId = checkpoint.RunId, + ParentRunId = checkpoint.ParentRunId, + DelegationDepth = checkpoint.DelegationDepth, + StablePrefixMessages = checkpoint.StablePrefixMessages, + Inputs = checkpoint.Inputs, + MaxIterations = maxIterations, + MaxModelAttempts = 3, + StartIteration = startIteration, + InitialSequence = Math.Max(lastJournalSequence, checkpoint.LastSequence), + Portability = checkpoint.ContextState.Portability, + DelegatedState = checkpoint.ContextState.DelegatedState is null ? [] : [.. checkpoint.ContextState.DelegatedState], + ActiveInvocationId = checkpoint.ActiveInvocationId, + PendingToolRequests = checkpoint.PendingToolRequests is null ? [] : [.. checkpoint.PendingToolRequests], + CompletedToolResults = checkpoint.CompletedToolResults is null ? [] : [.. checkpoint.CompletedToolResults], + CompletedModelIterations = checkpoint.CompletedModelIterations, + ReconciliationRequired = checkpoint.ReconciliationRequired, + ModelReconciliation = checkpoint.ModelReconciliation, + PendingOutput = checkpoint.PendingOutput, + FinalOutputReady = checkpoint.FinalOutputReady, + PendingModelResponse = checkpoint.PendingModelResponse, + PolicyAppliedForIteration = checkpoint.PolicyAppliedForIteration, + ReconciliationResolution = null, + ModelReconciliationResolution = null, + }; + } + + /// + /// Resume after the host resolves an indeterminate tool effect. Patches the checkpoint's + /// completed tool result and, if the conversation batch was already flushed, the matching + /// tool-result message, then resumes from the patched checkpoint. + /// + /// + /// The checkpoint does not require tool reconciliation, the resolved result is still + /// indeterminate, or the checkpoint has no matching indeterminate tool request. + /// + public static TurnEngineRequest ResumeAfterReconciliation( + EngineCheckpoint checkpoint, + int maxIterations, + long lastJournalSequence, + ModelToolResult resolvedResult) + { + if (!checkpoint.ReconciliationRequired) + { + throw new TurnEngineInvalidRequestException("checkpoint does not require reconciliation"); + } + + if (checkpoint.ModelReconciliation is not null) + { + throw new TurnEngineInvalidRequestException( + "checkpoint requires model reconciliation, not tool reconciliation"); + } + + if (resolvedResult.Outcome == ModelToolOutcome.Indeterminate) + { + throw new TurnEngineInvalidRequestException("resolved tool result must have a determinate outcome"); + } + + var resolved = CloneForPatch(checkpoint); + var results = resolved.CompletedToolResults ??= []; + var index = IndexOfToolResult(results, resolvedResult.RequestId); + if (index < 0) + { + throw new TurnEngineInvalidRequestException( + $"checkpoint does not contain indeterminate tool request '{resolvedResult.RequestId}'"); + } + + if (results[index].Outcome != ModelToolOutcome.Indeterminate) + { + throw new TurnEngineInvalidRequestException( + $"tool request '{resolvedResult.RequestId}' is already determinate"); + } + + results[index] = resolvedResult; + + if (resolved.PendingModelResponse is null) + { + var messageIndex = IndexOfToolResultMessage(resolved.Messages, resolvedResult.RequestId); + if (messageIndex < 0) + { + throw new TurnEngineInvalidRequestException( + $"checkpoint is missing the tool result message for '{resolvedResult.RequestId}'"); + } + + resolved.Messages[messageIndex] = Message.ToolResult(resolvedResult.RequestId, resolvedResult.ModelText()); + } + + resolved.ReconciliationRequired = false; + + var request = ResumeFrom(resolved, maxIterations, lastJournalSequence); + request.ReconciliationResolution = resolvedResult; + return request; + } + + /// + /// Resume after the host resolves an indeterminate model invocation, replaying the same + /// iteration with the resolved response instead of re-invoking the model. + /// + /// + /// The checkpoint does not require model reconciliation, or the reconciliation's invocation + /// id no longer matches the checkpoint's active invocation. + /// + public static TurnEngineRequest ResumeAfterModelReconciliation( + EngineCheckpoint checkpoint, + int maxIterations, + long lastJournalSequence, + ModelInvocationResponse resolvedResponse) + { + if (!checkpoint.ReconciliationRequired) + { + throw new TurnEngineInvalidRequestException("checkpoint does not require reconciliation"); + } + + var reconciliation = checkpoint.ModelReconciliation + ?? throw new TurnEngineInvalidRequestException( + "checkpoint requires tool reconciliation, not model reconciliation"); + + if (checkpoint.ActiveInvocationId != reconciliation.InvocationId) + { + throw new TurnEngineInvalidRequestException( + "model reconciliation identity does not match the active invocation"); + } + + var request = ResumeFrom(checkpoint, maxIterations, lastJournalSequence); + request.StartIteration = checkpoint.Iteration; + request.ReconciliationRequired = false; + request.ModelReconciliationResolution = resolvedResponse; + return request; + } + + /// + /// Build a resume request from the durable generated . Threads + /// from the durable record rather than + /// defaulting it. Use the reconciliation-specific overloads when the checkpoint is blocked + /// pending a resolved model or tool outcome. + /// + public static TurnEngineRequest FromResume(ResumeContext resume) + { + var request = ResumeFrom( + resume.Checkpoint, + Math.Max(resume.MaxIterations, 0), + Math.Max(resume.ResumeSequence(), 0)); + request.ApplyResumeAttempts(resume); + return request; + } + + /// + /// Build a resume request from a after the host resolves an + /// indeterminate tool effect recorded in the checkpoint. + /// + public static TurnEngineRequest FromResumeAfterReconciliation(ResumeContext resume, ModelToolResult resolvedResult) + { + var request = ResumeAfterReconciliation( + resume.Checkpoint, + Math.Max(resume.MaxIterations, 0), + Math.Max(resume.ResumeSequence(), 0), + resolvedResult); + request.ApplyResumeAttempts(resume); + return request; + } + + /// + /// Build a resume request from a after the host resolves an + /// indeterminate model invocation recorded in the checkpoint. + /// + public static TurnEngineRequest FromResumeAfterModelReconciliation( + ResumeContext resume, + ModelInvocationResponse resolvedResponse) + { + var request = ResumeAfterModelReconciliation( + resume.Checkpoint, + Math.Max(resume.MaxIterations, 0), + Math.Max(resume.ResumeSequence(), 0), + resolvedResponse); + request.ApplyResumeAttempts(resume); + return request; + } + + private void ApplyResumeAttempts(ResumeContext resume) + { + if (resume.MaxModelAttempts > 0) + { + MaxModelAttempts = resume.MaxModelAttempts; + } + } + + private static int IndexOfToolResult(IList results, string requestId) + { + for (var i = 0; i < results.Count; i++) + { + if (results[i].RequestId == requestId) + { + return i; + } + } + + return -1; + } + + private static int IndexOfToolResultMessage(IList messages, string requestId) + { + for (var i = 0; i < messages.Count; i++) + { + if (messages[i].Metadata.TryGetValue("tool_call_id", out var value) && + value is string toolCallId && toolCallId == requestId) + { + return i; + } + } + + return -1; + } + + /// + /// Shallow-clone a checkpoint's mutable collections so reconciliation patching never + /// mutates the caller's original checkpoint instance. + /// + private static EngineCheckpoint CloneForPatch(EngineCheckpoint checkpoint) => new() + { + Id = checkpoint.Id, + SessionId = checkpoint.SessionId, + TurnId = checkpoint.TurnId, + RunId = checkpoint.RunId, + ParentRunId = checkpoint.ParentRunId, + DelegationDepth = checkpoint.DelegationDepth, + Iteration = checkpoint.Iteration, + LastSequence = checkpoint.LastSequence, + Messages = [.. checkpoint.Messages], + StablePrefixMessages = checkpoint.StablePrefixMessages, + Inputs = checkpoint.Inputs, + ActiveInvocationId = checkpoint.ActiveInvocationId, + PendingToolRequests = checkpoint.PendingToolRequests is null ? null : [.. checkpoint.PendingToolRequests], + CompletedToolResults = checkpoint.CompletedToolResults is null ? null : [.. checkpoint.CompletedToolResults], + CompletedModelIterations = checkpoint.CompletedModelIterations, + ReconciliationRequired = checkpoint.ReconciliationRequired, + ModelReconciliation = checkpoint.ModelReconciliation, + PendingOutput = checkpoint.PendingOutput, + FinalOutputReady = checkpoint.FinalOutputReady, + PendingModelResponse = checkpoint.PendingModelResponse, + ResumeSameIteration = checkpoint.ResumeSameIteration, + PolicyAppliedForIteration = checkpoint.PolicyAppliedForIteration, + ContextState = checkpoint.ContextState, + Metadata = checkpoint.Metadata, + }; +} From ef73461727e6c30bf0c53052d313499a9a6d8349 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 01:47:44 -0700 Subject: [PATCH 07/19] feat(csharp): route turns through canonical engine Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../LiveTurnIntegrationTests.cs | 239 ++++++ .../Prompty.Core.Tests/ResilienceTests.cs | 10 +- runtime/csharp/Prompty.Core/LiveTurn.cs | 735 ++++++++++++++++++ runtime/csharp/Prompty.Core/Pipeline.cs | 518 +----------- 4 files changed, 1013 insertions(+), 489 deletions(-) create mode 100644 runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs create mode 100644 runtime/csharp/Prompty.Core/LiveTurn.cs diff --git a/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs b/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs new file mode 100644 index 000000000..ed8eb9d98 --- /dev/null +++ b/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Prompty.Core; + +namespace Prompty.Core.Tests; + +[Collection("InvokerRegistry")] +public sealed class LiveTurnIntegrationTests : IDisposable +{ + private const string Provider = "live-turn-test"; + private const string Format = "live-turn-format"; + + public LiveTurnIntegrationTests() + { + InvokerRegistry.Clear(); + InvokerRegistry.RegisterRenderer(Format, new PassthroughRenderer()); + InvokerRegistry.RegisterParser(Format, new PassthroughParser()); + InvokerRegistry.RegisterProcessor(Provider, new PassthroughProcessor()); + } + + public void Dispose() => InvokerRegistry.Clear(); + + [Fact] + public async Task TurnWithEngineRequest_PersistsAtomicCheckpoints_AndResumesWithoutReinvocation() + { + var executor = new QueueExecutor("durable result"); + InvokerRegistry.RegisterExecutor(Provider, executor); + var durability = new AtomicRecordingDurability(); + durability.FailAfter = EngineEventKind.ModelInvocationCompleted; + var options = new TurnEnginePipelineOptions { Durability = durability }; + + await Assert.ThrowsAsync(() => + Pipeline.TurnWithEngineRequestAsync( + NewAgent(), + NewRequest("durable-session", "durable-turn"), + options)); + Assert.NotEmpty(durability.AtomicAppends); + Assert.All(durability.AtomicAppends, append => + { + Assert.NotEmpty(append.Events); + var semanticTail = append.Events.Last(@event => @event.Kind != EngineEventKind.CheckpointCreated); + Assert.Equal(semanticTail.Sequence, append.Checkpoint.LastSequence); + }); + + var checkpoint = durability.AtomicAppends[^1].Checkpoint; + durability.FailAfter = null; + var resume = TurnEngineRequest.ResumeFrom(checkpoint, maxIterations: 3, checkpoint.LastSequence); + var resumed = await Pipeline.TurnWithEngineRequestAsync(NewAgent(), resume, options); + + Assert.Equal("durable result", resumed); + Assert.Equal(1, executor.InvocationCount); + } + + [Fact] + public async Task TurnWithEngineRequest_CancellationPersistsTerminalCheckpoint() + { + InvokerRegistry.RegisterExecutor(Provider, new QueueExecutor("unused")); + var durability = new AtomicRecordingDurability(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(() => + Pipeline.TurnWithEngineRequestAsync( + NewAgent(), + NewRequest("cancel-session", "cancel-turn"), + new TurnEnginePipelineOptions { Durability = durability }, + cancellation.Token)); + + var events = durability.StandaloneEvents + .Concat(durability.AtomicAppends.SelectMany(append => append.Events)); + Assert.Contains(events, @event => @event.Kind == EngineEventKind.TurnCancelled); + if (durability.AtomicAppends.Count > 0) + { + Assert.Equal( + durability.AtomicAppends[^1].Events[^1].Sequence, + durability.AtomicAppends[^1].Checkpoint.LastSequence); + } + } + + [Fact] + public async Task TurnAsync_RejectsParallelToolsBeforeInvokingProvider() + { + var executor = new QueueExecutor("unused"); + InvokerRegistry.RegisterExecutor(Provider, executor); + var agent = NewAgent(); + agent.Tools = [new FunctionTool { Name = "lookup", Kind = "function" }]; + + var error = await Assert.ThrowsAsync(() => + Pipeline.TurnAsync(agent, parallelToolCalls: true)); + + Assert.Contains("sequentially", error.Message); + Assert.Equal(0, executor.InvocationCount); + } + + [Fact] + public async Task TurnAsync_RawFinalSkipsOutputGuardrail() + { + var rawResponse = new object(); + InvokerRegistry.RegisterExecutor(Provider, new QueueExecutor(rawResponse)); + var outputGuardrailCalled = false; + var guardrails = new Guardrails( + output: _ => + { + outputGuardrailCalled = true; + return new GuardrailResult(false, "raw output must not be inspected"); + }); + + var result = await Pipeline.TurnAsync(NewAgent(), raw: true, guardrails: guardrails); + + Assert.Same(rawResponse, result); + Assert.False(outputGuardrailCalled); + } + + [Fact] + public async Task TurnAsync_ProcessesAccumulatedStreamingItems_AndProjectsTokens() + { + var stream = new PromptyStream(StreamItems("one", "two")); + InvokerRegistry.RegisterExecutor(Provider, new QueueExecutor(stream)); + InvokerRegistry.RegisterProcessor(Provider, new AccumulatedStreamProcessor()); + var tokens = new List(); + EventCallback onEvent = (type, data) => + { + if (type == AgentEventType.Token) + { + tokens.Add(data["token"]!.ToString()!); + } + }; + + var result = await Pipeline.TurnAsync(NewAgent(), onEvent: onEvent); + + Assert.Equal("onetwo", result); + Assert.Equal(["one", "two"], tokens); + Assert.Equal(["one", "two"], stream.Items); + } + + private static TurnEngineRequest NewRequest(string sessionId, string turnId) => + new(sessionId, turnId, []) + { + Inputs = new Dictionary(), + MaxIterations = 3, + MaxModelAttempts = 1, + }; + + private static Prompty NewAgent() => new() + { + Name = "live-turn-agent", + Instructions = "hello", + Model = new Model { Id = "test-model", Provider = Provider }, + Template = new Template + { + Format = new FormatConfig { Kind = Format }, + Parser = new ParserConfig { Kind = Format }, + }, + }; + + private sealed class QueueExecutor(params object[] responses) : IExecutor + { + private readonly Queue _responses = new(responses); + + public int InvocationCount { get; private set; } + + public Task ExecuteAsync(Prompty agent, List messages) + { + InvocationCount++; + return Task.FromResult(_responses.Dequeue()); + } + + public List FormatToolMessages( + object rawResponse, + List toolCalls, + List toolResults, + string? textContent = null) => throw new NotSupportedException(); + } + + private sealed class PassthroughRenderer : IRenderer + { + public Task RenderAsync(Prompty agent, string template, Dictionary inputs) => + Task.FromResult(template); + } + + private sealed class PassthroughParser : IParser + { + public Task> ParseAsync( + Prompty agent, + string rendered, + Dictionary? context) => + Task.FromResult(new List { Message.User(rendered) }); + } + + private sealed class PassthroughProcessor : IProcessor + { + public Task ProcessAsync(Prompty agent, object response) => Task.FromResult(response); + } + + private sealed class AccumulatedStreamProcessor : IProcessor + { + public Task ProcessAsync(Prompty agent, object response) + { + var stream = Assert.IsType(response); + return Task.FromResult(string.Concat(stream.Items.Cast())); + } + } + + private static async IAsyncEnumerable StreamItems(params object[] items) + { + foreach (var item in items) + { + await Task.Yield(); + yield return item; + } + } + + private sealed class AtomicRecordingDurability : IEngineDurabilityPort + { + public List<(IReadOnlyList Events, EngineCheckpoint Checkpoint)> AtomicAppends { get; } = []; + + public List StandaloneEvents { get; } = []; + + public EngineEventKind? FailAfter { get; set; } + + public Task AppendAsync(EngineEvent @event) + { + StandaloneEvents.Add(@event); + return Task.CompletedTask; + } + + public Task AppendWithCheckpointAsync( + IReadOnlyList events, + EngineCheckpoint checkpoint) + { + AtomicAppends.Add((events, checkpoint)); + if (FailAfter is not null && events.Any(@event => @event.Kind == FailAfter)) + { + throw PortError.Configuration("simulated durability crash"); + } + return Task.CompletedTask; + } + } +} diff --git a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs index a972bcfb3..966598261 100644 --- a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs @@ -379,7 +379,7 @@ public async Task ToolDispatch_ParseError_ReturnsErrorStringToLlm() } [Fact] - public async Task ToolDispatch_ParallelPath_HandlerThrows_LoopContinues() + public async Task ToolDispatch_ParallelPath_IsRejectedForDeterministicDurability() { var executor = ResilienceHelper.Register(); @@ -407,9 +407,11 @@ public async Task ToolDispatch_ParallelPath_HandlerThrows_LoopContinues() ["bad_tool"] = _ => throw new InvalidOperationException("parallel boom") }; - var result = await Pipeline.TurnAsync(agent, tools: tools, parallelToolCalls: true); - Assert.Equal("parallel recovered", result); - Assert.Equal(2, executor.Calls.Count); + var error = await Assert.ThrowsAsync( + () => Pipeline.TurnAsync(agent, tools: tools, parallelToolCalls: true)); + + Assert.Contains("sequentially", error.Message); + Assert.Empty(executor.Calls); } } diff --git a/runtime/csharp/Prompty.Core/LiveTurn.cs b/runtime/csharp/Prompty.Core/LiveTurn.cs new file mode 100644 index 000000000..bbc5df3bb --- /dev/null +++ b/runtime/csharp/Prompty.Core/LiveTurn.cs @@ -0,0 +1,735 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json; + +namespace Prompty.Core; + +/// Runtime-only configuration for a public turn backed by the canonical . +public sealed class TurnEnginePipelineOptions +{ + public Dictionary>>? Tools { get; init; } + + public bool Raw { get; init; } + + public EventCallback? OnEvent { get; init; } + + public int? ContextBudget { get; init; } + + public Guardrails? Guardrails { get; init; } + + public Steering? Steering { get; init; } + + public CompactionStrategy? Compaction { get; init; } + + public IEngineDurabilityPort? Durability { get; init; } + + public IEnginePermissionPort? Permission { get; init; } + + public IEnginePostCommitPort? PostCommit { get; init; } +} + +/// Adapts the public pipeline registry and hooks to the canonical turn engine ports. +internal static class LiveTurn +{ + private static long _turnId; + + public static Task RunAsync( + Prompty agent, + Dictionary? inputs, + Dictionary>>? tools, + int maxIterations, + bool raw, + EventCallback? onEvent, + CancellationToken cancellationToken, + int? contextBudget, + Guardrails? guardrails, + Steering? steering, + bool parallelToolCalls, + int maxLlmRetries, + CompactionStrategy? compaction) + { + if (parallelToolCalls) + { + throw new ArgumentException( + "parallelToolCalls=true is not supported by the canonical engine; " + + "tool effects execute sequentially for deterministic durable ordering.", + nameof(parallelToolCalls)); + } + + var id = Interlocked.Increment(ref _turnId); + var agentMode = (agent.Tools?.Count ?? 0) > 0 || (tools?.Count ?? 0) > 0 + || guardrails is not null || steering is not null || contextBudget is not null; + var request = new TurnEngineRequest($"pipeline-session-{id}", $"pipeline-turn-{id}", []) + { + Inputs = inputs ?? new Dictionary(), + MaxIterations = Math.Max(maxIterations, 1), + MaxModelAttempts = agentMode ? Math.Max(maxLlmRetries, 1) : 1, + }; + var options = new TurnEnginePipelineOptions + { + Tools = tools, + Raw = raw, + OnEvent = onEvent, + ContextBudget = contextBudget, + Guardrails = guardrails, + Steering = steering, + Compaction = compaction, + }; + return RunAsync(agent, request, options, cancellationToken); + } + + public static async Task RunAsync( + Prompty agent, + TurnEngineRequest request, + TurnEnginePipelineOptions? options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(agent); + ArgumentNullException.ThrowIfNull(request); + options ??= new TurnEnginePipelineOptions(); + + var provider = agent.Model?.Provider ?? "openai"; + var executor = InvokerRegistry.GetExecutor(provider); + var processor = InvokerRegistry.GetProcessor(provider); + var agentMode = (agent.Tools?.Count ?? 0) > 0 || (options.Tools?.Count ?? 0) > 0; + var failures = new LiveFailureState(); + var authorization = new LivePermissionPort(options.Permission, options.Guardrails); + var durability = new LiveDurabilityPort( + options.Durability ?? new NoopDurabilityPort(), + options.OnEvent, + agent, + agentMode, + request.MaxIterations); + var engine = new TurnEngine(new TurnEngineEffects + { + Model = new LiveModelPort(agent, executor, processor, options.Raw, agentMode, failures), + Tools = new LiveToolPort( + agent, + options.Tools, + request.Inputs as Dictionary, + authorization, + options.OnEvent), + Clock = new LiveClock(), + Ids = new LiveIds(), + Policy = new LivePolicyPort( + agent, + request.Inputs as Dictionary, + options.ContextBudget, + options.Guardrails, + options.Steering, + options.Compaction, + failures), + Retry = new LiveRetryPort(options.OnEvent), + Conversation = new LiveConversationPort(executor), + Permission = authorization, + Durability = durability, + PostCommit = options.PostCommit ?? new NoopPostCommitPort(), + Stream = new LiveStreamPort(options.OnEvent), + }); + + TurnEngineResult result; + try + { + result = await engine.RunAsync(request, cancellationToken).ConfigureAwait(false); + } + catch + { + durability.EmitUncommittedError(); + throw; + } + + return result.Commit.Status switch + { + EngineTurnStatus.Success => result.Commit.Output!, + EngineTurnStatus.Cancelled => throw new OperationCanceledException("Operation cancelled", cancellationToken), + EngineTurnStatus.Failed or EngineTurnStatus.ReconciliationRequired => + throw MapFailure(result, request.MaxModelAttempts, agentMode, failures), + _ => throw new InvalidOperationException($"Unsupported engine status '{result.Commit.Status}'."), + }; + } + + private static Exception MapFailure( + TurnEngineResult result, + int maxModelAttempts, + bool agentMode, + LiveFailureState failures) + { + var output = result.Commit.Output as IReadOnlyDictionary; + var kind = output?.GetValueOrDefault("errorKind")?.ToString() ?? "engine_error"; + var message = output?.GetValueOrDefault("message")?.ToString() ?? "TurnEngine failed."; + return kind switch + { + "input_guardrail_denied" or "output_guardrail_denied" => new GuardrailError(failures.GuardrailReason ?? message), + "model_error" when !agentMode && failures.InvokerError is not null => failures.InvokerError, + "model_error" => new ExecuteError( + $"LLM call failed after {maxModelAttempts} retries: {message}", + [.. result.Commit.Messages]), + "max_iterations" => new InvalidOperationException( + $"Agent loop exceeded maximum iterations ({result.Commit.Iterations})."), + "prepare_error" when failures.InvokerError is not null => failures.InvokerError, + _ => failures.InvokerError ?? new InvalidOperationException(message), + }; + } + + private static object? MetadataValue(IDictionary? metadata, string key) + => metadata is not null && metadata.TryGetValue(key, out var value) ? value : null; + + private sealed class LiveFailureState + { + public Exception? InvokerError { get; set; } + + public string? GuardrailReason { get; set; } + + public bool SkipOutputGuardrail { get; set; } + } + + private sealed class LiveModelPort( + Prompty agent, + IExecutor executor, + IProcessor processor, + bool raw, + bool agentMode, + LiveFailureState failures) : IEngineModelPort + { + public async Task InvokeAsync( + ModelInvocationRequest request, + CancellationToken cancellationToken, + IEngineModelStreamPort stream) + { + cancellationToken.ThrowIfCancellationRequested(); + object rawResponse; + try + { + rawResponse = await executor.ExecuteAsync(agent, [.. request.Context.Messages]).ConfigureAwait(false); + if (rawResponse is PromptyStream rawStream) + { + await foreach (var chunk in rawStream.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + if (chunk is string text && text.Length > 0) + { + await stream.EmitAsync(new ModelStreamChunk.Text(text)).ConfigureAwait(false); + } + } + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception error) + { + failures.InvokerError = error; + throw new PortError(error.Message); + } + + object processed; + try + { + failures.SkipOutputGuardrail = raw && !agentMode; + processed = raw && !agentMode + ? rawResponse + : await processor.ProcessAsync(agent, rawResponse).ConfigureAwait(false); + } + catch (Exception error) + { + failures.InvokerError = error; + throw new PortError(error.Message); + } + + if (processed is ToolCallResult toolResult && toolResult.ToolCalls.Count > 0) + { + return new ModelInvocationResponse + { + AssistantMessages = [], + ToolRequests = toolResult.ToolCalls.Select(call => new ModelToolRequest + { + Id = call.Id, + Name = call.Name, + Arguments = ParseArgumentsValue(call.Arguments), + Metadata = new Dictionary { ["argumentsText"] = call.Arguments }, + }).ToList(), + Metadata = new Dictionary + { + ["rawResponse"] = rawResponse, + ["textContent"] = toolResult.Content ?? string.Empty, + }, + }; + } + + return new ModelInvocationResponse + { + Output = processed, + AssistantMessages = [], + ToolRequests = [], + NextContextState = new InvocationContextState + { + Portability = InvocationContextPortability.Portable, + DelegatedState = [], + }, + Metadata = new Dictionary { ["rawResponse"] = rawResponse }, + }; + } + + private static object ParseArgumentsValue(string arguments) + { + try + { + return JsonSerializer.Deserialize(arguments) ?? arguments; + } + catch (JsonException) + { + return arguments; + } + } + } + + private sealed class LiveConversationPort(IExecutor executor) : IEngineConversationPort + { + public IList FormatToolExchange( + ModelInvocationResponse response, + IReadOnlyList results) + { + var requests = response.ToolRequests ?? []; + var calls = requests.Select(request => new ToolCall + { + Id = request.Id, + Name = request.Name, + Arguments = MetadataValue(request.Metadata, "argumentsText")?.ToString() + ?? request.ModelArgumentsText(), + }).ToList(); + var orderedResults = requests.Select(request => + results.First(result => result.RequestId == request.Id).ModelText()).ToList(); + var rawResponse = MetadataValue(response.Metadata, "rawResponse") + ?? throw PortError.Configuration("provider response metadata is missing rawResponse"); + var content = MetadataValue(response.Metadata, "textContent")?.ToString(); + try + { + return executor.FormatToolMessages(rawResponse, calls, orderedResults, content); + } + catch (Exception error) + { + throw PortError.Configuration(error.Message); + } + } + } + + private sealed class LiveToolPort( + Prompty agent, + Dictionary>>? tools, + Dictionary? inputs, + LivePermissionPort authorization, + EventCallback? onEvent) : IEngineToolPort + { + public async Task ExecuteAsync(ModelToolRequest request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var call = new ToolCall + { + Id = request.Id, + Name = request.Name, + Arguments = MetadataValue(request.Metadata, "argumentsText")?.ToString() + ?? request.ModelArgumentsText(), + }; + if (authorization.TakeRewrite(request.Id) is { } rewritten) + { + call.Arguments = JsonSerializer.Serialize(rewritten); + } + + string output; + try + { + output = await ToolDispatch.DispatchAsync(agent, call, tools, inputs).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception error) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.Error, new Dictionary + { + ["tool"] = request.Name, + ["error"] = error.Message, + }); + output = $"Error: Tool '{request.Name}' failed: {error.Message}"; + } + + cancellationToken.ThrowIfCancellationRequested(); + var failed = output.StartsWith("Error:", StringComparison.Ordinal); + return new ModelToolResult + { + RequestId = request.Id, + Name = request.Name, + Outcome = failed ? ModelToolOutcome.Failed : ModelToolOutcome.Success, + Output = output, + ErrorKind = failed ? "tool_error" : null, + }; + } + } + + private sealed class LivePermissionPort( + IEnginePermissionPort? inner, + Guardrails? guardrails) : IEnginePermissionPort + { + private readonly Dictionary> _rewrites = []; + + public async Task AuthorizeAsync( + ModelToolRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + if (inner is not null) + { + var decision = await inner.AuthorizeAsync(request, cancellationToken).ConfigureAwait(false); + if (!decision.Approved) + { + return decision; + } + } + if (guardrails is null) + { + return new EnginePermissionDecision { Approved = true }; + } + + var arguments = ToolDispatch.ParseArguments(request.ModelArgumentsText()); + var result = guardrails.CheckTool(request.Name, arguments); + if (!result.Allowed) + { + return new EnginePermissionDecision + { + Approved = false, + Reason = $"Error: Tool guardrail denied: {result.Reason ?? "Tool denied"}", + }; + } + if (result.Rewrite is Dictionary rewrite) + { + _rewrites[request.Id] = rewrite; + } + return new EnginePermissionDecision { Approved = true }; + } + + public Dictionary? TakeRewrite(string requestId) + => _rewrites.Remove(requestId, out var rewrite) ? rewrite : null; + } + + private sealed class LivePolicyPort( + Prompty agent, + Dictionary? inputs, + int? contextBudget, + Guardrails? guardrails, + Steering? steering, + CompactionStrategy? compaction, + LiveFailureState failures) : IEngineHostPolicyPort + { + private bool _prepared; + + public async Task BeforeModelAsync( + HostPolicyRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var messages = request.Messages.ToList(); + var stablePrefix = Math.Min(request.StablePrefixMessages, messages.Count); + if (!_prepared) + { + try + { + messages = await Pipeline.PrepareAsync(agent, inputs).ConfigureAwait(false); + } + catch (Exception error) + { + failures.InvokerError = error; + throw new HostPolicyException("prepare_error", error.Message); + } + stablePrefix = messages.Count; + _prepared = true; + } + + var steeringMessages = steering?.Drain() ?? []; + messages.AddRange(steeringMessages); + var trimmed = 0; + if (contextBudget is not null) + { + var before = messages.ToList(); + var (droppedCount, droppedMessages) = ContextWindow.TrimToContextWindow(messages, contextBudget.Value); + trimmed = droppedCount; + if (droppedCount > 0 && compaction is not null) + { + await Pipeline.ApplyCompactionAsync(compaction, droppedMessages, messages, onEvent: null).ConfigureAwait(false); + } + stablePrefix = Math.Min(stablePrefix, CommonPrefixLength(before, messages)); + } + + if (guardrails is not null) + { + var check = guardrails.CheckInput(messages); + if (!check.Allowed) + { + failures.GuardrailReason = check.Reason; + throw new HostPolicyException("input_guardrail_denied", check.Reason ?? "Input guardrail denied"); + } + if (check.Rewrite is List rewritten) + { + messages = rewritten; + stablePrefix = Math.Min(stablePrefix, messages.Count); + } + } + + return new HostPolicyResult + { + Messages = messages, + StablePrefixMessages = stablePrefix, + Metadata = new Dictionary + { + ["steeringCount"] = steeringMessages.Count, + ["trimmedCount"] = trimmed, + ["notifyMessagesUpdated"] = steeringMessages.Count > 0 || trimmed > 0, + }, + }; + } + + public Task BeforeCommitAsync( + FinalOutputPolicyRequest request, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + var output = request.Output; + if (guardrails is not null && !failures.SkipOutputGuardrail) + { + var message = output switch + { + Message existing => existing, + string text => Message.Assistant(text), + _ => Message.Assistant(output?.ToString() ?? string.Empty), + }; + var check = guardrails.CheckOutput(message); + if (!check.Allowed) + { + failures.GuardrailReason = check.Reason; + throw new HostPolicyException("output_guardrail_denied", check.Reason ?? "Output guardrail denied"); + } + if (check.Rewrite is not null) + { + output = check.Rewrite; + } + } + return Task.FromResult(new FinalOutputPolicyResult { Output = output }); + } + + private static int CommonPrefixLength(IList left, IList right) + { + var length = Math.Min(left.Count, right.Count); + var common = 0; + while (common < length && left[common].ToJson(indent: false) == right[common].ToJson(indent: false)) + { + common++; + } + return common; + } + } + + private sealed class LiveRetryPort(EventCallback? onEvent) : IEngineRetryPolicyPort + { + public async Task BackoffAsync(RetryPolicyRequest request, CancellationToken cancellationToken) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.Status, new Dictionary + { + ["message"] = $"LLM call failed, retrying (attempt {request.NextAttempt}/{request.MaxAttempts})...", + }); + AgentEvents.EmitEvent(onEvent, AgentEventType.Retry, new Dictionary + { + ["operation"] = "llm", + ["attempt"] = request.NextAttempt, + ["maxAttempts"] = request.MaxAttempts, + }); + var delay = Math.Min(Math.Pow(2, request.FailedAttempts) + Random.Shared.NextDouble(), 60); + await Task.Delay(TimeSpan.FromSeconds(delay), cancellationToken) + .ConfigureAwait(false); + } + } + + private sealed class LiveStreamPort(EventCallback? onEvent) : IEngineModelStreamPort + { + public Task EmitAsync(ModelStreamChunk chunk) + { + if (chunk is ModelStreamChunk.Text text) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.Token, new Dictionary { ["token"] = text.Value }); + } + else if (chunk is ModelStreamChunk.Thinking thinking) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.Thinking, new Dictionary { ["thinking"] = thinking.Value }); + } + return Task.CompletedTask; + } + } + + private sealed class LiveDurabilityPort( + IEngineDurabilityPort inner, + EventCallback? onEvent, + Prompty agent, + bool agentMode, + int maxIterations) : IEngineDurabilityPort + { + private IList _messages = []; + private int _iterations; + private bool _terminal; + + public async Task AppendAsync(EngineEvent @event) + { + await inner.AppendAsync(@event).ConfigureAwait(false); + Project(@event); + } + + public async Task AppendWithCheckpointAsync( + IReadOnlyList events, + EngineCheckpoint checkpoint) + { + await inner.AppendWithCheckpointAsync(events, checkpoint).ConfigureAwait(false); + _messages = checkpoint.Messages; + _iterations = checkpoint.CompletedModelIterations; + foreach (var @event in events) + { + Project(@event); + } + } + + public void EmitUncommittedError() => EmitTerminal("error", null); + + private void Project(EngineEvent @event) + { + var payload = @event.Payload as IReadOnlyDictionary; + switch (@event.Kind) + { + case EngineEventKind.TurnStarted: + AgentEvents.EmitEvent(onEvent, AgentEventType.TurnStart, new Dictionary + { + ["agent"] = agent.Name, + ["maxIterations"] = maxIterations, + }); + break; + case EngineEventKind.ModelInvocationStarted: + AgentEvents.EmitEvent(onEvent, AgentEventType.LlmStart, new Dictionary + { + ["provider"] = agent.Model?.Provider, + ["modelId"] = agent.Model?.Id, + ["messageCount"] = payload?.GetValueOrDefault("messageCount"), + ["attempt"] = payload?.GetValueOrDefault("attempt"), + ["iteration"] = @event.Iteration, + }); + break; + case EngineEventKind.ModelInvocationCompleted: + case EngineEventKind.ModelInvocationReconciled: + AgentEvents.EmitEvent(onEvent, AgentEventType.LlmComplete, new Dictionary + { + ["iteration"] = @event.Iteration, + }); + break; + case EngineEventKind.PermissionRequested: + AgentEvents.EmitEvent(onEvent, AgentEventType.PermissionRequested, new Dictionary + { + ["iteration"] = @event.Iteration, + ["request"] = payload?.GetValueOrDefault("toolRequest"), + }); + break; + case EngineEventKind.PermissionResolved: + AgentEvents.EmitEvent(onEvent, AgentEventType.PermissionCompleted, new Dictionary + { + ["iteration"] = @event.Iteration, + ["decision"] = payload?.GetValueOrDefault("decision"), + }); + break; + case EngineEventKind.ToolExecutionStarted: + if (payload?.GetValueOrDefault("toolRequest") is ModelToolRequest request) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallStart, new Dictionary + { + ["tool"] = request.Name, + ["arguments"] = request.ModelArgumentsText(), + }); + } + break; + case EngineEventKind.ToolExecutionCompleted: + if (payload?.GetValueOrDefault("toolResult") is ModelToolResult result) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, new Dictionary + { + ["name"] = result.Name, + ["success"] = result.Outcome == ModelToolOutcome.Success, + ["result"] = result.ModelText(), + ["errorKind"] = result.ErrorKind, + }); + } + break; + case EngineEventKind.ToolResultCommitted: + if (payload?.GetValueOrDefault("toolResult") is ModelToolResult committedResult) + { + AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, new Dictionary + { + ["tool"] = committedResult.Name, + ["result"] = committedResult.ModelText(), + }); + } + break; + case EngineEventKind.ConversationUpdated: + AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, new Dictionary + { + ["messages"] = _messages, + }); + break; + case EngineEventKind.TurnCommitted: + AgentEvents.EmitEvent(onEvent, AgentEventType.Done, new Dictionary + { + ["iterations"] = _iterations, + }); + EmitTerminal("success", payload?.GetValueOrDefault("output")); + break; + case EngineEventKind.TurnCancelled: + AgentEvents.EmitEvent(onEvent, AgentEventType.Cancelled, new Dictionary()); + EmitTerminal("cancelled", null); + break; + case EngineEventKind.TurnFailed: + case EngineEventKind.TurnReconciliationRequired: + EmitTerminal("error", null); + break; + } + } + + private void EmitTerminal(string status, object? response) + { + if (_terminal) + { + return; + } + _terminal = true; + AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, new Dictionary + { + ["iterations"] = agentMode ? _iterations : 0, + ["status"] = status, + ["response"] = response, + }); + } + } + + private sealed class LiveClock : IEngineClock + { + public string Now() => DateTimeOffset.UtcNow.ToString("O"); + } + + private sealed class LiveIds : IEngineIdGenerator + { + private long _id; + + public string NextId(string kind) => $"{kind}-{Interlocked.Increment(ref _id)}"; + } +} + +internal static class LiveTurnModelExtensions +{ + public static string ModelArgumentsText(this ModelToolRequest request) => request.Arguments switch + { + null => string.Empty, + string text => text, + var value => JsonSerializer.Serialize(value), + }; +} diff --git a/runtime/csharp/Prompty.Core/Pipeline.cs b/runtime/csharp/Prompty.Core/Pipeline.cs index a476ca3a7..a7a8664f3 100644 --- a/runtime/csharp/Prompty.Core/Pipeline.cs +++ b/runtime/csharp/Prompty.Core/Pipeline.cs @@ -10,20 +10,6 @@ namespace Prompty.Core; /// public static class Pipeline { - private static void EmitFailedTurnEnd(EventCallback? onEvent, Exception exception, int iterations, object? response = null) - { - var payload = new Dictionary - { - ["iterations"] = iterations, - ["status"] = exception is OperationCanceledException ? "cancelled" : "error" - }; - if (response is not null) - { - payload["response"] = response; - } - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, payload); - } - // ----------------------------------------------------------------------- // Input Validation // ----------------------------------------------------------------------- @@ -229,9 +215,8 @@ public static async Task InvokeAsync( // ----------------------------------------------------------------------- /// - /// Conversational round-trip: Prepare → [Execute → check tool_calls → execute tools → loop] → Process. - /// If no tools are provided, performs a single Prepare → Execute → Process pass. - /// Calls executor and processor directly (not via RunAsync). + /// Conversational round-trip through the canonical durable turn engine. + /// If no tools are provided, the engine performs a single Prepare → Execute → Process pass. /// public static async Task TurnAsync( Prompty agent, @@ -250,428 +235,43 @@ public static async Task TurnAsync( CompactionStrategy? compaction = null) { var label = turnNumber.HasValue ? $"turn {turnNumber.Value}" : "turn"; - return await Trace.TraceAsync($"prompty.turn", async (emit) => + return await Trace.TraceAsync("prompty.turn", async (emit) => { emit("signature", "prompty.turn"); - emit("inputs", new Dictionary { ["agent"] = agent.Name, ["label"] = label, ["maxIterations"] = maxIterations }); - var messages = await PrepareAsync(agent, inputs); - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnStart, - new Dictionary - { - ["agent"] = agent.Name, - ["inputs"] = inputs ?? new Dictionary(), - ["maxIterations"] = maxIterations - }); - - // Simple path: no tools on agent, no user tools, and no agent-loop features → single execute + process - var hasAgentTools = agent.Tools is not null && agent.Tools.Count > 0; - var hasUserTools = tools is not null && tools.Count > 0; - var hasAgentFeatures = guardrails is not null || steering is not null || contextBudget is not null; - if (!hasAgentTools && !hasUserTools && !hasAgentFeatures) + emit("inputs", new Dictionary { - AgentEvents.EmitEvent(onEvent, AgentEventType.LlmStart, - new Dictionary - { - ["provider"] = agent.Model?.Provider, - ["modelId"] = agent.Model?.Id, - ["messageCount"] = messages.Count, - ["attempt"] = 0 - }); - object response; - try - { - response = await ExecuteAsync(agent, messages); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, 0); - throw; - } - AgentEvents.EmitEvent(onEvent, AgentEventType.LlmComplete, new Dictionary()); - if (raw) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, - new Dictionary { ["iterations"] = 0, ["status"] = "success", ["response"] = response }); - return response; - } - object processed; - try - { - processed = await ProcessAsync(agent, response); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, 0, response); - throw; - } - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, - new Dictionary { ["iterations"] = 0, ["status"] = "success", ["response"] = processed }); - return processed; - } - - // Agent loop: execute → check tool_calls → dispatch tools → loop - var executor = InvokerRegistry.GetExecutor(agent.Model?.Provider ?? "openai"); - object? response2 = null; - int iteration = 0; - - while (true) - { - if (iteration >= maxIterations) - { - EmitFailedTurnEnd(onEvent, new InvalidOperationException("Agent loop exceeded maximum iterations."), iteration); - throw new InvalidOperationException( - $"Agent loop exceeded maximum iterations ({maxIterations})."); - } - - // Cancellation check at loop start - try - { - cancellationToken.ThrowIfCancellationRequested(); - } - catch (OperationCanceledException) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Cancelled, - new Dictionary { ["iteration"] = iteration, ["reason"] = "cancellation_requested" }); - EmitFailedTurnEnd(onEvent, new OperationCanceledException(), iteration); - throw; - } - - // Drain steering messages - if (steering is not null) - { - var steered = steering.Drain(); - if (steered.Count > 0) - { - messages.AddRange(steered); - AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, - new Dictionary { ["source"] = "steering", ["count"] = steered.Count }); - } - } - - // Context window trimming - if (contextBudget is not null) - { - var (droppedCount, droppedMessages) = ContextWindow.TrimToContextWindow(messages, contextBudget.Value); - if (droppedCount > 0) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, - new Dictionary { ["source"] = "context_trim", ["dropped"] = droppedCount }); - - if (compaction is not null) - { - await ApplyCompactionAsync(compaction, droppedMessages, messages, onEvent); - } - } - } - - // Input guardrail - if (guardrails is not null) - { - var inputCheck = guardrails.CheckInput(messages); - if (!inputCheck.Allowed) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Error, - new Dictionary { ["guardrail"] = "input", ["reason"] = inputCheck.Reason }); - EmitFailedTurnEnd(onEvent, new GuardrailError(inputCheck.Reason ?? "Input guardrail denied"), iteration); - throw new GuardrailError(inputCheck.Reason ?? "Input guardrail denied"); - } - if (inputCheck.Rewrite is List rewrittenMessages) - { - messages = rewrittenMessages; - } - } - - // Cancellation check before LLM call - try - { - cancellationToken.ThrowIfCancellationRequested(); - } - catch (OperationCanceledException) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Cancelled, - new Dictionary { ["iteration"] = iteration, ["reason"] = "cancelled_before_llm" }); - EmitFailedTurnEnd(onEvent, new OperationCanceledException(), iteration); - throw; - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.Status, - new Dictionary { ["iteration"] = iteration, ["phase"] = "executing" }); - - AgentEvents.EmitEvent(onEvent, AgentEventType.LlmStart, - new Dictionary - { - ["provider"] = agent.Model?.Provider, - ["modelId"] = agent.Model?.Id, - ["messageCount"] = messages.Count, - ["attempt"] = 0, - ["iteration"] = iteration - }); - try - { - response2 = await InvokeWithRetryAsync(agent, messages, maxLlmRetries, onEvent, cancellationToken); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration); - throw; - } - AgentEvents.EmitEvent(onEvent, AgentEventType.LlmComplete, - new Dictionary { ["iteration"] = iteration }); - - // If response is a stream, consume it fully before processing. - if (response2 is PromptyStream stream) - { - try - { - await foreach (var chunk in stream) - { - if (chunk is string tokenText && tokenText.Length > 0) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Token, - new Dictionary { ["token"] = tokenText }); - } - } - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration, response2); - throw; - } - response2 = stream; - } - - object result; - try - { - result = raw ? response2! : await ProcessAsync(agent, response2!); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration, response2); - throw; - } + ["agent"] = agent.Name, + ["label"] = label, + ["maxIterations"] = maxIterations, + }); + return await LiveTurn.RunAsync( + agent, + inputs, + tools, + maxIterations, + raw, + onEvent, + cancellationToken, + contextBudget, + guardrails, + steering, + parallelToolCalls, + maxLlmRetries, + compaction).ConfigureAwait(false); + }).ConfigureAwait(false); - if (result is ToolCallResult toolResult && toolResult.ToolCalls.Count > 0) - { - // Dispatch tool calls (parallel or sequential) - var toolResults = new List(); - - if (parallelToolCalls && toolResult.ToolCalls.Count > 1) - { - // Parallel dispatch via Task.WhenAll - var tasks = new List>(); - for (int ti = 0; ti < toolResult.ToolCalls.Count; ti++) - { - var call = toolResult.ToolCalls[ti]; - var capturedIndex = ti; - - // Tool guardrail (with rewrite support) - if (guardrails is not null) - { - var args = ToolDispatch.ParseArguments(call.Arguments); - var toolCheck = guardrails.CheckTool(call.Name, args); - if (!toolCheck.Allowed) - { - var deniedMsg = $"Tool denied by guardrail: {toolCheck.Reason}"; - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, - new Dictionary { ["tool"] = call.Name, ["result"] = deniedMsg }); - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, - new Dictionary { ["name"] = call.Name, ["success"] = false, ["result"] = deniedMsg, ["errorKind"] = "guardrail_denied" }); - tasks.Add(Task.FromResult((capturedIndex, deniedMsg))); - continue; - } - if (toolCheck.Rewrite is Dictionary rewrittenArgs) - { - call.Arguments = System.Text.Json.JsonSerializer.Serialize(rewrittenArgs); - } - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallStart, - new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - - tasks.Add(Task.Run(async () => - { - var toolStarted = DateTimeOffset.UtcNow; - string toolResponse; - try - { - toolResponse = await Trace.TraceAsync("Prompty.Core.ToolDispatch.Execute", async (toolEmit) => - { - toolEmit("inputs", new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - return await ToolDispatch.DispatchAsync(agent, call, tools, inputs) ?? string.Empty; - }); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - toolResponse = $"Error: Tool '{call.Name}' failed: {ex.Message}"; - AgentEvents.EmitEvent(onEvent, AgentEventType.Error, - new Dictionary { ["tool"] = call.Name, ["error"] = ex.Message }); - } - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, - new Dictionary - { - ["name"] = call.Name, - ["success"] = !toolResponse.StartsWith("Error:", StringComparison.Ordinal), - ["result"] = toolResponse, - ["durationMs"] = (DateTimeOffset.UtcNow - toolStarted).TotalMilliseconds, - ["errorKind"] = toolResponse.StartsWith("Error:", StringComparison.Ordinal) ? "tool_error" : null - }); - return (capturedIndex, toolResponse); - })); - } - - (int Index, string Result)[] completed; - try - { - completed = await Task.WhenAll(tasks); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration); - throw; - } - // Maintain order - var ordered = new string[toolResult.ToolCalls.Count]; - foreach (var (index, res) in completed) - { - ordered[index] = res; - } - toolResults.AddRange(ordered); - - for (int ti = 0; ti < toolResult.ToolCalls.Count; ti++) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, - new Dictionary { ["tool"] = toolResult.ToolCalls[ti].Name, ["result"] = toolResults[ti] }); - } - } - else - { - // Sequential dispatch - foreach (var call in toolResult.ToolCalls) - { - // Tool guardrail (with rewrite support) - if (guardrails is not null) - { - var args = ToolDispatch.ParseArguments(call.Arguments); - var toolCheck = guardrails.CheckTool(call.Name, args); - if (!toolCheck.Allowed) - { - var deniedMsg = $"Tool denied by guardrail: {toolCheck.Reason}"; - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, - new Dictionary { ["tool"] = call.Name, ["result"] = deniedMsg }); - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, - new Dictionary { ["name"] = call.Name, ["success"] = false, ["result"] = deniedMsg, ["errorKind"] = "guardrail_denied" }); - toolResults.Add(deniedMsg); - continue; - } - if (toolCheck.Rewrite is Dictionary rewrittenArgs) - { - call.Arguments = System.Text.Json.JsonSerializer.Serialize(rewrittenArgs); - } - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallStart, - new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - - var toolStarted = DateTimeOffset.UtcNow; - string toolResponse; - try - { - toolResponse = await Trace.TraceAsync("Prompty.Core.ToolDispatch.Execute", async (toolEmit) => - { - toolEmit("inputs", new Dictionary { ["tool"] = call.Name, ["arguments"] = call.Arguments }); - return await ToolDispatch.DispatchAsync(agent, call, tools, inputs); - }); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - toolResponse = $"Error: Tool '{call.Name}' failed: {ex.Message}"; - AgentEvents.EmitEvent(onEvent, AgentEventType.Error, - new Dictionary { ["tool"] = call.Name, ["error"] = ex.Message }); - } - catch (OperationCanceledException ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration); - throw; - } - toolResults.Add(toolResponse); - - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, - new Dictionary { ["tool"] = call.Name, ["result"] = toolResponse }); - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, - new Dictionary - { - ["name"] = call.Name, - ["success"] = !toolResponse.StartsWith("Error:", StringComparison.Ordinal), - ["result"] = toolResponse, - ["durationMs"] = (DateTimeOffset.UtcNow - toolStarted).TotalMilliseconds, - ["errorKind"] = toolResponse.StartsWith("Error:", StringComparison.Ordinal) ? "tool_error" : null - }); - } - } - - // Delegate message formatting to the executor (provider-specific) - List toolMessages; - try - { - toolMessages = executor.FormatToolMessages( - response2, toolResult.ToolCalls, toolResults, toolResult.Content); - } - catch (Exception ex) - { - EmitFailedTurnEnd(onEvent, ex, iteration, response2); - throw; - } - messages.AddRange(toolMessages); - - AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, - new Dictionary { ["source"] = "tool_results", ["count"] = toolMessages.Count }); - - iteration++; - continue; - } - - // Output guardrail on final response - if (guardrails is not null) - { - var outputMsg = result switch - { - string resultText => new Message - { - Role = Role.Assistant, - Parts = [new TextPart { Value = resultText }] - }, - Message msg => msg, - _ => new Message - { - Role = Role.Assistant, - Parts = [new TextPart { Value = result?.ToString() ?? "" }] - }, - }; - var outputCheck = guardrails.CheckOutput(outputMsg); - if (!outputCheck.Allowed) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.Error, - new Dictionary { ["guardrail"] = "output", ["reason"] = outputCheck.Reason }); - EmitFailedTurnEnd(onEvent, new GuardrailError(outputCheck.Reason ?? "Output guardrail denied"), iteration + 1, result); - throw new GuardrailError(outputCheck.Reason ?? "Output guardrail denied"); - } - if (outputCheck.Rewrite is not null) - { - result = outputCheck.Rewrite; - } - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.Done, - new Dictionary { ["iterations"] = iteration + 1 }); - AgentEvents.EmitEvent(onEvent, AgentEventType.TurnEnd, - new Dictionary { ["iterations"] = iteration + 1, ["status"] = "success", ["response"] = result }); - - return result!; - } - }); } + /// + /// Executes or resumes a turn through the canonical engine using caller-owned durability and effect ports. + /// + public static Task TurnWithEngineRequestAsync( + Prompty agent, + TurnEngineRequest request, + TurnEnginePipelineOptions? options = null, + CancellationToken cancellationToken = default) + => LiveTurn.RunAsync(agent, request, options, cancellationToken); + /// /// Conversational round-trip with path-based loading. /// @@ -823,58 +423,6 @@ internal static void ReplaceSummaryMessage(List messages, string newSum } } - // ----------------------------------------------------------------------- - // LLM Retry Helper (§9.10) - // ----------------------------------------------------------------------- - - /// - /// Invoke ExecuteAsync with exponential backoff retry on transient failures. - /// - private static async Task InvokeWithRetryAsync( - Prompty agent, - List messages, - int maxRetries, - EventCallback? onEvent, - CancellationToken cancellationToken) - { - var attempts = 0; - while (true) - { - try - { - return await ExecuteAsync(agent, messages); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - attempts++; - if (attempts >= maxRetries) - { - throw new ExecuteError( - $"LLM call failed after {maxRetries} retries: {ex.Message}", - new List(messages)); - } - - AgentEvents.EmitEvent(onEvent, AgentEventType.Status, - new Dictionary - { - ["message"] = $"LLM call failed, retrying (attempt {attempts + 1}/{maxRetries})..." - }); - AgentEvents.EmitEvent(onEvent, AgentEventType.Retry, - new Dictionary - { - ["operation"] = "llm", - ["attempt"] = attempts + 1, - ["maxAttempts"] = maxRetries, - ["reason"] = ex.Message - }); - - // Exponential backoff with jitter, capped at 60s - var backoff = Math.Min(Math.Pow(2, attempts) + Random.Shared.NextDouble(), 60); - await Task.Delay(TimeSpan.FromSeconds(backoff), cancellationToken); - } - } - } - // ----------------------------------------------------------------------- // Thread Expansion // ----------------------------------------------------------------------- From 94207f3d3fee80a76c7c312f542cd2c2a0f7c4c3 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 01:53:39 -0700 Subject: [PATCH 08/19] fix(csharp): commit tool results before cancellation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../LiveTurnIntegrationTests.cs | 95 ++++++++++++++++++- runtime/csharp/Prompty.Core/LiveTurn.cs | 17 +--- runtime/csharp/Prompty.Core/TurnEngine.cs | 10 +- .../SpecVectorAgentTests.cs | 11 ++- 4 files changed, 114 insertions(+), 19 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs b/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs index ed8eb9d98..dc398e192 100644 --- a/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs @@ -133,6 +133,94 @@ public async Task TurnAsync_ProcessesAccumulatedStreamingItems_AndProjectsTokens Assert.Equal(["one", "two"], stream.Items); } + [Fact] + public async Task TurnAsync_CancellationBetweenToolsCommitsFirstResultBeforeStopping() + { + var executor = new QueueExecutor(new ToolCallResult + { + ToolCalls = + [ + new ToolCall { Id = "call-1", Name = "first", Arguments = "{}" }, + new ToolCall { Id = "call-2", Name = "second", Arguments = "{}" }, + ], + }); + InvokerRegistry.RegisterExecutor(Provider, executor); + using var cancellation = new CancellationTokenSource(); + var executed = new List(); + var tools = new Dictionary>> + { + ["first"] = _ => + { + executed.Add("first"); + cancellation.Cancel(); + return Task.FromResult("first-result"); + }, + ["second"] = _ => + { + executed.Add("second"); + return Task.FromResult("second-result"); + }, + }; + var agent = NewAgent(); + agent.Tools = + [ + new FunctionTool { Name = "first", Kind = "function" }, + new FunctionTool { Name = "second", Kind = "function" }, + ]; + var toolResults = new List(); + EventCallback onEvent = (type, data) => + { + if (type == AgentEventType.ToolResult) + { + toolResults.Add(data["result"]!.ToString()!); + } + }; + + await Assert.ThrowsAnyAsync(() => + Pipeline.TurnAsync( + agent, + tools: tools, + onEvent: onEvent, + cancellationToken: cancellation.Token)); + + Assert.Equal(["first"], executed); + Assert.Equal(["first-result"], toolResults); + } + + [Fact] + public async Task TurnAsync_CancellationAfterFinalToolCommitsConversationBeforeStopping() + { + var executor = new QueueExecutor(new ToolCallResult + { + ToolCalls = [new ToolCall { Id = "call-1", Name = "only", Arguments = "{}" }], + }); + InvokerRegistry.RegisterExecutor(Provider, executor); + using var cancellation = new CancellationTokenSource(); + var tools = new Dictionary>> + { + ["only"] = _ => + { + cancellation.Cancel(); + return Task.FromResult("only-result"); + }, + }; + var agent = NewAgent(); + agent.Tools = [new FunctionTool { Name = "only", Kind = "function" }]; + var events = new List(); + EventCallback onEvent = (type, _) => events.Add(type); + + await Assert.ThrowsAnyAsync(() => + Pipeline.TurnAsync( + agent, + tools: tools, + onEvent: onEvent, + cancellationToken: cancellation.Token)); + + Assert.Contains(AgentEventType.ToolResult, events); + Assert.Contains(AgentEventType.MessagesUpdated, events); + Assert.Contains(AgentEventType.Cancelled, events); + } + private static TurnEngineRequest NewRequest(string sessionId, string turnId) => new(sessionId, turnId, []) { @@ -169,7 +257,12 @@ public List FormatToolMessages( object rawResponse, List toolCalls, List toolResults, - string? textContent = null) => throw new NotSupportedException(); + string? textContent = null) + { + var messages = new List { Message.Assistant(textContent ?? string.Empty) }; + messages.AddRange(toolCalls.Select((call, index) => Message.ToolResult(call.Id, toolResults[index]))); + return messages; + } } private sealed class PassthroughRenderer : IRenderer diff --git a/runtime/csharp/Prompty.Core/LiveTurn.cs b/runtime/csharp/Prompty.Core/LiveTurn.cs index bbc5df3bb..0114875ac 100644 --- a/runtime/csharp/Prompty.Core/LiveTurn.cs +++ b/runtime/csharp/Prompty.Core/LiveTurn.cs @@ -353,8 +353,6 @@ public async Task ExecuteAsync(ModelToolRequest request, Cancel }); output = $"Error: Tool '{request.Name}' failed: {error.Message}"; } - - cancellationToken.ThrowIfCancellationRequested(); var failed = output.StartsWith("Error:", StringComparison.Ordinal); return new ModelToolResult { @@ -652,6 +650,11 @@ private void Project(EngineEvent @event) case EngineEventKind.ToolExecutionCompleted: if (payload?.GetValueOrDefault("toolResult") is ModelToolResult result) { + AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, new Dictionary + { + ["tool"] = result.Name, + ["result"] = result.ModelText(), + }); AgentEvents.EmitEvent(onEvent, AgentEventType.ToolCallComplete, new Dictionary { ["name"] = result.Name, @@ -661,16 +664,6 @@ private void Project(EngineEvent @event) }); } break; - case EngineEventKind.ToolResultCommitted: - if (payload?.GetValueOrDefault("toolResult") is ModelToolResult committedResult) - { - AgentEvents.EmitEvent(onEvent, AgentEventType.ToolResult, new Dictionary - { - ["tool"] = committedResult.Name, - ["result"] = committedResult.ModelText(), - }); - } - break; case EngineEventKind.ConversationUpdated: AgentEvents.EmitEvent(onEvent, AgentEventType.MessagesUpdated, new Dictionary { diff --git a/runtime/csharp/Prompty.Core/TurnEngine.cs b/runtime/csharp/Prompty.Core/TurnEngine.cs index 3cb75b56e..f85a82378 100644 --- a/runtime/csharp/Prompty.Core/TurnEngine.cs +++ b/runtime/csharp/Prompty.Core/TurnEngine.cs @@ -141,11 +141,6 @@ public async Task RunAsync(TurnEngineRequest request, Cancella while (state.Iteration < state.MaxIterations) { - if (cancellationToken.IsCancellationRequested) - { - return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); - } - if (state.PendingToolRequests.Count == 0 && state.PendingModelResponse is not null) { var invocationId = state.ActiveInvocationId ?? _effects.Ids.NextId("invocation"); @@ -166,6 +161,11 @@ public async Task RunAsync(TurnEngineRequest request, Cancella continue; } + if (cancellationToken.IsCancellationRequested) + { + return await CommitCancelledAsync(state, cancellationToken).ConfigureAwait(false); + } + if (state.PendingToolRequests.Count > 0) { var invocationId = state.ActiveInvocationId ?? _effects.Ids.NextId("invocation"); diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs index a4a0af53d..d1a7a6b79 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs @@ -369,7 +369,16 @@ public async Task AgentLoop_ExtensionVectors(string _name, JsonElement input, Js } // Execute and verify - if (expected.TryGetProperty("error", out var errorProp)) + if (expected.TryGetProperty("rust_expected_error", out var rustError) + && rustError.GetString()?.Contains("parallel_tool_calls=true", StringComparison.Ordinal) == true) + { + var error = await Assert.ThrowsAsync(() => + Pipeline.TurnAsync(agent, tools: toolFunctions, onEvent: onEvent, + cancellationToken: cts?.Token ?? default, contextBudget: contextBudget, + guardrails: guardrails, steering: steering, parallelToolCalls: parallelToolCalls)); + Assert.Contains("sequentially", error.Message); + } + else if (expected.TryGetProperty("error", out var errorProp)) { var errorMsg = errorProp.GetString() ?? ""; if (errorMsg == "CancelledError" || errorMsg.Contains("cancelled", StringComparison.OrdinalIgnoreCase)) From fa6930f439e948b5b07d7344c4f58cd343671c0d Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 02:04:35 -0700 Subject: [PATCH 09/19] fix(csharp): preserve streamed Anthropic tool blocks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AnthropicExecutorTests.cs | 63 +++++++++++ .../Prompty.Anthropic/AnthropicExecutor.cs | 106 ++++++++++++++++++ 2 files changed, 169 insertions(+) diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs index 7c6c52926..4b11d207a 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs @@ -206,6 +206,59 @@ public void FormatToolMessages_RoundTripsCorrelatedBlocksIntoNextRequest() Assert.Equal("result", userContent[0]["content"]); } + [Fact] + public async Task FormatToolMessages_ReconstructsStreamingToolAndThinkingBlocks() + { + var executor = new Anthropic.AnthropicExecutor(); + var stream = new PromptyStream(StreamEvents( + """{"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"I should look."}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"signed"}}""", + """{"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}""", + """{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"Checking "}}""", + """{"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"now."}}""", + """{"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"call_1","name":"lookup","input":{}}}""", + """{"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"key\":\"value\"}"}}""")); + await foreach (var _ in stream) + { + } + + var messages = executor.FormatToolMessages( + stream, + [new ToolCall { Id = "call_1", Name = "lookup", Arguments = """{"key":"value"}""" }], + ["result"]); + var content = Assert.IsType>>(messages[0].Metadata["content"]); + + Assert.Equal("thinking", content[0]["type"]?.ToString()); + Assert.Equal("I should look.", content[0]["thinking"]); + Assert.Equal("signed", content[0]["signature"]); + Assert.Equal("text", content[1]["type"]?.ToString()); + Assert.Equal("Checking now.", content[1]["text"]); + Assert.Equal("tool_use", content[2]["type"]?.ToString()); + Assert.Equal("call_1", content[2]["id"]?.ToString()); + Assert.Equal("""{"key":"value"}""", Assert.IsType(content[2]["input"]).GetRawText()); + } + + [Fact] + public async Task FormatToolMessages_PreservesValidInitialInputWhenStreamEndsMidJson() + { + var executor = new Anthropic.AnthropicExecutor(); + var stream = new PromptyStream(StreamEvents( + """{"type":"content_block_start","index":0,"content_block":{"type":"tool_use","id":"call_1","name":"lookup","input":{}}}""", + """{"type":"content_block_delta","index":0,"delta":{"type":"input_json_delta","partial_json":"{\"key\":"}}""")); + await foreach (var _ in stream) + { + } + + var messages = executor.FormatToolMessages( + stream, + [new ToolCall { Id = "call_1", Name = "lookup", Arguments = """{"key":""" }], + ["invalid arguments"]); + var content = Assert.IsType>>(messages[0].Metadata["content"]); + + Assert.Equal("{}", Assert.IsType(content[0]["input"]).GetRawText()); + } + [Fact] public void BuildRequestBody_SingleToolResultMetadata_PreservesCorrelation() { @@ -224,4 +277,14 @@ public void BuildRequestBody_SingleToolResultMetadata_PreservesCorrelation() Assert.Equal("call_2", content[0]["tool_use_id"]); } + + private static async IAsyncEnumerable StreamEvents(params string[] events) + { + foreach (var json in events) + { + await Task.Yield(); + using var document = JsonDocument.Parse(json); + yield return document.RootElement.Clone(); + } + } } diff --git a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs index 2074ee61e..45e9c8285 100644 --- a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs +++ b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs @@ -372,6 +372,11 @@ public List FormatToolMessages( private static object GetRawContent(object rawResponse) { + if (rawResponse is PromptyStream stream) + { + return ReconstructStreamContent(stream.Items); + } + if (rawResponse is JsonElement { ValueKind: JsonValueKind.Object } element && element.TryGetProperty("content", out var content)) { @@ -387,4 +392,105 @@ private static object GetRawContent(object rawResponse) return Array.Empty(); } + + private static List> ReconstructStreamContent(IReadOnlyList items) + { + var blocks = new SortedDictionary>(); + var textDeltas = new Dictionary>(); + var thinkingDeltas = new Dictionary>(); + var signatureDeltas = new Dictionary>(); + var inputDeltas = new Dictionary>(); + + foreach (var item in items) + { + if (item is not JsonElement evt + || !evt.TryGetProperty("type", out var eventType)) + { + continue; + } + + if (eventType.GetString() == "content_block_start" + && evt.TryGetProperty("index", out var startIndex) + && evt.TryGetProperty("content_block", out var contentBlock)) + { + blocks[startIndex.GetInt32()] = + JsonSerializer.Deserialize>(contentBlock.GetRawText()) ?? []; + continue; + } + + if (eventType.GetString() != "content_block_delta" + || !evt.TryGetProperty("index", out var deltaIndex) + || !evt.TryGetProperty("delta", out var delta) + || !delta.TryGetProperty("type", out var deltaType)) + { + continue; + } + + var index = deltaIndex.GetInt32(); + switch (deltaType.GetString()) + { + case "text_delta": + AppendDelta(textDeltas, index, delta, "text"); + break; + case "thinking_delta": + AppendDelta(thinkingDeltas, index, delta, "thinking"); + break; + case "signature_delta": + AppendDelta(signatureDeltas, index, delta, "signature"); + break; + case "input_json_delta": + AppendDelta(inputDeltas, index, delta, "partial_json"); + break; + } + } + + foreach (var (index, block) in blocks) + { + if (textDeltas.TryGetValue(index, out var text)) + { + block["text"] = string.Concat(text); + } + if (thinkingDeltas.TryGetValue(index, out var thinking)) + { + block["thinking"] = string.Concat(thinking); + } + if (signatureDeltas.TryGetValue(index, out var signature)) + { + block["signature"] = string.Concat(signature); + } + if (inputDeltas.TryGetValue(index, out var input)) + { + var json = string.Concat(input); + try + { + block["input"] = JsonSerializer.Deserialize(json) ?? new Dictionary(); + } + catch (JsonException) + { + // Keep the valid initial input block when the provider stops mid-JSON. + } + } + } + + return blocks.Values.ToList(); + } + + private static void AppendDelta( + Dictionary> target, + int index, + JsonElement delta, + string property) + { + if (!delta.TryGetProperty(property, out var value) + || string.IsNullOrEmpty(value.GetString())) + { + return; + } + if (!target.TryGetValue(index, out var values)) + { + values = []; + target[index] = values; + } + values.Add(value.GetString()!); + } } From e0b136886622e8ca46877cddc446aeb4c5e759db Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 02:26:31 -0700 Subject: [PATCH 10/19] fix(csharp): synthesize streamed tool blocks Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AnthropicExecutorTests.cs | 23 +++++++++ .../Prompty.Anthropic/AnthropicExecutor.cs | 50 +++++++++++++++++-- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs index 4b11d207a..c83a37e01 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs @@ -259,6 +259,29 @@ public async Task FormatToolMessages_PreservesValidInitialInputWhenStreamEndsMid Assert.Equal("{}", Assert.IsType(content[0]["input"]).GetRawText()); } + [Fact] + public async Task FormatToolMessages_SynthesizesBlocksWhenStreamItemsAreUnavailable() + { + var executor = new Anthropic.AnthropicExecutor(); + var stream = new PromptyStream(StreamEvents()); + await foreach (var _ in stream) + { + } + + var messages = executor.FormatToolMessages( + stream, + [new ToolCall { Id = "call_1", Name = "lookup", Arguments = """{"key":"value"}""" }], + ["result"], + "Checking."); + var content = Assert.IsType>>(messages[0].Metadata["content"]); + + Assert.Equal("text", content[0]["type"]); + Assert.Equal("Checking.", content[0]["text"]); + Assert.Equal("tool_use", content[1]["type"]); + Assert.Equal("call_1", content[1]["id"]); + Assert.Equal("""{"key":"value"}""", Assert.IsType(content[1]["input"]).GetRawText()); + } + [Fact] public void BuildRequestBody_SingleToolResultMetadata_PreservesCorrelation() { diff --git a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs index 45e9c8285..a4945fc73 100644 --- a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs +++ b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs @@ -345,7 +345,10 @@ public List FormatToolMessages( Parts = !string.IsNullOrEmpty(textContent) ? [new TextPart { Value = textContent }] : [], - Metadata = new Dictionary { ["content"] = GetRawContent(rawResponse) }, + Metadata = new Dictionary + { + ["content"] = GetRawContent(rawResponse, toolCalls, textContent), + }, }); // --- Single user message with batched tool_result blocks --- @@ -370,11 +373,14 @@ public List FormatToolMessages( return messages; } - private static object GetRawContent(object rawResponse) + private static object GetRawContent( + object rawResponse, + IReadOnlyList toolCalls, + string? textContent) { if (rawResponse is PromptyStream stream) { - return ReconstructStreamContent(stream.Items); + return ReconstructStreamContent(stream.Items, toolCalls, textContent); } if (rawResponse is JsonElement { ValueKind: JsonValueKind.Object } element @@ -393,7 +399,10 @@ private static object GetRawContent(object rawResponse) return Array.Empty(); } - private static List> ReconstructStreamContent(IReadOnlyList items) + private static List> ReconstructStreamContent( + IReadOnlyList items, + IReadOnlyList toolCalls, + string? textContent) { var blocks = new SortedDictionary>(); var textDeltas = new Dictionary>(); @@ -472,6 +481,39 @@ private static object GetRawContent(object rawResponse) } } + if (blocks.Count == 0) + { + var index = 0; + if (!string.IsNullOrEmpty(textContent)) + { + blocks[index++] = new Dictionary + { + ["type"] = "text", + ["text"] = textContent, + }; + } + foreach (var toolCall in toolCalls) + { + object input; + try + { + input = JsonSerializer.Deserialize(toolCall.Arguments) + ?? new Dictionary(); + } + catch (JsonException) + { + input = new Dictionary(); + } + blocks[index++] = new Dictionary + { + ["type"] = "tool_use", + ["id"] = toolCall.Id, + ["name"] = toolCall.Name, + ["input"] = input, + }; + } + } + return blocks.Values.ToList(); } From 45e7c18fe393fff169b80af40d12ddd7bf717601 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 02:56:09 -0700 Subject: [PATCH 11/19] docs(csharp): clarify cancellation boundaries Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/csharp/Prompty.Core/Pipeline.cs | 9 +++++++++ runtime/csharp/Prompty.Core/TurnEnginePorts.cs | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/runtime/csharp/Prompty.Core/Pipeline.cs b/runtime/csharp/Prompty.Core/Pipeline.cs index a7a8664f3..5dd3397e2 100644 --- a/runtime/csharp/Prompty.Core/Pipeline.cs +++ b/runtime/csharp/Prompty.Core/Pipeline.cs @@ -218,6 +218,11 @@ public static async Task InvokeAsync( /// Conversational round-trip through the canonical durable turn engine. /// If no tools are provided, the engine performs a single Prepare → Execute → Process pass. /// + /// + /// Cancellation is cooperative at engine phase boundaries. The current generated provider and + /// tool protocols do not expose a native , so an in-flight + /// non-streaming provider or tool call may finish before cancellation is observed. + /// public static async Task TurnAsync( Prompty agent, Dictionary? inputs = null, @@ -265,6 +270,10 @@ public static async Task TurnAsync( /// /// Executes or resumes a turn through the canonical engine using caller-owned durability and effect ports. /// + /// + /// Runtime-local effect ports receive the native token. Adapters over generated protocols remain + /// boundary-cancellable until the schema/emitter supports language-native cancellation seams. + /// public static Task TurnWithEngineRequestAsync( Prompty agent, TurnEngineRequest request, diff --git a/runtime/csharp/Prompty.Core/TurnEnginePorts.cs b/runtime/csharp/Prompty.Core/TurnEnginePorts.cs index 1f194fc63..fe100d906 100644 --- a/runtime/csharp/Prompty.Core/TurnEnginePorts.cs +++ b/runtime/csharp/Prompty.Core/TurnEnginePorts.cs @@ -14,6 +14,13 @@ namespace Prompty.Core; // ModelToolRequest/Result, EnginePermissionDecision, EngineEvent, EngineCheckpoint, // TurnCommit, ...) directly. This mirrors the Rust reference engine's runtime-local // traits in runtime/rust/prompty/src/engine/ports.rs. +// +// Cancellation is intentionally a runtime seam, not serialized model data. The +// canonical engine checks it at semantic boundaries and passes the native token to +// these runtime-local ports. Generated protocols such as IExecutor do not yet accept +// native cancellation, so adapters over them can only observe cancellation before or +// after an in-flight call (and while draining a stream). Durability writes remain +// deliberately non-cancellable once persistence starts. // ----------------------------------------------------------------------- /// An ephemeral, non-durable chunk streamed while a model invocation is in flight. From 88d387445cef7990ef9b2f251c5499648edc4cd6 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 03:08:48 -0700 Subject: [PATCH 12/19] fix(csharp): clear handwritten nullable warnings Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../csharp/Prompty.Core.Tests/TurnRunnerTests.cs | 16 ++++++++-------- runtime/csharp/Prompty.Core/TurnEngine.cs | 12 ++++++------ runtime/csharp/Prompty.Core/TurnRunner.cs | 9 +++++++-- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs index ba5517400..e94b39706 100644 --- a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs @@ -24,7 +24,7 @@ public async Task ReferenceTurnRunner_EmitsJournalsAndCheckpoints() new FunctionHostToolExecutor(new Dictionary()), request => Task.FromResult(new TurnModelResponse { - Output = new Dictionary { ["text"] = $"hello {request.Inputs["name"]}" }, + Output = new Dictionary { ["text"] = $"hello {request.Inputs!["name"]}" }, CheckpointState = new Dictionary { ["stable"] = true } }), FixedClock(), @@ -97,7 +97,7 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() return Task.FromResult(new TurnModelResponse { - Output = new Dictionary { ["toolResult"] = request.ToolResults[0].Result } + Output = new Dictionary { ["toolResult"] = request.ToolResults![0].Result } }); }, FixedClock(), @@ -106,7 +106,7 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() var result = await runner.RunAsync(new RunTurnRequest { SessionId = "session-1", TurnId = "turn-1" }); Assert.Equal(5, Assert.IsType>(result.Output)["toolResult"]); - Assert.True(result.ToolResults[0].Success); + Assert.True(result.ToolResults![0].Success); Assert.Equal( [ TurnEventType.TurnStart, @@ -158,7 +158,7 @@ public async Task ReferenceTurnRunner_DeniedPermissionSkipsExecution() return Task.FromResult(new TurnModelResponse { - Output = new Dictionary { ["denied"] = request.ToolResults[0].ErrorKind } + Output = new Dictionary { ["denied"] = request.ToolResults![0].ErrorKind } }); }, FixedClock(), @@ -204,7 +204,7 @@ public async Task ReferenceTurnRunner_HostToolFailureIsReplayable() }); } - return Task.FromResult(new TurnModelResponse { Output = request.ToolResults[0].Save() }); + return Task.FromResult(new TurnModelResponse { Output = request.ToolResults![0].Save() }); }, FixedClock(), FixedIds()); @@ -318,7 +318,7 @@ private static Func> ModelForScenario( { return Task.FromResult(new TurnModelResponse { - Output = new Dictionary { ["text"] = $"hello {request.Inputs["name"]}" }, + Output = new Dictionary { ["text"] = $"hello {request.Inputs!["name"]}" }, CheckpointState = new Dictionary { ["stable"] = true } }); } @@ -344,8 +344,8 @@ private static Func> ModelForScenario( { Output = new Dictionary { - ["toolResult"] = request.ToolResults[0].Result, - ["errorKind"] = request.ToolResults[0].ErrorKind + ["toolResult"] = request.ToolResults![0].Result, + ["errorKind"] = request.ToolResults![0].ErrorKind } }); }; diff --git a/runtime/csharp/Prompty.Core/TurnEngine.cs b/runtime/csharp/Prompty.Core/TurnEngine.cs index f85a82378..ac96bfc6f 100644 --- a/runtime/csharp/Prompty.Core/TurnEngine.cs +++ b/runtime/csharp/Prompty.Core/TurnEngine.cs @@ -183,11 +183,11 @@ public async Task RunAsync(TurnEngineRequest request, Cancella } catch (ToolPermissionFailure error) { - return await CommitFailedAsync(state, "permission_error", error.Source.Message, cancellationToken).ConfigureAwait(false); + return await CommitFailedAsync(state, "permission_error", error.Error.Message, cancellationToken).ConfigureAwait(false); } catch (ToolConfigurationFailure error) { - return await CommitFailedAsync(state, "tool_configuration_error", error.Source.Message, cancellationToken).ConfigureAwait(false); + return await CommitFailedAsync(state, "tool_configuration_error", error.Error.Message, cancellationToken).ConfigureAwait(false); } var outcomeUnknown = toolResult.Outcome == ModelToolOutcome.Indeterminate; @@ -1073,15 +1073,15 @@ private async Task CommitAsync( } /// Signals that itself failed. - private sealed class ToolPermissionFailure(PortError source) : Exception(source.Message) + private sealed class ToolPermissionFailure(PortError error) : Exception(error.Message) { - public PortError Source { get; } = source; + public PortError Error { get; } = error; } /// Signals that failed with a configuration error. - private sealed class ToolConfigurationFailure(PortError source) : Exception(source.Message) + private sealed class ToolConfigurationFailure(PortError error) : Exception(error.Message) { - public PortError Source { get; } = source; + public PortError Error { get; } = error; } /// Prevents observational stream failures from changing semantic model execution. diff --git a/runtime/csharp/Prompty.Core/TurnRunner.cs b/runtime/csharp/Prompty.Core/TurnRunner.cs index c00460ef5..624d5e899 100644 --- a/runtime/csharp/Prompty.Core/TurnRunner.cs +++ b/runtime/csharp/Prompty.Core/TurnRunner.cs @@ -209,7 +209,12 @@ private async Task ResolveAndExecuteToolAsync(string turnId, int return result; } - private void RecordTurn(TurnEventType type, string turnId, int iteration, IDictionary payload) + private void RecordTurn( + TurnEventType type, + string turnId, + int iteration, + IDictionary payload) + where TPayload : class? { var turnEvent = new TurnEvent { @@ -218,7 +223,7 @@ private void RecordTurn(TurnEventType type, string turnId, int iteration, IDicti Timestamp = _now(), TurnId = turnId, Iteration = iteration, - Payload = payload + Payload = payload.ToDictionary(entry => entry.Key, entry => (object)entry.Value!) }; _eventSink.EmitTurn(turnEvent); _journal.AppendTurn(turnEvent); From 2d26f7c7700eb1f110beeab53e872513fd3ff7fa Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 03:27:05 -0700 Subject: [PATCH 13/19] fix(csharp): harden reviewed wire inputs Encode OpenAI audio patch values as JSON while preserving the shared verbatim source contract, and reject unsupported canonical turn input shapes instead of silently dropping them. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../LiveTurnIntegrationTests.cs | 15 ++++++ runtime/csharp/Prompty.Core/LiveTurn.cs | 20 +++++++- .../Prompty.OpenAI.Tests/WireFormatTests.cs | 50 +++++++++++++++++++ runtime/csharp/Prompty.OpenAI/WireFormat.cs | 4 +- 4 files changed, 85 insertions(+), 4 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs b/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs index dc398e192..a9dd0ee84 100644 --- a/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/LiveTurnIntegrationTests.cs @@ -77,6 +77,21 @@ await Assert.ThrowsAnyAsync(() => } } + [Fact] + public async Task TurnWithEngineRequest_RejectsNonObjectInputs() + { + var executor = new QueueExecutor("unused"); + InvokerRegistry.RegisterExecutor(Provider, executor); + var request = NewRequest("invalid-input-session", "invalid-input-turn"); + request.Inputs = "not-an-input-object"; + + var error = await Assert.ThrowsAsync(() => + Pipeline.TurnWithEngineRequestAsync(NewAgent(), request)); + + Assert.Contains("string-keyed dictionary or JSON object", error.Message, StringComparison.Ordinal); + Assert.Equal(0, executor.InvocationCount); + } + [Fact] public async Task TurnAsync_RejectsParallelToolsBeforeInvokingProvider() { diff --git a/runtime/csharp/Prompty.Core/LiveTurn.cs b/runtime/csharp/Prompty.Core/LiveTurn.cs index 0114875ac..144910d4d 100644 --- a/runtime/csharp/Prompty.Core/LiveTurn.cs +++ b/runtime/csharp/Prompty.Core/LiveTurn.cs @@ -87,6 +87,7 @@ public static async Task RunAsync( ArgumentNullException.ThrowIfNull(agent); ArgumentNullException.ThrowIfNull(request); options ??= new TurnEnginePipelineOptions(); + var inputs = NormalizeInputs(request.Inputs); var provider = agent.Model?.Provider ?? "openai"; var executor = InvokerRegistry.GetExecutor(provider); @@ -106,14 +107,14 @@ public static async Task RunAsync( Tools = new LiveToolPort( agent, options.Tools, - request.Inputs as Dictionary, + inputs, authorization, options.OnEvent), Clock = new LiveClock(), Ids = new LiveIds(), Policy = new LivePolicyPort( agent, - request.Inputs as Dictionary, + inputs, options.ContextBudget, options.Guardrails, options.Steering, @@ -148,6 +149,21 @@ public static async Task RunAsync( }; } + private static Dictionary? NormalizeInputs(object? inputs) + { + return inputs switch + { + null => null, + Dictionary values => values, + IReadOnlyDictionary values => values.ToDictionary(), + JsonElement { ValueKind: JsonValueKind.Object } value => + value.Deserialize>(), + _ => throw new ArgumentException( + "Turn engine inputs must be a string-keyed dictionary or JSON object.", + nameof(inputs)), + }; + } + private static Exception MapFailure( TurnEngineResult result, int maxModelAttempts, diff --git a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs index 8b412c0eb..80665ace1 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs @@ -74,6 +74,56 @@ public void MessageToWire_UserMultimodalMessage_ReturnsUserChatMessage() Assert.IsType(result); } + [Fact] + public void MessageToWire_AudioDataUri_PreservesSource() + { + var msg = new Message + { + Role = Role.User, + Parts = + [ + new AudioPart + { + Source = "data:audio/wav;base64,YXVkaW8=", + MediaType = "audio/wav", + }, + ], + }; + + var result = WireFormat.MessageToWire(msg); + var data = ModelReaderWriter.Write(result, ModelReaderWriterOptions.Json); + using var json = JsonDocument.Parse(data.ToStream()); + var inputAudio = json.RootElement.GetProperty("content")[0].GetProperty("input_audio"); + + Assert.Equal("data:audio/wav;base64,YXVkaW8=", inputAudio.GetProperty("data").GetString()); + Assert.Equal("wav", inputAudio.GetProperty("format").GetString()); + } + + [Fact] + public void MessageToWire_AudioUrl_PreservesSource() + { + var msg = new Message + { + Role = Role.User, + Parts = + [ + new AudioPart + { + Source = "https://example.com/audio.wav", + MediaType = "audio/wav", + }, + ], + }; + + var result = WireFormat.MessageToWire(msg); + var data = ModelReaderWriter.Write(result, ModelReaderWriterOptions.Json); + using var json = JsonDocument.Parse(data.ToStream()); + var inputAudio = json.RootElement.GetProperty("content")[0].GetProperty("input_audio"); + + Assert.Equal("https://example.com/audio.wav", inputAudio.GetProperty("data").GetString()); + Assert.Equal("wav", inputAudio.GetProperty("format").GetString()); + } + [Fact] public void MessageToWire_AssistantMessage_ReturnsAssistantChatMessage() { diff --git a/runtime/csharp/Prompty.OpenAI/WireFormat.cs b/runtime/csharp/Prompty.OpenAI/WireFormat.cs index 0788da29a..2837c8f83 100644 --- a/runtime/csharp/Prompty.OpenAI/WireFormat.cs +++ b/runtime/csharp/Prompty.OpenAI/WireFormat.cs @@ -94,8 +94,8 @@ private static ChatMessageContentPart BuildAudioContentPart(AudioPart audio) var part = ChatMessageContentPart.CreateInputAudioPart( BinaryData.FromBytes(bytes), format == "mp3" ? ChatInputAudioFormat.Mp3 : ChatInputAudioFormat.Wav); - part.Patch.Set("$.input_audio.data"u8, audio.Source); - part.Patch.Set("$.input_audio.format"u8, format); + part.Patch.Set("$.input_audio.data"u8, BinaryData.FromObjectAsJson(audio.Source).ToMemory().Span); + part.Patch.Set("$.input_audio.format"u8, BinaryData.FromObjectAsJson(format).ToMemory().Span); return part; } From 13b04f62b768517b2cf6c3c1c09ccbfc30fff9c8 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 13:05:54 -0700 Subject: [PATCH 14/19] Migrate unknown record boundaries to nullable values Align handwritten C# Record consumers with explicit-null schema semantics and add regressions across loading, providers, turns, durability, replay, and discovery. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Prompty.Anthropic.Tests/TestHelpers.cs | 4 +- .../Prompty.Anthropic/AnthropicExecutor.cs | 4 +- .../AgentLoopIntegrationTests.cs | 4 +- .../Prompty.Core.Tests/CompactionTests.cs | 2 +- .../HarnessAdaptersTests.cs | 31 ++++++++- .../csharp/Prompty.Core.Tests/LoaderTests.cs | 24 +++++++ .../Prompty.Core.Tests/PipelineTests.cs | 21 +++--- .../Prompty.Core.Tests/ResilienceTests.cs | 4 +- .../TurnEngineReconciliationTests.cs | 10 ++- .../TurnEngineTestDoubles.cs | 6 +- .../Prompty.Core.Tests/TurnRunnerTests.cs | 69 +++++++++++++++---- .../csharp/Prompty.Core/HarnessAdapters.cs | 4 +- runtime/csharp/Prompty.Core/ModelDiscovery.cs | 4 +- runtime/csharp/Prompty.Core/Pipeline.cs | 4 +- .../csharp/Prompty.Core/PromptyChatParser.cs | 2 +- runtime/csharp/Prompty.Core/PromptyLoader.cs | 2 +- runtime/csharp/Prompty.Core/TurnRunner.cs | 27 ++++---- .../FoundryModelDiscoveryTests.cs | 17 +++++ .../Prompty.Foundry.Tests/TestHelpers.cs | 4 +- .../Prompty.OpenAI.Tests/AgentLoopTests.cs | 4 +- .../Integration/IntegrationTestBase.cs | 6 +- .../Integration/ResponsesApiTests.cs | 2 +- .../Integration/StreamingAgentTests.cs | 2 +- .../Integration/StreamingTests.cs | 2 +- .../Prompty.OpenAI.Tests/ResponsesApiTests.cs | 2 +- .../SpecVectorAgentTests.cs | 4 +- .../Prompty.OpenAI.Tests/StreamingTests.cs | 4 +- .../Prompty.OpenAI.Tests/TestHelpers.cs | 4 +- .../Prompty.OpenAI.Tests/WireFormatTests.cs | 4 +- .../csharp/Prompty.OpenAI/OpenAIExecutor.cs | 8 +-- 30 files changed, 204 insertions(+), 81 deletions(-) diff --git a/runtime/csharp/Prompty.Anthropic.Tests/TestHelpers.cs b/runtime/csharp/Prompty.Anthropic.Tests/TestHelpers.cs index 080dba9fe..370f15abd 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/TestHelpers.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/TestHelpers.cs @@ -97,7 +97,7 @@ internal static Message CreateAssistantWithToolCalls(string text, List { Role = Role.Assistant, Parts = [new TextPart { Value = text }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = toolCalls, }, @@ -113,7 +113,7 @@ internal static Message CreateToolMessage(string toolCallId, string content) { Role = Role.Tool, Parts = [new TextPart { Value = content }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCallId, }, diff --git a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs index a4945fc73..a7f6088a1 100644 --- a/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs +++ b/runtime/csharp/Prompty.Anthropic/AnthropicExecutor.cs @@ -345,7 +345,7 @@ public List FormatToolMessages( Parts = !string.IsNullOrEmpty(textContent) ? [new TextPart { Value = textContent }] : [], - Metadata = new Dictionary + Metadata = new Dictionary { ["content"] = GetRawContent(rawResponse, toolCalls, textContent), }, @@ -367,7 +367,7 @@ public List FormatToolMessages( { Role = Role.User, Parts = toolResults.Select(r => (ContentPart)new TextPart { Value = r }).ToList(), - Metadata = new Dictionary { ["tool_results"] = toolResultBlocks }, + Metadata = new Dictionary { ["tool_results"] = toolResultBlocks }, }); return messages; diff --git a/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs b/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs index ce0bcb19a..b8eadf5e4 100644 --- a/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/AgentLoopIntegrationTests.cs @@ -83,7 +83,7 @@ public List FormatToolMessages( { Role = Role.Assistant, Parts = [new TextPart { Value = textContent ?? "" }], - Metadata = new Dictionary { ["tool_calls"] = toolCalls } + Metadata = new Dictionary { ["tool_calls"] = toolCalls } }); // one tool-result message per call for (int i = 0; i < toolCalls.Count; i++) @@ -92,7 +92,7 @@ public List FormatToolMessages( { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], - Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); } return msgs; diff --git a/runtime/csharp/Prompty.Core.Tests/CompactionTests.cs b/runtime/csharp/Prompty.Core.Tests/CompactionTests.cs index 628b53cf2..e20dc9e7e 100644 --- a/runtime/csharp/Prompty.Core.Tests/CompactionTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/CompactionTests.cs @@ -14,7 +14,7 @@ private static Message ToolCallMsg(string name, string args) => { Role = Role.Assistant, Parts = [new TextPart { Value = "" }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = new List { diff --git a/runtime/csharp/Prompty.Core.Tests/HarnessAdaptersTests.cs b/runtime/csharp/Prompty.Core.Tests/HarnessAdaptersTests.cs index 1884d3f75..8721c6318 100644 --- a/runtime/csharp/Prompty.Core.Tests/HarnessAdaptersTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/HarnessAdaptersTests.cs @@ -39,6 +39,8 @@ public void JsonlEventJournalWriter_WritesRecords() Assert.Equal(new[] { "turn", "session", "summary" }, lines.Select(line => line["kind"].GetString()).ToArray()); Assert.Equal("turn-event", lines[0]["event"].GetProperty("id").GetString()); Assert.Equal("session-event", lines[1]["event"].GetProperty("id").GetString()); + Assert.Equal(JsonValueKind.Null, lines[0]["event"].GetProperty("payload").GetProperty("nullable").ValueKind); + Assert.Equal(JsonValueKind.Null, lines[1]["event"].GetProperty("payload").GetProperty("nullable").ValueKind); Assert.Equal("session-1", lines[2]["summary"].GetProperty("sessionId").GetString()); Assert.DoesNotContain("\r\n", File.ReadAllText(path)); } @@ -121,7 +123,7 @@ public async Task FunctionHostToolExecutor_ExecutesRegisteredHandlers() { RequestId = "exec-1", ToolName = "add", - Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } + Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } }); Assert.True(result.Success); @@ -144,6 +146,29 @@ public async Task FunctionHostToolExecutor_PassesEmptyArguments() Assert.Equal(0, result.Result); } + [Fact] + public async Task FunctionHostToolExecutor_PreservesExplicitNullArguments() + { + var executor = new FunctionHostToolExecutor(new Dictionary + { + ["inspect"] = (args, _) => + { + Assert.True(args.ContainsKey("nullable")); + Assert.Null(args["nullable"]); + return Task.FromResult("observed"); + } + }); + + var result = await executor.ExecuteAsync(new HostToolRequest + { + ToolName = "inspect", + Arguments = new Dictionary { ["nullable"] = null } + }); + + Assert.True(result.Success); + Assert.Equal("observed", result.Result); + } + [Fact] public async Task FunctionHostToolExecutor_ReturnsFailureResults() { @@ -168,7 +193,7 @@ public async Task FunctionHostToolExecutor_ReturnsFailureResults() Id = "turn-event", Type = TurnEventType.TurnStart, Timestamp = "2026-06-10T00:00:00Z", - Payload = new Dictionary { ["phase"] = "start" } + Payload = new Dictionary { ["phase"] = "start", ["nullable"] = null } }; private static SessionEvent SessionEvent() => new() @@ -177,6 +202,6 @@ public async Task FunctionHostToolExecutor_ReturnsFailureResults() Type = SessionEventType.SessionStart, Timestamp = "2026-06-10T00:00:00Z", SessionId = "session-1", - Payload = new Dictionary { ["phase"] = "start" } + Payload = new Dictionary { ["phase"] = "start", ["nullable"] = null } }; } diff --git a/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs b/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs index 29317693b..a70a2101e 100644 --- a/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/LoaderTests.cs @@ -485,6 +485,30 @@ public void Load_SetsSourcePath() Assert.EndsWith("minimal.prompty", sourcePath); } + [Fact] + public void Load_PreservesExplicitNullMetadata() + { + var root = Directory.CreateTempSubdirectory("prompty-loader-null-"); + try + { + var path = Path.Combine(root.FullName, "nullable-metadata.prompty"); + File.WriteAllText( + path, + "---\nname: nullable-metadata\nmetadata:\n nullable: null\nmodel: gpt-4\n---\nsystem:\nHello."); + + var agent = PromptyLoader.Load(path); + + Assert.NotNull(agent.Metadata); + Assert.True(agent.Metadata.ContainsKey("nullable")); + Assert.Null(agent.Metadata["nullable"]); + Assert.True(agent.Metadata.ContainsKey("__source_path")); + } + finally + { + root.Delete(recursive: true); + } + } + // --- Error cases --- [Fact] diff --git a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs index 4c111edcb..c008d8cbe 100644 --- a/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/PipelineTests.cs @@ -537,17 +537,22 @@ public void ExpandThreadMarkers_CopiesMetadata() new() { Role = Role.System, - Parts = [new TextPart { Value = "Prefix __PROMPTY_THREAD_abcd1234_conv__" }], - Metadata = new Dictionary { ["source"] = "test" }, + Parts = [new TextPart { Value = "Prefix __PROMPTY_THREAD_abcd1234_conv__ Suffix" }], + Metadata = new Dictionary { ["source"] = "test", ["nullable"] = null }, }, }; var inputs = new Dictionary { ["conv"] = threadMessages }; var result = Pipeline.ExpandThreadMarkers(messages, inputs); - // The "before" fragment should carry the original message's metadata - Assert.Equal(2, result.Count); + // Both fragments should carry all metadata, including explicit null values. + Assert.Equal(3, result.Count); Assert.Equal("test", result[0].Metadata["source"]); + Assert.True(result[0].Metadata.ContainsKey("nullable")); + Assert.Null(result[0].Metadata["nullable"]); + Assert.Equal("test", result[2].Metadata["source"]); + Assert.True(result[2].Metadata.ContainsKey("nullable")); + Assert.Null(result[2].Metadata["nullable"]); } // ----------------------------------------------------------------------- @@ -608,10 +613,10 @@ public List FormatToolMessages(object rawResponse, List toolC { var messages = new List { - new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, + new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, }; for (var i = 0; i < toolCalls.Count; i++) - messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); + messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); return messages; } } @@ -643,10 +648,10 @@ public List FormatToolMessages(object rawResponse, List toolC { var messages = new List { - new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, + new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, }; for (var i = 0; i < toolCalls.Count; i++) - messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); + messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); return messages; } } diff --git a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs index 966598261..95bbb1410 100644 --- a/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ResilienceTests.cs @@ -48,7 +48,7 @@ public List FormatToolMessages( { Role = Role.Assistant, Parts = [new TextPart { Value = textContent ?? "" }], - Metadata = new Dictionary { ["tool_calls"] = toolCalls } + Metadata = new Dictionary { ["tool_calls"] = toolCalls } }); for (int i = 0; i < toolCalls.Count; i++) { @@ -56,7 +56,7 @@ public List FormatToolMessages( { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], - Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id } }); } return msgs; diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs index 0b2fd3226..33f14b5ba 100644 --- a/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineReconciliationTests.cs @@ -86,9 +86,10 @@ public async Task IndeterminateToolEffect_BlocksTurnUntilExplicitReconciliation( public async Task IndeterminateModelEffect_BlocksTurnUntilExplicitReconciliation() { var durability = new RecordingDurabilityPort(); + var failureMetadata = new Dictionary { ["nullable"] = null }; var effects = new TurnEngineEffects { - Model = new IndeterminateModelPort(), + Model = new IndeterminateModelPort(failureMetadata), Tools = new EchoToolPort(), Clock = new FakeEngineClock(), Ids = new FakeEngineIdGenerator(), @@ -108,6 +109,13 @@ public async Task IndeterminateModelEffect_BlocksTurnUntilExplicitReconciliation Assert.True(checkpoint.ReconciliationRequired); Assert.NotNull(checkpoint.ModelReconciliation); Assert.Equal(checkpoint.ActiveInvocationId, checkpoint.ModelReconciliation!.InvocationId); + Assert.NotNull(checkpoint.ModelReconciliation.Metadata); + Assert.True(checkpoint.ModelReconciliation.Metadata.ContainsKey("nullable")); + Assert.Null(checkpoint.ModelReconciliation.Metadata["nullable"]); + var reloadedReconciliation = ModelReconciliationState.Load(checkpoint.ModelReconciliation.Save()); + Assert.NotNull(reloadedReconciliation.Metadata); + Assert.True(reloadedReconciliation.Metadata.ContainsKey("nullable")); + Assert.Null(reloadedReconciliation.Metadata["nullable"]); // Resolving with an explicit model response resumes without re-invoking the model port. var resolvedResponse = new ModelInvocationResponse { Output = "resolved output", AssistantMessages = [], ToolRequests = [] }; diff --git a/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs b/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs index 2cf0784f0..855a877ec 100644 --- a/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs +++ b/runtime/csharp/Prompty.Core.Tests/TurnEngineTestDoubles.cs @@ -98,6 +98,10 @@ public Task InvokeAsync( /// Model port whose single scripted attempt always fails with an indeterminate (outcome-unknown) error. internal sealed class IndeterminateModelPort : IEngineModelPort { + private readonly IDictionary? _metadata; + + public IndeterminateModelPort(IDictionary? metadata = null) => _metadata = metadata; + public int InvocationCount { get; private set; } public Task InvokeAsync( @@ -106,7 +110,7 @@ public Task InvokeAsync( IEngineModelStreamPort stream) { InvocationCount++; - throw PortError.Indeterminate("model invocation outcome is unknown"); + throw PortError.Indeterminate("model invocation outcome is unknown", _metadata); } } diff --git a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs index e94b39706..68b6f0073 100644 --- a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs @@ -22,11 +22,16 @@ public async Task ReferenceTurnRunner_EmitsJournalsAndCheckpoints() checkpointStore, new AllowAllPermissionResolver(), new FunctionHostToolExecutor(new Dictionary()), - request => Task.FromResult(new TurnModelResponse + request => { - Output = new Dictionary { ["text"] = $"hello {request.Inputs!["name"]}" }, - CheckpointState = new Dictionary { ["stable"] = true } - }), + Assert.True(request.Inputs!.ContainsKey("nullable")); + Assert.Null(request.Inputs["nullable"]); + return Task.FromResult(new TurnModelResponse + { + Output = new Dictionary { ["text"] = $"hello {request.Inputs["name"]}" }, + CheckpointState = new Dictionary { ["stable"] = true, ["nullable"] = null } + }); + }, FixedClock(), FixedIds()); @@ -34,7 +39,7 @@ public async Task ReferenceTurnRunner_EmitsJournalsAndCheckpoints() { SessionId = "session-1", TurnId = "turn-1", - Inputs = new Dictionary { ["name"] = "Ada" }, + Inputs = new Dictionary { ["name"] = "Ada", ["nullable"] = null }, Options = new TurnOptions { MaxIterations = 3 } }); @@ -50,6 +55,12 @@ public async Task ReferenceTurnRunner_EmitsJournalsAndCheckpoints() var checkpoint = await checkpointStore.LoadAsync("session-1", "turn-1-checkpoint-0"); Assert.NotNull(checkpoint); Assert.True((bool)checkpoint.State!["stable"]); + Assert.True(checkpoint.State.ContainsKey("nullable")); + Assert.Null(checkpoint.State["nullable"]); + var turnStart = sink.TurnEvents.Single(turnEvent => turnEvent.Type == TurnEventType.TurnStart); + var turnInputs = Assert.IsAssignableFrom>(turnStart.Payload["inputs"]); + Assert.True(turnInputs.ContainsKey("nullable")); + Assert.Null(turnInputs["nullable"]); Assert.Equal( ["session", "turn", "turn", "turn", "session", "turn", "session", "summary"], JournalKinds(journalPath)); @@ -67,14 +78,20 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() try { var sink = new CollectingEventSink(); + var permissionResolver = new CapturingPermissionResolver(); var runner = new ReferenceTurnRunner( sink, new JsonlEventJournalWriter(Path.Combine(directory.FullName, "trace.jsonl")), new InMemoryCheckpointStore(), - new AllowAllPermissionResolver(), + permissionResolver, new FunctionHostToolExecutor(new Dictionary { - ["add"] = (args, _) => Task.FromResult(Convert.ToInt32(args["a"]) + Convert.ToInt32(args["b"])) + ["add"] = (args, _) => + { + Assert.True(args.ContainsKey("nullable")); + Assert.Null(args["nullable"]); + return Task.FromResult(Convert.ToInt32(args["a"]) + Convert.ToInt32(args["b"])); + } }), request => { @@ -89,7 +106,7 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() RequestId = "exec-1", ToolCallId = "call-1", ToolName = "add", - Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } + Arguments = new Dictionary { ["a"] = 2, ["b"] = 3, ["nullable"] = null } } ] }); @@ -107,6 +124,12 @@ public async Task ReferenceTurnRunner_ExecutesHostTools() Assert.Equal(5, Assert.IsType>(result.Output)["toolResult"]); Assert.True(result.ToolResults![0].Success); + Assert.NotNull(permissionResolver.Request); + Assert.NotNull(permissionResolver.Request.Details); + var permissionArguments = + Assert.IsAssignableFrom>(permissionResolver.Request.Details["arguments"]); + Assert.True(permissionArguments.ContainsKey("nullable")); + Assert.Null(permissionArguments["nullable"]); Assert.Equal( [ TurnEventType.TurnStart, @@ -281,7 +304,7 @@ await runner.RunAsync(new RunTurnRequest { SessionId = vectors.SessionId, TurnId = vectors.TurnId, - Inputs = scenario.Inputs ?? new Dictionary(), + Inputs = scenario.Inputs ?? new Dictionary(), Options = new TurnOptions { MaxIterations = scenario.MaxIterations } }); @@ -296,6 +319,24 @@ await runner.RunAsync(new RunTurnRequest private static Func FixedClock() => () => "2026-06-28T00:00:00Z"; + private sealed class CapturingPermissionResolver : IPermissionResolver + { + public PermissionRequest? Request { get; private set; } + + public Task RequestAsync(PermissionRequest request) + { + Request = request; + return Task.FromResult(new PermissionDecision + { + RequestId = request.RequestId, + ToolCallId = request.ToolCallId, + Permission = request.Permission, + Approved = true, + Reason = "captured" + }); + } + } + private static Func FixedIds() { var index = 0; @@ -319,7 +360,7 @@ private static Func> ModelForScenario( return Task.FromResult(new TurnModelResponse { Output = new Dictionary { ["text"] = $"hello {request.Inputs!["name"]}" }, - CheckpointState = new Dictionary { ["stable"] = true } + CheckpointState = new Dictionary { ["stable"] = true } }); } @@ -334,7 +375,7 @@ private static Func> ModelForScenario( RequestId = "exec-1", ToolCallId = "call-1", ToolName = name == "tool_failure" ? "fail" : "add", - Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } + Arguments = new Dictionary { ["a"] = 2, ["b"] = 3 } } ] }); @@ -432,7 +473,7 @@ public static ReplayVectors Load() private sealed record ReplayScenario( string Name, - Dictionary? Inputs, + Dictionary? Inputs, int? MaxIterations, string[] Expected) { @@ -446,9 +487,9 @@ public static ReplayScenario Load(JsonElement element) } } - private static Dictionary ToDictionary(JsonElement element) + private static Dictionary ToDictionary(JsonElement element) { - return element.EnumerateObject().ToDictionary(property => property.Name, property => ToObject(property.Value)!); + return element.EnumerateObject().ToDictionary(property => property.Name, property => ToObject(property.Value)); } private static object? ToObject(JsonElement element) diff --git a/runtime/csharp/Prompty.Core/HarnessAdapters.cs b/runtime/csharp/Prompty.Core/HarnessAdapters.cs index ef0348751..92a3aab4e 100644 --- a/runtime/csharp/Prompty.Core/HarnessAdapters.cs +++ b/runtime/csharp/Prompty.Core/HarnessAdapters.cs @@ -234,7 +234,7 @@ public Task RequestAsync(PermissionRequest request) } } -public delegate Task HostToolHandler(IDictionary arguments, HostToolRequest request); +public delegate Task HostToolHandler(IDictionary arguments, HostToolRequest request); /// /// Dispatches host tool requests to registered local functions. @@ -267,7 +267,7 @@ public async Task ExecuteAsync(HostToolRequest request) try { - var result = await handler(request.Arguments ?? new Dictionary(), request); + var result = await handler(request.Arguments ?? new Dictionary(), request); return new HostToolResult { RequestId = request.RequestId, diff --git a/runtime/csharp/Prompty.Core/ModelDiscovery.cs b/runtime/csharp/Prompty.Core/ModelDiscovery.cs index d51ace6b4..6710bc79e 100644 --- a/runtime/csharp/Prompty.Core/ModelDiscovery.cs +++ b/runtime/csharp/Prompty.Core/ModelDiscovery.cs @@ -34,13 +34,13 @@ public static void Enrich(string provider, ModelInfo info) /// /// Convert a provider JSON object to a dictionary while preserving nested raw values. /// - public static IDictionary PreserveRaw(JsonElement value) + public static IDictionary PreserveRaw(JsonElement value) { if (value.ValueKind != JsonValueKind.Object) throw new ArgumentException("Provider model payload must be a JSON object.", nameof(value)); return value.EnumerateObject() - .ToDictionary(property => property.Name, property => (object)property.Value.Clone()); + .ToDictionary(property => property.Name, property => (object?)property.Value.Clone()); } private static bool PrefixMatches(string id, string prefix) diff --git a/runtime/csharp/Prompty.Core/Pipeline.cs b/runtime/csharp/Prompty.Core/Pipeline.cs index 5dd3397e2..c0bba3d2d 100644 --- a/runtime/csharp/Prompty.Core/Pipeline.cs +++ b/runtime/csharp/Prompty.Core/Pipeline.cs @@ -480,7 +480,7 @@ internal static List ExpandThreadMarkers( { Role = msg.Role, Parts = [new TextPart { Value = before }], - Metadata = msg.Metadata is not null ? new Dictionary(msg.Metadata) : new Dictionary(), + Metadata = msg.Metadata is not null ? new Dictionary(msg.Metadata) : new Dictionary(), }); } @@ -494,7 +494,7 @@ internal static List ExpandThreadMarkers( { Role = msg.Role, Parts = [new TextPart { Value = after }], - Metadata = msg.Metadata is not null ? new Dictionary(msg.Metadata) : new Dictionary(), + Metadata = msg.Metadata is not null ? new Dictionary(msg.Metadata) : new Dictionary(), }); } } diff --git a/runtime/csharp/Prompty.Core/PromptyChatParser.cs b/runtime/csharp/Prompty.Core/PromptyChatParser.cs index 3fd9cd2d1..bca45d508 100644 --- a/runtime/csharp/Prompty.Core/PromptyChatParser.cs +++ b/runtime/csharp/Prompty.Core/PromptyChatParser.cs @@ -176,7 +176,7 @@ private Message CreateMessage( if (attrs is not null && attrs.Count > 0) { - message.Metadata ??= new Dictionary(); + message.Metadata ??= new Dictionary(); foreach (var kvp in attrs) message.Metadata[kvp.Key] = kvp.Value; } diff --git a/runtime/csharp/Prompty.Core/PromptyLoader.cs b/runtime/csharp/Prompty.Core/PromptyLoader.cs index 186e66d8a..b1c5754a2 100644 --- a/runtime/csharp/Prompty.Core/PromptyLoader.cs +++ b/runtime/csharp/Prompty.Core/PromptyLoader.cs @@ -70,7 +70,7 @@ private static Prompty Build(string contents, string fullPath, PromptyLoadOption var agent = Prompty.Load(data, new LoadContext()); // 3. Attach source path in metadata - agent.Metadata ??= new Dictionary(); + agent.Metadata ??= new Dictionary(); agent.Metadata["__source_path"] = fullPath; return agent; diff --git a/runtime/csharp/Prompty.Core/TurnRunner.cs b/runtime/csharp/Prompty.Core/TurnRunner.cs index 624d5e899..bf0423f3a 100644 --- a/runtime/csharp/Prompty.Core/TurnRunner.cs +++ b/runtime/csharp/Prompty.Core/TurnRunner.cs @@ -37,7 +37,7 @@ public ReferenceTurnRunner( public async Task RunAsync(RunTurnRequest request) { var options = request.Options ?? new TurnOptions(); - var inputs = request.Inputs ?? new Dictionary(); + var inputs = request.Inputs ?? new Dictionary(); var maxIterations = options.MaxIterations ?? 10; var checkpoints = new List(); var allToolResults = new List(); @@ -47,12 +47,12 @@ public async Task RunAsync(RunTurnRequest request) var status = RunTurnStatus.Success; var iterations = 0; - RecordSession(SessionEventType.SessionStart, request.SessionId, request.TurnId, new Dictionary + RecordSession(SessionEventType.SessionStart, request.SessionId, request.TurnId, new Dictionary { ["sessionId"] = request.SessionId, ["schemaVersion"] = "1" }); - RecordTurn(TurnEventType.TurnStart, request.TurnId, 0, new Dictionary + RecordTurn(TurnEventType.TurnStart, request.TurnId, 0, new Dictionary { ["inputs"] = inputs, ["maxIterations"] = maxIterations @@ -61,7 +61,7 @@ public async Task RunAsync(RunTurnRequest request) for (var iteration = 0; iteration < maxIterations; iteration++) { iterations = iteration + 1; - RecordTurn(TurnEventType.LlmStart, request.TurnId, iteration, new Dictionary { ["attempt"] = 0 }); + RecordTurn(TurnEventType.LlmStart, request.TurnId, iteration, new Dictionary { ["attempt"] = 0 }); var modelResponse = await _invokeModel(new TurnModelRequest { SessionId = request.SessionId, @@ -71,7 +71,7 @@ public async Task RunAsync(RunTurnRequest request) Options = options, ToolResults = pendingToolResults }); - RecordTurn(TurnEventType.LlmComplete, request.TurnId, iteration, new Dictionary()); + RecordTurn(TurnEventType.LlmComplete, request.TurnId, iteration, new Dictionary()); var checkpoint = await SaveCheckpointAsync(request.SessionId, request.TurnId, iteration, modelResponse); checkpoints.Add(checkpoint); @@ -92,7 +92,7 @@ public async Task RunAsync(RunTurnRequest request) allToolResults.Add(toolResult); } - RecordTurn(TurnEventType.MessagesUpdated, request.TurnId, iteration, new Dictionary + RecordTurn(TurnEventType.MessagesUpdated, request.TurnId, iteration, new Dictionary { ["toolResults"] = pendingToolResults.Select(result => result.Save()).ToList() }); @@ -102,7 +102,7 @@ public async Task RunAsync(RunTurnRequest request) { status = RunTurnStatus.Error; output = new Dictionary { ["message"] = "Maximum turn iterations reached" }; - RecordTurn(TurnEventType.Error, request.TurnId, iterations, new Dictionary + RecordTurn(TurnEventType.Error, request.TurnId, iterations, new Dictionary { ["errorKind"] = "max_iterations", ["message"] = "Maximum turn iterations reached" @@ -115,7 +115,7 @@ public async Task RunAsync(RunTurnRequest request) ["status"] = status == RunTurnStatus.Success ? "success" : "error", ["response"] = output }!); - RecordSession(SessionEventType.SessionEnd, request.SessionId, request.TurnId, new Dictionary + RecordSession(SessionEventType.SessionEnd, request.SessionId, request.TurnId, new Dictionary { ["sessionId"] = request.SessionId, ["status"] = status == RunTurnStatus.Success ? "success" : "error", @@ -149,7 +149,7 @@ private async Task SaveCheckpointAsync(string sessionId, string turn ["output"] = response.Output, ["toolRequests"] = (response.ToolRequests ?? []).Select(request => request.Save()).ToList() }; - foreach (var (key, value) in response.CheckpointState ?? new Dictionary()) + foreach (var (key, value) in response.CheckpointState ?? new Dictionary()) { state[key] = value; } @@ -209,12 +209,11 @@ private async Task ResolveAndExecuteToolAsync(string turnId, int return result; } - private void RecordTurn( + private void RecordTurn( TurnEventType type, string turnId, int iteration, - IDictionary payload) - where TPayload : class? + IDictionary payload) { var turnEvent = new TurnEvent { @@ -223,13 +222,13 @@ private void RecordTurn( Timestamp = _now(), TurnId = turnId, Iteration = iteration, - Payload = payload.ToDictionary(entry => entry.Key, entry => (object)entry.Value!) + Payload = payload }; _eventSink.EmitTurn(turnEvent); _journal.AppendTurn(turnEvent); } - private void RecordSession(SessionEventType type, string sessionId, string turnId, IDictionary payload) + private void RecordSession(SessionEventType type, string sessionId, string turnId, IDictionary payload) { var sessionEvent = new SessionEvent { diff --git a/runtime/csharp/Prompty.Foundry.Tests/FoundryModelDiscoveryTests.cs b/runtime/csharp/Prompty.Foundry.Tests/FoundryModelDiscoveryTests.cs index 2c3b59ccb..364e65613 100644 --- a/runtime/csharp/Prompty.Foundry.Tests/FoundryModelDiscoveryTests.cs +++ b/runtime/csharp/Prompty.Foundry.Tests/FoundryModelDiscoveryTests.cs @@ -87,4 +87,21 @@ public async Task ListModelsAsync_ReferenceDeploymentClient_ReturnsDeploymentsWi Assert.Equal(new[] { "text", "json" }, model.OutputModalities); Assert.NotNull(model.AdditionalProperties); } + + [Fact] + public void MapCatalogModel_PreservesExplicitJsonNull() + { + using var document = System.Text.Json.JsonDocument.Parse( + """{"id":"gpt-test","owned_by":"contoso","nullable":null}"""); + + var model = FoundryModels.MapCatalogModel(document.RootElement); + + Assert.NotNull(model.AdditionalProperties); + var nullable = Assert.IsType(model.AdditionalProperties["nullable"]); + Assert.Equal(System.Text.Json.JsonValueKind.Null, nullable.ValueKind); + using var saved = System.Text.Json.JsonDocument.Parse(model.ToJson(indent: false)); + Assert.Equal( + System.Text.Json.JsonValueKind.Null, + saved.RootElement.GetProperty("additionalProperties").GetProperty("nullable").ValueKind); + } } diff --git a/runtime/csharp/Prompty.Foundry.Tests/TestHelpers.cs b/runtime/csharp/Prompty.Foundry.Tests/TestHelpers.cs index 51ef13ab5..59690f74c 100644 --- a/runtime/csharp/Prompty.Foundry.Tests/TestHelpers.cs +++ b/runtime/csharp/Prompty.Foundry.Tests/TestHelpers.cs @@ -97,7 +97,7 @@ internal static Message CreateAssistantWithToolCalls(string text, List { Role = Role.Assistant, Parts = [new TextPart { Value = text }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = toolCalls, }, @@ -113,7 +113,7 @@ internal static Message CreateToolMessage(string toolCallId, string content) { Role = Role.Tool, Parts = [new TextPart { Value = content }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCallId, }, diff --git a/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs index 563759e14..133ff23c6 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/AgentLoopTests.cs @@ -327,10 +327,10 @@ private static List DefaultFormatToolMessages(List toolCalls, { var messages = new List { - new() { Role = Role.Assistant, Parts = string.IsNullOrEmpty(textContent) ? [] : [new TextPart { Value = textContent }], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, + new() { Role = Role.Assistant, Parts = string.IsNullOrEmpty(textContent) ? [] : [new TextPart { Value = textContent }], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, }; for (var i = 0; i < toolCalls.Count; i++) - messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name } }); + messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name } }); return messages; } } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/Integration/IntegrationTestBase.cs b/runtime/csharp/Prompty.OpenAI.Tests/Integration/IntegrationTestBase.cs index e0d32acfd..fad91d480 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/Integration/IntegrationTestBase.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/Integration/IntegrationTestBase.cs @@ -77,7 +77,7 @@ protected static Core.Prompty MakeOpenAIAgent( ModelOptions? options = null, IList? tools = null, IList? outputs = null, - IDictionary? metadata = null) + IDictionary? metadata = null) { var apiKey = GetEnvOrSkip("OPENAI_API_KEY"); model ??= OpenAIModel; @@ -129,7 +129,7 @@ protected static Core.Prompty MakeFoundryAgent( ModelOptions? options = null, IList? tools = null, IList? outputs = null, - IDictionary? metadata = null) + IDictionary? metadata = null) { var apiKey = GetEnvOrSkip("AZURE_OPENAI_API_KEY"); var endpoint = GetEnvOrSkip("AZURE_OPENAI_ENDPOINT"); @@ -180,7 +180,7 @@ protected static Core.Prompty MakeAnthropicAgent( ModelOptions? options = null, IList? tools = null, IList? outputs = null, - IDictionary? metadata = null) + IDictionary? metadata = null) { var apiKey = GetEnvOrSkip("ANTHROPIC_API_KEY"); diff --git a/runtime/csharp/Prompty.OpenAI.Tests/Integration/ResponsesApiTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/Integration/ResponsesApiTests.cs index 115c9838b..8e02b9c09 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/Integration/ResponsesApiTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/Integration/ResponsesApiTests.cs @@ -64,7 +64,7 @@ public async Task OpenAI_ResponsesApi_Streaming() var agent = MakeOpenAIAgent( apiType: "responses", options: new ModelOptions { Temperature = 0.5f, MaxOutputTokens = 200 }, - metadata: new Dictionary { ["stream"] = true }); + metadata: new Dictionary { ["stream"] = true }); var messages = new List { diff --git a/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingAgentTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingAgentTests.cs index f05430da5..3a4030312 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingAgentTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingAgentTests.cs @@ -37,7 +37,7 @@ private static void RegisterProviders() private static Dictionary>> WeatherToolFunctions() => new() { ["get_weather"] = GetWeatherAsync }; - private static Dictionary StreamingMetadata() => + private static Dictionary StreamingMetadata() => new() { ["stream"] = true }; // ----------------------------------------------------------------------- diff --git a/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingTests.cs index ad10a94f5..3678582e3 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/Integration/StreamingTests.cs @@ -15,7 +15,7 @@ public class StreamingTests : IntegrationTestBase { private static void EnableStreaming(Core.Prompty agent) { - agent.Metadata ??= new Dictionary(); + agent.Metadata ??= new Dictionary(); agent.Metadata["stream"] = true; } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/ResponsesApiTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/ResponsesApiTests.cs index d84620320..fdac335ab 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/ResponsesApiTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/ResponsesApiTests.cs @@ -167,7 +167,7 @@ public void MessageToResponsesInput_FunctionCallPassthrough_ReturnsStoredItem() { Role = Role.Assistant, Parts = [new TextPart { Value = "" }], - Metadata = new Dictionary + Metadata = new Dictionary { ["responses_function_call"] = storedItem, }, diff --git a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs index d1a7a6b79..93c987758 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/SpecVectorAgentTests.cs @@ -731,10 +731,10 @@ public List FormatToolMessages(object rawResponse, List toolC { var messages = new List { - new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, + new() { Role = Role.Assistant, Parts = [], Metadata = new Dictionary { ["tool_calls"] = toolCalls } }, }; for (var i = 0; i < toolCalls.Count; i++) - messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name } }); + messages.Add(new() { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name } }); return messages; } } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/StreamingTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/StreamingTests.cs index 8a12b745c..8be5571de 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/StreamingTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/StreamingTests.cs @@ -108,7 +108,7 @@ public async Task ProcessedStream_ExtractsDeltaText() // The processor should wrap it in a ProcessedStream var agent = TestHelpers.CreateAgent(); - agent.Metadata = new Dictionary { ["stream"] = true }; + agent.Metadata = new Dictionary { ["stream"] = true }; var result = await processor.ProcessAsync(agent, rawStream); Assert.IsType(result); @@ -195,7 +195,7 @@ public async Task AnthropicProcessor_HandlesStreamInput() public void StreamMetadata_EnablesStreaming() { var agent = TestHelpers.CreateAgent(); - agent.Metadata = new Dictionary { ["stream"] = true }; + agent.Metadata = new Dictionary { ["stream"] = true }; var streamVal = agent.Metadata.TryGetValue("stream", out var val) && val is true; Assert.True(streamVal); diff --git a/runtime/csharp/Prompty.OpenAI.Tests/TestHelpers.cs b/runtime/csharp/Prompty.OpenAI.Tests/TestHelpers.cs index 64961ef40..798150417 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/TestHelpers.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/TestHelpers.cs @@ -97,7 +97,7 @@ internal static Message CreateAssistantWithToolCalls(string text, List { Role = Role.Assistant, Parts = [new TextPart { Value = text }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = toolCalls, }, @@ -113,7 +113,7 @@ internal static Message CreateToolMessage(string toolCallId, string content) { Role = Role.Tool, Parts = [new TextPart { Value = content }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCallId, }, diff --git a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs index 80665ace1..dcf912484 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs @@ -145,7 +145,7 @@ public void MessageToWire_AssistantWithToolCalls_PreservesToolCalls() { Role = Role.Assistant, Parts = [new TextPart { Value = "" }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_calls"] = new List { @@ -167,7 +167,7 @@ public void MessageToWire_ToolMessage_ReturnsToolChatMessage() { Role = Role.Tool, Parts = [new TextPart { Value = "72°F and sunny" }], - Metadata = new Dictionary { ["tool_call_id"] = "call_1" }, + Metadata = new Dictionary { ["tool_call_id"] = "call_1" }, }; var result = WireFormat.MessageToWire(msg); diff --git a/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs b/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs index 335b635bd..6c3dc640b 100644 --- a/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs +++ b/runtime/csharp/Prompty.OpenAI/OpenAIExecutor.cs @@ -193,7 +193,7 @@ public virtual List FormatToolMessages( { Role = Role.Assistant, Parts = [], - Metadata = new Dictionary { ["responses_function_call"] = (ResponseItem)fc }, + Metadata = new Dictionary { ["responses_function_call"] = (ResponseItem)fc }, }); } @@ -204,7 +204,7 @@ public virtual List FormatToolMessages( { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name, @@ -224,7 +224,7 @@ public virtual List FormatToolMessages( { Role = Role.Assistant, Parts = assistantParts, - Metadata = new Dictionary { ["tool_calls"] = toolCalls.ToList() }, + Metadata = new Dictionary { ["tool_calls"] = toolCalls.ToList() }, }); // --- One tool message per result --- @@ -234,7 +234,7 @@ public virtual List FormatToolMessages( { Role = Role.Tool, Parts = [new TextPart { Value = toolResults[i] }], - Metadata = new Dictionary + Metadata = new Dictionary { ["tool_call_id"] = toolCalls[i].Id, ["name"] = toolCalls[i].Name, From 2832883b35cae2f5229390de5a3e86d45e21f75b Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 13:47:51 -0700 Subject: [PATCH 15/19] Complete nullable metadata boundaries Align remaining live-turn, engine, discovery, and test metadata consumers with explicit-null Record values exposed by the accepted Typra candidate gate. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Prompty.Anthropic.Tests/AnthropicExecutorTests.cs | 2 +- .../AnthropicModelDiscoveryTests.cs | 2 +- runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs | 2 +- runtime/csharp/Prompty.Core/LiveTurn.cs | 10 +++++----- runtime/csharp/Prompty.Core/TurnEngine.cs | 2 +- .../csharp/Prompty.Core/TurnEngineModelExtensions.cs | 2 +- runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs | 4 ++-- runtime/csharp/Prompty.OpenAI/Models.cs | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs index c83a37e01..11628abdb 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicExecutorTests.cs @@ -291,7 +291,7 @@ public void BuildRequestBody_SingleToolResultMetadata_PreservesCorrelation() { Role = Role.Tool, Parts = [new TextPart { Value = "result" }], - Metadata = new Dictionary { ["tool_use_id"] = "call_2" }, + Metadata = new Dictionary { ["tool_use_id"] = "call_2" }, }; var body = executor.BuildRequestBody(agent, [message], stream: false); diff --git a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs index c56a90feb..51b2486ab 100644 --- a/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs +++ b/runtime/csharp/Prompty.Anthropic.Tests/AnthropicModelDiscoveryTests.cs @@ -72,7 +72,7 @@ public async Task ListModelsAsync_FollowsPaginationAndPreservesProviderPayload() await server; Assert.Equal(2, models.Count); Assert.Equal("Claude First", models[0].DisplayName); - Assert.Equal("2025-01-01T00:00:00Z", models[1].AdditionalProperties!["created_at"].ToString()); + Assert.Equal("2025-01-01T00:00:00Z", models[1].AdditionalProperties!["created_at"]?.ToString()); Assert.Equal("?limit=100", requests[0].Query); Assert.Equal("?limit=100&after_id=claude-first", requests[1].Query); Assert.All(requests, request => diff --git a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs index 68b6f0073..fed8a8215 100644 --- a/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/TurnRunnerTests.cs @@ -54,7 +54,7 @@ public async Task ReferenceTurnRunner_EmitsJournalsAndCheckpoints() sink.SessionEvents.Select(sessionEvent => sessionEvent.Type).ToArray()); var checkpoint = await checkpointStore.LoadAsync("session-1", "turn-1-checkpoint-0"); Assert.NotNull(checkpoint); - Assert.True((bool)checkpoint.State!["stable"]); + Assert.True(Assert.IsType(checkpoint.State!["stable"])); Assert.True(checkpoint.State.ContainsKey("nullable")); Assert.Null(checkpoint.State["nullable"]); var turnStart = sink.TurnEvents.Single(turnEvent => turnEvent.Type == TurnEventType.TurnStart); diff --git a/runtime/csharp/Prompty.Core/LiveTurn.cs b/runtime/csharp/Prompty.Core/LiveTurn.cs index 144910d4d..21b76d9ed 100644 --- a/runtime/csharp/Prompty.Core/LiveTurn.cs +++ b/runtime/csharp/Prompty.Core/LiveTurn.cs @@ -187,7 +187,7 @@ private static Exception MapFailure( }; } - private static object? MetadataValue(IDictionary? metadata, string key) + private static object? MetadataValue(IDictionary? metadata, string key) => metadata is not null && metadata.TryGetValue(key, out var value) ? value : null; private sealed class LiveFailureState @@ -262,9 +262,9 @@ public async Task InvokeAsync( Id = call.Id, Name = call.Name, Arguments = ParseArgumentsValue(call.Arguments), - Metadata = new Dictionary { ["argumentsText"] = call.Arguments }, + Metadata = new Dictionary { ["argumentsText"] = call.Arguments }, }).ToList(), - Metadata = new Dictionary + Metadata = new Dictionary { ["rawResponse"] = rawResponse, ["textContent"] = toolResult.Content ?? string.Empty, @@ -282,7 +282,7 @@ public async Task InvokeAsync( Portability = InvocationContextPortability.Portable, DelegatedState = [], }, - Metadata = new Dictionary { ["rawResponse"] = rawResponse }, + Metadata = new Dictionary { ["rawResponse"] = rawResponse }, }; } @@ -493,7 +493,7 @@ public async Task BeforeModelAsync( { Messages = messages, StablePrefixMessages = stablePrefix, - Metadata = new Dictionary + Metadata = new Dictionary { ["steeringCount"] = steeringMessages.Count, ["trimmedCount"] = trimmed, diff --git a/runtime/csharp/Prompty.Core/TurnEngine.cs b/runtime/csharp/Prompty.Core/TurnEngine.cs index ac96bfc6f..f4edd6b2f 100644 --- a/runtime/csharp/Prompty.Core/TurnEngine.cs +++ b/runtime/csharp/Prompty.Core/TurnEngine.cs @@ -685,7 +685,7 @@ private async Task PersistCheckpointAsync( private async Task PersistPolicyUpdateAsync( TurnState state, string invocationId, - IDictionary? metadata) + IDictionary? metadata) { var sequence = state.Sequence + 1; state.PolicyAppliedForIteration = true; diff --git a/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs b/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs index c63872e07..6b69a2f63 100644 --- a/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs +++ b/runtime/csharp/Prompty.Core/TurnEngineModelExtensions.cs @@ -24,7 +24,7 @@ public partial class Message { Role = Role.Tool, Parts = [new TextPart { Value = text }], - Metadata = new Dictionary { ["tool_call_id"] = requestId }, + Metadata = new Dictionary { ["tool_call_id"] = requestId }, }; } diff --git a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs index dcf912484..33e9194e0 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs @@ -492,7 +492,7 @@ public void BuildOptions_AdditionalProperties_DoNotOverrideCanonicalOptions() Options = new ModelOptions { Temperature = 0.2f, - AdditionalProperties = new Dictionary + AdditionalProperties = new Dictionary { ["temperature"] = 0.9, ["logprobs"] = true, @@ -518,7 +518,7 @@ public void BuildOptions_UnsafeAdditionalPropertyName_Throws() { Options = new ModelOptions { - AdditionalProperties = new Dictionary { ["nested.option"] = true }, + AdditionalProperties = new Dictionary { ["nested.option"] = true }, }, }, }; diff --git a/runtime/csharp/Prompty.OpenAI/Models.cs b/runtime/csharp/Prompty.OpenAI/Models.cs index 5e3f725d4..c2b85bef7 100644 --- a/runtime/csharp/Prompty.OpenAI/Models.cs +++ b/runtime/csharp/Prompty.OpenAI/Models.cs @@ -57,7 +57,7 @@ public static async Task> ListModelsAsync( { Id = m.Id, OwnedBy = m.OwnedBy, - AdditionalProperties = new Dictionary + AdditionalProperties = new Dictionary { ["id"] = m.Id, ["object"] = "model", From bc9635bc7578463a04fe7dee55461dde10e12763 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 15:21:07 -0700 Subject: [PATCH 16/19] Clear parser nonce after each parse Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../csharp/Prompty.Core.Tests/ParserTests.cs | 84 ++++++++++++++++- .../csharp/Prompty.Core/PromptyChatParser.cs | 93 ++++++++++--------- 2 files changed, 131 insertions(+), 46 deletions(-) diff --git a/runtime/csharp/Prompty.Core.Tests/ParserTests.cs b/runtime/csharp/Prompty.Core.Tests/ParserTests.cs index 4e6bf4bb1..c36bf10e1 100644 --- a/runtime/csharp/Prompty.Core.Tests/ParserTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ParserTests.cs @@ -184,6 +184,56 @@ public async Task PreRender_Then_Parse_ValidatesNonce() Assert.False(messages[0].Metadata?.ContainsKey("nonce") ?? false); } + [Fact] + public void PreRender_Then_Parse_ClearsNonceForSubsequentSyncParse() + { + var parser = new PromptyChatParser(); + var (sanitized, _) = parser.PreRender("system:\nStrict"); + + var strictMessages = parser.Parse(sanitized); + var plainMessages = parser.Parse("user:\nPlain"); + + Assert.Single(strictMessages); + Assert.Single(plainMessages); + Assert.Equal("Plain", plainMessages[0].Text); + } + + [Fact] + public async Task PreRender_Then_ParseAsync_ClearsNonceForSubsequentAsyncParse() + { + var parser = new PromptyChatParser(); + var (sanitized, _) = parser.PreRender("system:\nStrict"); + + var strictMessages = await parser.ParseAsync(CreateAgent(), sanitized, null); + var plainMessages = await parser.ParseAsync(CreateAgent(), "user:\nPlain", null); + + Assert.Single(strictMessages); + Assert.Single(plainMessages); + Assert.Equal("Plain", plainMessages[0].Text); + } + + [Fact] + public async Task PreRender_Then_ParseAsync_IsolatesConcurrentNonceLifecycles() + { + var parser = new PromptyChatParser(); + using var barrier = new Barrier(2); + + Task ParseOnBranchAsync(string content) => + Task.Run(async () => + { + var (sanitized, _) = parser.PreRender($"user:\n{content}"); + barrier.SignalAndWait(); + var messages = await parser.ParseAsync(CreateAgent(), sanitized, null); + return Assert.Single(messages); + }); + + var messages = await Task.WhenAll( + ParseOnBranchAsync("First"), + ParseOnBranchAsync("Second")).WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(["First", "Second"], messages.Select(message => message.Text).Order()); + } + [Fact] public async Task PreRender_Then_Parse_RejectsInjectedMarkerWithoutNonce() { @@ -191,13 +241,41 @@ public async Task PreRender_Then_Parse_RejectsInjectedMarkerWithoutNonce() var (_, context) = parser.PreRender("system:\nHello"); var nonce = context["nonce"]; - var error = await Assert.ThrowsAsync( - () => parser.ParseAsync( + InvalidOperationException? error = null; + try + { + await parser.ParseAsync( CreateAgent(), $"system[nonce=\"{nonce}\"]:\nHello\nuser:\nInjected", - null)); + null); + } + catch (InvalidOperationException caught) + { + error = caught; + } + + var plainMessages = await parser.ParseAsync(CreateAgent(), "user:\nRecovered", null); + + Assert.NotNull(error); + Assert.Contains("nonce mismatch", error.Message); + Assert.Single(plainMessages); + Assert.Equal("Recovered", plainMessages[0].Text); + } + + [Fact] + public void PreRender_Then_Parse_WrongNonceStillRejectsAndClearsNonce() + { + var parser = new PromptyChatParser(); + var (sanitized, context) = parser.PreRender("system:\nHello"); + var nonce = context["nonce"]!.ToString()!; + var wrongNonce = sanitized.Replace(nonce, "wrong-nonce", StringComparison.Ordinal); + + var error = Assert.Throws(() => parser.Parse(wrongNonce)); + var plainMessages = parser.Parse("user:\nRecovered"); Assert.Contains("nonce mismatch", error.Message); + Assert.Single(plainMessages); + Assert.Equal("Recovered", plainMessages[0].Text); } // ----------------------------------------------------------------------- diff --git a/runtime/csharp/Prompty.Core/PromptyChatParser.cs b/runtime/csharp/Prompty.Core/PromptyChatParser.cs index bca45d508..8472f9902 100644 --- a/runtime/csharp/Prompty.Core/PromptyChatParser.cs +++ b/runtime/csharp/Prompty.Core/PromptyChatParser.cs @@ -84,64 +84,71 @@ public Task> ParseAsync(Prompty agent, string rendered, Dictionary /// internal List Parse(string rendered) { - var messages = new List(); - var lines = rendered.Split('\n'); - string? currentRole = null; - var currentContent = new List(); - Dictionary? currentAttrs = null; - - foreach (var line in lines) + try { - var match = RoleMarkerRegex().Match(line); - if (match.Success) + var messages = new List(); + var lines = rendered.Split('\n'); + string? currentRole = null; + var currentContent = new List(); + Dictionary? currentAttrs = null; + + foreach (var line in lines) { - if (currentRole is not null) - { - messages.Add(CreateMessage(currentRole, currentContent, currentAttrs, validateNonce: true)); - } - else if (currentContent.Count > 0) + var match = RoleMarkerRegex().Match(line); + if (match.Success) { - var leadingText = string.Join("\n", currentContent).Trim('\r', '\n'); - if (!string.IsNullOrEmpty(leadingText)) + if (currentRole is not null) + { + messages.Add(CreateMessage(currentRole, currentContent, currentAttrs, validateNonce: true)); + } + else if (currentContent.Count > 0) { - messages.Add(new Message + var leadingText = string.Join("\n", currentContent).Trim('\r', '\n'); + if (!string.IsNullOrEmpty(leadingText)) { - Role = Role.System, - Parts = [new TextPart { Value = leadingText }] - }); + messages.Add(new Message + { + Role = Role.System, + Parts = [new TextPart { Value = leadingText }] + }); + } } + + currentRole = match.Groups[1].Value; + currentAttrs = ParseAttributes(match.Groups[2].Value); + currentContent = []; + } + else + { + currentContent.Add(line); } + } - currentRole = match.Groups[1].Value; - currentAttrs = ParseAttributes(match.Groups[2].Value); - currentContent = []; + // Flush last message + if (currentRole is not null) + { + messages.Add(CreateMessage(currentRole, currentContent, currentAttrs, validateNonce: true)); } - else + else if (currentContent.Count > 0) { - currentContent.Add(line); + // No role markers at all — treat as system message + var text = string.Join("\n", currentContent).Trim(); + if (!string.IsNullOrEmpty(text)) + { + messages.Add(new Message + { + Role = Role.System, + Parts = [new TextPart { Value = text }] + }); + } } - } - // Flush last message - if (currentRole is not null) - { - messages.Add(CreateMessage(currentRole, currentContent, currentAttrs, validateNonce: true)); + return messages; } - else if (currentContent.Count > 0) + finally { - // No role markers at all — treat as system message - var text = string.Join("\n", currentContent).Trim(); - if (!string.IsNullOrEmpty(text)) - { - messages.Add(new Message - { - Role = Role.System, - Parts = [new TextPart { Value = text }] - }); - } + _renderNonce.Value = null; } - - return messages; } // ----------------------------------------------------------------------- From 4dc63dc0b36f4ce44c7c034ebe26a41ca92c6721 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 15:28:44 -0700 Subject: [PATCH 17/19] Narrow parser test exception null state Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/csharp/Prompty.Core.Tests/ParserTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/csharp/Prompty.Core.Tests/ParserTests.cs b/runtime/csharp/Prompty.Core.Tests/ParserTests.cs index c36bf10e1..27f2b215e 100644 --- a/runtime/csharp/Prompty.Core.Tests/ParserTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/ParserTests.cs @@ -257,7 +257,8 @@ await parser.ParseAsync( var plainMessages = await parser.ParseAsync(CreateAgent(), "user:\nRecovered", null); Assert.NotNull(error); - Assert.Contains("nonce mismatch", error.Message); + var nonNullError = error!; + Assert.Contains("nonce mismatch", nonNullError.Message); Assert.Single(plainMessages); Assert.Equal("Recovered", plainMessages[0].Text); } From ada7f324e6f581eb695c911fb9044536a02286f0 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 15:39:52 -0700 Subject: [PATCH 18/19] Guard FunctionTool binding expectations Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Prompty.Core.Tests/SpecVectorTests.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs index cb556860b..1d451674c 100644 --- a/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs +++ b/runtime/csharp/Prompty.Core.Tests/SpecVectorTests.cs @@ -285,6 +285,26 @@ public void LoadVectors_AllPass() } } + [Fact] + public void FunctionToolLoadVector_DeclaresBindingsExpectation() + { + var vector = LoadVectors("load") + .Single(vector => vector.GetProperty("name").GetString() == "tools_function_load"); + var expectedTool = vector.GetProperty("expected") + .GetProperty("tools") + .EnumerateArray() + .Single(tool => + tool.GetProperty("kind").GetString() == "function" && + tool.GetProperty("name").GetString() == "get_weather"); + + Assert.True( + expectedTool.TryGetProperty("bindings", out var bindings), + "tools_function_load must declare expected FunctionTool bindings"); + Assert.True( + bindings.ValueKind is JsonValueKind.Object or JsonValueKind.Array, + "FunctionTool bindings expectations must use the equivalent map or list wire form"); + } + // ========================================================================= // Helpers // ========================================================================= From 539703f7dbbf5803a6abc14ab2609e712b568937 Mon Sep 17 00:00:00 2001 From: Seth Juarez Date: Tue, 4 Aug 2026 15:59:08 -0700 Subject: [PATCH 19/19] Reject empty OpenAI option names Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs | 9 ++++++--- runtime/csharp/Prompty.OpenAI/WireFormat.cs | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs index 33e9194e0..80b454a5a 100644 --- a/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs +++ b/runtime/csharp/Prompty.OpenAI.Tests/WireFormatTests.cs @@ -509,8 +509,11 @@ public void BuildOptions_AdditionalProperties_DoNotOverrideCanonicalOptions() Assert.True(json.GetProperty("logprobs").GetBoolean()); } - [Fact] - public void BuildOptions_UnsafeAdditionalPropertyName_Throws() + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("nested.option")] + public void BuildOptions_UnsafeAdditionalPropertyName_Throws(string optionName) { var agent = new Core.Prompty { @@ -518,7 +521,7 @@ public void BuildOptions_UnsafeAdditionalPropertyName_Throws() { Options = new ModelOptions { - AdditionalProperties = new Dictionary { ["nested.option"] = true }, + AdditionalProperties = new Dictionary { [optionName] = true }, }, }, }; diff --git a/runtime/csharp/Prompty.OpenAI/WireFormat.cs b/runtime/csharp/Prompty.OpenAI/WireFormat.cs index 2837c8f83..9ac1bbf1f 100644 --- a/runtime/csharp/Prompty.OpenAI/WireFormat.cs +++ b/runtime/csharp/Prompty.OpenAI/WireFormat.cs @@ -218,7 +218,8 @@ public static ChatCompletionOptions BuildOptions(Core.Prompty agent) { if (CanonicalOptionIsSet(name, agent, opts)) continue; - if (name.Any(character => !char.IsAsciiLetterOrDigit(character) && character != '_')) + if (string.IsNullOrWhiteSpace(name) || + name.Any(character => !char.IsAsciiLetterOrDigit(character) && character != '_')) throw new ArgumentException( $"OpenAI additional option name '{name}' cannot be represented safely by the SDK.", nameof(agent));