diff --git a/AgenticPatterns.Tests/AgenticPatterns.Tests.csproj b/AgenticPatterns.Tests/AgenticPatterns.Tests.csproj index 5142e47..8851ea2 100644 --- a/AgenticPatterns.Tests/AgenticPatterns.Tests.csproj +++ b/AgenticPatterns.Tests/AgenticPatterns.Tests.csproj @@ -20,6 +20,11 @@ + + X{Exact cache hit?} X -->|yes| R[Cached response] X -->|no| E[Embed query] - E --> C{Cosine similarity
at least 0.9?} + E --> P[Look up namespace
partition] + P --> C{Cosine similarity
at least 0.9?} C -->|yes| R C -->|no| M[Real model call] - M --> ST[Store embedding
plus response] + M --> ST[Store embedding + response
with expiry; evict oldest
if over the bound] ST --> R ``` @@ -64,6 +97,8 @@ reads to classify each call. - `IEmbeddingGenerator.GenerateVectorAsync(query)` — one embedding per uncached question. - `TensorPrimitives.CosineSimilarity(cached, incoming)` — the similarity scan, an O(n) list walk that a persistent vector store would replace in production. +- `CacheNamespace` — the six required isolation dimensions (tenant, authorization scope, system + prompt, tool schema, model version, data revision) that key every partition. ## What to watch in the output diff --git a/PatternExplorer/patterns/SkillLearning.md b/PatternExplorer/patterns/SkillLearning.md index d725c52..dc0e04f 100644 --- a/PatternExplorer/patterns/SkillLearning.md +++ b/PatternExplorer/patterns/SkillLearning.md @@ -74,8 +74,17 @@ flowchart LR model call with the trajectory in the prompt. - `AIFunctionFactory.Create(...)` for `read_skill`, plus instance-method tools bound from the fake provisioning system. -- `SkillLifecycle` persists a versioned manifest and enforces legal promotion transitions. -- `ProvisionEmployeeSkillTests.Pass(...)` verifies the learned formats before review. +- `SkillLifecycle` persists a versioned manifest and enforces legal promotion transitions. Every + read and transition re-hashes the on-disk `SKILL.md` against the SHA-256 recorded at candidate + creation and refuses to load it on mismatch, so an approved file edited in place is **detected**, + not prevented — whoever can write `SKILL.md` can also write `manifest.json` and update the digest + to match. Closing that gap means signing the approved manifest or keeping the manifest store + outside the agent's write scope. +- `ProvisionEmployeeSkillTests.Pass(...)` verifies the learned formats before review. It is a + substring-order check on the markdown — it confirms the four facts appear in the right order, + not that the skill actually works. A real behavioural test would run the procedure against the + fake provisioning system (or a sandboxed copy) and assert the resulting account has the right + username, license, and team, the way an integration test would. ## What to watch in the output diff --git a/SemanticCaching.AgentFramework/Program.cs b/SemanticCaching.AgentFramework/Program.cs index f94204d..101f3bd 100644 --- a/SemanticCaching.AgentFramework/Program.cs +++ b/SemanticCaching.AgentFramework/Program.cs @@ -1,5 +1,7 @@ using System.ClientModel; using System.Diagnostics; +using System.Security.Cryptography; +using System.Text; using Azure.AI.OpenAI; using Microsoft.Agents.AI; using Microsoft.Extensions.AI; @@ -16,17 +18,32 @@ .GetEmbeddingClient(Settings.AzureOpenAi.EmbeddingModelDeployment) .AsIEmbeddingGenerator(); +const string systemPrompt = "You are a concise assistant. Answer in one or two sentences."; + +// Every dimension a real deployment must not forget: which tenant, under which authorization +// scope, which system prompt, which tool policy, which model, and which revision of the +// underlying data the answer was drawn from. This sample has one caller, no tools and one +// static document set, so most of these are constants — a real deployment reads TenantId and +// PrincipalScopeHash from the caller's auth context per request. +var cacheNamespace = new CacheNamespace( + TenantId: "sample-tenant", + PrincipalScopeHash: "sample-principal", + SystemPromptHash: Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(systemPrompt))), + ToolSchemaHash: "no-tools", // CachingAgent exposes no tools; hash the registered schema once it does + ModelVersion: Settings.AzureOpenAi.ChatModelDeployment, + DataRevision: "v1"); // bump whenever the knowledge this agent answers from changes + // Cheapest check first: exact-match cache (free hash lookup) is outermost, then the // semantic cache (costs one embedding call), then the real model. SemanticCachingChatClient semanticCache = null!; var client = new ChatClientBuilder(Settings.ChatClient) .UseDistributedCache(new MemoryDistributedCache(Options.Create(new MemoryDistributedCacheOptions()))) - .Use(inner => semanticCache = new SemanticCachingChatClient(inner, embeddingGenerator)) + .Use(inner => semanticCache = new SemanticCachingChatClient( + inner, embeddingGenerator, cacheNamespace, + entryLifetime: TimeSpan.FromMinutes(10), maxEntriesPerPartition: 500)) .Build(); -var agent = new ChatClientAgent(client, - "You are a concise assistant. Answer in one or two sentences.", - "CachingAgent"); +var agent = new ChatClientAgent(client, systemPrompt, "CachingAgent"); (string Label, string Query)[] calls = [ diff --git a/SemanticCaching.AgentFramework/SemanticCachingChatClient.cs b/SemanticCaching.AgentFramework/SemanticCachingChatClient.cs index ae94d5f..2c7b212 100644 --- a/SemanticCaching.AgentFramework/SemanticCachingChatClient.cs +++ b/SemanticCaching.AgentFramework/SemanticCachingChatClient.cs @@ -1,21 +1,48 @@ using System.Numerics.Tensors; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; using Microsoft.Extensions.AI; namespace SemanticCaching.AgentFramework; +/// +/// Every dimension that must isolate one cached answer from another. All six are required: +/// a semantic cache keyed on conversation shape alone will happily serve tenant A's answer to +/// tenant B, or an answer generated under a stale tool policy or a stale data revision — a +/// cross-tenant data leak waiting to happen. +/// +public sealed record CacheNamespace( + string TenantId, + string PrincipalScopeHash, + string SystemPromptHash, + string ToolSchemaHash, + string ModelVersion, + string DataRevision); + /// Serves cached responses for queries semantically similar to previously answered ones. public sealed class SemanticCachingChatClient( IChatClient innerClient, - IEmbeddingGenerator> embeddingGenerator) + IEmbeddingGenerator> embeddingGenerator, + CacheNamespace ns, + TimeSpan entryLifetime, + int maxEntriesPerPartition) : DelegatingChatClient(innerClient) { // 0.9 accepts close paraphrases while rejecting merely related questions private const float SimilarityThreshold = 0.9f; - // Partitioned by context: similar user text under a different system prompt, model, - // or options must never reuse another context's answer. - // ponytail: in-memory dictionary with O(n) scan per partition — swap for a persistent vector store in production - private readonly Dictionary> _cache = []; + // ponytail: one lock; shard it if this ever leaves a sample + private readonly object _lock = new(); + + // Partitioned by namespace + context: similar user text under a different tenant, system + // prompt, model, tool policy, data revision, or options must never reuse another partition's + // answer. + // ponytail: in-memory dictionary with O(n) scan per partition, and expired entries are only + // reclaimed when their partition is next read (a partition nobody ever queries again holds + // its expired entries for the process lifetime, bounded only by maxEntriesPerPartition) — + // swap for a persistent vector store with its own TTL sweep in production + private readonly Dictionary> _cache = []; public int Hits { get; private set; } public int Misses { get; private set; } @@ -26,50 +53,133 @@ public override async Task GetResponseAsync( ChatOptions? options = null, CancellationToken cancellationToken = default) { - var query = messages.LastOrDefault(m => m.Role == ChatRole.User)?.Text; + var messageList = messages as IReadOnlyList ?? messages.ToList(); + var query = messageList.LastOrDefault(m => m.Role == ChatRole.User)?.Text; if (string.IsNullOrWhiteSpace(query)) - return await base.GetResponseAsync(messages, options, cancellationToken); + return await base.GetResponseAsync(messageList, options, cancellationToken); var embedding = await embeddingGenerator.GenerateVectorAsync(query, cancellationToken: cancellationToken); + var key = PartitionKey(ns, messageList, options); - var key = ContextKey(messages, options); - if (!_cache.TryGetValue(key, out var partition)) - _cache[key] = partition = []; - - var best = (Similarity: -1f, Response: (ChatResponse?)null); - foreach (var (cachedEmbedding, cachedResponse) in partition) + lock (_lock) { - var similarity = TensorPrimitives.CosineSimilarity(cachedEmbedding, embedding.Span); - if (similarity > best.Similarity) - best = (similarity, cachedResponse); + var best = (Similarity: -1f, Response: (ChatResponse?)null); + if (_cache.TryGetValue(key, out var partition)) + { + var now = DateTimeOffset.UtcNow; + partition.RemoveAll(e => e.ExpiresAt <= now); + + foreach (var (cachedEmbedding, cachedResponse, _) in partition) + { + var similarity = TensorPrimitives.CosineSimilarity(cachedEmbedding, embedding.Span); + if (similarity > best.Similarity) + best = (similarity, cachedResponse); + } + } + + LastSimilarity = best.Similarity; + if (best.Response is not null && best.Similarity >= SimilarityThreshold) + { + Hits++; + // Hand out a copy, not the shared cached instance (its Usage/ResponseId belong + // to the original call — a cache hit costs no tokens). + return new ChatResponse([.. best.Response.Messages.Select(m => m.Clone())]) + { + ModelId = best.Response.ModelId + }; + } + + Misses++; } - LastSimilarity = best.Similarity; - if (best.Response is not null && best.Similarity >= SimilarityThreshold) + // The model call happens outside the lock — it can be slow, and must not serialize + // every other concurrent caller behind it. + var response = await base.GetResponseAsync(messageList, options, cancellationToken); + + // Clone on store too: the caller's copy of `response` is theirs to mutate freely, so the + // cache must not keep a reference to the exact object handed back to them. + var stored = new ChatResponse([.. response.Messages.Select(m => m.Clone())]) { ModelId = response.ModelId }; + var expiresAt = DateTimeOffset.UtcNow + entryLifetime; + + lock (_lock) { - Hits++; - // Hand out a copy, not the shared cached instance (its Usage/ResponseId belong - // to the original call — a cache hit costs no tokens). - return new ChatResponse([.. best.Response.Messages.Select(m => m.Clone())]) - { - ModelId = best.Response.ModelId - }; + if (!_cache.TryGetValue(key, out var partition)) + _cache[key] = partition = []; + + partition.Add((embedding.ToArray(), stored, expiresAt)); + if (partition.Count > maxEntriesPerPartition) + partition.RemoveAt(0); } - Misses++; - var response = await base.GetResponseAsync(messages, options, cancellationToken); - partition.Add((embedding.ToArray(), response)); return response; } - // Everything that changes what a valid answer looks like belongs in the key: the system - // prompt, every prior turn, and the options. Only the final user message (the embedded - // query) is excluded — that's what the similarity search matches on. - private static string ContextKey(IEnumerable messages, ChatOptions? options) + // Everything that changes what a valid answer looks like belongs in the key: the namespace + // (tenant, authorization scope, system prompt, tool policy, model, data revision), every + // prior turn, and the options. Only the final user message (the embedded query) is excluded + // — that's what the similarity search matches on. + public static string PartitionKey(CacheNamespace ns, IEnumerable messages, ChatOptions? options) { var list = messages.ToList(); var lastUser = list.FindLastIndex(m => m.Role == ChatRole.User); - return string.Join("\n", list.Where((m, i) => i != lastUser).Select(m => $"{m.Role}:{m.Text}")) + - $"|{options?.ModelId}|{options?.Temperature}|{options?.ResponseFormat}"; + var priorTurns = string.Join("\n", list.Where((m, i) => i != lastUser).Select(DigestMessage)); + var canonicalOptions = CanonicalOptions(options); + + // Every component is hashed to a fixed-length digest before joining. A raw '|'-join of + // the raw fields would let a delimiter inside a field (e.g. TenantId "a|b") shift the + // boundary and collide with an unrelated namespace whose fields split differently — a + // hash has no delimiter to smuggle across the join, so no combination of field values + // can produce another combination's key. + return string.Join('|', + Hash(ns.TenantId), Hash(ns.PrincipalScopeHash), Hash(ns.SystemPromptHash), + Hash(ns.ToolSchemaHash), Hash(ns.ModelVersion), Hash(ns.DataRevision), + Hash(priorTurns), Hash(canonicalOptions)); + } + + // Every option that changes what a valid answer looks like belongs in the key, not just + // ModelId/Temperature/ResponseFormat — a runtime ChatOptions.Tools that diverges from the + // namespace's declared ToolSchemaHash, or a different MaxOutputTokens/Seed/StopSequences + // etc., must not collide with an unrelated request. Deliberately the same shape as + // EvaluationAndMonitoring.AgentFramework/TraceReplay.cs's TraceStore.CanonicalOptions (the + // two projects don't reference each other, so this is the pattern copied, not code shared). + // ConversationId, AllowBackgroundResponses, ContinuationToken and RawRepresentationFactory + // are excluded: none of them changes what a valid answer looks like. + private static string CanonicalOptions(ChatOptions? options) + { + var tools = string.Join(";", options?.Tools?.Select(tool => tool is AIFunctionDeclaration function + ? $"{function.Name}:{function.JsonSchema.GetRawText()}" + : $"{tool.Name}:{tool.Description}") ?? []); + var toolMode = options?.ToolMode is RequiredChatToolMode required + ? $"Required:{required.RequiredFunctionName}" + : options?.ToolMode?.GetType().Name ?? ""; + var additionalProperties = options?.AdditionalProperties is { } props + ? JsonSerializer.Serialize(props.OrderBy(p => p.Key, StringComparer.Ordinal)) + : ""; + + return string.Join('|', + $"model:{options?.ModelId}", $"temperature:{options?.Temperature}", $"format:{options?.ResponseFormat}", + $"tools:{tools}", $"toolMode:{toolMode}", $"allowMultipleToolCalls:{options?.AllowMultipleToolCalls}", + $"instructions:{options?.Instructions}", $"maxOutputTokens:{options?.MaxOutputTokens}", + $"topP:{options?.TopP}", $"topK:{options?.TopK}", $"seed:{options?.Seed}", + $"stopSequences:{string.Join(",", options?.StopSequences ?? [])}", + $"frequencyPenalty:{options?.FrequencyPenalty}", $"presencePenalty:{options?.PresencePenalty}", + $"reasoningEffort:{options?.Reasoning?.Effort}", $"reasoningOutput:{options?.Reasoning?.Output}", + $"additionalProperties:{additionalProperties}"); } + + private static string Hash(string s) => Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(s))); + + // Digest every AIContent kind, not just TextContent — a prior function call or its result + // changes what a valid cached answer looks like just as much as prior text does, and a + // digest keyed on `.Text` alone silently drops both. + private static string DigestMessage(ChatMessage m) => + $"{m.Role}:{string.Join(",", m.Contents.Select(DigestContent))}"; + + private static string DigestContent(AIContent content) => content switch + { + TextContent t => $"text:{t.Text}", + FunctionCallContent c => $"call:{c.CallId}:{c.Name}:{string.Join(",", c.Arguments?.Select(a => $"{a.Key}={a.Value}") ?? [])}", + FunctionResultContent r => $"result:{r.CallId}:{r.Result}", + _ => $"{content.GetType().Name}:{content}" + }; } diff --git a/SkillLearning.AgentFramework/SkillLifecycle.cs b/SkillLearning.AgentFramework/SkillLifecycle.cs index c2a2f4e..db3be2b 100644 --- a/SkillLearning.AgentFramework/SkillLifecycle.cs +++ b/SkillLearning.AgentFramework/SkillLifecycle.cs @@ -1,3 +1,4 @@ +using System.Security.Cryptography; using System.Text.Json; namespace SkillLearning.AgentFramework; @@ -9,6 +10,7 @@ public sealed record SkillManifest( int Version, SkillStage Stage, DateTimeOffset CreatedAt, + string ContentSha256, string? ApprovedBy = null); public sealed class SkillLifecycle(string skillsDirectory) @@ -22,9 +24,10 @@ public SkillManifest CreateCandidate(string name, string markdown) if (existing is not null && existing.Stage != SkillStage.Retired) throw new InvalidOperationException("The current skill version must be retired before creating another."); var manifest = new SkillManifest(name, (existing?.Version ?? 0) + 1, SkillStage.Candidate, - DateTimeOffset.UtcNow); + DateTimeOffset.UtcNow, ContentSha256: ""); Directory.CreateDirectory(VersionDirectory(manifest)); File.WriteAllText(SkillPath(manifest), markdown); + manifest = manifest with { ContentSha256 = Digest(SkillPath(manifest)) }; Save(manifest); return manifest; } @@ -32,7 +35,7 @@ public SkillManifest CreateCandidate(string name, string markdown) public SkillManifest Validate(string name) { var manifest = Require(name, SkillStage.Candidate); - var markdown = File.ReadAllText(SkillPath(manifest)); + var markdown = ReadVerified(manifest); var lines = markdown.Replace("\r\n", "\n").Split('\n'); var closingFence = Array.IndexOf(lines, "---", 1); var frontmatter = closingFence > 0 ? lines[1..closingFence] : []; @@ -47,7 +50,7 @@ public SkillManifest Validate(string name) public SkillManifest MarkTested(string name, Func test) { var manifest = Require(name, SkillStage.Validated); - if (!test(File.ReadAllText(SkillPath(manifest)))) + if (!test(ReadVerified(manifest))) throw new InvalidDataException("Skill contract tests failed; candidate was not promoted."); return Transition(manifest, SkillStage.Tested); } @@ -56,22 +59,36 @@ public SkillManifest Approve(string name, string reviewer) { var manifest = Require(name, SkillStage.Tested); if (string.IsNullOrWhiteSpace(reviewer)) throw new ArgumentException("A trusted reviewer is required."); + ReadVerified(manifest); return Transition(manifest with { ApprovedBy = reviewer }, SkillStage.Approved); } - public SkillManifest Activate(string name) => Transition(Require(name, SkillStage.Approved), SkillStage.Active); + public SkillManifest Activate(string name) + { + var manifest = Require(name, SkillStage.Approved); + ReadVerified(manifest); + return Transition(manifest, SkillStage.Active); + } - public SkillManifest Retire(string name) => Transition(Require(name, SkillStage.Active), SkillStage.Retired); + public SkillManifest Retire(string name) + { + var manifest = Require(name, SkillStage.Active); + ReadVerified(manifest); + return Transition(manifest, SkillStage.Retired); + } public string? ReadActive(string name) { var manifest = Load(SafeName(name)); - return manifest?.Stage == SkillStage.Active ? File.ReadAllText(SkillPath(manifest)) : null; + return manifest?.Stage == SkillStage.Active ? ReadVerified(manifest) : null; } public SkillManifest? Load(string name) { var path = ManifestPath(SafeName(name)); + // ponytail: a manifest.json written before ContentSha256 existed deserializes with a null + // digest and then fails ReadVerified with the tamper message, not a migration message. No + // migration path for a sample; add one if this ever needs to read pre-existing manifests. return File.Exists(path) ? JsonSerializer.Deserialize(File.ReadAllText(path), Json) : null; } @@ -96,6 +113,21 @@ private void Save(SkillManifest manifest) File.WriteAllText(ManifestPath(manifest.Name), JsonSerializer.Serialize(manifest, Json)); } + private static string Digest(string path) => + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(path))); + + // Every transition and every read re-verifies. A version directory is immutable by policy + // once the candidate is created (nothing here enforces that on disk); the only legal way to + // change a skill is a new version. + private string ReadVerified(SkillManifest manifest) + { + var path = SkillPath(manifest); + if (Digest(path) != manifest.ContentSha256) + throw new InvalidDataException( + $"Skill '{manifest.Name}' v{manifest.Version} was modified after approval; refusing to load it."); + return File.ReadAllText(path); + } + private string VersionDirectory(SkillManifest manifest) => Path.Combine(skillsDirectory, manifest.Name, "versions", manifest.Version.ToString()); private string SkillPath(SkillManifest manifest) => Path.Combine(VersionDirectory(manifest), "SKILL.md");