From d808c2c7aa7db12ae769454b6234fd6dea9f0886 Mon Sep 17 00:00:00 2001 From: Marcel Neidinger Date: Mon, 27 Jul 2026 10:08:47 +0200 Subject: [PATCH] feat(bb-agent): opt-in Bedrock prompt caching via ModelConfig.cacheConfig Add an optional cacheConfig ({ strategy: 'auto' | 'anthropic' }) to the Agent's ModelConfig, threaded into Strands' BedrockModel. Off by default and additive; ignored by the openai-api and canned providers. --- .changeset/bb-agent-prompt-caching.md | 17 +++++++++++++++++ packages/bb-agent/API.md | 2 ++ packages/bb-agent/README.md | 16 ++++++++++++++++ packages/bb-agent/src/index.test.ts | 13 +++++++++++++ packages/bb-agent/src/model-factory.ts | 1 + packages/bb-agent/src/types.ts | 24 ++++++++++++++++++++++++ 6 files changed, 73 insertions(+) create mode 100644 .changeset/bb-agent-prompt-caching.md diff --git a/.changeset/bb-agent-prompt-caching.md b/.changeset/bb-agent-prompt-caching.md new file mode 100644 index 00000000..2280010f --- /dev/null +++ b/.changeset/bb-agent-prompt-caching.md @@ -0,0 +1,17 @@ +--- +"@aws-blocks/bb-agent": minor +--- + +feat(bb-agent): opt-in Bedrock prompt caching via `ModelConfig.cacheConfig` + +Agents can now enable Strands prompt caching for the `bedrock` provider by +setting `cacheConfig: { strategy: 'auto' | 'anthropic' }` on a model config. +Caching reuses the cached request prefix (tools + system prompt + prior turns) +across requests, cutting input-token cost and latency for agents with long +system prompts, many tools, or multi-turn conversations. + +Off by default and additive — existing configs are unaffected. Use +`'auto'` for the `BedrockModels` presets (Strands places cache points for +known model IDs) and `'anthropic'` when `modelId` is an ARN application +inference profile that `'auto'` can't detect. Ignored by the `openai-api` +and `canned` providers. diff --git a/packages/bb-agent/API.md b/packages/bb-agent/API.md index 0b84559f..4c2c2c92 100644 --- a/packages/bb-agent/API.md +++ b/packages/bb-agent/API.md @@ -198,6 +198,8 @@ export interface Message { // @public (undocumented) export interface ModelConfig { apiKey?: string | (() => Promise); + // Warning: (ae-forgotten-export) The symbol "CacheConfig" needs to be exported by the entry point index.aws.d.ts + cacheConfig?: CacheConfig; // (undocumented) endpoint?: string; // Warning: (ae-forgotten-export) The symbol "GuardrailsConfig" needs to be exported by the entry point index.aws.d.ts diff --git a/packages/bb-agent/README.md b/packages/bb-agent/README.md index ff859fa7..2ac3d637 100644 --- a/packages/bb-agent/README.md +++ b/packages/bb-agent/README.md @@ -148,6 +148,7 @@ Model configuration is optional. When omitted, the agent defaults to `BedrockMod | `endpoint` | `string` | API endpoint. For openai-api (defaults to api.openai.com). | | `apiKey` | `string \| () => Promise` | API key for openai-api. Accepts a string or async resolver. Falls back to `OPENAI_API_KEY` env var. | | `inferenceConfig` | `{ temperature?, topP?, maxTokens?, stopSequences? }` | Optional inference parameters. | +| `cacheConfig` | `{ strategy: 'auto' \| 'anthropic' }` | Optional prompt caching (bedrock only). See [Prompt Caching](#prompt-caching). | ```typescript import { Agent } from '@aws-blocks/bb-agent'; @@ -217,6 +218,21 @@ Override inference settings with spread: model: { deployed: { ...BedrockModels.BALANCED, inferenceConfig: { temperature: 0.9, maxTokens: 8192 } } } ``` +#### Prompt Caching + +Prompt caching reuses the cached request prefix (tools + system prompt + prior turns) across requests, cutting input-token cost and latency — especially for agents with long system prompts, many tools, or multi-turn conversations. It's off by default and applies only to the `bedrock` provider (ignored by `openai-api` and `canned`). + +```typescript +model: { deployed: { ...BedrockModels.BALANCED, cacheConfig: { strategy: 'auto' } } } +``` + +| Strategy | When to use | +|----------|-------------| +| `'auto'` | Recommended default. Strands places cache points automatically for known Bedrock model IDs — including all [`BedrockModels`](#bedrock-presets) presets. | +| `'anthropic'` | Use when `modelId` is an ARN application inference profile, where `'auto'` cannot detect the underlying model and won't enable caching. Same performance, no extra permissions. | + +> Caching has per-model minimum token thresholds (e.g. ~1,024 tokens for Claude Sonnet) and cache entries expire after ~5 minutes of inactivity. See the [Strands caching docs](https://strandsagents.com/docs/user-guide/concepts/model-providers/amazon-bedrock/#caching). + #### Ollama Presets Convenience shortcuts for local development using [Ollama](https://ollama.com/). Requires Ollama installed and running (`ollama serve`), model pulled (`ollama pull `). Uses the default endpoint `http://localhost:11434/v1`. diff --git a/packages/bb-agent/src/index.test.ts b/packages/bb-agent/src/index.test.ts index 5fb0d7b7..0f1084af 100644 --- a/packages/bb-agent/src/index.test.ts +++ b/packages/bb-agent/src/index.test.ts @@ -1041,6 +1041,19 @@ describe('BedrockModels presets', () => { const model = await createStrandsModel(BedrockModels.BALANCED); assert.ok(model, 'should create a model instance'); }); + + test('cacheConfig flows through createStrandsModel to BedrockModel', async () => { + const model = await createStrandsModel({ ...BedrockModels.BALANCED, cacheConfig: { strategy: 'auto' } }); + assert.ok(model, 'should create a model instance'); + const config = (model as { getConfig(): { cacheConfig?: { strategy: string } } }).getConfig(); + assert.deepStrictEqual(config.cacheConfig, { strategy: 'auto' }, 'cacheConfig should reach the BedrockModel'); + }); + + test('omitting cacheConfig leaves caching off', async () => { + const model = await createStrandsModel(BedrockModels.BALANCED); + const config = (model as { getConfig(): { cacheConfig?: { strategy: string } } }).getConfig(); + assert.strictEqual(config.cacheConfig, undefined, 'caching should be off by default'); + }); }); describe('OllamaModels presets', () => { diff --git a/packages/bb-agent/src/model-factory.ts b/packages/bb-agent/src/model-factory.ts index 735774ad..278fb9c4 100644 --- a/packages/bb-agent/src/model-factory.ts +++ b/packages/bb-agent/src/model-factory.ts @@ -178,6 +178,7 @@ export async function createStrandsModel(config?: ModelConfig, log?: ChildLogger maxTokens: config.inferenceConfig.maxTokens, stopSequences: config.inferenceConfig.stopSequences, }), + ...(config.cacheConfig && { cacheConfig: config.cacheConfig }), }); } diff --git a/packages/bb-agent/src/types.ts b/packages/bb-agent/src/types.ts index 464a742e..53cad0f0 100644 --- a/packages/bb-agent/src/types.ts +++ b/packages/bb-agent/src/types.ts @@ -21,6 +21,30 @@ export interface ModelConfig { apiKey?: string | (() => Promise); inferenceConfig?: InferenceConfig; guardrails?: GuardrailsConfig; + /** + * Prompt caching for the `bedrock` provider. Off by default. Caching reuses the + * cached prefix (tools + system prompt + prior turns) across requests, cutting + * input-token cost and latency on multi-turn or long-system-prompt agents. + * Ignored by the `openai-api` and `canned` providers. + * + * - `{ strategy: 'auto' }` — let Strands place cache points at optimal positions + * (after tools, after the last user message) for known Bedrock model IDs. This is + * the recommended default for the {@link BedrockModels} presets. + * - `{ strategy: 'anthropic' }` — force-enable Anthropic-style caching. Use this + * when `modelId` is an ARN application inference profile, where `'auto'` cannot + * detect the underlying model and therefore won't enable caching. + * + * Caching has per-model minimum token thresholds (e.g. ~1,024 for Claude Sonnet) + * and cache entries expire after ~5 minutes of inactivity. + * + * @see https://strandsagents.com/docs/user-guide/concepts/model-providers/amazon-bedrock/#caching + */ + cacheConfig?: CacheConfig; +} + +/** Prompt-caching strategy for the `bedrock` provider. @see {@link ModelConfig.cacheConfig} */ +export interface CacheConfig { + strategy: 'auto' | 'anthropic'; } export interface InferenceConfig {