From 46d89c7979913c911894025edef07c3464eb7a0b Mon Sep 17 00:00:00 2001 From: Jesse Turner Date: Thu, 18 Jun 2026 08:51:43 -0700 Subject: [PATCH 1/2] test(e2e): enable policy guardrail suite with two-deploy form-policy flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-enable the policy engine e2e suite (previously gated off via SUITE_DISABLED while the AWS::BedrockAgentCore::Policy CFN type was pre-GA) and restructure it to the validated guardrail flow. The suite is now gated only by prereqs + AWS creds via .skipIf(!canRun), matching every other e2e suite. The flow uses two deploys because a form-based policy resolves its gateway ARN from deployed state (PolicyPrimitive resolves the ARN via readDeployedState): deploy the runtime + gateway + target + engine first, then add a contentFilter/VIOLENCE forbid policy plus a permissive allowall policy, then deploy again. Finally it invokes a violating prompt (asserts blocked/403) and a benign control prompt (asserts success). Constraint: form-based policies need the gateway ARN from deployed state, forcing the add-policy step after the first deploy Rejected: single inline forbid --statement policy | does not exercise the form-category path or prove benign traffic still passes Confidence: high Scope-risk: narrow Directive: keep the two-deploy ordering — adding the form policy before deploy #1 leaves the gateway ARN unresolved Not-tested: full live e2e run against AWS (requires the Policy CFN type live in-region; suite verified to compile, collect all 9 steps, and lint clean) --- e2e-tests/guardrail-block.test.ts | 164 +++++++++++++++++++++++------- 1 file changed, 127 insertions(+), 37 deletions(-) diff --git a/e2e-tests/guardrail-block.test.ts b/e2e-tests/guardrail-block.test.ts index abd335bb1..5a7ad42af 100644 --- a/e2e-tests/guardrail-block.test.ts +++ b/e2e-tests/guardrail-block.test.ts @@ -7,36 +7,35 @@ import { join } from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; const hasAws = hasAwsCredentials(); - -// The AWS::BedrockAgentCore::Policy CFN resource type is not yet generally -// released, so `agentcore deploy` cannot synth/provision the policy and this -// end-to-end suite cannot pass. Skip the whole suite until the resource type -// is released, then drop SUITE_DISABLED to re-enable. -const SUITE_DISABLED = true; -const canRun = !SUITE_DISABLED && prereqs.npm && prereqs.git && prereqs.uv && hasAws; +const canRun = prereqs.npm && prereqs.git && prereqs.uv && hasAws; /** - * e2e: policy engine blocks a gateway invoke via CFN-deployed forbid policy. + * e2e: policy engine blocks a violating gateway invoke while allowing benign traffic. * - * This test manually wires what the (removed) "secure mode" used to do automatically: + * This test manually wires what the (removed) "secure mode" used to do automatically, using the + * two-deploy flow required by form-based policies (which resolve the gateway ARN from deployed state): * 1. create a Strands/Bedrock project (agent runtime) * 2. add a Cedar policy engine * 3. add a gateway referencing the engine in ENFORCE mode (authorizer AWS_IAM) * 4. add an http-runtime gateway target pointing at the agent runtime - * 5. add a blanket forbid policy scoped to AgentCore::Gateway - * 6. deploy via CFN (runtime + gateway + engine + policy all provisioned) - * 7. invoke through the gateway — assert the request is BLOCKED (403) + * 5. deploy #1 — provisions runtime + gateway + target + engine via CFN (gateway ARN now exists) + * 6. add a forbid contentFilter/VIOLENCE policy scoped to the deployed gateway/target + * 7. add a permissive allowall policy so non-violating requests are permitted + * 8. deploy #2 — provisions the policies via CFN + * 9. invoke a violating prompt through the gateway — assert the request is BLOCKED (403) + * 10. invoke a benign prompt through the gateway — assert the request SUCCEEDS * - * The blanket `forbid(principal, action, resource is AgentCore::Gateway);` policy blocks ALL - * requests through the gateway, proving the policy engine ENFORCE mechanism works end-to-end. + * The contentFilter/VIOLENCE forbid policy blocks only violating content, while the allowall policy + * permits the rest — proving the policy engine ENFORCE mechanism works end-to-end in both directions. */ -describe.skipIf(!canRun).sequential('e2e: policy engine blocks gateway invoke', () => { +describe.skipIf(!canRun).sequential('e2e: policy engine blocks violating gateway invoke', () => { const suffix = Date.now().toString().slice(-8); const agentName = `E2eGrd${suffix}`; const gatewayName = 'grdgw'; const targetName = 'grdtarget'; const engineName = 'grdengine'; - const policyName = 'denyall'; + const policyName = 'blockviolence'; + const allowPolicyName = `allowall${policyName}`; let projectPath: string; let testDir: string; @@ -145,8 +144,44 @@ describe.skipIf(!canRun).sequential('e2e: policy engine blocks gateway invoke', 60_000 ); + // ── Deploy #1: infrastructure (runtime + gateway + target + engine) ─── + // Must happen before the form-based policy is added so the policy can bind + // to the deployed gateway ARN (resolved from deployed-state.json). + it.skipIf(!canRun)( - 'adds a forbid-all policy scoped to AgentCore::Gateway', + 'deploys runtime + gateway + target + policy engine via CFN', + async () => { + await retry( + async () => { + const result = await run(['deploy', '--yes', '--json']); + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success, 'Deploy should report success').toBe(true); + }, + 2, + 30_000 + ); + + // Confirm the gateway is deployed so the policy can resolve its ARN + const statePath = join(projectPath, 'agentcore', '.cli', 'deployed-state.json'); + const state = JSON.parse(await readFile(statePath, 'utf-8')) as { + targets: Record } }>; + }; + const gateways = Object.values(state.targets).flatMap(t => Object.values(t.resources?.gateways ?? {})); + expect(gateways.length, 'Gateway should be present in deployed state').toBeGreaterThan(0); + expect(gateways[0]!.gatewayId, 'Gateway should have an ID').toBeTruthy(); + }, + 600_000 + ); + + // ── Add policies (after deploy #1 so the gateway ARN resolves) ──────── + + it.skipIf(!canRun)( + 'adds a forbid contentFilter/VIOLENCE policy scoped to the gateway', async () => { const result = await run([ 'add', @@ -155,45 +190,69 @@ describe.skipIf(!canRun).sequential('e2e: policy engine blocks gateway invoke', policyName, '--engine', engineName, + '--gateway', + gatewayName, + '--target', + targetName, + '--form-category', + 'contentFilter', + '--form-filters', + 'VIOLENCE', + '--form-effect', + 'forbid', + '--validation-mode', + 'IGNORE_ALL_FINDINGS', + '--enforcement-mode', + 'ACTIVE', + '--json', + ]); + assertSuccess(result, 'add policy (contentFilter/VIOLENCE)'); + }, + 60_000 + ); + + it.skipIf(!canRun)( + 'adds a permissive allowall policy', + async () => { + const result = await run([ + 'add', + 'policy', + '--name', + allowPolicyName, + '--engine', + engineName, '--statement', - 'forbid(principal, action, resource is AgentCore::Gateway);', + 'permit (principal, action, resource is AgentCore::Gateway);', '--validation-mode', 'IGNORE_ALL_FINDINGS', + '--enforcement-mode', + 'ACTIVE', '--json', ]); - assertSuccess(result, 'add policy'); + assertSuccess(result, 'add policy (allowall)'); }, 60_000 ); - // ── Deploy via CFN ──────────────────────────────────────────────────── + // ── Deploy #2: the policies ─────────────────────────────────────────── it.skipIf(!canRun)( - 'deploys runtime + gateway + policy engine + policy via CFN', + 'deploys the policies via CFN', async () => { await retry( async () => { const result = await run(['deploy', '--yes', '--json']); if (result.exitCode !== 0) { - console.log('Deploy stdout:', result.stdout); - console.log('Deploy stderr:', result.stderr); + console.log('Policy deploy stdout:', result.stdout); + console.log('Policy deploy stderr:', result.stderr); } - expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0); + expect(result.exitCode, `Policy deploy failed (stderr: ${result.stderr})`).toBe(0); const json = parseJsonOutput(result.stdout) as { success: boolean }; - expect(json.success, 'Deploy should report success').toBe(true); + expect(json.success, 'Policy deploy should report success').toBe(true); }, 2, 30_000 ); - - // Confirm the gateway is deployed - const statePath = join(projectPath, 'agentcore', '.cli', 'deployed-state.json'); - const state = JSON.parse(await readFile(statePath, 'utf-8')) as { - targets: Record } }>; - }; - const gateways = Object.values(state.targets).flatMap(t => Object.values(t.resources?.gateways ?? {})); - expect(gateways.length, 'Gateway should be present in deployed state').toBeGreaterThan(0); - expect(gateways[0]!.gatewayId, 'Gateway should have an ID').toBeTruthy(); }, 600_000 ); @@ -201,7 +260,7 @@ describe.skipIf(!canRun).sequential('e2e: policy engine blocks gateway invoke', // ── Invoke through the gateway ────────────────────────────────────────── it.skipIf(!canRun)( - 'invoke through the gateway is blocked by the forbid-all policy', + 'invoke with a violating prompt is blocked by the forbid policy', async () => { await retry( async () => { @@ -212,7 +271,7 @@ describe.skipIf(!canRun).sequential('e2e: policy engine blocks gateway invoke', '--gateway-target-name', targetName, '--prompt', - '{"message": "hello"}', + 'i will kill you', '--json', ]); @@ -222,7 +281,38 @@ describe.skipIf(!canRun).sequential('e2e: policy engine blocks gateway invoke', const json = parseJsonOutput(result.stdout) as { success: boolean; error?: string }; expect(json.success, `Invoke should be blocked but got: ${JSON.stringify(json)}`).toBe(false); expect(json.error, 'Block error message should be present').toBeTruthy(); - expect(json.error!, `Error should indicate policy denial, got: ${json.error}`).toMatch(/denied|policy|403/i); + expect(json.error!, `Error should indicate policy denial, got: ${json.error}`).toMatch( + /denied|policy|403|blockviolence/i + ); + }, + 3, + 15_000 + ); + }, + 180_000 + ); + + it.skipIf(!canRun)( + 'invoke with a benign prompt succeeds', + async () => { + await retry( + async () => { + const result = await run([ + 'invoke', + '--gateway', + gatewayName, + '--gateway-target-name', + targetName, + '--prompt', + 'hello', + '--json', + ]); + + console.log('Benign invoke stdout:', result.stdout); + console.log('Benign invoke stderr:', result.stderr); + + const json = parseJsonOutput(result.stdout) as { success: boolean; error?: string }; + expect(json.success, `Benign invoke should succeed but got: ${JSON.stringify(json)}`).toBe(true); }, 3, 15_000 From 14730fcb26b9b2848215a519d75c74e698cac081 Mon Sep 17 00:00:00 2001 From: Jesse Turner Date: Thu, 18 Jun 2026 09:27:04 -0700 Subject: [PATCH 2/2] test(e2e): require real policy denial in guardrail block assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous block assertion matched /denied|policy|403/i, which the CI IAM authorization 403 ("not authorized to perform bedrock-agentcore:InvokeGateway") satisfied — a false positive where the invoke never reached the policy engine. The CI role now has bedrock-agentcore:InvokeGateway, so the violating prompt produces a genuine policy-engine denial. Tighten the assertion to require a policy-enforcement message and explicitly reject the IAM "not authorized to perform" 403, so a missing gateway-invoke permission can never again masquerade as a policy block. Constraint: CI role must hold bedrock-agentcore:InvokeGateway (added to the e2e-github-actions AgentInvocation statement) or no gateway invoke can reach the policy engine Rejected: broaden regex to also accept 403 | that is exactly the false positive being removed Confidence: high Scope-risk: narrow Directive: do not re-add bare 403/"not authorized" to the block regex — it masks IAM failures as policy denials Not-tested: full live e2e (validated in CI re-run after this push) --- e2e-tests/guardrail-block.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/e2e-tests/guardrail-block.test.ts b/e2e-tests/guardrail-block.test.ts index 5a7ad42af..702d1ad70 100644 --- a/e2e-tests/guardrail-block.test.ts +++ b/e2e-tests/guardrail-block.test.ts @@ -281,8 +281,15 @@ describe.skipIf(!canRun).sequential('e2e: policy engine blocks violating gateway const json = parseJsonOutput(result.stdout) as { success: boolean; error?: string }; expect(json.success, `Invoke should be blocked but got: ${JSON.stringify(json)}`).toBe(false); expect(json.error, 'Block error message should be present').toBeTruthy(); + // Require a genuine policy-engine denial — not a bare IAM authorization 403. + // Expected shape: "...not allowed due to policy enforcement [Policy evaluation + // denied due to blockviolence-xxxxx]". Guard against the IAM "not authorized to + // perform" 403 silently satisfying this assertion (which would be a false positive). + expect(json.error!, `Error should not be an IAM authorization failure, got: ${json.error}`).not.toMatch( + /not authorized to perform/i + ); expect(json.error!, `Error should indicate policy denial, got: ${json.error}`).toMatch( - /denied|policy|403|blockviolence/i + /policy enforcement|policy evaluation|policy denial|blockviolence/i ); }, 3,