From c7dacb6dbba41256053b5d0933e43088d6318ac3 Mon Sep 17 00:00:00 2001 From: bradAGI <46579244+bradAGI@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:08:41 -0400 Subject: [PATCH] test(rules): mirror and cover CSDK-021, CSDK-022 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine half of a coordinated pair with trustabl/trustabl-rules#90, on a branch of the same name so the rules-sync job resolves the matching pack rather than main. Neither half should merge alone — check-rules-sync.sh fails if they do. Mirrors claude_sdk/tool_definition.yaml into testdata/rules-fixture and adds cases to policyRuleCases, as TestPolicyRules_AllRulesCovered requires. CSDK-017/018 cover Python; the TypeScript half was missing. Five cases rather than four. CSDK-022 pairs description_length_lt with has_docstring so an empty description stays CSDK-014's finding instead of double-reporting, and the fifth case pins that guard. --- internal/rules/policies_test.go | 33 ++++++++++- .../claude_sdk/tool_definition.yaml | 58 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/internal/rules/policies_test.go b/internal/rules/policies_test.go index 65af6c0a..2dc012d5 100644 --- a/internal/rules/policies_test.go +++ b/internal/rules/policies_test.go @@ -1986,6 +1986,38 @@ def fetch_data(x: str) -> dict: return {} `, toolConfig: nil, wantFires: false}, + + // ─── CSDK-021 / CSDK-022: TS Claude SDK description quality ───────────── + { + name: "CSDK-021 fires on placeholder description", ruleID: "CSDK-021", + kind: models.KindClaudeSDKTool, lang: models.LanguageTypeScript, wantFires: true, + src: "import { tool } from \"@anthropic-ai/claude-agent-sdk\";\n" + + "export const t = tool(\"lookup_order\", \"TODO: describe this tool.\", {}, async () => ({ content: [] }));\n", + }, + { + name: "CSDK-021 silent on a real description", ruleID: "CSDK-021", + kind: models.KindClaudeSDKTool, lang: models.LanguageTypeScript, wantFires: false, + src: "import { tool } from \"@anthropic-ai/claude-agent-sdk\";\n" + + "export const t = tool(\"lookup_order\", \"Look up one order by its number and return its fulfillment status.\", {}, async () => ({ content: [] }));\n", + }, + { + name: "CSDK-022 fires on a too-short description", ruleID: "CSDK-022", + kind: models.KindClaudeSDKTool, lang: models.LanguageTypeScript, wantFires: true, + src: "import { tool } from \"@anthropic-ai/claude-agent-sdk\";\n" + + "export const t = tool(\"list_orders\", \"Gets data.\", {}, async () => ({ content: [] }));\n", + }, + { + name: "CSDK-022 silent on a full description", ruleID: "CSDK-022", + kind: models.KindClaudeSDKTool, lang: models.LanguageTypeScript, wantFires: false, + src: "import { tool } from \"@anthropic-ai/claude-agent-sdk\";\n" + + "export const t = tool(\"list_orders\", \"List every order belonging to one customer, most recent first.\", {}, async () => ({ content: [] }));\n", + }, + { + name: "CSDK-022 silent when the description is absent (CSDK-014's case)", ruleID: "CSDK-022", + kind: models.KindClaudeSDKTool, lang: models.LanguageTypeScript, wantFires: false, + src: "import { tool } from \"@anthropic-ai/claude-agent-sdk\";\n" + + "export const t = tool(\"list_orders\", \"\", {}, async () => ({ content: [] }));\n", + }, } // policyRepoRuleCases covers repo-scoped rules. @@ -2246,7 +2278,6 @@ var policyRepoRuleCases = []policyRepoCase{ }, models.RepoInventory{SDKsDetected: []models.SDK{models.SDKOpenAIAgents}}, false}, - } // optionsWithPermissionMode builds a ClaudeAgentOptionsDef whose captured diff --git a/testdata/rules-fixture/claude_sdk/tool_definition.yaml b/testdata/rules-fixture/claude_sdk/tool_definition.yaml index 766b5dc0..a2e52647 100644 --- a/testdata/rules-fixture/claude_sdk/tool_definition.yaml +++ b/testdata/rules-fixture/claude_sdk/tool_definition.yaml @@ -183,3 +183,61 @@ rules: Expand the description to at least a full sentence covering inputs, outputs, and the situation in which this tool should be used over alternatives. + + - id: CSDK-021 + title: TypeScript Claude SDK tool description is a placeholder + severity: low + confidence: 0.85 + language: typescript + applies_to: + - claude_sdk_tool + scope: tool + match: + has_description_text: + - todo + - tbd + - fixme + - placeholder + - no description + - does stuff + explanation: > + The tool sets a description, so it passes CSDK-014, but the string is a + placeholder rather than real content. That leaves the tool in exactly the + state CSDK-014 exists to prevent: the TypeScript SDK has no docstring + fallback, so this argument is the entire account of the tool in the + prompt, and "TODO: describe this tool" tells the model nothing the tool + name did not. Nothing reports it — the SDK does not warn, the Zod schema + still validates, and the only symptom is a tool the model calls at the + wrong moment or never reaches for at all. + fix: > + Replace the placeholder with a real description covering what the tool + does, what it returns, and when the model should call it rather than a + neighboring tool. The SDK passes the string to the model verbatim, so + write it for the model rather than a human maintainer. + + - id: CSDK-022 + title: TypeScript Claude SDK tool description is too short to guide model selection + severity: low + confidence: 0.8 + language: typescript + applies_to: + - claude_sdk_tool + scope: tool + match: + all: + - has_docstring: true + - description_length_lt: 40 + explanation: > + A description under 40 characters is rarely enough to convey what a tool + does, what it returns, and when to call it rather than a similarly named + neighbor. With no docstring fallback in the TypeScript SDK, a stub like + "Gets data." is the whole prompt-side account of the tool, so scope and + preconditions are left to guesswork. The Zod input schema does not close + the gap: it constrains the shape of the arguments once the model has + decided to call this tool, and says nothing about whether calling it was + the right move. + fix: > + Expand the description to at least a full sentence covering inputs, + outputs, and the situation in which this tool should be used over the + alternatives. Where two tools are easy to confuse, say in each which one + the other case belongs to.