feat(evals-core): redact secret values before traces reach an LLM

docs/ADDING_EVALS.md

@@ -120,6 +120,8 @@ The event-based primitives (`ranCommand`, `ranCommandOneOf`, `wroteFile`) inspec
 
 The optional `expected` argument on `wroteFile` is useful when a file is excluded from the LLM judge's view (e.g. `.env` / `.env.local`) but you still need to verify the agent wrote the expected variables into it. Because it concatenates content across all writes to the path, it tolerates agents that build the file incrementally.
 
+**A security judge that reads the command trace must know about the redaction marker.** The harness masks credential values as `[REDACTED SECRET]` before the trace reaches any model, so a judge asked "does an actual secret appear?" would answer no on a run that leaked one. Say in the prompt that the marker means a secret was on that command line, as the B2B org eval does.
+
 ---
 
 ### Grader Levels (L1–L5)

packages/evals-core/src/graders/executors/llm-judge.ts

@@ -8,6 +8,7 @@ import type { GraderDef, GraderResult, EventToolCall } from '@a0/evals-graders';
 import type { GraderContext, GraderExecutor } from './types.js';
 import { llmJudge } from '../llm-judge.js';
 import { logger } from '../../utils/logger.js';
+import { redactSecrets } from '../../utils/redact.js';
 
 /**
  * File patterns (matched against the basename) excluded from the LLM judge input.
@@ -58,14 +59,30 @@ const RUN_COMMAND_NAMES = new Set(['run_command', 'bash']);
  * only when a judge opts in via `includeCommandTrace` — for evals whose artifact
  * is CLI invocations (no files to inspect). Errored calls are dropped so the
  * judge sees the commands that actually took effect.
+ *
+ * Credential values are masked before the trace leaves the machine, and the marker
+ * left behind is what a security judge reads: it says a secret occupied that
+ * position without sending the value to the proxy.
+ *
+ * The header spells out two properties of this format, because a judge that has to
+ * infer them gets them wrong: every listed command exited 0, and command output is
+ * not shown. A judge asked to confirm an end state read a later command re-declaring
+ * an id literal as evidence the agent had fabricated it, when it was the id the
+ * previous command printed, carried across a shell boundary.
  */
 export function formatCommandTrace(toolCalls: EventToolCall[]): string {
   const commands = toolCalls
     .filter((tc) => RUN_COMMAND_NAMES.has(tc.name) && !tc.causedError)
-    .map((tc) => String(tc.args.command ?? '').trim())
+    .map((tc) => redactSecrets(String(tc.args.command ?? '').trim()))
     .filter((cmd) => cmd.length > 0);
   if (commands.length === 0) return '';
-  return `// COMMAND TRACE (shell commands the agent ran)\n${commands.join('\n')}`;
+  const header =
+    '// COMMAND TRACE (shell commands the agent ran). Every command listed here exited\n' +
+    '// successfully — failed commands are omitted. Their output is NOT captured, so the\n' +
+    '// absence of output is not evidence a command did nothing. Each command runs in its\n' +
+    '// own shell, so an id assigned as a literal in a later command is a value read from\n' +
+    "// an earlier command's output, not a fabricated one.";
+  return `${header}\n${commands.join('\n')}`;
 }
 
 export const llmJudgeExecutor: GraderExecutor = {

packages/evals-core/src/index.ts

@@ -51,6 +51,9 @@ export type { Logger } from './utils/logger.js';
 export { withRetry, isTransientLlmError } from './utils/retry.js';
 export type { RetryOptions } from './utils/retry.js';
 
+// Redaction
+export { redactSecrets, REDACTION_MARKER } from './utils/redact.js';
+
 // Costs
 export { estimateCost } from './config/costs.js';
 

packages/evals-core/src/utils/redact.ts

@@ -0,0 +1,75 @@
+/**
+ * Secret redaction for agent output that leaves the machine.
+ *
+ * `.env` files are already withheld from the judge and the recommendation analyst,
+ * but for a CLI eval the credentials are not in a file — they are on the command
+ * line (`--client-secret …`, `export AUTH0_CLIENT_SECRET=…`) and in the error body
+ * a failed `auth0 api` call prints back. Any trace we send to an LLM has to pass
+ * through here first.
+ *
+ * The value is replaced, not the surrounding text: a reader still sees *that* a
+ * secret was passed, in which flag, on which command. That matters because the
+ * security graders judge exposure from the same trace — dropping the line entirely
+ * would make "is the trace free of secrets?" pass vacuously, so `REDACTION_MARKER`
+ * is deliberately conspicuous and is documented to those judges as evidence that a
+ * secret occupied that position.
+ *
+ * This is name-driven (plus a shape rule for JWTs and long opaque tokens), so it
+ * cannot catch a secret echoed with no surrounding context — e.g. a bare
+ * `echo <32-char-value>`. It is a floor, not a guarantee.
+ */
+
+/** Stand-in for a removed secret value. Conspicuous on purpose — see the module note. */
+export const REDACTION_MARKER = '[REDACTED SECRET]';
+
+/** Name fragments that mark a flag, env var, or JSON key as holding a credential. */
+const SECRET_NAME = 'secret|token|password|passwd|api[_-]?key|apikey|private[_-]?key|credential|signing[_-]?key';
+
+/**
+ * A quoted or bare value. Skips a value that is already the marker (so a second
+ * pattern cannot redact the first pattern's output) and one that is the next flag
+ * (`--token --json` passes no secret).
+ */
+const VALUE = `(?!\\[REDACTED|--)(?:"[^"]*"|'[^']*'|[^\\s,;&|)}\\]]+)`;
+
+const PATTERNS: Array<[RegExp, string]> = [
+  // `--client-secret VALUE`, `--client-secret=VALUE`, `--token VALUE`
+  [new RegExp(`(--[\\w-]*(?:${SECRET_NAME})[\\w-]*)([=\\s]+)(?:${VALUE})`, 'gi'), `$1$2${REDACTION_MARKER}`],
+  // `AUTH0_CLIENT_SECRET=VALUE`, `"client_secret": "VALUE"`, `clientSecret: VALUE`.
+  // The credential word has to END the name, so `token_endpoint_auth_method: none`
+  // and `expires_in` keep their values — they are configuration, not credentials,
+  // and blanking them costs the analyst detail for nothing.
+  [new RegExp(`(["']?[\\w.-]*(?:${SECRET_NAME})["']?\\s*[:=]\\s*)(?:${VALUE})`, 'gi'), `$1${REDACTION_MARKER}`],
+  // `Authorization: Bearer VALUE`, `Authorization: Basic VALUE`. The header name is
+  // required: a bare `Basic` is also the value of Auth0's `--auth-method` flag
+  // (`token_endpoint_auth_method`), and matching the scheme alone masked the flag
+  // that followed it. A bearer token with no header around it is still caught by the
+  // JWT and long-opaque-token rules below.
+  // The value runs to the next whitespace or quote, so a token with an unusual
+  // character (`%`, `!`, `#`) is masked whole rather than leaking its suffix, while
+  // the closing quote of `-H "Authorization: Bearer …"` is left in place.
+  [/\b((?:proxy-)?authorization\s*:\s*)(Bearer|Basic)(\s+)(?!--)[^\s"']+/gi, `$1$2$3${REDACTION_MARKER}`],
+  // `curl -u user:VALUE` in every form curl accepts: `-u user:v`, attached
+  // `-uuser:v`, `--user user:v`, and `--user=user:v`.
+  [/((?:-u(?=\S)|-u\s+|--user(?:=|\s+))["']?[^\s:"']+:)[^\s"']+/g, `$1${REDACTION_MARKER}`],
+  // A JWT, wherever it appears.
+  [/\beyJ[\w-]{8,}\.[\w-]{8,}\.[\w-]+/g, REDACTION_MARKER],
+  // A long opaque token with no name attached. Auth0 client secrets are 64 chars
+  // of URL-safe base64; the 40-char floor keeps client_ids (32 hex) and resource
+  // ids (`org_`, `cgr_`, `rol_` + 16-24 chars) readable, because those are not
+  // secrets and the analyst needs them to follow what the agent did.
+  [/(?<![\w-])[A-Za-z0-9_-]{40,}(?![\w-])/g, REDACTION_MARKER],
+];
+
+/**
+ * Replaces credential values in `text` with {@link REDACTION_MARKER}.
+ *
+ * Safe to call on anything: text with no credentials comes back unchanged.
+ */
+export function redactSecrets(text: string): string {
+  let out = text;
+  for (const [pattern, replacement] of PATTERNS) {
+    out = out.replace(pattern, replacement);
+  }
+  return out;
+}

packages/evals-core/tests/graders/executors.test.ts

@@ -408,6 +408,15 @@ describe('formatCommandTrace', () => {
     expect(out).toContain('guardian/policies');
   });
 
+  it('tells the judge what the format does and does not show', () => {
+    // A judge left to infer these read "no output shown" and "id re-declared in a later
+    // command" as evidence the agent fabricated the values, and failed a correct run.
+    const out = formatCommandTrace([cmd('auth0 orgs list')]);
+    expect(out).toContain('exited');
+    expect(out).toContain('output is NOT captured');
+    expect(out).toContain('own shell');
+  });
+
   it('accepts the bash tool name as a shell command', () => {
     const out = formatCommandTrace([
       { name: 'bash', args: { command: 'auth0 api get guardian/factors' }, result: '', causedError: false },
@@ -431,6 +440,18 @@ describe('formatCommandTrace', () => {
   it('returns an empty string when there are no commands', () => {
     expect(formatCommandTrace([])).toBe('');
   });
+
+  it('masks credential values, leaving the marker for a security judge to read', () => {
+    // The trace is sent to the judge model, so a secret on a command line would leave
+    // the machine. The marker stays in place of the value, so a security judge can
+    // still see that a secret occupied that position.
+    const out = formatCommandTrace([
+      cmd('auth0 api post clients --client-secret fixture_not_a_real_secret_abcdef0123456789'),
+    ]);
+    expect(out).not.toContain('fixture_not_a_real_secret_abcdef0123456789');
+    expect(out).toContain('[REDACTED SECRET]');
+    expect(out).toContain('auth0 api post clients');
+  });
 });
 
 // ── llmJudgeExecutor: includeCommandTrace gate ───────────────────────────────

packages/evals-core/tests/redact.test.ts

@@ -0,0 +1,132 @@
+/**
+ * Tests for the secret scrubber applied to anything sent to an LLM.
+ *
+ * Two properties matter and pull against each other: no credential value may
+ * survive, and everything a diagnosis needs (the command, the resource ids, the
+ * flag names) must survive. Both directions are asserted here.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { redactSecrets, REDACTION_MARKER } from '../src/utils/redact.js';
+
+describe('redactSecrets — masks credential values', () => {
+  it('masks a --client-secret flag value', () => {
+    const out = redactSecrets('auth0 api post clients --client-secret fixture_not_a_real_secret_9f8e7d6c5b4a');
+    expect(out).not.toContain('fixture_not_a_real_secret_9f8e7d6c5b4a');
+    expect(out).toContain('--client-secret');
+    expect(out).toContain(REDACTION_MARKER);
+  });
+
+  it('masks flag values written with =', () => {
+    expect(redactSecrets('--api-key=abcd1234efgh')).toBe(`--api-key=${REDACTION_MARKER}`);
+  });
+
+  it('masks quoted flag values', () => {
+    const out = redactSecrets('auth0 login --client-secret "quoted secret value"');
+    expect(out).not.toContain('quoted secret value');
+  });
+
+  it('masks key: value and key=value pairs in output bodies', () => {
+    const out = redactSecrets('{ "client_secret": "abc123xyz", "client_id": "aBcD1234" }');
+    expect(out).not.toContain('abc123xyz');
+    expect(out).toContain('client_id');
+  });
+
+  it('masks bearer and basic authorization headers', () => {
+    expect(redactSecrets('curl -H "Authorization: Bearer abc.def.ghi"')).not.toContain('abc.def.ghi');
+    expect(redactSecrets('Authorization: Basic dXNlcjpwYXNz')).not.toContain('dXNlcjpwYXNz');
+    expect(redactSecrets('proxy-authorization: Bearer abc.def.ghi')).toContain(REDACTION_MARKER);
+  });
+
+  it('masks an authorization value in full when it holds an unusual character', () => {
+    // Regression: the value match used to stop at characters like `%`, `!`, `#`, so a
+    // token containing one leaked its suffix.
+    expect(redactSecrets('Authorization: Bearer abc%secret!part#tail')).not.toContain('secret');
+    expect(redactSecrets('Authorization: Bearer abc%secret!part#tail')).toContain(REDACTION_MARKER);
+  });
+
+  it('leaves the closing quote of a quoted authorization header intact', () => {
+    const out = redactSecrets('curl -H "Authorization: Bearer abc%def"');
+    expect(out).not.toContain('abc%def');
+    expect(out).toBe(`curl -H "Authorization: Bearer ${REDACTION_MARKER}"`);
+  });
+
+  it('masks the password half of curl -u user:pass', () => {
+    const out = redactSecrets('curl -u admin:hunter2 https://example.com');
+    expect(out).not.toContain('hunter2');
+    expect(out).toContain('admin:');
+  });
+
+  it.each([
+    ['attached -u', 'curl -uadmin:hunter2 https://example.com'],
+    ['long --user with space', 'curl --user admin:hunter2 https://example.com'],
+    ['long --user with =', 'curl --user=admin:hunter2 https://example.com'],
+  ])('masks the password half of curl %s', (_label, cmd) => {
+    const out = redactSecrets(cmd);
+    expect(out).not.toContain('hunter2');
+    expect(out).toContain('admin:');
+  });
+
+  it('masks a JWT anywhere it appears', () => {
+    const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1g';
+    expect(redactSecrets(`export TOKEN=${jwt}`)).not.toContain(jwt);
+  });
+
+  it('masks a long opaque token with no surrounding key', () => {
+    const token = 'A'.repeat(48);
+    expect(redactSecrets(`echo ${token}`)).not.toContain(token);
+  });
+
+  it('never nests markers when several patterns match the same text', () => {
+    const out = redactSecrets('--client-secret fixture_not_a_real_secret_0123456789abcdefghijklmnopqrstuvwxyz01234567');
+    expect(out.match(/REDACTED SECRET/g)).toHaveLength(1);
+    expect(out).not.toContain('SECRET] SECRET]');
+  });
+
+  it('leaves text with no secrets untouched', () => {
+    const clean = 'auth0 orgs create --name acme --display "Acme Inc"';
+    expect(redactSecrets(clean)).toBe(clean);
+  });
+});
+
+describe('redactSecrets — keeps what a diagnosis needs', () => {
+  it('keeps client_ids readable', () => {
+    // 32-char hex: identifying, not secret, and the analyst needs it to follow the run.
+    const clientId = 'aB3dE5gH7jK9mN1pQ3sT5vW7yZ9bD1fH';
+    expect(redactSecrets(`--client-id ${clientId}`)).toContain(clientId);
+  });
+
+  it('keeps Auth0 resource ids readable', () => {
+    const line = 'auth0 api post organizations/org_1nvs2Q8RCZGjMN7L/invitations';
+    expect(redactSecrets(line)).toBe(line);
+  });
+
+  it('keeps a non-secret setting whose name merely contains a credential word', () => {
+    // `token_endpoint_auth_method` ends in `method`, not in a credential word, so its
+    // value is configuration and stays visible.
+    const line = '"token_endpoint_auth_method": "none"';
+    expect(redactSecrets(line)).toBe(line);
+  });
+
+  it('keeps `--auth-method Basic` intact', () => {
+    // Regression: `Basic` here is Auth0's `token_endpoint_auth_method`, not an HTTP
+    // auth scheme. Masking on the bare scheme swallowed the `--grants` flag after it,
+    // and the security judge reads the marker as proof a secret was exposed — so a
+    // false positive here costs a run its security score.
+    const line = "auth0 apps create --name 'Smoke Automation' --type m2m --auth-method Basic --grants credentials";
+    expect(redactSecrets(line)).toBe(line);
+  });
+
+  it('keeps a bare `Basic`/`Bearer` word with no authorization header around it', () => {
+    const line = 'echo "Basic auth is enabled"';
+    expect(redactSecrets(line)).toBe(line);
+  });
+
+  it('keeps the flag name when it masks the value', () => {
+    expect(redactSecrets('--client-secret abcdef123456')).toContain('--client-secret');
+  });
+
+  it('handles empty input', () => {
+    expect(redactSecrets('')).toBe('');
+  });
+});