FailproofAI
diff --git a/‎.github/workflows/ci.yml‎
Lines changed: 251 additions & 0 deletions b/‎.github/workflows/ci.yml‎
Lines changed: 251 additions & 0 deletions
diff --git a/‎.github/workflows/osv-scanner.yml‎
Lines changed: 2 additions & 1 deletion b/‎.github/workflows/osv-scanner.yml‎
Lines changed: 2 additions & 1 deletion
@@ -484,6 +484,257 @@ jobs:
           AGENTEYE_TESTS_REQUIRE_FRAMEWORKS: "1"
         run: uv run pytest tests/integrations -q
 
+  # The TypeScript telemetry SDK (`@failproofai/sdk`). Separate from `quality`
+  # and `test` because it is its own npm package with its own lockfile, its own
+  # tsconfig and its own vitest config — running it inside the root project's
+  # jobs would mean the root's dependency tree decided whether this package's
+  # zero-dependency claim holds.
+  #
+  # The matrix is Node's supported majors, not a single version. This package's
+  # floor is 20.9 and its `using` support, `AsyncLocalStorage.enterWith`,
+  # `worker_threads` resource limits and `Symbol.dispose` shim all behave
+  # differently across that range — which is precisely the range a customer's
+  # agent runs on.
+  failproofai-ts-sdk:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: sdk/typescript
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # The `engines` FLOOR. This leg proves the published artifact runs on
+          # it, and deliberately does not run the suite: vitest 5 pulls vite 8,
+          # which pulls rolldown, which imports `styleText` from `node:util` —
+          # added in Node 20.12. The TEST RUNNER's floor is not the PACKAGE's
+          # floor, and the honest way to say so is to keep the floor at 20.9 and
+          # prove it with the thing a consumer actually gets.
+          #
+          # Pinning the runner back to something 20.9 can load is the tail
+          # wagging the dog: it means carrying the CVEs vitest 5 fixed (one
+          # Critical, one High) so that a test runner can start on a Node
+          # release nobody runs the tests on.
+          - node-version: "20.9"
+            suite: false
+          - node-version: "20.x"
+            suite: true
+          - node-version: "22.x"
+            suite: true
+          - node-version: "24.x"
+            suite: true
+    steps:
+      - uses: actions/checkout@v7.0.1
+
+      - uses: actions/setup-node@v5
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: npm
+          cache-dependency-path: sdk/typescript/package-lock.json
+
+      - name: Install dependencies
+        uses: nick-fields/retry@v4
+        with:
+          max_attempts: 3
+          timeout_minutes: 5
+          command: cd sdk/typescript && npm ci --no-audit --no-fund
+
+      - name: Typecheck
+        if: matrix.suite
+        run: npm run typecheck
+
+      - name: Lint
+        if: matrix.suite
+        run: npm run lint
+
+      # Runs on EVERY leg, including the floor: `tsc` is the thing that produces
+      # what ships, so "it builds on 20.9" is a claim worth checking there.
+      - name: Build
+        run: npm run build
+
+      # The sandbox suite needs `dist/` present — the evaluator sandbox is a
+      # real `worker_threads` entry and cannot load a `.ts` file — and it
+      # asserts rather than skips when it is missing, so the build above is a
+      # prerequisite rather than a duplicate.
+      - name: Test
+        if: matrix.suite
+        run: npx vitest run
+
+      # Everything above ran against the source tree. These two steps run
+      # against the ARTIFACT, because the failures they catch — a missing export
+      # condition, a CommonJS build Node reads as ESM, a `dist/` path the files
+      # list does not ship — are invisible from inside the package and total
+      # from outside it.
+      - name: Pack
+        run: npm pack --pack-destination /tmp
+
+      - name: Smoke-test the packed tarball with no dependencies
+        run: |
+          mkdir -p /tmp/ts-sdk-smoke && cd /tmp/ts-sdk-smoke
+          npm init -y >/dev/null
+          # `--omit=optional --omit=peer` is the assertion, not an optimisation.
+          # "Zero dependencies" is the reason this package is safe to drop into
+          # someone else's agent, so prove it against the built artifact: install
+          # it with nothing else present, emit real events, and read them back
+          # off disk.
+          npm install --no-audit --no-fund --omit=optional --omit=peer /tmp/failproofai-sdk-*.tgz
+          test ! -d node_modules/@failproofai/sdk/node_modules \
+            || { echo "the published package brought transitive dependencies"; exit 1; }
+
+          cat > esm.mjs <<'EOF'
+          import * as fp from "@failproofai/sdk";
+          fp.configure({ baseDir: process.env.SPOOL });
+          await fp.agent("smoke", { goal: "ci" }, async () => {
+            await fp.toolCall("t", { input: { q: 1 } }, () => "ok");
+          });
+          await fp.flush();
+          console.log(fp.version);
+          EOF
+          SPOOL=/tmp/ts-sdk-spool node esm.mjs
+
+          cat > cjs.cjs <<'EOF'
+          const fp = require("@failproofai/sdk");
+          fp.configure({ baseDir: process.env.SPOOL });
+          fp.event.agentStart({ sessionId: "cjs", goal: "ci" });
+          fp.flushSync();
+          console.log(fp.version);
+          EOF
+          SPOOL=/tmp/ts-sdk-spool node cjs.cjs
+
+          # The evaluator loads from its own subpath, and its CLI has to be
+          # executable — a `bin` that is not fails on every platform where the
+          # installer links rather than copies.
+          node -e "const e = require('@failproofai/sdk/evaluator'); if (typeof e.Evaluator !== 'function') throw new Error('evaluator subpath is broken')"
+          npx --no-install failproofai-evaluator --help > /dev/null
+
+          node - <<'EOF'
+          const { readdirSync, readFileSync } = require("node:fs");
+          const { join } = require("node:path");
+          const dir = "/tmp/ts-sdk-spool/events";
+          const events = readdirSync(dir)
+            .filter((f) => f.endsWith(".jsonl"))
+            .flatMap((f) => readFileSync(join(dir, f), "utf8").split("\n").filter(Boolean))
+            .map((line) => JSON.parse(line));
+          const types = new Set(events.map((e) => e.type));
+          const want = ["agent_start", "agent_end", "tool_use", "tool_result"];
+          for (const type of want) {
+            if (!types.has(type)) throw new Error(`the installed artifact never wrote ${type}`);
+          }
+          // Emitted by the artifact, so this also proves the wire format
+          // survived packaging rather than only surviving an in-tree import.
+          if (!events.every((e) => "environment" in e && "session_id" in e)) {
+            throw new Error("an event reached disk missing a required field");
+          }
+          console.log(`${events.length} events written by the installed artifact`);
+          EOF
+
+      # The evaluator sandbox in an INSTALLED package resolves its worker
+      # through the package's own `./sandbox-worker` export, with no env
+      # override in sight. That resolution is the one part of the sandbox that
+      # cannot be exercised from inside this repository, and a failure in it
+      # means managed evaluations refuse to run for every customer.
+      - name: Verify the evaluator sandbox resolves from an installed package
+        run: |
+          cd /tmp/ts-sdk-smoke
+          node - <<'EOF'
+          const { compileEvaluator, sessionTranscriptFromWire } = require("@failproofai/sdk/evaluator");
+          const session = sessionTranscriptFromWire({
+            schema_version: "2",
+            assignment_id: "a", session_id: "s", session_revision_id: "r",
+            agent_id: "main", environment: "dev",
+            started_at: "2026-01-01T00:00:00.000000Z",
+            ended_at: "2026-01-01T00:01:00.000000Z",
+            event_count: 1,
+            events: [{ id: "1", ts: "2026-01-01T00:00:01.000000Z", event_type: "tool_use", payload: {} }],
+          });
+          compileEvaluator("EvalResult({ score: Score(session.count('tool_use') > 0 ? 1 : 0) })", { evalKey: "k" })(session)
+            .then((result) => {
+              if (result.score.value !== 1) throw new Error(`unexpected score ${result.score.value}`);
+              console.log("sandbox resolved and evaluated from the installed package");
+            })
+            .catch((error) => { console.error(error); process.exit(1); });
+          EOF
+
+  # The TypeScript SDK's framework adapters, against REAL framework releases —
+  # the counterpart of `failproofai-sdk-integrations` above. `failproofai-ts-sdk`
+  # runs the adapters with no framework installed, which proves their logic and
+  # nothing about whether it ever reaches a framework: the first release passed
+  # 242 unit tests with every adapter recording nothing in an ES-module app.
+  #
+  # Each fixture under `sdk/typescript/integration/fixtures/` is a consumer
+  # project with its own lockfile pinning one framework release. The packed
+  # tarball is extracted into each, and one agent is run as BOTH an ES module and
+  # CommonJS, because the two module systems load different copies of a
+  # dual-published framework. A fixture that fails to install fails the job —
+  # there is no skip path to read as green.
+  failproofai-ts-sdk-integrations:
+    name: failproofai-ts-sdk-integrations (${{ matrix.shard }}, node ${{ matrix.node-version }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    defaults:
+      run:
+        working-directory: sdk/typescript
+    strategy:
+      fail-fast: false
+      matrix:
+        # Sharded by what a shard needs installed, because the whole suite —
+        # four frameworks at both ends of their ranges, Bun and Deno parity over
+        # every fixture, and five real `next build`s — is ~23 CPU-minutes, too
+        # much for one runner inside a timeout. Each shard installs only its own
+        # fixtures (FAILPROOFAI_IT_FIXTURES) and runs only its own files.
+        #
+        # frameworks: Node's oldest and newest supported majors — the ESM/CJS
+        # split this job exists for behaves differently once `require(esm)` is
+        # unflagged. runtimes and nextjs: one Node, because what they vary is
+        # the runtime or the bundler, not Node.
+        include:
+          - shard: frameworks
+            node-version: "20.x"
+            fixtures: ai-4,ai-5,ai-6,ai-7,langchain-0.3,langchain-1,langchain-dup-core,mastra-0,mastra-1,llamaindex-0.11,llamaindex-0.12,types,vanilla
+            files: integration/ai.test.ts integration/langchain.test.ts integration/mastra.test.ts integration/mastra-coverage.test.ts integration/llamaindex.test.ts integration/types.test.ts integration/vanilla.test.ts
+          - shard: frameworks
+            node-version: "24.x"
+            fixtures: ai-4,ai-5,ai-6,ai-7,langchain-0.3,langchain-1,langchain-dup-core,mastra-0,mastra-1,llamaindex-0.11,llamaindex-0.12,types,vanilla
+            files: integration/ai.test.ts integration/langchain.test.ts integration/mastra.test.ts integration/mastra-coverage.test.ts integration/llamaindex.test.ts integration/types.test.ts integration/vanilla.test.ts
+          - shard: runtimes
+            node-version: "24.x"
+            fixtures: ai-4,ai-5,ai-6,ai-7,langchain-0.3,langchain-1,mastra-0,mastra-1,llamaindex-0.11,llamaindex-0.12,runtimes
+            files: integration/runtimes.core.test.ts integration/runtimes.bun.test.ts integration/runtimes.deno.test.ts
+          - shard: nextjs
+            node-version: "24.x"
+            fixtures: nextjs,langchain-1,ai-7,mastra-1,llamaindex-0.12
+            files: integration/nextjs.test.ts
+    steps:
+      - uses: actions/checkout@v7.0.1
+
+      - uses: actions/setup-node@v5
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: npm
+          cache-dependency-path: |
+            sdk/typescript/package-lock.json
+            sdk/typescript/integration/fixtures/*/package-lock.json
+
+      - name: Install dependencies
+        uses: nick-fields/retry@v4
+        with:
+          max_attempts: 3
+          timeout_minutes: 5
+          command: cd sdk/typescript && npm ci --no-audit --no-fund
+
+      # `test:integration` builds, packs, `npm ci`s this shard's fixtures and
+      # runs its files; the fixture installs are the network-heavy part, so
+      # retry them as a whole rather than failing the job on a registry blip.
+      - name: Test against real releases (${{ matrix.shard }})
+        uses: nick-fields/retry@v4
+        env:
+          FAILPROOFAI_IT_FIXTURES: ${{ matrix.fixtures }}
+        with:
+          max_attempts: 2
+          timeout_minutes: 25
+          command: cd sdk/typescript && npm run test:integration -- ${{ matrix.files }}
+
   test:
     runs-on: ubuntu-latest
     # The retry above nominally allows 3 attempts x 10 minutes. Capping the job
 
@@ -62,7 +62,7 @@ jobs:
       # nothing at all — not this job, which was only ever given `bun.lock`, and
       # not Dependabot, which had no `cargo` ecosystem — for a TLS stack that
       # compiles into a root-installed system service.
-      - name: Scan bun.lock and Cargo.lock for known-vulnerable / malicious dependencies
+      - name: Scan every lockfile for known-vulnerable / malicious dependencies
         id: scan
         uses: google/osv-scanner-action/osv-scanner-action@f4cfcc01edc9c8b756a9b873b7a623ca674da51e # v2.3.8
         with:
@@ -79,6 +79,7 @@ jobs:
             --lockfile=Cargo.lock
             --lockfile=fp-cloud-cli/uv.lock
             --lockfile=sdk/python/uv.lock
+            --lockfile=sdk/typescript/package-lock.json
       # Only the schedule run notifies — nothing on main touched the lockfile,
       # so nobody is watching it the way a PR author watches their own checks
       # or a push failure shows up against the commit they just merged. Same