Speed scores passing tasks only, CI, MCP version

.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - pkg: cli
+            run: npm run typecheck && npm test
+          - pkg: mcp
+            run: npm run typecheck && npm run build
+          - pkg: backend
+            run: npm run typecheck
+          - pkg: web
+            run: npm run lint && npx tsc --noEmit
+    name: ${{ matrix.pkg }}
+    defaults:
+      run:
+        working-directory: ${{ matrix.pkg }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: npm
+          cache-dependency-path: ${{ matrix.pkg }}/package-lock.json
+      - run: npm ci
+      - run: ${{ matrix.run }}

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to PipelineScore will be documented here. Format roughly follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) + [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.2] — 2026-09-23
+
+### Fixed
+- **Speed only counts correct answers.** Throughput was measured on every task, so a model that answered
+  instantly and got everything wrong still scored Speed 100. Speed is now the median tokens/sec across
+  *passing* tasks only; with fewer than 3 passing tasks it is unscored and drops out of the composite.
+  For strong models the change is small (their speed was already measured mostly on passing tasks), but
+  speed scores from CLI versions before 0.4.2 are not strictly comparable. Every submission records its
+  `cli_version`.
+- `@pipelinescore/mcp` 0.1.2 reports its real version to MCP clients (it was hardcoded to `0.1.0`).
+
+### Added
+- CI on every push and pull request: typecheck + tests for the CLI, typecheck/build for the MCP server
+  and backend, lint + typecheck for the site.
+
 ## [0.4.1] — 2026-07-27
 
 ### Fixed

cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@pipelinescore/cli",
-  "version": "0.4.1",
-  "description": "PipelineScore CLI \u2014 benchmark LLMs on your own hardware. Zero-config: `npx @pipelinescore/cli` auto-detects Ollama / LM Studio / llama.cpp / MLX and walks you through a run. Fully deterministic suite (no API key required), scored locally, published to the hardware-aware public leaderboard at pipelinescore.ai.",
+  "version": "0.4.2",
+  "description": "PipelineScore CLI — benchmark LLMs on your own hardware. Zero-config: `npx @pipelinescore/cli` auto-detects Ollama / LM Studio / llama.cpp / MLX and walks you through a run. Fully deterministic suite (no API key required), scored locally, published to the hardware-aware public leaderboard at pipelinescore.ai.",
   "type": "module",
   "main": "./dist/index.js",
   "bin": {

cli/src/score.ts

@@ -74,13 +74,14 @@ export function categoryScore(points: TaskPoint[]): CategoryScore & { se: number
   };
 }
 
-// Throughput speed: tokens/sec, length-independent. Unscored when too few task
-// calls report token counts.
+// Throughput speed: tokens/sec, length-independent. Only passing tasks count,
+// so a model that answers fast but wrong earns no speed. Unscored when too few
+// passing task calls report token counts.
 export function speedScore(results: TaskResult[], taxonomy: Taxonomy): SpeedDetail {
   const target = taxonomy.speed?.tps_target ?? 100;
   const minSamples = taxonomy.speed?.min_samples ?? 3;
   const tps = results
-    .filter((r) => (r.tokens_out ?? 0) > 0 && r.latency_ms > 0)
+    .filter((r) => r.passed && (r.tokens_out ?? 0) > 0 && r.latency_ms > 0)
     .map((r) => (r.tokens_out as number) / (r.latency_ms / 1000));
   if (tps.length < minSamples) {
     return { scored: false, tps_p50: null, speed_score: null, samples: tps.length };

cli/test/score-v2.test.ts

@@ -73,6 +73,21 @@ check('speed is length-independent (rate)', spdLong.speed_score === spd.speed_sc
 const spdNone = speedScore([tr('code', 8), tr('code', 8)], taxonomy);
 check('speed unscored without token data', spdNone.scored === false && spdNone.speed_score === null);
 
+// 7b. fast but wrong earns no speed: failed tasks don't count toward throughput
+const spdWrong = speedScore(
+  Array.from({ length: 5 }, () => tr('code', 0, { tokens_out: 100, latency_ms: 10 })),
+  taxonomy,
+);
+check('speed unscored when every task failed', spdWrong.scored === false && spdWrong.speed_score === null, JSON.stringify(spdWrong));
+const spdMixed = speedScore(
+  [
+    ...Array.from({ length: 3 }, () => tr('code', 8, { tokens_out: 100, latency_ms: 1000 })),
+    ...Array.from({ length: 5 }, () => tr('code', 0, { tokens_out: 100, latency_ms: 10 })),
+  ],
+  taxonomy,
+);
+check('only passing tasks set the speed', spdMixed.tps_p50 === 100 && spdMixed.samples === 3, JSON.stringify(spdMixed));
+
 // 8. per-profile composites diverge by use case
 const results: TaskResult[] = [
   ...Array.from({ length: 3 }, () => tr('code', 10, { tokens_out: 50, latency_ms: 1000 })),

mcp/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@pipelinescore/mcp",
-  "version": "0.1.1",
-  "description": "MCP server for the PipelineScore LLM benchmark \u2014 exposes run_benchmark / get_user_leaderboard / get_user_profile tools to any MCP-compatible client (Claude Code, Codex, Cursor, Continue, Cline). Lets your AI drive local-first LLM benchmarking on your hardware.",
+  "version": "0.1.2",
+  "description": "MCP server for the PipelineScore LLM benchmark — exposes run_benchmark / get_user_leaderboard / get_user_profile tools to any MCP-compatible client (Claude Code, Codex, Cursor, Continue, Cline). Lets your AI drive local-first LLM benchmarking on your hardware.",
   "type": "module",
   "main": "./dist/index.js",
   "bin": {
@@ -59,4 +59,4 @@
     "tsx": "^4.19.0",
     "typescript": "^5.6.0"
   }
-}
+}

mcp/src/index.ts

@@ -19,13 +19,14 @@ import {
   ListToolsRequestSchema,
 } from '@modelcontextprotocol/sdk/types.js';
 import { spawn } from 'node:child_process';
+import { readFileSync } from 'node:fs';
 
 const BACKEND = process.env.PIPELINESCORE_BACKEND ?? 'https://api.pipelinescore.ai';
 
 const server = new Server(
   {
     name: 'pipelinescore-mcp',
-    version: '0.1.0',
+    version: (JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) as { version: string }).version,
   },
   {
     capabilities: {

web/eslint.config.mjs

@@ -12,6 +12,9 @@ const eslintConfig = defineConfig([
     "out/**",
     "build/**",
     "next-env.d.ts",
+    // OpenNext / Wrangler build output (cf:build)
+    ".open-next/**",
+    ".wrangler/**",
   ]),
 ]);