From 4ab4f8f7c0aa99fd7ac8e4f8cabc9c7737ae01fd Mon Sep 17 00:00:00 2001 From: 1bcMax <195689928+1bcMax@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:43:29 -0500 Subject: [PATCH 1/2] fix(models): retire dead model ids and reconcile the picker against the gateway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The in-session /model picker rendered a hand-maintained array and never talked to the gateway, so it drifted on every upstream change. (`franklin models` was already live-fetching — two surfaces, one stale.) Dead ids, verified by probing the chat endpoint (402 = exists, 400 = unknown; the /api/v1/models listing is NOT authoritative — it hides working models like grok-3 and gpt-5-nano): - anthropic/claude-haiku-4.5-20251001 -> 400. Used in 9 places, incl. compact.ts, which selected it for every Sonnet/GPT-5.4/5.5/Gemini-Pro session — so long sessions compacted against a model that 400s. The gateway wants the undated id, which the repo already used elsewhere. - openai/gpt-5 -> 400 in the tool-use fallback chain; now gpt-5.4. Both ids stay in pricing.ts/tokens.ts: those are keyed by whatever id a past session recorded, and the file already keeps retired ids for that. nvidia/llama-4-maverick retired everywhere. It does not 410 — it answers 200 while the free pool silently serves something else (probed live: it and qwen3-next both came back as nemotron-3-super-120b-a12b-free). That made the free "diverse-family" fallback chains fake: primary and fallback resolved to the same backend, so the fallback could never rescue a turn the primary failed. Replaced with mistral-nemotron, which verifiably serves itself. Also dropped it from the vision allowlist, which contradicted routeRequest()'s own "maverick is text-only" comment — free vision turns were routed to a text-only model. Picker now reconciles against the live catalog via reconcilePicker(): rows the gateway stopped listing drop themselves, prices refresh, curated labels/order stay (gateway names overflow the tuned columns), and it falls back to the static list when offline. Adds Grok 4.5 — the gateway calls it xAI's flagship, so `grok` follows it per the bare-alias-tracks- flagship convention; 4.3 stays pinned (cheaper, 1M ctx vs 4.5's 500K). Verified: 559/559 tests (5 new reconciliation tests), picker driven end-to-end, offline fallback exercised, and all 20 models the picker now offers probed live — every one resolves. --- src/agent/compact.ts | 2 +- src/agent/context.ts | 2 +- src/agent/loop.ts | 4 +- src/agent/optimize.ts | 2 +- src/agent/tokens.ts | 2 + src/commands/init.ts | 2 +- src/learnings/extractor.ts | 2 +- src/plugin-sdk/workflow.ts | 4 +- src/pricing.ts | 5 +- src/router/index.ts | 24 +++-- src/router/vision.ts | 9 +- src/tools/moa.ts | 2 +- src/ui/app.tsx | 77 +++++++++++++--- src/ui/model-picker.ts | 180 +++++++++++++++++++++++++++++++------ test/free-model-matrix.mjs | 4 +- test/local.mjs | 128 ++++++++++++++++++++++---- test/model-resolver.mjs | 10 ++- 17 files changed, 376 insertions(+), 83 deletions(-) diff --git a/src/agent/compact.ts b/src/agent/compact.ts index fb3e64b..5cce0e8 100644 --- a/src/agent/compact.ts +++ b/src/agent/compact.ts @@ -552,7 +552,7 @@ function pickCompactionModel(primaryModel: string): string { return 'anthropic/claude-sonnet-4.6'; } if (primaryModel.includes('sonnet') || primaryModel.includes('gpt-5.4') || primaryModel.includes('gpt-5.5') || primaryModel.includes('gemini-2.5-pro')) { - return 'anthropic/claude-haiku-4.5-20251001'; + return 'anthropic/claude-haiku-4.5'; } if (primaryModel.includes('haiku') || primaryModel.includes('mini') || primaryModel.includes('nano')) { return 'google/gemini-2.5-flash'; // Cheapest capable model diff --git a/src/agent/context.ts b/src/agent/context.ts index 8f1ea53..6f8f04a 100644 --- a/src/agent/context.ts +++ b/src/agent/context.ts @@ -254,7 +254,7 @@ You run on the BlockRun AI Gateway. When the user asks you to "test the BlockRun - \`GET /.well-known/x402\` — x402 resource list with prices **LLM (POST, x402-paid)** -- \`POST /v1/chat/completions\` — OpenAI-compatible. Body: \`{ model, messages, stream?, tools?, max_tokens?, temperature? }\`. \`model\` MUST come from \`GET /v1/models\` (real frontier examples on the gateway as of 2026-06: \`anthropic/claude-sonnet-4.6\`, \`anthropic/claude-opus-4.8\`, \`deepseek/deepseek-v4-pro\`, \`zai/glm-5.2\`, \`nvidia/llama-4-maverick\`, \`openai/gpt-5-nano\`). Do NOT invent versions like \`openai/gpt-5.1\` or \`xai/grok-5\` — those don't exist; the gateway 400s with the valid list in the error body, so when in doubt fetch \`GET /v1/models\` first. +- \`POST /v1/chat/completions\` — OpenAI-compatible. Body: \`{ model, messages, stream?, tools?, max_tokens?, temperature? }\`. \`model\` MUST come from \`GET /v1/models\` (real frontier examples on the gateway, verified live 2026-07-14: \`anthropic/claude-sonnet-5\`, \`anthropic/claude-opus-4.8\`, \`deepseek/deepseek-v4-pro\`, \`zai/glm-5.2\`, \`xai/grok-4.5\`, \`nvidia/qwen3-next-80b-a3b-instruct\` (free)). Do NOT invent versions like \`openai/gpt-5.1\` or \`xai/grok-5\` — those don't exist; the gateway 400s with the valid list in the error body, so when in doubt fetch \`GET /v1/models\` first. - \`POST /v1/messages\` — Anthropic-compatible. Body: \`{ model, messages, max_tokens, system?, tools? }\`. **Media (POST, x402-paid; GET to poll async jobs)** diff --git a/src/agent/loop.ts b/src/agent/loop.ts index f47eef1..e5959ff 100644 --- a/src/agent/loop.ts +++ b/src/agent/loop.ts @@ -1563,7 +1563,7 @@ export async function interactiveSession( // Free-only recovery chain — a free/empty-response session must NEVER // fall back to a paid model (would silently charge the wallet). Both // entries are $0 nvidia models. - const EMPTY_FALLBACK_MODELS = ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/llama-4-maverick']; + const EMPTY_FALLBACK_MODELS = ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/mistral-nemotron']; const nextModel = EMPTY_FALLBACK_MODELS.find(m => m !== config.model && !turnFailedModels.has(m)); if (nextModel && recoveryAttempts < 2 && !config.disableModelFallback) { recoveryAttempts++; @@ -1607,7 +1607,7 @@ export async function interactiveSession( const TOOL_USE_FALLBACK_MODELS = [ 'anthropic/claude-haiku-4.5', 'moonshot/kimi-k2.7', - 'openai/gpt-5', + 'openai/gpt-5.4', 'anthropic/claude-sonnet-4.6', ]; const nextModel = TOOL_USE_FALLBACK_MODELS.find( diff --git a/src/agent/optimize.ts b/src/agent/optimize.ts index ff2fea5..98eeb41 100644 --- a/src/agent/optimize.ts +++ b/src/agent/optimize.ts @@ -43,7 +43,7 @@ const MODEL_MAX_OUTPUT: Record = { 'anthropic/claude-sonnet-5': 128_000, 'anthropic/claude-sonnet-4.6': 64_000, 'anthropic/claude-sonnet-4.5': 64_000, - 'anthropic/claude-haiku-4.5-20251001': 16_384, + 'anthropic/claude-haiku-4.5': 16_384, 'openai/gpt-5.6-sol': 128_000, 'openai/gpt-5.6-terra': 128_000, 'openai/gpt-5.6-luna': 128_000, diff --git a/src/agent/tokens.ts b/src/agent/tokens.ts index 0151c2f..804b169 100644 --- a/src/agent/tokens.ts +++ b/src/agent/tokens.ts @@ -227,6 +227,8 @@ const MODEL_CONTEXT_WINDOWS: Record = { 'anthropic/claude-sonnet-4.5': 200_000, 'anthropic/claude-sonnet-4': 200_000, 'anthropic/claude-haiku-4.5': 200_000, + // Retired 2026-07-14 (gateway 400s on it) — kept so replayed sessions that + // recorded the dated id still resolve a context window. 'anthropic/claude-haiku-4.5-20251001': 200_000, // OpenAI // gpt-5.5 advertises 1.05M context at the gateway, but Franklin keeps the diff --git a/src/commands/init.ts b/src/commands/init.ts index 071079d..f4fcc79 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -33,7 +33,7 @@ export async function initCommand(options: { port?: string }) { ANTHROPIC_MODEL: 'blockrun/auto', ANTHROPIC_DEFAULT_SONNET_MODEL: 'anthropic/claude-sonnet-4.6', ANTHROPIC_DEFAULT_OPUS_MODEL: 'anthropic/claude-opus-4.8', - ANTHROPIC_DEFAULT_HAIKU_MODEL: 'anthropic/claude-haiku-4.5-20251001', + ANTHROPIC_DEFAULT_HAIKU_MODEL: 'anthropic/claude-haiku-4.5', }; fs.mkdirSync(path.dirname(CLAUDE_SETTINGS_FILE), { recursive: true }); diff --git a/src/learnings/extractor.ts b/src/learnings/extractor.ts index 2e51823..802ee98 100644 --- a/src/learnings/extractor.ts +++ b/src/learnings/extractor.ts @@ -16,7 +16,7 @@ import type { Skill } from './types.js'; // Ordered by reliability: try the best free model first, fall back to others. const EXTRACTION_MODELS = [ 'nvidia/qwen3-next-80b-a3b-instruct', // Clean JSON output, no thinking leak (verified live) - 'nvidia/llama-4-maverick', // Diverse-family free fallback + 'nvidia/mistral-nemotron', // Diverse-family free fallback (serves itself; maverick is pooled) ]; const VALID_CATEGORIES = new Set([ diff --git a/src/plugin-sdk/workflow.ts b/src/plugin-sdk/workflow.ts index 725c718..8884612 100644 --- a/src/plugin-sdk/workflow.ts +++ b/src/plugin-sdk/workflow.ts @@ -15,9 +15,9 @@ export type ModelTier = 'free' | 'cheap' | 'premium' | 'none'; /** Maps tier names to actual model identifiers */ export interface ModelTierConfig { - free: string; // e.g. "nvidia/llama-4-maverick" + free: string; // e.g. "nvidia/qwen3-next-80b-a3b-instruct" cheap: string; // e.g. "zai/glm-5.1" - premium: string; // e.g. "anthropic/claude-sonnet-4.6" + premium: string; // e.g. "anthropic/claude-sonnet-5" } export const DEFAULT_MODEL_TIERS: ModelTierConfig = { diff --git a/src/pricing.ts b/src/pricing.ts index e77422c..b15fbd3 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -46,6 +46,8 @@ export const MODEL_PRICING: Record = { - coding: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/llama-4-maverick'], - trading: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/llama-4-maverick'], - research: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/llama-4-maverick'], - reasoning: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/llama-4-maverick'], - chat: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/llama-4-maverick'], - creative: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/llama-4-maverick'], + coding: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/mistral-nemotron'], + trading: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/mistral-nemotron'], + research: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/mistral-nemotron'], + reasoning: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/mistral-nemotron'], + chat: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/mistral-nemotron'], + creative: ['nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/mistral-nemotron'], }; const DEFAULT_FREE_CHAIN: string[] = [ 'nvidia/qwen3-next-80b-a3b-instruct', - 'nvidia/llama-4-maverick', + 'nvidia/mistral-nemotron', ]; /** diff --git a/src/router/vision.ts b/src/router/vision.ts index 3430777..02f4cb1 100644 --- a/src/router/vision.ts +++ b/src/router/vision.ts @@ -29,7 +29,7 @@ const VISION_MODELS = new Set([ 'anthropic/claude-sonnet-5', 'anthropic/claude-sonnet-4.6', 'anthropic/claude-sonnet-4.5', - 'anthropic/claude-haiku-4.5-20251001', + 'anthropic/claude-haiku-4.5', // OpenAI — multimodal flagships + o3 (Codex 5.3 is text-only, excluded). // GPT-5.6 family + 5.4-mini are vision; 5.4-nano is text-only. 'openai/gpt-5.6-sol', @@ -55,9 +55,10 @@ const VISION_MODELS = new Set([ // Moonshot — K2.7 (flagship) + K2.6 are multimodal (image + video input) 'moonshot/kimi-k2.7', 'moonshot/kimi-k2.6', - // NVIDIA inference — Llama 4 Maverick + Nemotron Nano VL are multimodal; - // deepseek/qwen-coder are not. - 'nvidia/llama-4-maverick', + // NVIDIA inference — Nemotron Nano VL is multimodal; deepseek/qwen-coder are + // not. Llama 4 Maverick dropped 2026-07-14: it left the gateway catalog, and + // listing it here contradicted routeRequest()'s own "maverick is text-only" + // note — the free profile would route a vision turn to a text-only model. 'nvidia/nemotron-nano-12b-v2-vl', ]); diff --git a/src/tools/moa.ts b/src/tools/moa.ts index d64a566..48015e8 100644 --- a/src/tools/moa.ts +++ b/src/tools/moa.ts @@ -19,7 +19,7 @@ import { ModelClient } from '../agent/llm.js'; /** Reference models — diverse, cheap/free models for parallel queries. */ const REFERENCE_MODELS = [ 'nvidia/qwen3-next-80b-a3b-instruct', // Free, clean instruction-follower - 'nvidia/llama-4-maverick', // Free, agent-tested general chat (diverse family) + 'nvidia/mistral-nemotron', // Free, diverse family (maverick left the catalog 2026-07-14) 'google/gemini-2.5-flash', // Fast, cheap 'deepseek/deepseek-chat', // Cheap, good reasoning ]; diff --git a/src/ui/app.tsx b/src/ui/app.tsx index 9b6b591..99adb70 100644 --- a/src/ui/app.tsx +++ b/src/ui/app.tsx @@ -8,7 +8,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { execFileSync } from 'node:child_process'; -import React, { useState, useEffect, useCallback, useRef } from 'react'; +import React, { useState, useEffect, useCallback, useMemo, useRef } from 'react'; import { render, Static, Box, Text, useApp, useInput, useStdout } from 'ink'; import Spinner from 'ink-spinner'; import TextInput from 'ink-text-input'; @@ -17,8 +17,10 @@ import type { StreamEvent } from '../agent/types.js'; import { renderMarkdown, renderMarkdownStreaming } from './markdown.js'; import { resolveModel, + getPickerCategories, PICKER_CATEGORIES, PICKER_MODELS_FLAT, + type ModelCategory, } from './model-picker.js'; import { estimateCost } from '../pricing.js'; import { formatTokens, shortModelName } from '../stats/format.js'; @@ -786,9 +788,13 @@ function formatAgentErrorForDisplay(error: string): string { return out.join('\n'); } -// Picker model list is imported from ./model-picker.js (single source of truth). -// PICKER_CATEGORIES provides grouped data for rendering; PICKER_MODELS_FLAT -// provides a flat array for pickerIdx navigation. +// Picker model list is imported from ./model-picker.js (single source of truth +// for curation). PICKER_CATEGORIES is the static editorial list, used as the +// initial paint and the offline fallback; getPickerCategories() reconciles it +// against the live gateway catalog (fresh labels/prices, retired rows dropped) +// and the result lands in `pickerCats` state. Render and pickerIdx navigation +// both read `pickerCats` / `pickerFlat` — never the static consts — so the +// cursor and the visible rows can't disagree after hydration. interface ToolStatus { name: string; @@ -869,6 +875,16 @@ function RunCodeApp({ // Short preview of latest response shown in dynamic area (last ~5 lines, cleared on next turn) const [responsePreview, setResponsePreview] = useState(''); const [currentModel, setCurrentModel] = useState(initialModel || PICKER_MODELS_FLAT[0].id); + // Gateway-reconciled picker list. Seeded with the static curation so the + // first paint is instant and an offline session still gets a usable picker; + // replaced by the hydrated list once the catalog fetch lands. + const [pickerCats, setPickerCats] = useState(PICKER_CATEGORIES); + const [pickerMoreCount, setPickerMoreCount] = useState(0); + const pickerFlat = useMemo(() => pickerCats.flatMap(c => c.models), [pickerCats]); + // Track the live model without re-firing the hydration effect on every model + // change — the effect should run when the picker opens, not when the model + // switches, but it still needs the current id to re-anchor the cursor. + const currentModelRef = useRef(initialModel); const [ready, setReady] = useState(!startWithPicker); const [mode, setMode] = useState(startWithPicker ? 'model-picker' : 'input'); const [pickerIdx, setPickerIdx] = useState(0); @@ -905,6 +921,35 @@ function RunCodeApp({ // tab and the agent stops to ask for approval — verified 2026-05-04 // from a real screenshot where the user missed the dialog because the // input box still read "Working...". Opt-out via FRANKLIN_NO_BELL=1. + // Single sync point for currentModelRef — cheaper to keep correct than + // updating the ref at every setCurrentModel call site. + useEffect(() => { currentModelRef.current = currentModel; }, [currentModel]); + + // Reconcile the picker against the live gateway catalog whenever it opens — + // covers both /model with no args and --model-picker startup. Runs in the + // background: the picker has already painted from the static curation (or the + // 5-min cached catalog), so a slow gateway delays nothing. gateway-models.ts + // dedupes concurrent fetches and serves stale-on-error, so re-opening the + // picker is cheap and an offline session simply keeps the static list. + useEffect(() => { + if (mode !== 'model-picker') return; + let cancelled = false; + void getPickerCategories() + .then(({ categories, moreCount, live }) => { + if (cancelled || !live) return; + setPickerCats(categories); + setPickerMoreCount(moreCount); + // Rows may have dropped out from under the cursor — re-anchor on the + // model actually in use rather than leaving the highlight on whatever + // slid into that index. + const flat = categories.flatMap(c => c.models); + const at = flat.findIndex(m => m.id === currentModelRef.current); + setPickerIdx(at >= 0 ? at : 0); + }) + .catch(() => { /* keep the static list — the picker must stay usable */ }); + return () => { cancelled = true; }; + }, [mode]); + const bellPlayedRef = useRef(false); useEffect(() => { const dialogActive = !!permissionRequest || !!askUserRequest; @@ -1101,9 +1146,12 @@ function RunCodeApp({ // Arrow key navigation for model picker if (mode !== 'model-picker') return; if (key.upArrow) setPickerIdx(i => Math.max(0, i - 1)); - else if (key.downArrow) setPickerIdx(i => Math.min(PICKER_MODELS_FLAT.length - 1, i + 1)); + else if (key.downArrow) setPickerIdx(i => Math.min(pickerFlat.length - 1, i + 1)); else if (key.return) { - const selected = PICKER_MODELS_FLAT[pickerIdx]; + // Hydration can shrink the list under the cursor (a retired row drops + // out), so treat the index as untrusted rather than assuming a hit. + const selected = pickerFlat[pickerIdx] ?? pickerFlat[0]; + if (!selected) return; setCurrentModel(selected.id); onModelChange(selected.id, 'user'); showStatus(`Model → ${selected.label}`, 'success', 3000); @@ -1192,8 +1240,10 @@ function RunCodeApp({ onModelChange(resolved, 'user'); showStatus(`Model → ${resolved}`, 'success', 3000); } else { - const idx = PICKER_MODELS_FLAT.findIndex(m => m.id === currentModel); + const idx = pickerFlat.findIndex(m => m.id === currentModel); setPickerIdx(idx >= 0 ? idx : 0); + // Gateway reconciliation is kicked off by an effect on picker mode + // (see below), so it covers this path and --model-picker startup. // Defensive: ensure no draft text survives into the picker — // closing handlers clear input too, so both ends are covered. setInput(''); @@ -1973,7 +2023,7 @@ function RunCodeApp({ markers. Same reason as streamText — Ink wipes scrollback the moment dynamic output exceeds the terminal height. */} {inPicker && (() => { - const totalModels = PICKER_MODELS_FLAT.length; + const totalModels = pickerFlat.length; const maxModels = Math.max(6, termRows - 12); let start = Math.max(0, pickerIdx - Math.floor(maxModels / 2)); let end = Math.min(totalModels, start + maxModels); @@ -1985,7 +2035,7 @@ function RunCodeApp({ // Pre-compute each category's base offset into the flat model list so // we can map (cat, localIdx) → globalIdx in one pass without re-walking. let cursor = 0; - const catBases = PICKER_CATEGORIES.map((cat) => { + const catBases = pickerCats.map((cat) => { const base = cursor; cursor += cat.models.length; return base; @@ -2001,7 +2051,7 @@ function RunCodeApp({ ↑ {hiddenAbove} more above )} - {PICKER_CATEGORIES.map((cat, catIdx) => { + {pickerCats.map((cat, catIdx) => { const base = catBases[catIdx]; const visible = cat.models .map((m, localIdx) => ({ m, globalIdx: base + localIdx })) @@ -2044,6 +2094,13 @@ function RunCodeApp({ ↓ {hiddenBelow} more below )} + {pickerMoreCount > 0 && ( + + + + {pickerMoreCount} more on gateway — `franklin models` to list, or /model <id> to pick one + + + )} Your conversation stays above — picking a model keeps all history intact. diff --git a/src/ui/model-picker.ts b/src/ui/model-picker.ts index 6d6052c..5b9e3c5 100644 --- a/src/ui/model-picker.ts +++ b/src/ui/model-picker.ts @@ -5,6 +5,7 @@ import readline from 'node:readline'; import chalk from 'chalk'; +import { getGatewayModels, type GatewayModel } from '../gateway-models.js'; // ─── Model Shortcuts (same as proxy) ─────────────────────────────────────── @@ -32,8 +33,8 @@ export const MODEL_SHORTCUTS: Record = { 'opus-4.7': 'anthropic/claude-opus-4.7', 'opus-4.6': 'anthropic/claude-opus-4.6', 'opus-4.5': 'anthropic/claude-opus-4.5', - haiku: 'anthropic/claude-haiku-4.5-20251001', - 'haiku-4.5': 'anthropic/claude-haiku-4.5-20251001', + haiku: 'anthropic/claude-haiku-4.5', + 'haiku-4.5': 'anthropic/claude-haiku-4.5', // OpenAI // `gpt` / `gpt5` / `gpt-5` follow the gateway's flagship — currently 5.6 Sol. gpt: 'openai/gpt-5.6-sol', @@ -67,9 +68,13 @@ export const MODEL_SHORTCUTS: Record = { 'gemini-3.5-flash': 'google/gemini-3.5-flash', 'gemini-3': 'google/gemini-3.1-pro', 'gemini-3.1': 'google/gemini-3.1-pro', - // xAI — grok-4.3 is the public flagship since 2026-06-04 (grok-3 and the - // fast families are hidden on the gateway; explicit IDs still resolve). - grok: 'xai/grok-4.3', + // xAI — grok-4.5 is the public flagship since 2026-07-14; `grok` follows it + // per the bare-alias-tracks-flagship convention. 4.3 stays pinned: it's + // cheaper ($1.5/$4 vs $2.5/$9) and carries 1M context to 4.5's 500K, so it's + // the better pick for long-context work. (grok-3 and the fast families are + // hidden on the gateway; explicit IDs still resolve.) + grok: 'xai/grok-4.5', + 'grok-4.5': 'xai/grok-4.5', 'grok-4.3': 'xai/grok-4.3', 'grok-build': 'xai/grok-build-0.1', 'grok-3': 'xai/grok-3', @@ -101,7 +106,6 @@ export const MODEL_SHORTCUTS: Record = { qwen3: 'nvidia/qwen3-next-80b-a3b-instruct', 'qwen3-next': 'nvidia/qwen3-next-80b-a3b-instruct', 'qwen3.5': 'nvidia/qwen3.5-122b-a10b', - maverick: 'nvidia/llama-4-maverick', glm4: 'nvidia/qwen3-next-80b-a3b-instruct', 'deepseek-free': 'nvidia/qwen3-next-80b-a3b-instruct', 'qwen-coder': 'nvidia/qwen3-next-80b-a3b-instruct', @@ -110,19 +114,24 @@ export const MODEL_SHORTCUTS: Record = { 'gpt-oss-small': 'nvidia/qwen3-next-80b-a3b-instruct', 'mistral-small': 'nvidia/qwen3-next-80b-a3b-instruct', 'mistral-nemotron': 'nvidia/mistral-nemotron', - // llama shortcuts point at the still-live free maverick model (not the free - // default — that's qwen3-next above). Kept as explicit aliases so the name - // resolves for users who type it. - llama: 'nvidia/llama-4-maverick', - 'llama-4': 'nvidia/llama-4-maverick', - 'llama-4-maverick': 'nvidia/llama-4-maverick', + // Maverick left the gateway catalog on/before 2026-07-14. The id still + // answers, but only because the free pool silently substitutes another model + // for it — so pointing users at it would be promising a model they don't get. + // Follow the established retired-free-id pattern instead: resolve to the + // current free default so muscle memory keeps working. + llama: 'nvidia/qwen3-next-80b-a3b-instruct', + 'llama-4': 'nvidia/qwen3-next-80b-a3b-instruct', + 'llama-4-maverick': 'nvidia/qwen3-next-80b-a3b-instruct', + maverick: 'nvidia/qwen3-next-80b-a3b-instruct', // Backward-compatibility aliases for models the gateway retired or exposes // unreliably on /v1/messages. Map to agent-tested free models so shortcuts // keep working without silent paid fallback or empty tool-use turns. // Map to the closest current free model so old session records + user // muscle memory keep working. - nemotron: 'nvidia/llama-4-maverick', - devstral: 'nvidia/llama-4-maverick', + // `nemotron` now resolves to the real Mistral Nemotron (in-catalog, and one + // of the few free ids that verifiably serves itself) rather than to Maverick. + nemotron: 'nvidia/mistral-nemotron', + devstral: 'nvidia/qwen3-next-80b-a3b-instruct', // Others minimax: 'minimax/minimax-m3', 'm3': 'minimax/minimax-m3', @@ -250,7 +259,7 @@ export const PICKER_CATEGORIES: ModelCategory[] = [ { id: 'openai/gpt-5.6-sol', shortcut: 'gpt', label: 'GPT-5.6 Sol', price: '$5/$30', highlight: true }, { id: 'google/gemini-3.1-pro', shortcut: 'gemini-3', label: 'Gemini 3.1 Pro', price: '$2/$12' }, { id: 'google/gemini-2.5-pro', shortcut: 'gemini', label: 'Gemini 2.5 Pro', price: '$1.25/$10' }, - { id: 'xai/grok-4.3', shortcut: 'grok', label: 'Grok 4.3', price: '$1.5/$4' }, + { id: 'xai/grok-4.5', shortcut: 'grok', label: 'Grok 4.5', price: '$2.5/$9' }, ], }, { @@ -272,7 +281,7 @@ export const PICKER_CATEGORIES: ModelCategory[] = [ { category: '💰 Budget', models: [ - { id: 'anthropic/claude-haiku-4.5-20251001', shortcut: 'haiku', label: 'Claude Haiku 4.5', price: '$1/$5' }, + { id: 'anthropic/claude-haiku-4.5', shortcut: 'haiku', label: 'Claude Haiku 4.5', price: '$1/$5' }, { id: 'openai/gpt-5-mini', shortcut: 'mini', label: 'GPT-5 Mini', price: '$0.25/$2' }, { id: 'google/gemini-2.5-flash', shortcut: 'flash', label: 'Gemini 2.5 Flash', price: '$0.3/$2.5' }, // Re-aliased to V4 Flash Chat upstream — context 1M, price 30% lower. @@ -290,14 +299,20 @@ export const PICKER_CATEGORIES: ModelCategory[] = [ { category: '🆓 Free (no USDC needed)', models: [ - // Qwen3-Next 80B leads: cleanest free instruction-follower (no thinking - // leak / markdown fences — verified live 2026-07-11) and the default the - // `free` shortcut + free routing profile resolve to. Llama 4 Maverick is - // the diverse-family secondary. (nvidia/deepseek-v4-flash removed - // 2026-07-11: the gateway 410s on it.) Both are $0 — the free tier never - // falls back to a paid model. - { id: 'nvidia/qwen3-next-80b-a3b-instruct', shortcut: 'free', label: 'Qwen3-Next 80B', price: 'FREE', highlight: true }, - { id: 'nvidia/llama-4-maverick', shortcut: 'maverick', label: 'Llama 4 Maverick', price: 'FREE' }, + // Qwen3-Next 80B leads: it's what the `free` shortcut + free routing + // profile resolve to. Mistral Nemotron is the diverse-family secondary, + // promoted 2026-07-14 when nvidia/llama-4-maverick fell out of the + // gateway catalog. Both are $0 — the free tier never falls back to paid. + // + // Caveat worth knowing before editing this list: the NVIDIA free pool + // silently substitutes. Verified live 2026-07-14 — qwen3-next and + // mistral-large-3-675b both came back served by + // `nvidia/nemotron-3-super-120b-a12b-free`, so the free id you request + // is not necessarily the one you get. mistral-nemotron was picked as the + // secondary precisely because it does serve itself, consistently, and + // answers without leaking reasoning prose. + { id: 'nvidia/qwen3-next-80b-a3b-instruct', shortcut: 'free', label: 'Qwen3-Next 80B', price: 'FREE', highlight: true }, + { id: 'nvidia/mistral-nemotron', shortcut: 'nemotron', label: 'Mistral Nemotron', price: 'FREE' }, ], }, ]; @@ -305,14 +320,121 @@ export const PICKER_CATEGORIES: ModelCategory[] = [ /** Flat list of all picker models (for index-based navigation). */ export const PICKER_MODELS_FLAT: ModelEntry[] = PICKER_CATEGORIES.flatMap(c => c.models); -// Kept for backward compatibility with the readline pickModel() below. -const PICKER_MODELS = PICKER_CATEGORIES; +// ─── Live hydration against the gateway catalog ──────────────────────────── + +/** + * Synthetic ids that are Franklin concepts, not gateway models (`blockrun/auto` + * is a routing profile resolved client-side). They never appear in the catalog, + * so they're exempt from the drop-if-absent rule below. + */ +function isSyntheticId(id: string): boolean { + return id.startsWith('blockrun/'); +} + +/** Render a gateway price object into the picker's display format. */ +function formatPrice(m: GatewayModel): string { + const p = m.pricing as unknown as Record; + switch (m.billing_mode) { + case 'free': + return 'FREE'; + case 'paid': + return `$${p.input ?? 0}/$${p.output ?? 0}`; + case 'flat': + return `$${p.flat ?? 0} flat`; + default: + // Non-chat billing modes (per_image/per_second/…) shouldn't reach the + // picker, but don't invent a number if one does. + return '—'; + } +} + +export interface HydratedPicker { + categories: ModelCategory[]; + /** Chat models live on the gateway that aren't curated into the picker. */ + moreCount: number; + /** False when the gateway was unreachable and this is the static fallback. */ + live: boolean; +} + +/** + * The picker list, reconciled against the live gateway catalog. + * + * {@link PICKER_CATEGORIES} stays the editorial layer — which models are worth + * featuring, in what order, under which heading, with which shortcut. The + * gateway is the source of truth for everything factual: whether a model still + * exists, its display name, and its price. That split is deliberate: the + * gateway lists 57 chat models and the picker deliberately shows ~21 (see the + * v3.9.3 trim), so we can't just render the catalog. + * + * Reconciliation rules: + * - Curated row present in the catalog → refresh its label + price from the + * gateway, so a price change upstream can't leave a stale number on screen. + * - Curated row absent → drop it. This is the self-healing half: an id the + * gateway retired (the `claude-haiku-4.5-20251001` case) stops being + * offered without waiting on a Franklin release. Its MODEL_SHORTCUTS alias + * survives, matching the long-standing "hide the row, keep the shortcut" + * pattern. + * - Gateway unreachable → serve the static list verbatim (`live: false`). + * An offline picker showing a slightly stale list beats an empty one. + * + * Note the catalog hides some models that still resolve (grok-3 and the xAI + * fast family, for instance). Hidden-but-working ids are therefore dropped from + * the *visible* list while remaining reachable by typing the shortcut — which + * is the intended behavior, not a bug. + */ +export async function getPickerCategories(): Promise { + try { + return reconcilePicker(await getGatewayModels()); + } catch { + return { categories: PICKER_CATEGORIES, moreCount: 0, live: false }; + } +} + +/** + * Pure reconciliation step behind {@link getPickerCategories} — exported so the + * rules can be tested against a synthetic catalog without touching the network. + */ +export function reconcilePicker(catalog: GatewayModel[]): HydratedPicker { + const byId = new Map(catalog.map(m => [m.id, m])); + const categories: ModelCategory[] = []; + const curated = new Set(); + + for (const cat of PICKER_CATEGORIES) { + const models: ModelEntry[] = []; + for (const row of cat.models) { + curated.add(row.id); + if (isSyntheticId(row.id)) { + models.push(row); + continue; + } + const live = byId.get(row.id); + if (!live) continue; // retired upstream — drop the row, keep the alias + // Price comes from the gateway (it's factual, it drifts, and it's the + // user's money). The label stays curated: gateway names are longer than + // the picker's tuned columns ("Qwen3-Next 80B Instruct (Free)" is 30 + // chars against a 26-wide field) and carry redundant "(Free)" suffixes + // that the 🆓 heading already says. + models.push({ ...row, price: formatPrice(live) }); + } + if (models.length > 0) categories.push({ ...cat, models }); + } + + const moreCount = catalog.filter( + m => m.categories?.includes('chat') && !curated.has(m.id), + ).length; + + return { categories, moreCount, live: true }; +} /** * Show interactive model picker. Returns the selected model ID. * Falls back to text input if terminal doesn't support raw mode. */ export async function pickModel(currentModel?: string): Promise { + // Same gateway reconciliation the Ink picker gets — falls back to the static + // curation when the catalog is unreachable. + const { categories: PICKER_MODELS, moreCount } = await getPickerCategories(); + // Flatten for numbering const allModels: ModelEntry[] = []; for (const cat of PICKER_MODELS) { @@ -337,6 +459,12 @@ export async function pickModel(currentModel?: string): Promise { console.error(''); } + if (moreCount > 0) { + console.error( + chalk.dim(` + ${moreCount} more on gateway — run \`franklin models\` to list them.\n`) + ); + } + console.error(chalk.dim(' Enter number, shortcut, or full model ID:')); // Read input diff --git a/test/free-model-matrix.mjs b/test/free-model-matrix.mjs index 64ca34c..86c19fc 100644 --- a/test/free-model-matrix.mjs +++ b/test/free-model-matrix.mjs @@ -3,7 +3,7 @@ * * Run: * npm run test:free-models - * FREE_MODEL_MATRIX=nvidia/qwen3-coder-480b,nvidia/llama-4-maverick npm run test:free-models + * FREE_MODEL_MATRIX=nvidia/qwen3-next-80b-a3b-instruct,nvidia/mistral-nemotron npm run test:free-models * FREE_MODEL_MATRIX=all npm run test:free-models * FREE_MODEL_MATRIX_PROBES=echo npm run test:free-models * @@ -35,7 +35,7 @@ const requestedModels = (process.env.FREE_MODEL_MATRIX || '') const defaultMatrixModels = new Set([ 'nvidia/qwen3-next-80b-a3b-instruct', - 'nvidia/llama-4-maverick', + 'nvidia/mistral-nemotron', ]); const selectedModels = process.env.FREE_MODEL_MATRIX === 'all' diff --git a/test/local.mjs b/test/local.mjs index 30da923..d101d2f 100644 --- a/test/local.mjs +++ b/test/local.mjs @@ -387,13 +387,13 @@ test('proxy server handles OPTIONS and local model switching without backend cal glm4: 'nvidia/qwen3-next-80b-a3b-instruct', 'qwen-think': 'nvidia/qwen3-next-80b-a3b-instruct', 'qwen-coder': 'nvidia/qwen3-next-80b-a3b-instruct', - maverick: 'nvidia/llama-4-maverick', + maverick: 'nvidia/qwen3-next-80b-a3b-instruct', 'deepseek-free': 'nvidia/qwen3-next-80b-a3b-instruct', 'gpt-oss': 'nvidia/qwen3-next-80b-a3b-instruct', 'gpt-oss-small': 'nvidia/qwen3-next-80b-a3b-instruct', 'mistral-small': 'nvidia/qwen3-next-80b-a3b-instruct', - nemotron: 'nvidia/llama-4-maverick', - devstral: 'nvidia/llama-4-maverick', + nemotron: 'nvidia/mistral-nemotron', + devstral: 'nvidia/qwen3-next-80b-a3b-instruct', }; for (const [shortcut, expectedModel] of Object.entries(freeSwitches)) { const freeSwitchRes = await fetch(`http://127.0.0.1:${port}/api/messages`, { @@ -2941,12 +2941,15 @@ test('agent context: chat-completions example uses real model names (no fictiona path.join(process.cwd(), 'dist', 'agent', 'context.js'), 'utf-8', ); - // Real names that should appear as illustrative examples. + // Real names that should appear as illustrative examples. Refreshed + // 2026-07-14: every id below returned 402 (exists, needs payment) rather + // than 400 on a live POST /api/v1/chat/completions probe. for (const real of [ - 'anthropic/claude-sonnet-4.6', + 'anthropic/claude-sonnet-5', 'anthropic/claude-opus-4.8', 'deepseek/deepseek-v4-pro', 'zai/glm-5.2', + 'xai/grok-4.5', ]) { assert.ok(src.includes(real), `chat-completions example list must include real model "${real}"`); } @@ -5667,8 +5670,10 @@ test('free model catalog: picker, shortcuts, pricing, and weak-model guard stay 'gpt-oss': 'nvidia/qwen3-next-80b-a3b-instruct', 'gpt-oss-small': 'nvidia/qwen3-next-80b-a3b-instruct', 'mistral-small': 'nvidia/qwen3-next-80b-a3b-instruct', - nemotron: 'nvidia/llama-4-maverick', - devstral: 'nvidia/llama-4-maverick', + nemotron: 'nvidia/mistral-nemotron', + devstral: 'nvidia/qwen3-next-80b-a3b-instruct', + maverick: 'nvidia/qwen3-next-80b-a3b-instruct', + llama: 'nvidia/qwen3-next-80b-a3b-instruct', }; for (const [shortcut, expectedModel] of Object.entries(freeAliases)) { @@ -5725,7 +5730,7 @@ test('free routing profile stays free across router entry points', async () => { const FREE_GATEWAY_MODELS = new Set([ 'nvidia/qwen3-next-80b-a3b-instruct', 'nvidia/qwen3.5-122b-a10b', - 'nvidia/llama-4-maverick', + 'nvidia/mistral-nemotron', 'nvidia/mistral-large-3-675b', ]); for (const tier of ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING']) { @@ -6612,9 +6617,10 @@ test('picker trim: shortcuts for hidden models still resolve (muscle-memory pres assert.equal(resolveModel('o1'), 'openai/o1'); assert.equal(resolveModel('o4'), 'openai/o4-mini'); assert.equal(resolveModel('nano'), 'openai/gpt-5-nano'); - // grok promoted to the public flagship grok-4.3 (2026-06-04) — same + // grok promoted to the public flagship grok-4.5 (2026-07-14) — same // flagship-promotion pattern as kimi → k2.7. Explicit pins still resolve. - assert.equal(resolveModel('grok'), 'xai/grok-4.3'); + assert.equal(resolveModel('grok'), 'xai/grok-4.5'); + assert.equal(resolveModel('grok-4.3'), 'xai/grok-4.3'); assert.equal(resolveModel('grok-3'), 'xai/grok-3'); assert.equal(resolveModel('grok-4'), 'xai/grok-4-0709'); assert.equal(resolveModel('grok-build'), 'xai/grok-build-0.1'); @@ -6628,7 +6634,7 @@ test('picker trim: hero shortcuts (opus, sonnet, gpt, gemini-3, grok) still in v assert.ok(ids.includes('openai/gpt-5.6-sol')); assert.ok(ids.includes('google/gemini-3.1-pro')); assert.ok(ids.includes('google/gemini-2.5-pro')); - assert.ok(ids.includes('xai/grok-4.3')); // grok-4-0709 hidden on gateway; 4.3 is the public flagship row + assert.ok(ids.includes('xai/grok-4.5')); // grok-4-0709 hidden on gateway; 4.5 is the public flagship row }); test('picker trim: total visible entries dropped meaningfully', async () => { @@ -6641,6 +6647,88 @@ test('picker trim: total visible entries dropped meaningfully', async () => { assert.ok(total <= 24, `expected <= 24 visible entries (33 → ~21), got ${total}`); }); +// ─── picker ↔ gateway reconciliation ────────────────────────────────────── +// +// These drive reconcilePicker() with a synthetic catalog so the rules are +// pinned without a network call. The live counterpart is `franklin models`. + +/** Minimal gateway record shaped like GET /api/v1/models returns. */ +function gwModel(id, pricing, extra = {}) { + return { + id, + name: `${id} (gateway name)`, + billing_mode: pricing.input !== undefined ? 'paid' : 'free', + categories: ['chat'], + pricing, + ...extra, + }; +} + +test('reconcilePicker: drops curated rows the gateway no longer lists', async () => { + const { reconcilePicker, PICKER_CATEGORIES } = await import('../dist/ui/model-picker.js'); + const staticIds = PICKER_CATEGORIES.flatMap((c) => c.models.map((m) => m.id)); + const realIds = staticIds.filter((id) => !id.startsWith('blockrun/')); + // Catalog carrying every curated model except the first real one. + const dropped = realIds[0]; + const catalog = realIds.slice(1).map((id) => gwModel(id, { input: 1, output: 2 })); + + const { categories, live } = reconcilePicker(catalog); + const ids = categories.flatMap((c) => c.models.map((m) => m.id)); + assert.equal(live, true); + assert.ok(!ids.includes(dropped), `${dropped} left the catalog and must not be offered`); + for (const id of realIds.slice(1)) { + assert.ok(ids.includes(id), `${id} is still in the catalog and must survive`); + } +}); + +test('reconcilePicker: every offered row exists in the catalog (the haiku-4.5-20251001 class)', async () => { + const { reconcilePicker, PICKER_CATEGORIES } = await import('../dist/ui/model-picker.js'); + const realIds = PICKER_CATEGORIES.flatMap((c) => c.models.map((m) => m.id)) + .filter((id) => !id.startsWith('blockrun/')); + const catalog = realIds.map((id) => gwModel(id, { input: 1, output: 2 })); + const { categories } = reconcilePicker(catalog); + const known = new Set(catalog.map((m) => m.id)); + for (const m of categories.flatMap((c) => c.models)) { + if (m.id.startsWith('blockrun/')) continue; + assert.ok(known.has(m.id), `picker offered ${m.id}, which the gateway doesn't serve`); + } +}); + +test('reconcilePicker: keeps synthetic routing profiles that are not gateway models', async () => { + const { reconcilePicker } = await import('../dist/ui/model-picker.js'); + // blockrun/auto is resolved client-side and never appears in the catalog — + // an empty catalog must not delete it. + const { categories } = reconcilePicker([]); + const ids = categories.flatMap((c) => c.models.map((m) => m.id)); + assert.deepEqual(ids, ['blockrun/auto']); +}); + +test('reconcilePicker: refreshes price from the gateway but keeps the curated label', async () => { + const { reconcilePicker, PICKER_CATEGORIES } = await import('../dist/ui/model-picker.js'); + const opus = PICKER_CATEGORIES.flatMap((c) => c.models).find((m) => m.shortcut === 'opus'); + // Gateway reports a price that disagrees with the hardcoded display string. + const catalog = [gwModel(opus.id, { input: 99, output: 123 })]; + const { categories } = reconcilePicker(catalog); + const row = categories.flatMap((c) => c.models).find((m) => m.id === opus.id); + assert.equal(row.price, '$99/$123', 'price must track the gateway, not the hardcoded string'); + assert.equal(row.label, opus.label, 'label stays curated — gateway names overflow the column'); +}); + +test('reconcilePicker: free models render FREE, and moreCount counts uncurated chat models', async () => { + const { reconcilePicker, PICKER_CATEGORIES } = await import('../dist/ui/model-picker.js'); + const free = PICKER_CATEGORIES.flatMap((c) => c.models).find((m) => m.shortcut === 'free'); + const catalog = [ + gwModel(free.id, {}), + gwModel('someprovider/brand-new-flagship', { input: 5, output: 25 }), + gwModel('someprovider/an-image-model', { per_image: 0.02 }, { categories: ['image'] }), + ]; + const { categories, moreCount } = reconcilePicker(catalog); + const row = categories.flatMap((c) => c.models).find((m) => m.id === free.id); + assert.equal(row.price, 'FREE'); + // Only the uncurated *chat* model counts — the image model isn't picker material. + assert.equal(moreCount, 1, 'moreCount should count uncurated chat models only'); +}); + // ─── nemotron prose stripper ────────────────────────────────────────────── test('nemotron prose stripper: only matches Nemotron Omni model id', async () => { @@ -8001,12 +8089,13 @@ test('pickFreeFallback: research / chat / creative also skip coder first', async test('pickFreeFallback: respects alreadyFailed set', async () => { const { pickFreeFallback } = await import('../dist/router/index.js'); - // Coding starts with qwen3-next. After it fails, next is the maverick secondary. + // Coding starts with qwen3-next. After it fails, next is the mistral-nemotron + // secondary (replaced maverick 2026-07-14 — it left the gateway catalog). const failed = new Set(['nvidia/qwen3-next-80b-a3b-instruct']); const pick = pickFreeFallback('coding', failed); assert.notEqual(pick, 'nvidia/qwen3-next-80b-a3b-instruct'); - assert.equal(pick, 'nvidia/llama-4-maverick', - `after qwen3-next fails, coding should fall to maverick, got ${pick}`); + assert.equal(pick, 'nvidia/mistral-nemotron', + `after qwen3-next fails, coding should fall to mistral-nemotron, got ${pick}`); }); test('pickFreeFallback: unknown category uses default chain (general model first)', async () => { @@ -8021,7 +8110,7 @@ test('pickFreeFallback: returns undefined when every candidate failed', async () const { pickFreeFallback } = await import('../dist/router/index.js'); const failed = new Set([ 'nvidia/qwen3-next-80b-a3b-instruct', - 'nvidia/llama-4-maverick', + 'nvidia/mistral-nemotron', ]); const pick = pickFreeFallback('trading', failed); assert.equal(pick, undefined); @@ -9644,7 +9733,7 @@ test('vision helpers: isVisionModel allowlist matches curated set', async () => for (const m of [ 'anthropic/claude-opus-4.7', 'anthropic/claude-sonnet-4.6', - 'anthropic/claude-haiku-4.5-20251001', + 'anthropic/claude-haiku-4.5', 'openai/gpt-5.5', 'openai/gpt-5-mini', 'openai/o3', @@ -9652,14 +9741,16 @@ test('vision helpers: isVisionModel allowlist matches curated set', async () => 'google/gemini-2.5-flash', 'xai/grok-4-0709', 'moonshot/kimi-k2.6', - 'nvidia/llama-4-maverick', + 'nvidia/nemotron-nano-12b-v2-vl', ]) { assert.equal(isVisionModel(m), true, `${m} should be vision-capable`); } // Text-only — should all return false. Includes the failure modes the // user reported: deepseek family is entirely text-only; grok-4.1 fast - // reasoning dropped vision; codex 5.3 is text-only. + // reasoning dropped vision; codex 5.3 is text-only. maverick joined this + // list 2026-07-14 — routeRequest() always treated it as text-only, so the + // old vision entry was the side that was wrong. for (const m of [ 'deepseek/deepseek-v4-pro', 'deepseek/deepseek-chat', @@ -9668,6 +9759,7 @@ test('vision helpers: isVisionModel allowlist matches curated set', async () => 'xai/grok-4-1-fast-reasoning', 'openai/gpt-5.3-codex', 'nvidia/qwen3-coder-480b', + 'nvidia/llama-4-maverick', ]) { assert.equal(isVisionModel(m), false, `${m} should be text-only`); } diff --git a/test/model-resolver.mjs b/test/model-resolver.mjs index 2577ed4..6411200 100644 --- a/test/model-resolver.mjs +++ b/test/model-resolver.mjs @@ -13,8 +13,12 @@ test('resolveModel: shortcut maps to canonical id', () => { assert.equal(resolveModel('sonnet'), 'anthropic/claude-sonnet-5'); assert.equal(resolveModel(' SONNET '), 'anthropic/claude-sonnet-5'); assert.equal(resolveModel('free'), 'nvidia/qwen3-next-80b-a3b-instruct'); - assert.equal(resolveModel('llama'), 'nvidia/llama-4-maverick'); - assert.equal(resolveModel('llama-4-maverick'), 'nvidia/llama-4-maverick'); + // Maverick left the gateway catalog 2026-07-14 — its aliases now follow the + // retired-free-id pattern and resolve to the current free default. + assert.equal(resolveModel('llama'), 'nvidia/qwen3-next-80b-a3b-instruct'); + assert.equal(resolveModel('llama-4-maverick'), 'nvidia/qwen3-next-80b-a3b-instruct'); + assert.equal(resolveModel('maverick'), 'nvidia/qwen3-next-80b-a3b-instruct'); + assert.equal(resolveModel('nemotron'), 'nvidia/mistral-nemotron'); }); test('resolveModel: full provider/model id is passed through', () => { @@ -36,7 +40,7 @@ test('resolveModelStrict: shortcut maps to canonical id and flags viaShortcut', const r = resolveModelStrict('llama'); assert.equal(r.ok, true); if (r.ok) { - assert.equal(r.id, 'nvidia/llama-4-maverick'); + assert.equal(r.id, 'nvidia/qwen3-next-80b-a3b-instruct'); assert.equal(r.viaShortcut, true); } }); From 3be14321383893f1073f61085db0916d2de58a50 Mon Sep 17 00:00:00 2001 From: 1bcMax <195689928+1bcMax@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:44:41 -0500 Subject: [PATCH 2/2] fix(models): price each billing mode correctly in `franklin models` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The command modelled every model as { input, output } and printed `$0.00/M` for anything not token-metered — so the entire image / video / music / speech catalog (22 paid models) rendered as free, under a "Paid Models" heading. On a price list, that is the worst direction to be wrong in. The gateway actually bills seven different ways. Rewritten on top of gateway-models.ts instead of its own fetch + wrong local interface — it already types every billing mode, caches 5 minutes, and serves stale-on-error. Each mode now renders in its own unit, and unknown modes are still listed with raw pricing rather than a wrong unit: silently dropping a model is how $0.00/M stayed invisible. Same root cause, latent: per_character and per_generation are real gateway billing modes but were missing from BillingMode, so estimateCostUsd fell through the switch and returned 0 — a paid ElevenLabs speech/sfx call estimated as free. No speech tool calls it today, so nothing leaked; fixed before one does. Also: the Context column had a header but was never populated (the gateway returns context_window), and the sort keyed off pricing.input, floating every media model to the top. Adds the first tests for this file — the precision assertions target the bug's exact shape ($0.435 must not round to $0.44; a real per-image price must never render as $0.00). 562/562 pass; output verified live. --- src/commands/models.ts | 235 ++++++++++++++++++++++++++++++++--------- src/gateway-models.ts | 34 +++++- test/local.mjs | 63 +++++++++++ 3 files changed, 278 insertions(+), 54 deletions(-) diff --git a/src/commands/models.ts b/src/commands/models.ts index b30eefb..d3e1598 100644 --- a/src/commands/models.ts +++ b/src/commands/models.ts @@ -1,11 +1,120 @@ import chalk from 'chalk'; import { loadChain, API_URLS } from '../config.js'; +import { + getGatewayModels, + GATEWAY_MARGIN, + type GatewayModel, + type BillingMode, +} from '../gateway-models.js'; -interface Model { - id: string; - owned_by: string; - billing_mode: string; - pricing: { input: number; output: number }; +/** + * `franklin models` — the live catalog. + * + * Reuses gateway-models.ts rather than re-fetching: it already types every + * billing mode, caches for 5 minutes, and serves stale-on-error. + * + * The old version modelled pricing as `{ input, output }` for every model and + * printed `$0.00/M` for anything that isn't token-metered — so the entire + * image/video/music/speech catalog rendered as free under a "Paid Models" + * heading. The gateway actually bills seven different ways, and each needs its + * own unit; that's what BILLING_GROUPS below encodes. + */ + +const RULE_WIDTH = 74; + +/** + * Compact USD — keeps sub-cent prices honest ($0.435/M must not round to + * $0.44). Exported for tests. + */ +export function money(n: number): string { + if (n === 0) return '$0'; + const decimals = n < 0.01 ? 4 : n < 1 ? 3 : 2; + return `$${n.toFixed(decimals).replace(/(\.\d*?)0+$/, '$1').replace(/\.$/, '')}`; +} + +/** + * 1000000 → 1M, 1048576 → 1M, 1050000 → 1.1M, 262144 → 262K. + * Both 1000000 and 1048576 are "1M context" to a reader — don't render one as + * `1M` and the other as `1.0M` in the same column. Exported for tests. + */ +export function formatContext(n: number | undefined): string { + if (!n) return ''; + if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1).replace(/\.0$/, '')}M`; + return `${Math.round(n / 1000)}K`; +} + +function pricingOf(m: GatewayModel): Record { + return m.pricing as unknown as Record; +} + +interface BillingGroup { + mode: BillingMode; + heading: string; + /** Right-hand price/unit string for one model. */ + render: (m: GatewayModel) => string; +} + +// Ordered as printed. `free` and `paid` are handled separately — free gets its +// own promoted section, and paid is the only one with an Input/Output/Context +// table rather than a single price column. +const BILLING_GROUPS: BillingGroup[] = [ + { + mode: 'per_image', + heading: 'Image generation (per image)', + render: m => `${money(pricingOf(m).per_image ?? 0)}/image`, + }, + { + mode: 'per_second', + heading: 'Video generation (per second)', + render: m => { + const p = pricingOf(m); + const per = p.per_second ?? 0; + const def = p.default_duration_seconds; + const max = p.max_duration_seconds; + const parts: string[] = [`${money(per)}/sec`]; + if (def) parts.push(`${def}s default ≈ ${money(per * def)}`); + if (max) parts.push(`max ${max}s`); + return parts.join(' · '); + }, + }, + { + mode: 'per_character', + heading: 'Speech / TTS (per 1K characters)', + render: m => { + const p = pricingOf(m); + const s = `${money(p.per_1k_chars ?? 0)}/1K chars`; + return p.max_input_chars ? `${s} · max ${formatContext(p.max_input_chars)} chars` : s; + }, + }, + { + mode: 'per_track', + heading: 'Music (per track)', + render: m => `${money(pricingOf(m).per_track ?? 0)}/track`, + }, + { + mode: 'per_generation', + heading: 'Sound effects (per generation)', + render: m => { + const p = pricingOf(m); + const s = `${money(p.per_generation ?? 0)}/generation`; + return p.max_duration_seconds ? `${s} · max ${p.max_duration_seconds}s` : s; + }, + }, + { + mode: 'flat', + heading: 'Flat rate (per call)', + render: m => `${money(pricingOf(m).flat ?? 0)}/call`, + }, +]; + +function printSection(heading: string, models: GatewayModel[], render: (m: GatewayModel) => string) { + if (models.length === 0) return; + console.log(chalk.yellow.bold(heading)); + console.log(chalk.dim('─'.repeat(RULE_WIDTH))); + for (const m of models) { + console.log(` ${chalk.cyan(m.id.padEnd(44))} ${render(m)}`); + } + console.log(''); } export async function modelsCommand() { @@ -15,64 +124,86 @@ export async function modelsCommand() { console.log(chalk.bold('Available Models\n')); console.log(`Chain: ${chalk.magenta(chain)} — ${chalk.dim(apiUrl)}\n`); + let models: GatewayModel[]; try { - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 15_000); - const response = await fetch(`${apiUrl}/v1/models`, { signal: controller.signal }); - clearTimeout(timeout); - if (!response.ok) { - console.log(chalk.red(`Failed to fetch models: ${response.status}`)); - return; + models = await getGatewayModels(); + } catch (err) { + const msg = err instanceof Error ? err.message : 'unknown error'; + if (/fetch|ECONNREFUSED|ENOTFOUND|abort/i.test(msg)) { + console.log(chalk.red(`Cannot reach BlockRun API at ${apiUrl}`)); + console.log(chalk.dim('Check your internet connection or try again later.')); + } else { + console.log(chalk.red(`Error: ${msg}`)); } + return; + } - const data = (await response.json()) as { data: Model[] }; - if (!data.data || data.data.length === 0) { - console.log(chalk.yellow('No models returned from API.')); - return; - } - const models = data.data - .sort((a: Model, b: Model) => (a.pricing?.input ?? 0) - (b.pricing?.input ?? 0)); - - const free = models.filter((m: Model) => m.billing_mode === 'free'); - const paid = models.filter((m: Model) => m.billing_mode !== 'free'); - - if (free.length > 0) { - console.log(chalk.green.bold('Free Models (no USDC needed)')); - console.log(chalk.dim('─'.repeat(70))); - for (const m of free) { - console.log(` ${chalk.cyan(m.id)}`); - } - console.log(''); + if (models.length === 0) { + console.log(chalk.yellow('No models returned from API.')); + return; + } + + // Free first — it's the "no USDC needed" on-ramp and the reason many people + // install Franklin at all. + const free = models.filter(m => m.billing_mode === 'free'); + if (free.length > 0) { + console.log(chalk.green.bold('Free Models (no USDC needed)')); + console.log(chalk.dim('─'.repeat(RULE_WIDTH))); + for (const m of free) { + const ctx = formatContext(m.context_window); + console.log(` ${chalk.cyan(m.id.padEnd(44))} ${chalk.dim(ctx ? `${ctx} ctx` : '')}`); } + console.log(''); + } + + // Token-metered — the only group with a two-price table. + const paid = models + .filter(m => m.billing_mode === 'paid') + .sort((a, b) => { + const ap = a.pricing as { input?: number }; + const bp = b.pricing as { input?: number }; + return (ap.input ?? 0) - (bp.input ?? 0); + }); - console.log(chalk.yellow.bold('Paid Models')); - console.log(chalk.dim('─'.repeat(70))); + if (paid.length > 0) { + console.log(chalk.yellow.bold('Chat & Reasoning (per 1M tokens)')); + console.log(chalk.dim('─'.repeat(RULE_WIDTH))); console.log( - chalk.dim( - ` ${'Model'.padEnd(35)} ${'Input'.padEnd(12)} ${'Output'.padEnd(12)} Context` - ) + chalk.dim(` ${'Model'.padEnd(44)} ${'Input'.padEnd(11)} ${'Output'.padEnd(11)} Context`) ); - console.log(chalk.dim('─'.repeat(70))); - + console.log(chalk.dim('─'.repeat(RULE_WIDTH))); for (const m of paid) { - const input = `$${(m.pricing?.input ?? 0).toFixed(2)}/M`; - const output = `$${(m.pricing?.output ?? 0).toFixed(2)}/M`; - const ctx = ''; + const p = pricingOf(m); + const input = `${money(p.input ?? 0)}/M`; + const output = `${money(p.output ?? 0)}/M`; console.log( - ` ${chalk.cyan(m.id.padEnd(35))} ${input.padEnd(12)} ${output.padEnd(12)} ${ctx}` + ` ${chalk.cyan(m.id.padEnd(44))} ${input.padEnd(11)} ${output.padEnd(11)} ${chalk.dim(formatContext(m.context_window))}` ); } + console.log(''); + } - console.log( - `\n${chalk.dim(`${models.length} models available. Use:`)} ${chalk.bold('franklin start --model ')}` - ); - } catch (err) { - const msg = err instanceof Error ? err.message : 'unknown error'; - if (msg.includes('fetch') || msg.includes('ECONNREFUSED') || msg.includes('ENOTFOUND')) { - console.log(chalk.red(`Cannot reach BlockRun API at ${apiUrl}`)); - console.log(chalk.dim('Check your internet connection or try again later.')); - } else { - console.log(chalk.red(`Error: ${msg}`)); - } + for (const group of BILLING_GROUPS) { + const inGroup = models.filter(m => m.billing_mode === group.mode); + printSection(group.heading, inGroup, group.render); } + + // Anything the gateway starts billing a way this build doesn't know about + // must still be listed — silently dropping a model is how the $0.00/M bug + // stayed invisible. Show it with its raw pricing rather than a wrong unit. + const known = new Set(['free', 'paid', ...BILLING_GROUPS.map(g => g.mode)]); + const unknown = models.filter(m => !known.has(m.billing_mode)); + printSection( + 'Other billing modes (not yet modelled by this Franklin build)', + unknown, + m => chalk.dim(`${m.billing_mode}: ${JSON.stringify(m.pricing)}`) + ); + + const margin = Math.round((GATEWAY_MARGIN - 1) * 100); + console.log( + chalk.dim( + `${models.length} models available. Prices are gateway list — x402 settlement adds ~${margin}%.` + ) + ); + console.log(`${chalk.dim('Use:')} ${chalk.bold('franklin start --model ')}`); } diff --git a/src/gateway-models.ts b/src/gateway-models.ts index 7c2a605..e9783a3 100644 --- a/src/gateway-models.ts +++ b/src/gateway-models.ts @@ -18,7 +18,15 @@ import { loadChain, API_URLS, USER_AGENT } from './config.js'; // ─── Types ────────────────────────────────────────────────────────────── -export type BillingMode = 'paid' | 'free' | 'flat' | 'per_image' | 'per_second' | 'per_track'; +export type BillingMode = + | 'paid' + | 'free' + | 'flat' + | 'per_image' + | 'per_second' + | 'per_track' + | 'per_character' + | 'per_generation'; export interface PaidPricing { input: number; output: number; } export interface FlatPricing { flat: number; } @@ -29,13 +37,25 @@ export interface PerSecondPricing { max_duration_seconds?: number; } export interface PerTrackPricing { per_track: number; } +/** ElevenLabs / ByteDance speech — billed per 1K characters of input text. */ +export interface PerCharacterPricing { + per_1k_chars: number; + max_input_chars?: number; +} +/** ElevenLabs sound effects — flat charge per generation. */ +export interface PerGenerationPricing { + per_generation: number; + max_duration_seconds?: number; +} export type ModelPricing = | PaidPricing | FlatPricing | PerImagePricing | PerSecondPricing - | PerTrackPricing; + | PerTrackPricing + | PerCharacterPricing + | PerGenerationPricing; export interface GatewayModel { id: string; @@ -135,6 +155,8 @@ export interface EstimateContext { quantity?: number; /** Clip length in seconds (per_second). Falls back to model's default_duration_seconds, then 8. */ duration_seconds?: number; + /** Input text length (per_character). Required for a meaningful speech estimate. */ + characters?: number; } /** @@ -157,6 +179,14 @@ export function estimateCostUsd(model: GatewayModel, ctx: EstimateContext = {}): case 'per_track': base = p.per_track ?? 0; break; + case 'per_character': + // Priced per 1K characters of input text. Without a length there's no + // meaningful estimate, so fall through to 0 rather than invent one. + base = ((p.per_1k_chars ?? 0) * (ctx.characters ?? 0)) / 1000; + break; + case 'per_generation': + base = p.per_generation ?? 0; + break; case 'flat': base = p.flat ?? 0; break; diff --git a/test/local.mjs b/test/local.mjs index d101d2f..00265a7 100644 --- a/test/local.mjs +++ b/test/local.mjs @@ -6729,6 +6729,69 @@ test('reconcilePicker: free models render FREE, and moreCount counts uncurated c assert.equal(moreCount, 1, 'moreCount should count uncurated chat models only'); }); +// ─── franklin models: per-billing-mode rendering ────────────────────────── +// +// Regression net for the "$0.00/M" bug: the command modelled every model as +// { input, output }, so the whole image/video/music/speech catalog printed as +// $0.00/M under a "Paid Models" heading — i.e. the paid media catalog looked +// free. Nothing tested this file, which is why it survived. + +test('models money(): sub-cent prices keep their precision (no $0.00)', async () => { + const { money } = await import('../dist/commands/models.js'); + assert.equal(money(0), '$0'); + assert.equal(money(5), '$5'); + assert.equal(money(2.5), '$2.5'); + assert.equal(money(0.435), '$0.435', 'V4 Pro input must not round to $0.44'); + assert.equal(money(0.098), '$0.098', 'seedance per-second must not round to $0.10'); + assert.equal(money(0.015), '$0.015', 'cogview per-image must not render as $0.02'); + // The actual old-bug shape: a real per-image price must never print as $0.00. + for (const p of [0.02, 0.045, 0.05, 0.06, 0.1]) { + assert.notEqual(money(p), '$0.00', `${p} must not render as $0.00`); + assert.notEqual(money(p), '$0', `${p} must not render as $0`); + } +}); + +test('models formatContext(): 1000000 and 1048576 both read as 1M', async () => { + const { formatContext } = await import('../dist/commands/models.js'); + assert.equal(formatContext(undefined), ''); + assert.equal(formatContext(0), ''); + assert.equal(formatContext(131_072), '131K'); + assert.equal(formatContext(262_144), '262K'); + assert.equal(formatContext(200_000), '200K'); + // Same column must not mix "1M" and "1.0M". + assert.equal(formatContext(1_000_000), '1M'); + assert.equal(formatContext(1_048_576), '1M'); + assert.equal(formatContext(1_050_000), '1.1M'); +}); + +test('estimateCostUsd: per_character and per_generation are priced, not silently $0', async () => { + const { estimateCostUsd, GATEWAY_MARGIN } = await import('../dist/gateway-models.js'); + // Both modes are live on the gateway (elevenlabs speech + sound effects) but + // were missing from BillingMode, so the switch fell through and returned 0 — + // a paid call estimated as free. + const speech = { + id: 'elevenlabs/flash-v2.5', + name: 'Flash v2.5', + billing_mode: 'per_character', + categories: ['speech'], + pricing: { per_1k_chars: 0.05, max_input_chars: 40000 }, + }; + // 2000 chars × $0.05/1K = $0.10 base, +5% gateway margin. + assert.equal(estimateCostUsd(speech, { characters: 2000 }), +(0.1 * GATEWAY_MARGIN).toFixed(6)); + // No length given → no meaningful estimate rather than an invented one. + assert.equal(estimateCostUsd(speech, {}), 0); + + const sfx = { + id: 'elevenlabs/sound-effects', + name: 'Sound Effects', + billing_mode: 'per_generation', + categories: ['sound_effect'], + pricing: { per_generation: 0.05, max_duration_seconds: 22 }, + }; + assert.equal(estimateCostUsd(sfx, {}), +(0.05 * GATEWAY_MARGIN).toFixed(6)); + assert.ok(estimateCostUsd(sfx, {}) > 0, 'a paid sound-effect call must not estimate as free'); +}); + // ─── nemotron prose stripper ────────────────────────────────────────────── test('nemotron prose stripper: only matches Nemotron Omni model id', async () => {