From a61248fa2d551b457f811de1b9f799e720bba212 Mon Sep 17 00:00:00 2001 From: Amir Moradi Date: Sun, 27 Sep 2026 23:16:07 +0200 Subject: [PATCH] feat(notes): local-first voice memo transcription into notes (ODIAA-2331) Add `tb notes transcribe ` and MCP tool `note_transcribe`, transcribing audio to text with a locally installed speech-to-text engine (whisper.cpp or faster-whisper, auto-detected on PATH or via TB_WHISPER_CPP_BIN/TB_FASTER_WHISPER_BIN) and saving the result as a note with source/engine metadata in its front matter. No audio or text ever leaves the machine: there is no cloud STT path and no API keys. When no local engine is found, the error names both supported engines and how to install them. Cloud STT is a separate vendor decision left for the PR to route through CTO/CEO approval. Tests cover the stt.js engine-selection/error paths and the CLI/MCP surfaces using stub engine binaries, with no real audio processing required. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 12 +++ SPEC.md | 21 ++++- cli/src/cli.js | 30 +++++- cli/src/notes.js | 11 ++- cli/src/stt.js | 124 +++++++++++++++++++++++++ docs/COMMANDS.md | 3 + mcp/src/notes.js | 11 ++- mcp/src/stt.js | 127 ++++++++++++++++++++++++++ mcp/src/tools.js | 40 +++++++- skills/thunderbird-cli/SKILL.md | 6 ++ test/docs.test.mjs | 8 +- test/fixtures/stub-faster-whisper.mjs | 22 +++++ test/fixtures/stub-whisper-cpp.mjs | 18 ++++ test/mcp-test.mjs | 28 +++++- test/notes-test.mjs | 79 +++++++++++++++- 15 files changed, 529 insertions(+), 11 deletions(-) create mode 100644 cli/src/stt.js create mode 100644 mcp/src/stt.js create mode 100755 test/fixtures/stub-faster-whisper.mjs create mode 100755 test/fixtures/stub-whisper-cpp.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8673873..b82bab8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 actively maintained) and sanitizes with [`sanitize-html`](https://www.npmjs.com/package/sanitize-html) (parser-driven, not regex-based) before the HTML ever reaches an email draft. +- **`tb notes transcribe `** / MCP `note_transcribe` — voice memo transcription into + Notes. Local-first: at runtime it detects a locally-installed speech-to-text engine — + [whisper.cpp](https://github.com/ggml-org/whisper.cpp) (`whisper-cli` on PATH, or + `TB_WHISPER_CPP_BIN`; needs a model via `--model`/`TB_WHISPER_CPP_MODEL`) or + [faster-whisper](https://github.com/SYSTRAN/faster-whisper)'s `whisper-ctranslate2` CLI wrapper + (on PATH, or `TB_FASTER_WHISPER_BIN`) — and shells out to it; no audio or text ever leaves the + machine, and no API key is read. Absence of either engine is a clean `NO_ENGINE` error with + install hints, not a stack trace. The transcript is saved as a note with the source audio + filename and engine used recorded in front matter (`source`, `engine`). No cloud STT provider + is implemented — that's a separate vendor/stack decision requiring its own sign-off before any + network path ships (see the ODIAA-2331 PR for the option A vs B trade-off). Tested against a + stubbed engine (`test/fixtures/stub-*.mjs`). - `--from ` on `tb forward` (`from` on MCP `email_forward`); `--from` on reply/forward may now name an identity from any account (unknown identities are still rejected). - `--subject ` on `tb reply` (`subject` on MCP `email_reply`). - `npm run test:draft` — draft-routing regression suite (identity resolution, cross-account `--from`, Gmail dual-drafts warning, unconfirmed-save errors, reply subject/quote/HTML handling), also part of `test:extension`. diff --git a/SPEC.md b/SPEC.md index dee71bc..54e7446 100644 --- a/SPEC.md +++ b/SPEC.md @@ -580,6 +580,12 @@ tb notes delete --confirm tb notes search # Returns: [{name, title, snippet}] +# Transcribe a local audio file to text with a local speech-to-text engine +# and save the transcript as a note. Local-first: no audio or text ever +# leaves this machine. +tb notes transcribe [--engine whisper-cpp|faster-whisper] [--model ] + [--language ] [--save-as ] [--title ] [--append] + # Render a note's Markdown to sanitized HTML and open it as an email draft. # Uses the same /compose route as `tb compose` — never sends. tb notes to-draft --to
[--cc ] [--bcc ] [--subject ] [--from ] [--open] @@ -589,6 +595,18 @@ Markdown → HTML uses `marked` (rendering) and `sanitize-html` (stripping scripts, event handlers, and unsafe URL schemes) before the HTML ever reaches a compose draft. +`tb notes transcribe` detects a locally-installed speech-to-text engine at +runtime — [whisper.cpp](https://github.com/ggml-org/whisper.cpp) (`whisper-cli` +on PATH, or `TB_WHISPER_CPP_BIN`; requires a model file via `--model` or +`TB_WHISPER_CPP_MODEL`) or [faster-whisper](https://github.com/SYSTRAN/faster-whisper)'s +`whisper-ctranslate2` CLI wrapper (on PATH, or `TB_FASTER_WHISPER_BIN`) — and +shells out to it. If neither is found it fails with a clean error and install +hints rather than a stack trace. No cloud STT provider is implemented: that's +a separate vendor/stack decision (see the ODIAA-2331 PR for the option A vs B +trade-off) and would need its own sign-off before any network path ships. +The engine used and the source audio filename are recorded in the saved +note's front matter (`engine`, `source`). + ### 19. Calendars Read-only, experimental — requires the `calendar` Experiment API vendored into @@ -1220,6 +1238,7 @@ The 16 MCP tools are **curated** for AI agent use cases. Bulk admin operations ( | `note_read` | `tb notes read` ("Use as Context") | | `note_save` | `tb notes save` ("Save to Notes") | | `note_append` | `tb notes append` | +| `note_transcribe` | `tb notes transcribe` (local speech-to-text engine, auto-detected) | | `note_to_draft` | `tb notes to-draft` (draft/open modes, never sends) | | `contact_search` | `tb contacts`, `tb contacts-search` | | `contact_create` | `tb contacts create` (requires `contactsWrite`) | @@ -1231,7 +1250,7 @@ local notes workspace directly. ### Why fewer MCP tools than CLI commands? -| | CLI (43 commands) | MCP (21 tools) | +| | CLI (44 commands) | MCP (22 tools) | |---|---|---| | Audience | Humans + scripts | AI agents | | Discovery | `tb --help` | Tool descriptions in LLM context | diff --git a/cli/src/cli.js b/cli/src/cli.js index 6564f5d..f80b1a3 100755 --- a/cli/src/cli.js +++ b/cli/src/cli.js @@ -10,7 +10,7 @@ import { createRequire } from "module"; import { Command } from "commander"; import { api, output, outputError, getConfig, parseRelativeDate } from "./client.js"; -import { extname } from "path"; +import { basename, extname } from "path"; import { listNotes, readNote, @@ -20,6 +20,7 @@ import { searchNotes, renderNoteHtml, } from "./notes.js"; +import { transcribeAudio } from "./stt.js"; const MIME_TO_EXT = { "application/pdf": ".pdf", @@ -1300,6 +1301,33 @@ notes output(searchNotes(query), getFormat(g), getOutputOpts(g)); })); +notes + .command("transcribe ") + .description( + "Transcribe a local audio file to text with a local speech-to-text engine and save it as a note " + + "(no audio ever leaves this machine)" + ) + .option("--engine ", "STT engine to use: whisper-cpp or faster-whisper (default: auto-detect)") + .option("--model ", "model file path (required for whisper.cpp; optional override for faster-whisper)") + .option("--language ", "spoken language hint, e.g. en") + .option("--save-as ", "note name to save as (defaults to the audio file's base name)") + .option("--title ", "note title (defaults to the note name)") + .option("--append", "append to an existing note instead of overwriting") + .action(run(async (audioFile, opts) => { + const g = program.opts(); + const { text, engine } = transcribeAudio(audioFile, { + engine: opts.engine, + model: opts.model, + language: opts.language, + }); + const name = opts.saveAs || basename(audioFile, extname(audioFile)); + const source = `voice-memo:${basename(audioFile)}`; + const result = opts.append + ? appendNote(name, text, { title: opts.title, source, engine }) + : saveNote(name, text, { title: opts.title, source, engine }); + output({ ...result, engine, transcript: text }, getFormat(g), getOutputOpts(g)); + })); + notes .command("to-draft ") .description("Render a note to an HTML email draft (never sends)") diff --git a/cli/src/notes.js b/cli/src/notes.js index d58039c..a070de1 100644 --- a/cli/src/notes.js +++ b/cli/src/notes.js @@ -137,6 +137,7 @@ export function listNotes(opts = {}) { title: meta.title || name, created: meta.created || stat.birthtime.toISOString(), source: meta.source ?? null, + engine: meta.engine ?? null, size: stat.size, modified: stat.mtime.toISOString(), }; @@ -158,6 +159,7 @@ export function readNote(name, opts = {}) { title: meta.title || safeName, created: meta.created || stat.birthtime.toISOString(), source: meta.source ?? null, + engine: meta.engine ?? null, modified: stat.mtime.toISOString(), body: body.trimEnd(), }; @@ -171,6 +173,7 @@ export function saveNote(name, body, opts = {}) { title: opts.title || safeName, created: new Date().toISOString(), source: opts.source ?? undefined, + engine: opts.engine ?? undefined, }; writeFileSync(filePath, serializeFrontMatter(meta) + String(body ?? "").trimEnd() + "\n", "utf-8"); return { name: safeName, title: meta.title, path: filePath }; @@ -186,12 +189,18 @@ export function appendNote(name, body, opts = {}) { const nextMeta = { ...meta }; if (opts.title) nextMeta.title = opts.title; if (opts.source !== undefined) nextMeta.source = opts.source; + if (opts.engine !== undefined) nextMeta.engine = opts.engine; const combined = `${existingBody.trimEnd()}\n\n${String(body ?? "").trimEnd()}\n`; writeFileSync(filePath, serializeFrontMatter(nextMeta) + combined, "utf-8"); return { name: safeName, title: nextMeta.title || safeName, path: filePath, created: false }; } - const meta = { title: opts.title || safeName, created: new Date().toISOString(), source: opts.source ?? undefined }; + const meta = { + title: opts.title || safeName, + created: new Date().toISOString(), + source: opts.source ?? undefined, + engine: opts.engine ?? undefined, + }; writeFileSync(filePath, serializeFrontMatter(meta) + String(body ?? "").trimEnd() + "\n", "utf-8"); return { name: safeName, title: meta.title, path: filePath, created: true }; } diff --git a/cli/src/stt.js b/cli/src/stt.js new file mode 100644 index 0000000..ccbdb75 --- /dev/null +++ b/cli/src/stt.js @@ -0,0 +1,124 @@ +/** + * Local speech-to-text for voice memo transcription. + * + * Local-first by design: every engine here shells out to a binary the user + * installs themselves and runs entirely on their machine. No audio or text + * ever leaves the machine, no network call is made, and no API key is read + * or required. A cloud STT provider is a separate vendor/stack decision and + * is intentionally not implemented here (see the ODIAA-2331 PR description + * for the option A vs B trade-off). + * + * Engines are detected at runtime (env var override, then well-known PATH + * binary names) so the CLI works whether the user has whisper.cpp, + * faster-whisper, both, or neither installed — absence produces a clean + * error with an install hint rather than a stack trace. + */ + +import { existsSync, mkdtempSync, readFileSync, rmSync } from "fs"; +import { execFileSync } from "child_process"; +import { tmpdir } from "os"; +import { basename, delimiter, extname, join, resolve } from "path"; + +function invalidArgs(message) { + return Object.assign(new Error(`INVALID_ARGS: ${message}`), { code: "INVALID_ARGS" }); +} + +// Default cap on how long an engine may run before we give up and kill it — +// guards against a hung or runaway process rather than any expectation of +// how long real transcription takes. +const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000; + +function which(names) { + const dirs = (process.env.PATH || "").split(delimiter); + for (const dir of dirs) { + if (!dir) continue; + for (const name of names) { + const candidate = join(dir, name); + if (existsSync(candidate)) return candidate; + } + } + return null; +} + +const ENGINES = [ + { + id: "whisper-cpp", + label: "whisper.cpp", + resolveBin: () => process.env.TB_WHISPER_CPP_BIN || which(["whisper-cli", "whisper-cpp"]), + installHint: + "whisper.cpp: install it (https://github.com/ggml-org/whisper.cpp), put `whisper-cli` on PATH " + + "or set TB_WHISPER_CPP_BIN to its path, and pass --model (or set TB_WHISPER_CPP_MODEL) to a " + + "downloaded ggml model file.", + transcribe(bin, audioFile, outDir, opts) { + const model = opts.model || process.env.TB_WHISPER_CPP_MODEL; + if (!model) throw invalidArgs("whisper.cpp requires a model file: pass --model or set TB_WHISPER_CPP_MODEL"); + if (!existsSync(model)) throw invalidArgs(`model file not found: ${model}`); + const outPrefix = join(outDir, "transcript"); + const args = ["-m", model, "-f", audioFile, "-otxt", "-of", outPrefix, "-nt"]; + if (opts.language) args.push("-l", opts.language); + execFileSync(bin, args, { stdio: ["ignore", "pipe", "pipe"], timeout: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS }); + return readFileSync(`${outPrefix}.txt`, "utf-8"); + }, + }, + { + id: "faster-whisper", + label: "faster-whisper", + resolveBin: () => process.env.TB_FASTER_WHISPER_BIN || which(["whisper-ctranslate2"]), + installHint: + "faster-whisper: `pip install whisper-ctranslate2` (its CLI wrapper), ensure `whisper-ctranslate2` " + + "is on PATH, or set TB_FASTER_WHISPER_BIN to its path.", + transcribe(bin, audioFile, outDir, opts) { + const args = [audioFile, "--output_format", "txt", "--output_dir", outDir]; + if (opts.model) args.push("--model", opts.model); + if (opts.language) args.push("--language", opts.language); + execFileSync(bin, args, { stdio: ["ignore", "pipe", "pipe"], timeout: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS }); + const base = basename(audioFile, extname(audioFile)); + return readFileSync(join(outDir, `${base}.txt`), "utf-8"); + }, + }, +]; + +export function listEngines() { + return ENGINES.map((e) => ({ id: e.id, label: e.label, available: Boolean(e.resolveBin()) })); +} + +function resolveEngine(preferredId) { + const candidates = preferredId ? ENGINES.filter((e) => e.id === preferredId) : ENGINES; + if (preferredId && candidates.length === 0) { + throw invalidArgs(`unknown engine '${preferredId}'; expected one of: ${ENGINES.map((e) => e.id).join(", ")}`); + } + for (const engine of candidates) { + const bin = engine.resolveBin(); + if (bin) return { engine, bin }; + } + const hints = candidates.map((e) => ` - ${e.installHint}`).join("\n"); + throw Object.assign( + new Error( + "NO_ENGINE: no local speech-to-text engine found.\n" + + `${hints}\n` + + "Cloud speech-to-text is not supported here (local-first by design) — no audio ever leaves this machine." + ), + { code: "NO_ENGINE" } + ); +} + +export function transcribeAudio(audioFile, opts = {}) { + if (typeof audioFile !== "string" || !audioFile.trim()) throw invalidArgs("audio file path is required"); + const resolvedAudio = resolve(audioFile); + if (!existsSync(resolvedAudio)) { + throw Object.assign(new Error(`NOT_FOUND: audio file '${audioFile}' does not exist`), { code: "NOT_FOUND" }); + } + + const { engine, bin } = resolveEngine(opts.engine); + const outDir = mkdtempSync(join(tmpdir(), "tb-transcribe-")); + try { + const text = engine.transcribe(bin, resolvedAudio, outDir, opts).trim(); + return { text, engine: engine.id, audioFile: resolvedAudio }; + } catch (err) { + if (err.code === "INVALID_ARGS" || err.code === "NOT_FOUND") throw err; + const detail = err.stderr ? String(err.stderr).slice(0, 2000).trim() : err.message; + throw Object.assign(new Error(`ENGINE_FAILED: ${engine.label} failed: ${detail}`), { code: "ENGINE_FAILED" }); + } finally { + rmSync(outDir, { recursive: true, force: true }); + } +} diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index d5c2ebe..15f6330 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -271,6 +271,9 @@ tb notes append --body "more text" [--body-file ] [--title ] [-- # append, creates if missing tb notes delete --confirm # delete a note tb notes search # search titles and bodies +tb notes transcribe [--engine whisper-cpp|faster-whisper] [--model ] + [--language ] [--save-as ] [--title ] [--append] + # local speech-to-text, save transcript as a note tb notes to-draft --to
[--cc ] [--bcc ] [--subject ] [--from ] [--open] # render Markdown to sanitized HTML, open as draft (never sends) ``` diff --git a/mcp/src/notes.js b/mcp/src/notes.js index 44bc6f7..04fb93e 100644 --- a/mcp/src/notes.js +++ b/mcp/src/notes.js @@ -140,6 +140,7 @@ export function listNotes(opts = {}) { title: meta.title || name, created: meta.created || stat.birthtime.toISOString(), source: meta.source ?? null, + engine: meta.engine ?? null, size: stat.size, modified: stat.mtime.toISOString(), }; @@ -161,6 +162,7 @@ export function readNote(name, opts = {}) { title: meta.title || safeName, created: meta.created || stat.birthtime.toISOString(), source: meta.source ?? null, + engine: meta.engine ?? null, modified: stat.mtime.toISOString(), body: body.trimEnd(), }; @@ -174,6 +176,7 @@ export function saveNote(name, body, opts = {}) { title: opts.title || safeName, created: new Date().toISOString(), source: opts.source ?? undefined, + engine: opts.engine ?? undefined, }; writeFileSync(filePath, serializeFrontMatter(meta) + String(body ?? "").trimEnd() + "\n", "utf-8"); return { name: safeName, title: meta.title, path: filePath }; @@ -189,12 +192,18 @@ export function appendNote(name, body, opts = {}) { const nextMeta = { ...meta }; if (opts.title) nextMeta.title = opts.title; if (opts.source !== undefined) nextMeta.source = opts.source; + if (opts.engine !== undefined) nextMeta.engine = opts.engine; const combined = `${existingBody.trimEnd()}\n\n${String(body ?? "").trimEnd()}\n`; writeFileSync(filePath, serializeFrontMatter(nextMeta) + combined, "utf-8"); return { name: safeName, title: nextMeta.title || safeName, path: filePath, created: false }; } - const meta = { title: opts.title || safeName, created: new Date().toISOString(), source: opts.source ?? undefined }; + const meta = { + title: opts.title || safeName, + created: new Date().toISOString(), + source: opts.source ?? undefined, + engine: opts.engine ?? undefined, + }; writeFileSync(filePath, serializeFrontMatter(meta) + String(body ?? "").trimEnd() + "\n", "utf-8"); return { name: safeName, title: meta.title, path: filePath, created: true }; } diff --git a/mcp/src/stt.js b/mcp/src/stt.js new file mode 100644 index 0000000..bd94c9b --- /dev/null +++ b/mcp/src/stt.js @@ -0,0 +1,127 @@ +/** + * Local speech-to-text for voice memo transcription. + * + * Local-first by design: every engine here shells out to a binary the user + * installs themselves and runs entirely on their machine. No audio or text + * ever leaves the machine, no network call is made, and no API key is read + * or required. A cloud STT provider is a separate vendor/stack decision and + * is intentionally not implemented here (see the ODIAA-2331 PR description + * for the option A vs B trade-off). + * + * Engines are detected at runtime (env var override, then well-known PATH + * binary names) so the CLI works whether the user has whisper.cpp, + * faster-whisper, both, or neither installed — absence produces a clean + * error with an install hint rather than a stack trace. + * + * Self-contained copy of cli/src/stt.js, so the mcp package has no runtime + * dependency on the cli package (see notes.js). Keep in sync. + */ + +import { existsSync, mkdtempSync, readFileSync, rmSync } from "fs"; +import { execFileSync } from "child_process"; +import { tmpdir } from "os"; +import { basename, delimiter, extname, join, resolve } from "path"; + +function invalidArgs(message) { + return Object.assign(new Error(`INVALID_ARGS: ${message}`), { code: "INVALID_ARGS" }); +} + +// Default cap on how long an engine may run before we give up and kill it — +// guards against a hung or runaway process rather than any expectation of +// how long real transcription takes. +const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000; + +function which(names) { + const dirs = (process.env.PATH || "").split(delimiter); + for (const dir of dirs) { + if (!dir) continue; + for (const name of names) { + const candidate = join(dir, name); + if (existsSync(candidate)) return candidate; + } + } + return null; +} + +const ENGINES = [ + { + id: "whisper-cpp", + label: "whisper.cpp", + resolveBin: () => process.env.TB_WHISPER_CPP_BIN || which(["whisper-cli", "whisper-cpp"]), + installHint: + "whisper.cpp: install it (https://github.com/ggml-org/whisper.cpp), put `whisper-cli` on PATH " + + "or set TB_WHISPER_CPP_BIN to its path, and pass --model (or set TB_WHISPER_CPP_MODEL) to a " + + "downloaded ggml model file.", + transcribe(bin, audioFile, outDir, opts) { + const model = opts.model || process.env.TB_WHISPER_CPP_MODEL; + if (!model) throw invalidArgs("whisper.cpp requires a model file: pass --model or set TB_WHISPER_CPP_MODEL"); + if (!existsSync(model)) throw invalidArgs(`model file not found: ${model}`); + const outPrefix = join(outDir, "transcript"); + const args = ["-m", model, "-f", audioFile, "-otxt", "-of", outPrefix, "-nt"]; + if (opts.language) args.push("-l", opts.language); + execFileSync(bin, args, { stdio: ["ignore", "pipe", "pipe"], timeout: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS }); + return readFileSync(`${outPrefix}.txt`, "utf-8"); + }, + }, + { + id: "faster-whisper", + label: "faster-whisper", + resolveBin: () => process.env.TB_FASTER_WHISPER_BIN || which(["whisper-ctranslate2"]), + installHint: + "faster-whisper: `pip install whisper-ctranslate2` (its CLI wrapper), ensure `whisper-ctranslate2` " + + "is on PATH, or set TB_FASTER_WHISPER_BIN to its path.", + transcribe(bin, audioFile, outDir, opts) { + const args = [audioFile, "--output_format", "txt", "--output_dir", outDir]; + if (opts.model) args.push("--model", opts.model); + if (opts.language) args.push("--language", opts.language); + execFileSync(bin, args, { stdio: ["ignore", "pipe", "pipe"], timeout: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS }); + const base = basename(audioFile, extname(audioFile)); + return readFileSync(join(outDir, `${base}.txt`), "utf-8"); + }, + }, +]; + +export function listEngines() { + return ENGINES.map((e) => ({ id: e.id, label: e.label, available: Boolean(e.resolveBin()) })); +} + +function resolveEngine(preferredId) { + const candidates = preferredId ? ENGINES.filter((e) => e.id === preferredId) : ENGINES; + if (preferredId && candidates.length === 0) { + throw invalidArgs(`unknown engine '${preferredId}'; expected one of: ${ENGINES.map((e) => e.id).join(", ")}`); + } + for (const engine of candidates) { + const bin = engine.resolveBin(); + if (bin) return { engine, bin }; + } + const hints = candidates.map((e) => ` - ${e.installHint}`).join("\n"); + throw Object.assign( + new Error( + "NO_ENGINE: no local speech-to-text engine found.\n" + + `${hints}\n` + + "Cloud speech-to-text is not supported here (local-first by design) — no audio ever leaves this machine." + ), + { code: "NO_ENGINE" } + ); +} + +export function transcribeAudio(audioFile, opts = {}) { + if (typeof audioFile !== "string" || !audioFile.trim()) throw invalidArgs("audio file path is required"); + const resolvedAudio = resolve(audioFile); + if (!existsSync(resolvedAudio)) { + throw Object.assign(new Error(`NOT_FOUND: audio file '${audioFile}' does not exist`), { code: "NOT_FOUND" }); + } + + const { engine, bin } = resolveEngine(opts.engine); + const outDir = mkdtempSync(join(tmpdir(), "tb-transcribe-")); + try { + const text = engine.transcribe(bin, resolvedAudio, outDir, opts).trim(); + return { text, engine: engine.id, audioFile: resolvedAudio }; + } catch (err) { + if (err.code === "INVALID_ARGS" || err.code === "NOT_FOUND") throw err; + const detail = err.stderr ? String(err.stderr).slice(0, 2000).trim() : err.message; + throw Object.assign(new Error(`ENGINE_FAILED: ${engine.label} failed: ${detail}`), { code: "ENGINE_FAILED" }); + } finally { + rmSync(outDir, { recursive: true, force: true }); + } +} diff --git a/mcp/src/tools.js b/mcp/src/tools.js index 3163856..929c9bc 100644 --- a/mcp/src/tools.js +++ b/mcp/src/tools.js @@ -12,6 +12,8 @@ import { parseRelativeDate } from "./client.js"; import { listNotes, readNote, saveNote, appendNote, renderNoteHtml } from "./notes.js"; +import { transcribeAudio } from "./stt.js"; +import { basename, extname } from "path"; export const tools = [ // ─── 1. Stats ────────────────────────────────────────────────── @@ -755,7 +757,43 @@ export const tools = [ handler: async (args) => appendNote(args.name, args.body, { title: args.title, source: args.source }), }, - // ─── 21. Notes: to draft ───────────────────────────────────────── + // ─── 21. Notes: transcribe ──────────────────────────────────────── + { + name: "note_transcribe", + description: + "Transcribe a local audio file to text with a local speech-to-text engine (whisper.cpp or " + + "faster-whisper, auto-detected on this machine) and save the transcript as a note. Local-first: " + + "no audio or text ever leaves this machine, and no cloud STT provider is used. Source metadata " + + "(the audio file name and engine used) is recorded in the note's front matter.", + inputSchema: { + type: "object", + properties: { + audioFile: { type: "string", description: "Path to a local audio file (format support depends on the detected engine)" }, + name: { type: "string", description: "Note name to save as (defaults to the audio file's base name); letters, numbers, spaces, '.', '_', '-' only" }, + title: { type: "string", description: "Note title (defaults to the note name)" }, + engine: { type: "string", description: "STT engine to use: 'whisper-cpp' or 'faster-whisper' (default: auto-detect)" }, + model: { type: "string", description: "Model file path (required for whisper.cpp; optional override for faster-whisper)" }, + language: { type: "string", description: "Spoken language hint, e.g. 'en'" }, + append: { type: "boolean", description: "Append to an existing note instead of overwriting it" }, + }, + required: ["audioFile"], + }, + handler: async (args) => { + const { text, engine } = transcribeAudio(args.audioFile, { + engine: args.engine, + model: args.model, + language: args.language, + }); + const name = args.name || basename(args.audioFile, extname(args.audioFile)); + const source = `voice-memo:${basename(args.audioFile)}`; + const result = args.append + ? appendNote(name, text, { title: args.title, source, engine }) + : saveNote(name, text, { title: args.title, source, engine }); + return { ...result, engine, transcript: text }; + }, + }, + + // ─── 22. Notes: to draft ──────────────────────────────────────── { name: "note_to_draft", description: diff --git a/skills/thunderbird-cli/SKILL.md b/skills/thunderbird-cli/SKILL.md index 1040770..7c5dac8 100644 --- a/skills/thunderbird-cli/SKILL.md +++ b/skills/thunderbird-cli/SKILL.md @@ -61,6 +61,7 @@ Use these; don't reach for the 43-command CLI unless the user explicitly asks fo | `note_read` | Read a note's Markdown body — "Use as Context" | ✅ read-only | | `note_save` | Save/overwrite a note — "Save to Notes" | ✅ local file only | | `note_append` | Append to a note, creating it if missing | ✅ local file only | +| `note_transcribe` | Transcribe a local audio file with a local speech-to-text engine (whisper.cpp or faster-whisper, auto-detected) and save it as a note | ✅ local file + local process only | | `note_to_draft` | Render a note's Markdown to sanitized HTML and open it as an email draft. `mode: draft` / `open`. Never sends | ✅ draft by default | | `contact_search` | Search/list address book contacts across all books, matching name or any email | ✅ read-only | | `contact_create` | Create a contact in an address book | ⚠️ requires `contactsWrite` access switch (default off) | @@ -69,6 +70,10 @@ Use these; don't reach for the 43-command CLI unless the user explicitly asks fo Notes live entirely on disk (`~/.config/thunderbird-cli/notes` by default) — no Thunderbird round-trip except `note_to_draft`, which reuses the same compose route as `email_compose` and never exposes a `send` mode. +`note_transcribe` is also local-only: it shells out to a speech-to-text +engine the user installed themselves (no audio ever leaves the machine, no +cloud provider, no API key) and errors with an install hint if none is +found. ## Core patterns — always apply these @@ -289,6 +294,7 @@ MCP tool → CLI command mapping: | `note_read` | `tb notes read ` | | `note_save` | `tb notes save --body "..."` | | `note_append` | `tb notes append --body "..."` | +| `note_transcribe` | `tb notes transcribe ` | | `note_to_draft` | `tb notes to-draft --to X` | | `contact_search` | `tb contacts-search ` | | `contact_create` | `tb contacts create --book ...` | diff --git a/test/docs.test.mjs b/test/docs.test.mjs index 238b8bb..771836b 100644 --- a/test/docs.test.mjs +++ b/test/docs.test.mjs @@ -42,8 +42,14 @@ function section(text, title) { const m = text.match(new RegExp(`\\n${title}:\\n([\\s\\S]*?)(\\n\\n|$)`)); return m ? m[1].split("\n") : []; } +// Commander wraps long descriptions onto indented continuation lines (e.g. " transcribe\n" + +// " [options] Transcribe...\n as a note)"); only lines +// starting with exactly two leading spaces are real command rows, not description wrap. const commandsOf = (text) => - section(text, "Commands").map((l) => l.trim().split(/\s/)[0]).filter((c) => c && c !== "help"); + section(text, "Commands") + .filter((l) => /^ {2}\S/.test(l)) + .map((l) => l.trim().split(/\s/)[0]) + .filter((c) => c && c !== "help"); const flagsOf = (text) => section(text, "Options").flatMap((l) => l.match(/--[a-z][a-z-]*/g) || []).filter((f) => f !== "--help"); diff --git a/test/fixtures/stub-faster-whisper.mjs b/test/fixtures/stub-faster-whisper.mjs new file mode 100755 index 0000000..e4c41a2 --- /dev/null +++ b/test/fixtures/stub-faster-whisper.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +/** + * Stub faster-whisper (whisper-ctranslate2) CLI for tests (see test/notes-test.mjs and + * test/mcp-test.mjs). + * + * Mimics just enough of `whisper-ctranslate2