diff --git a/packages/redrob/src/redrob/transcription.ts b/packages/redrob/src/redrob/transcription.ts new file mode 100644 index 0000000000..0273e80a73 --- /dev/null +++ b/packages/redrob/src/redrob/transcription.ts @@ -0,0 +1,163 @@ +import { Effect, Schema } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import { Auth } from "@/auth" +import { gatewayMessage, gatewayUrl, redrobKey } from "@/tool/image_generate" + +/** + * Transcription through the Redrob gateway's `POST /v1/audio/transcriptions`, on the engine's own + * Redrob credential, for the reason speech gives: the key is the engine's and stays here. Its caller + * is the engine's `POST /redrob/transcribe` route, which is how Redrob Cowork's push-to-talk turns a + * recording into text without the app ever holding the key. + * + * Everything that can be refused is refused before anything is sent, so a bad request costs nothing. + */ + +/** Scribe handles Korean, English and the two mixed in one sentence, which is how users speak. */ +export const DEFAULT_TRANSCRIPTION_MODEL = "elevenlabs/scribe-v2" + +/** The formats the gateway accepts, which are the ones its upstream documents. */ +export const TRANSCRIPTION_FORMATS = ["wav", "mp3", "flac", "m4a", "ogg", "webm", "aac"] as const +export type TranscriptionFormat = (typeof TRANSCRIPTION_FORMATS)[number] + +/** The gateway's limit for one request, decoded, checked here so an over-size clip costs nothing. */ +export const MAX_TRANSCRIPTION_BYTES = 10 * 1024 * 1024 + +const TIMEOUT_MS = 120_000 +const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/ +const LANGUAGE = /^[a-z]{2}$/ + +/** + * Why audio could not be transcribed. `invalid` and `no_key` are found before anything is sent and + * cost nothing; `upstream` and `timeout` are the gateway's. + */ +export class TranscriptionError extends Schema.TaggedErrorClass()("TranscriptionError", { + reason: Schema.Literals(["invalid", "no_key", "upstream", "timeout"]), + message: Schema.String, + status: Schema.optional(Schema.Number), +}) {} + +export type PreparedTranscription = { + audio: string + format: TranscriptionFormat + model: string + language?: string + bytes: number +} + +export type Transcript = { text: string; seconds?: number; costUsd?: number } + +/** What base64 decodes to, without decoding it: three bytes per four characters, less the padding. */ +function decodedBytes(base64: string): number { + const padding = base64.endsWith("==") ? 2 : base64.endsWith("=") ? 1 : 0 + return (base64.length / 4) * 3 - padding +} + +/** The request as it will be sent, or why it would be refused, with nothing sent either way. */ +export const prepare = Effect.fn("RedrobTranscription.prepare")(function* (input: { + audio: string + format: TranscriptionFormat + model?: string + language?: string +}) { + const audio = input.audio.replace(/\s+/g, "") + if (!audio) return yield* new TranscriptionError({ reason: "invalid", message: "Give the audio to transcribe." }) + if (audio.length % 4 !== 0 || !BASE64.test(audio)) { + return yield* new TranscriptionError({ + reason: "invalid", + message: "The audio must be base64: the raw bytes, not a data URL.", + }) + } + const bytes = decodedBytes(audio) + if (bytes > MAX_TRANSCRIPTION_BYTES) { + return yield* new TranscriptionError({ + reason: "invalid", + message: `The audio is ${bytes} bytes; transcribe at most ${MAX_TRANSCRIPTION_BYTES} per call.`, + }) + } + const language = input.language?.trim().toLowerCase() || undefined + if (language && !LANGUAGE.test(language)) { + return yield* new TranscriptionError({ + reason: "invalid", + message: `"${input.language}" is not an ISO-639-1 language such as "ko" or "en".`, + }) + } + const prepared: PreparedTranscription = { + audio, + format: input.format, + model: input.model?.trim() || DEFAULT_TRANSCRIPTION_MODEL, + ...(language ? { language } : {}), + bytes, + } + return prepared +}) + +/** The engine's Redrob key, or a `no_key` failure, which is found before anything is sent. */ +export const key = Effect.fn("RedrobTranscription.key")(function* (auth: Auth.Interface) { + const found = redrobKey(yield* auth.get("redrob").pipe(Effect.orElseSucceed(() => undefined))) + if (!found) { + return yield* new TranscriptionError({ + reason: "no_key", + message: "Connect Redrob to transcribe speech: no Redrob API key is available.", + }) + } + return found +}) + +const GatewayTranscript = Schema.Struct({ + text: Schema.String, + seconds: Schema.optional(Schema.Number), + costUsd: Schema.optional(Schema.Number), +}) +const decodeTranscript = Schema.decodeUnknownEffect(Schema.fromJsonString(GatewayTranscript)) + +/** One call to the gateway. The answer is checked for being a transcript before anyone reads it. */ +export const request = Effect.fn("RedrobTranscription.request")(function* ( + http: HttpClient.HttpClient, + key: string, + transcription: PreparedTranscription, +) { + const outgoing = yield* HttpClientRequest.post(`${gatewayUrl()}/audio/transcriptions`).pipe( + HttpClientRequest.bearerToken(key), + HttpClientRequest.bodyJson({ + model: transcription.model, + input_audio: { data: transcription.audio, format: transcription.format }, + response_format: "json", + ...(transcription.language ? { language: transcription.language } : {}), + }), + Effect.orDie, + ) + const response = yield* http.execute(outgoing).pipe( + Effect.mapError( + () => new TranscriptionError({ reason: "upstream", message: "The Redrob gateway could not be reached." }), + ), + Effect.timeoutOrElse({ + duration: TIMEOUT_MS, + orElse: () => Effect.fail(new TranscriptionError({ reason: "timeout", message: "Transcription timed out." })), + }), + ) + const body = yield* response.text.pipe( + Effect.mapError( + () => new TranscriptionError({ reason: "upstream", message: "The Redrob gateway's answer was cut off." }), + ), + ) + if (response.status < 200 || response.status >= 300) { + return yield* new TranscriptionError({ + reason: "upstream", + status: response.status, + message: `The Redrob gateway answered ${response.status}: ${gatewayMessage(body)}`, + }) + } + const parsed = yield* decodeTranscript(body).pipe( + Effect.mapError( + () => new TranscriptionError({ reason: "upstream", message: "The Redrob gateway answered without a transcript." }), + ), + ) + const transcript: Transcript = { + text: parsed.text.trim(), + ...(parsed.seconds !== undefined && Number.isFinite(parsed.seconds) ? { seconds: parsed.seconds } : {}), + ...(parsed.costUsd !== undefined && Number.isFinite(parsed.costUsd) ? { costUsd: parsed.costUsd } : {}), + } + return transcript +}) + +export * as RedrobTranscription from "./transcription" diff --git a/packages/redrob/src/server/routes/instance/httpapi/groups/control.ts b/packages/redrob/src/server/routes/instance/httpapi/groups/control.ts index 11c5315093..c3598bcfcb 100644 --- a/packages/redrob/src/server/routes/instance/httpapi/groups/control.ts +++ b/packages/redrob/src/server/routes/instance/httpapi/groups/control.ts @@ -1,9 +1,11 @@ import { Auth } from "@/auth" import { Schema } from "effect" -import { HttpApi, HttpApiEndpoint, HttpApiError, HttpApiGroup, OpenApi } from "effect/unstable/httpapi" +import { HttpApi, HttpApiEndpoint, HttpApiError, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi" +import { InvalidRequestError, ServiceUnavailableError, TimeoutError, UpstreamError } from "../errors" import { described } from "./metadata" import { ProviderV2 } from "@redrob-code/core/provider" +import { TRANSCRIPTION_FORMATS } from "@/redrob/transcription" const AuthParams = Schema.Struct({ providerID: ProviderV2.ID, @@ -28,9 +30,36 @@ export const LogInput = Schema.Struct({ }), }) +export const SpeechInput = Schema.Struct({ + text: Schema.String.annotate({ description: "The words to speak, at most 4096 characters" }), + voice: Schema.optional(Schema.String).annotate({ description: "A voice of the chosen model" }), + model: Schema.optional(Schema.String).annotate({ + description: "A speech model id served by the Redrob gateway. Defaults to elevenlabs/eleven-flash-v2.5, for playback.", + }), +}) + +export const TranscribeInput = Schema.Struct({ + audio: Schema.String.annotate({ description: "Base64-encoded audio, the raw bytes rather than a data URL, at most 10 MB" }), + format: Schema.Literals(TRANSCRIPTION_FORMATS).annotate({ description: "The audio's container format" }), + language: Schema.optional(Schema.String).annotate({ + description: 'ISO-639-1 language, e.g. "ko". Detected when omitted.', + }), + model: Schema.optional(Schema.String).annotate({ + description: "A transcription model id served by the Redrob gateway", + }), +}) + +export const TranscribeOutput = Schema.Struct({ + text: Schema.String.annotate({ description: "The transcript, trimmed; empty when nothing was said" }), + seconds: Schema.optional(Schema.Number).annotate({ description: "Seconds of audio billed" }), + costUsd: Schema.optional(Schema.Number).annotate({ description: "What the request cost, in USD" }), +}) + export const ControlPaths = { auth: "/auth/:providerID", log: "/log", + speech: "/redrob/speech", + transcribe: "/redrob/transcribe", } as const export const ControlApi = HttpApi.make("control").add( @@ -59,6 +88,33 @@ export const ControlApi = HttpApi.make("control").add( description: "Remove authentication credentials", }), ), + /* + * Speech on the engine's own Redrob credential, for a client that has to play audio and must + * not hold the key: Redrob Cowork's read-aloud. The key never leaves this process. + */ + HttpApiEndpoint.post("speech", ControlPaths.speech, { + payload: SpeechInput, + success: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array({ contentType: "audio/mpeg" })), + error: [InvalidRequestError, ServiceUnavailableError, UpstreamError, TimeoutError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "redrob.speech", + summary: "Speak text", + description: "Turn text into mp3 speech through the Redrob gateway, billed to the connected Redrob account.", + }), + ), + /* Transcription on the same credential, for Redrob Cowork's push-to-talk, for the same reason. */ + HttpApiEndpoint.post("transcribe", ControlPaths.transcribe, { + payload: TranscribeInput, + success: described(TranscribeOutput, "The transcript"), + error: [InvalidRequestError, ServiceUnavailableError, UpstreamError, TimeoutError], + }).annotateMerge( + OpenApi.annotations({ + identifier: "redrob.transcribe", + summary: "Transcribe speech", + description: "Turn recorded speech into text through the Redrob gateway, billed to the connected Redrob account.", + }), + ), HttpApiEndpoint.post("log", ControlPaths.log, { query: LogQuery, payload: LogInput, diff --git a/packages/redrob/src/server/routes/instance/httpapi/handlers/control.ts b/packages/redrob/src/server/routes/instance/httpapi/handlers/control.ts index e120749f18..07a2f7079b 100644 --- a/packages/redrob/src/server/routes/instance/httpapi/handlers/control.ts +++ b/packages/redrob/src/server/routes/instance/httpapi/handlers/control.ts @@ -3,12 +3,49 @@ import { Auth } from "@/auth" import { Effect } from "effect" import { HttpApiBuilder } from "effect/unstable/httpapi" import { RootHttpApi } from "../api" -import { LogInput } from "../groups/control" +import { LogInput, SpeechInput, TranscribeInput } from "../groups/control" +import { RedrobTranscription, type TranscriptionError } from "@/redrob/transcription" +import { InvalidRequestError, ServiceUnavailableError, TimeoutError, UpstreamError } from "../errors" +import { PLAYBACK_DEFAULTS, prepareSpeech, requestSpeech, speechKey, type SpeechError } from "@/tool/speech_generate" +import { HttpClient } from "effect/unstable/http" import { ProviderV2 } from "@redrob-code/core/provider" export const controlHandlers = HttpApiBuilder.group(RootHttpApi, "control", (handlers) => Effect.gen(function* () { const auth = yield* Auth.Service + const http = yield* HttpClient.HttpClient + + /** + * Each gateway failure as the endpoints declare it; no_key is 503 because the engine, not the + * caller, lacks it. Speech and transcription fail for the same four reasons. + */ + const gatewayFailure = (operation: string) => (error: SpeechError | TranscriptionError) => + error.reason === "invalid" + ? new InvalidRequestError({ message: error.message }) + : error.reason === "no_key" + ? new ServiceUnavailableError({ message: error.message, service: "redrob" }) + : error.reason === "timeout" + ? new TimeoutError({ message: error.message, operation }) + : new UpstreamError({ + message: error.message, + service: "redrob", + ...(error.status ? { status: error.status } : {}), + }) + + const speech = Effect.fn("ControlHttpApi.speech")(function* (ctx: { payload: typeof SpeechInput.Type }) { + // This route is for audio that is played at once, so it defaults to the fast playback model. + const prepared = yield* prepareSpeech(ctx.payload, PLAYBACK_DEFAULTS) + const key = yield* speechKey(auth) + return (yield* requestSpeech(http, key, prepared)).audio + }, Effect.mapError(gatewayFailure("speech"))) + + const transcribe = Effect.fn("ControlHttpApi.transcribe")(function* (ctx: { + payload: typeof TranscribeInput.Type + }) { + const prepared = yield* RedrobTranscription.prepare(ctx.payload) + const key = yield* RedrobTranscription.key(auth) + return yield* RedrobTranscription.request(http, key, prepared) + }, Effect.mapError(gatewayFailure("transcribe"))) const authSet = Effect.fn("ControlHttpApi.authSet")(function* (ctx: { params: { providerID: ProviderV2.ID } @@ -38,6 +75,11 @@ export const controlHandlers = HttpApiBuilder.group(RootHttpApi, "control", (han return true }) - return handlers.handle("authSet", authSet).handle("authRemove", authRemove).handle("log", log) + return handlers + .handle("authSet", authSet) + .handle("authRemove", authRemove) + .handle("speech", speech) + .handle("transcribe", transcribe) + .handle("log", log) }), ) diff --git a/packages/redrob/src/tool/image_generate.ts b/packages/redrob/src/tool/image_generate.ts new file mode 100644 index 0000000000..12ff427f09 --- /dev/null +++ b/packages/redrob/src/tool/image_generate.ts @@ -0,0 +1,218 @@ +import { Effect, Schema } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import * as path from "path" +import * as Tool from "./tool" +import DESCRIPTION from "./image_generate.txt" +import { Auth } from "@/auth" +import { EventV2Bridge } from "@/event-v2-bridge" +import { InstanceState } from "@/effect/instance-state" +import { FileSystem } from "@redrob-code/core/filesystem" +import { Watcher } from "@redrob-code/core/filesystem/watcher" +import { FSUtil } from "@redrob-code/core/fs-util" +import { CONSOLE_URL } from "@redrob-code/core/plugin/provider/redrob-constants" +import { assertExternalDirectoryEffect } from "./external-directory" + +/** + * Image generation through the Redrob gateway, on the Redrob credential this engine already holds. + * + * A built-in rather than a plugin tool because the credential is the engine's: it lives in the auth + * store (or `REDROB_API_KEY`), the desktop app never reads it, and no plugin API hands it out. Calling + * the gateway from here keeps the key where it is and bills the image to the same account as the chat. + * + * The gateway relays an image model's pictures in OpenRouter's shape, `choices[0].message.images`, + * when asked with `modalities: ["image", "text"]`. The picture is written under `artifacts/` and also + * returned as an attachment, which is what puts it inline in the conversation and in front of the model. + */ + +/** A good, mid-priced default. Any image-output model the gateway serves can be named instead. */ +export const DEFAULT_IMAGE_MODEL = "google/gemini-2.5-flash-image" + +/** + * The gateway, read per call. `REDROB_CONSOLE_URL` is the same developer seam `CONSOLE_URL` honours + * (never set in a shipped build); reading it here rather than only at module load is what lets a test + * point one call at a stand-in without depending on which module loaded first. + */ +export function gatewayUrl(): string { + return (process.env["REDROB_CONSOLE_URL"] ?? CONSOLE_URL).replace(/\/+$/, "") +} + +const TIMEOUT_MS = 180_000 +const MAX_IMAGE_BYTES = 20 * 1024 * 1024 + +const EXTENSIONS: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/webp": "webp", + "image/gif": "gif", +} + +export const Parameters = Schema.Struct({ + prompt: Schema.String.annotate({ description: "A full visual description of the image to generate" }), + filename: Schema.optional(Schema.String).annotate({ + description: "Optional file name without an extension. Derived from the prompt when omitted.", + }), + model: Schema.optional(Schema.String).annotate({ + description: `Optional image model id served by the Redrob gateway. Defaults to ${DEFAULT_IMAGE_MODEL}.`, + }), +}) + +const GatewayResponse = Schema.Struct({ + choices: Schema.Array( + Schema.Struct({ + message: Schema.Struct({ + content: Schema.optional(Schema.NullOr(Schema.String)), + images: Schema.optional(Schema.Array(Schema.Struct({ image_url: Schema.Struct({ url: Schema.String }) }))), + }), + }), + ), + redrob: Schema.optional(Schema.Struct({ costUsd: Schema.optional(Schema.Number) })), +}) + +const decodeResponse = Schema.decodeUnknownEffect(Schema.fromJsonString(GatewayResponse)) + +/** The Redrob key: the engine's stored credential first, then the environment, as the provider resolves it. */ +export function redrobKey(stored: Auth.Info | undefined): string | undefined { + if (stored?.type === "api" && stored.key.trim()) return stored.key.trim() + return process.env.REDROB_API_KEY?.trim() || undefined +} + +/** A file-name stem from free text: lower case, ASCII letters and digits, at most 48 characters. */ +export function slug(value: string): string { + const stem = value + .normalize("NFKD") + .toLowerCase() + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 48) + .replace(/-+$/, "") + return stem || "image" +} + +/** A data URL's type and bytes, or undefined when it is not an inline image of a type we write. */ +export function decodeImageDataUrl(url: string): { mime: string; bytes: Uint8Array } | undefined { + const match = /^data:(image\/[a-z+.-]+);base64,([A-Za-z0-9+/=\s]+)$/i.exec(url) + if (!match) return undefined + const mime = match[1].toLowerCase() + if (!EXTENSIONS[mime]) return undefined + const bytes = Uint8Array.from(Buffer.from(match[2], "base64")) + if (bytes.byteLength === 0 || bytes.byteLength > MAX_IMAGE_BYTES) return undefined + return { mime, bytes } +} + +export const ImageGenerateTool = Tool.define( + "image_generate", + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient + const auth = yield* Auth.Service + const fs = yield* FSUtil.Service + const events = yield* EventV2Bridge.Service + + return { + description: DESCRIPTION, + parameters: Parameters, + execute: (params: Schema.Schema.Type, ctx: Tool.Context) => + Effect.gen(function* () { + const prompt = params.prompt.trim() + if (!prompt) throw new Error("Describe the image to generate.") + const key = redrobKey(yield* auth.get("redrob").pipe(Effect.orElseSucceed(() => undefined))) + if (!key) throw new Error("Connect Redrob to generate images: no Redrob API key is available.") + const model = params.model?.trim() || DEFAULT_IMAGE_MODEL + + const instance = yield* InstanceState.context + const directory = path.join(instance.directory, "artifacts") + const stem = slug(params.filename?.trim() || prompt) + + /* + * The path is chosen before anything is spent, so the request is refused for free where a file + * cannot be written. Asked as `edit`, like `write`, because writing a file is what this does: + * plan mode, which denies edits, denies this too. + */ + const extension = "png" + let filepath = path.join(directory, `${stem}.${extension}`) + for (let n = 2; yield* fs.existsSafe(filepath); n += 1) { + filepath = path.join(directory, `${stem}-${n}.${extension}`) + } + yield* assertExternalDirectoryEffect(ctx, filepath) + yield* ctx.ask({ + permission: "edit", + patterns: [path.relative(instance.worktree, filepath)], + always: ["*"], + metadata: { filepath, prompt, model }, + }) + + const request = yield* HttpClientRequest.post(`${gatewayUrl()}/chat/completions`).pipe( + HttpClientRequest.bearerToken(key), + HttpClientRequest.acceptJson, + HttpClientRequest.bodyJson({ + model, + messages: [{ role: "user", content: prompt }], + modalities: ["image", "text"], + }), + ) + const response = yield* http.execute(request).pipe( + Effect.timeoutOrElse({ + duration: TIMEOUT_MS, + orElse: () => Effect.die(new Error("Image generation timed out")), + }), + ) + const body = yield* response.text + if (response.status < 200 || response.status >= 300) { + throw new Error(`The Redrob gateway answered ${response.status}: ${gatewayMessage(body)}`) + } + const payload = yield* decodeResponse(body) + const message = payload.choices[0]?.message + const image = (message?.images ?? []).map((item) => decodeImageDataUrl(item.image_url.url)).find(Boolean) + if (!image) { + const said = message?.content?.trim() + throw new Error(`${model} returned no image${said ? `. It said: ${said.slice(0, 500)}` : "."}`) + } + + /* A model that answered in another format keeps the chosen name with the right extension. */ + if (EXTENSIONS[image.mime] !== extension) { + const base = filepath.slice(0, -extension.length - 1) + filepath = `${base}.${EXTENSIONS[image.mime]}` + for (let n = 2; yield* fs.existsSafe(filepath); n += 1) { + filepath = `${base}-${n}.${EXTENSIONS[image.mime]}` + } + } + yield* fs.writeWithDirs(filepath, image.bytes) + yield* events.publish(FileSystem.Event.Edited, { file: filepath }) + yield* events.publish(Watcher.Event.Updated, { file: filepath, event: "add" }) + + const relative = path.relative(instance.worktree, filepath) + const costUsd = payload.redrob?.costUsd + const caption = message?.content?.trim() + return { + title: relative, + output: [`Generated ${relative} with ${model}.`, caption ? `The model said: ${caption}` : undefined] + .filter(Boolean) + .join("\n"), + metadata: { filepath, model, ...(costUsd === undefined ? {} : { costUsd }) }, + attachments: [ + { + type: "file" as const, + mime: image.mime, + filename: path.basename(filepath), + url: `data:${image.mime};base64,${Buffer.from(image.bytes).toString("base64")}`, + }, + ], + } + }).pipe(Effect.orDie), + } + }), +) + +/** The gateway's error message, from OpenAI's envelope or Nest's, without echoing a whole body. */ +export function gatewayMessage(body: string): string { + try { + const parsed: unknown = JSON.parse(body) + if (parsed && typeof parsed === "object") { + const error = "error" in parsed ? parsed.error : undefined + if (error && typeof error === "object" && "message" in error && typeof error.message === "string") { + return error.message + } + if ("message" in parsed && typeof parsed.message === "string") return parsed.message + } + } catch {} + return body.slice(0, 200) || "no details" +} diff --git a/packages/redrob/src/tool/image_generate.txt b/packages/redrob/src/tool/image_generate.txt new file mode 100644 index 0000000000..bf065a045d --- /dev/null +++ b/packages/redrob/src/tool/image_generate.txt @@ -0,0 +1,9 @@ +- Generates an image from a text description and saves it in the project's `artifacts/` folder +- Use this when the user asks for a picture, illustration, logo, diagram-like artwork, mockup, or any other image +- The image is shown to the user in the conversation and returned to you, so you can see what was made and refine it with another call +- Billed to the user's Redrob account per image, so make one image per request unless the user asks for variations + +Usage notes: + - Write the prompt as a full visual description: subject, setting, style, composition, colors, and any text that must appear in the image + - `filename` is optional and has no extension; a name is derived from the prompt when it is omitted, and an existing file is never overwritten + - Leave `model` unset unless the user names an image model diff --git a/packages/redrob/src/tool/registry.ts b/packages/redrob/src/tool/registry.ts index e5c3bde3c2..8b1c35417b 100644 --- a/packages/redrob/src/tool/registry.ts +++ b/packages/redrob/src/tool/registry.ts @@ -26,6 +26,9 @@ import { Plugin } from "../plugin" import { Provider } from "@/provider/provider" import { WebSearchTool } from "./websearch" +import { ImageGenerateTool, redrobKey } from "./image_generate" +import { SpeechGenerateTool } from "./speech_generate" +import { Auth } from "@/auth" import { LspTool } from "./lsp" import * as Truncate from "./truncate" import { ApplyPatchTool } from "./apply_patch" @@ -58,10 +61,7 @@ import { McpCatalog } from "@/mcp/catalog" export function webSearchEnabled(providerID: ProviderV2.ID, flags = { exa: false, parallel: false }) { return ( - providerID === ProviderV2.ID.redrob || - providerID === ProviderV2.ID.make("redrob-go") || - flags.exa || - flags.parallel + providerID === ProviderV2.ID.redrob || providerID === ProviderV2.ID.make("redrob-go") || flags.exa || flags.parallel ) } @@ -108,6 +108,9 @@ const layer = Layer.effect( const plan = yield* PlanExitTool const webfetch = yield* WebFetchTool const websearch = yield* WebSearchTool + const imageGenerate = yield* ImageGenerateTool + const speechGenerate = yield* SpeechGenerateTool + const auth = yield* Auth.Service const shell = yield* ShellTool const globtool = yield* GlobTool const writetool = yield* WriteTool @@ -219,6 +222,8 @@ const layer = Layer.effect( fetch: Tool.init(webfetch), todo: Tool.init(todo), search: Tool.init(websearch), + image: Tool.init(imageGenerate), + speech: Tool.init(speechGenerate), skill: Tool.init(skilltool), patch: Tool.init(patchtool), question: Tool.init(question), @@ -242,6 +247,8 @@ const layer = Layer.effect( tool.fetch, tool.todo, tool.search, + tool.image, + tool.speech, tool.skill, tool.patch, ...(tool.execute ? [tool.execute] : []), @@ -290,7 +297,14 @@ const layer = Layer.effect( }) const tools: Interface["tools"] = Effect.fn("ToolRegistry.tools")(function* (input) { + /* + * Image and speech generation are offered only where they can run: they bill the Redrob account, so without a + * Redrob credential each would be a tool that fails on every call. Read per request because the + * credential can be connected or removed while the engine runs. + */ + const canGenerateMedia = Boolean(redrobKey(yield* auth.get("redrob").pipe(Effect.orElseSucceed(() => undefined)))) const filtered = (yield* all()).filter((tool) => { + if (tool.id === ImageGenerateTool.id || tool.id === SpeechGenerateTool.id) return canGenerateMedia if (tool.id === WebSearchTool.id) { return webSearchEnabled(input.providerID, { exa: flags.enableExa, parallel: flags.enableParallel }) } @@ -438,6 +452,7 @@ export const node = LayerNode.make({ Session.node, BackgroundJob.node, Provider.node, + Auth.node, LSP.node, Instruction.node, FSUtil.node, diff --git a/packages/redrob/src/tool/speech_generate.ts b/packages/redrob/src/tool/speech_generate.ts new file mode 100644 index 0000000000..e358522802 --- /dev/null +++ b/packages/redrob/src/tool/speech_generate.ts @@ -0,0 +1,221 @@ +import { Effect, Schema } from "effect" +import { HttpClient, HttpClientRequest } from "effect/unstable/http" +import * as path from "path" +import * as Tool from "./tool" +import DESCRIPTION from "./speech_generate.txt" +import { Auth } from "@/auth" +import { EventV2Bridge } from "@/event-v2-bridge" +import { InstanceState } from "@/effect/instance-state" +import { FileSystem } from "@redrob-code/core/filesystem" +import { Watcher } from "@redrob-code/core/filesystem/watcher" +import { FSUtil } from "@redrob-code/core/fs-util" +import { assertExternalDirectoryEffect } from "./external-directory" +import { gatewayMessage, gatewayUrl, redrobKey, slug } from "./image_generate" + +/** + * Speech through the Redrob gateway's `POST /v1/audio/speech`, on the engine's own Redrob credential, + * for the reason image_generate gives: the key is the engine's and stays here. + * + * Two callers share `prepareSpeech` and `requestSpeech`: the `speech_generate` tool, which saves an + * mp3, and the engine's `POST /redrob/speech` route, which hands the audio to Redrob Cowork to play + * a reply aloud without the app ever holding the key. + * + * The tool saves its audio and does NOT return it as an attachment. An attachment goes back to the + * model with the next turn, and audio input would steer `auto` to an audio-input model and bill the + * clip as input on every turn after, to tell the model nothing it does not already know: it wrote + * the words. + */ + +/** Multilingual, so a Korean script is spoken as Korean. Any speech model the gateway serves can be named. */ +export const DEFAULT_SPEECH_MODEL = "elevenlabs/eleven-multilingual-v2" +export const DEFAULT_SPEECH_VOICE = "sarah" + +/** + * For speech that is played as it arrives, a reply read aloud or a voice conversation's answer, where + * the wait before the first word matters more than the last bit of quality. Same voice, so a saved file + * and a reply read aloud sound like the same speaker. Multilingual, so Korean is spoken as Korean. + */ +export const PLAYBACK_SPEECH_MODEL = "elevenlabs/eleven-flash-v2.5" + +export type SpeechDefaults = { model: string; voice: string } +const FILE_DEFAULTS: SpeechDefaults = { model: DEFAULT_SPEECH_MODEL, voice: DEFAULT_SPEECH_VOICE } +export const PLAYBACK_DEFAULTS: SpeechDefaults = { model: PLAYBACK_SPEECH_MODEL, voice: DEFAULT_SPEECH_VOICE } + +/** The gateway's limit for one request, checked here so an over-long request costs nothing. */ +export const MAX_SPEECH_CHARACTERS = 4_096 + +const TIMEOUT_MS = 180_000 +const MAX_AUDIO_BYTES = 50 * 1024 * 1024 + +/** + * Why speech could not be made. `invalid` and `no_key` are found before anything is sent and cost + * nothing; `upstream` and `timeout` are the gateway's. + */ +export class SpeechError extends Schema.TaggedErrorClass()("SpeechError", { + reason: Schema.Literals(["invalid", "no_key", "upstream", "timeout"]), + message: Schema.String, + status: Schema.optional(Schema.Number), +}) {} + +export type PreparedSpeech = { text: string; model: string; voice?: string; characters: number } + +/** Characters as the gateway bills them: code points, so the limit here agrees with its limit there. */ +function characters(text: string): number { + let count = 0 + for (const _ of text) count += 1 + return count +} + +/** The request as it will be sent, or why it would be refused, with nothing sent either way. */ +export const prepareSpeech = Effect.fn("Speech.prepare")(function* ( + input: { text: string; model?: string; voice?: string }, + defaults: SpeechDefaults = FILE_DEFAULTS, +) { + const text = input.text.trim() + if (!text) return yield* new SpeechError({ reason: "invalid", message: "Give the words to speak." }) + const length = characters(text) + if (length > MAX_SPEECH_CHARACTERS) { + return yield* new SpeechError({ + reason: "invalid", + message: `The text is ${length} characters; speak at most ${MAX_SPEECH_CHARACTERS} per call and save longer material in parts.`, + }) + } + const model = input.model?.trim() || defaults.model + /* Voices belong to a model, so the default voice only goes with the default model. */ + const voice = input.voice?.trim() || (model === defaults.model ? defaults.voice : undefined) + const prepared: PreparedSpeech = { text, model, ...(voice ? { voice } : {}), characters: length } + return prepared +}) + +/** The engine's Redrob key, or a `no_key` failure, which is found before anything is sent. */ +export const speechKey = Effect.fn("Speech.key")(function* (auth: Auth.Interface) { + const key = redrobKey(yield* auth.get("redrob").pipe(Effect.orElseSucceed(() => undefined))) + if (!key) { + return yield* new SpeechError({ + reason: "no_key", + message: "Connect Redrob to generate speech: no Redrob API key is available.", + }) + } + return key +}) + +/** One call to the gateway. The audio is checked for being audio before anyone keeps or plays it. */ +export const requestSpeech = Effect.fn("Speech.request")(function* ( + http: HttpClient.HttpClient, + key: string, + speech: PreparedSpeech, +) { + const request = yield* HttpClientRequest.post(`${gatewayUrl()}/audio/speech`).pipe( + HttpClientRequest.bearerToken(key), + HttpClientRequest.bodyJson({ + model: speech.model, + input: speech.text, + response_format: "mp3", + ...(speech.voice ? { voice: speech.voice } : {}), + }), + Effect.orDie, + ) + const response = yield* http.execute(request).pipe( + Effect.mapError(() => new SpeechError({ reason: "upstream", message: "The Redrob gateway could not be reached." })), + Effect.timeoutOrElse({ + duration: TIMEOUT_MS, + orElse: () => Effect.fail(new SpeechError({ reason: "timeout", message: "Speech generation timed out." })), + }), + ) + if (response.status < 200 || response.status >= 300) { + const body = yield* response.text.pipe(Effect.orElseSucceed(() => "")) + return yield* new SpeechError({ + reason: "upstream", + status: response.status, + message: `The Redrob gateway answered ${response.status}: ${gatewayMessage(body)}`, + }) + } + const contentType = response.headers["content-type"]?.split(";")[0]?.trim().toLowerCase() ?? "" + if (!contentType.startsWith("audio/")) { + return yield* new SpeechError({ + reason: "upstream", + message: `The Redrob gateway answered with ${contentType || "no content type"}, not audio.`, + }) + } + const buffer = yield* response.arrayBuffer.pipe( + Effect.mapError(() => new SpeechError({ reason: "upstream", message: "The Redrob gateway's audio was cut off." })), + ) + const audio = new Uint8Array(buffer) + if (audio.byteLength === 0) { + return yield* new SpeechError({ reason: "upstream", message: "The Redrob gateway returned no audio." }) + } + if (audio.byteLength > MAX_AUDIO_BYTES) { + return yield* new SpeechError({ reason: "upstream", message: "The audio is over 50 MB." }) + } + const cost = Number(response.headers["x-redrob-cost-usd"]) + return { audio, contentType, ...(Number.isFinite(cost) ? { costUsd: cost } : {}) } +}) + +export const Parameters = Schema.Struct({ + text: Schema.String.annotate({ description: `The words to speak, at most ${MAX_SPEECH_CHARACTERS} characters` }), + voice: Schema.optional(Schema.String).annotate({ + description: `Optional voice of the chosen model. Defaults to ${DEFAULT_SPEECH_VOICE} with the default model.`, + }), + model: Schema.optional(Schema.String).annotate({ + description: `Optional speech model id served by the Redrob gateway. Defaults to ${DEFAULT_SPEECH_MODEL}.`, + }), + filename: Schema.optional(Schema.String).annotate({ + description: "Optional file name without an extension. Derived from the text when omitted.", + }), +}) + +export const SpeechGenerateTool = Tool.define( + "speech_generate", + Effect.gen(function* () { + const http = yield* HttpClient.HttpClient + const auth = yield* Auth.Service + const fs = yield* FSUtil.Service + const events = yield* EventV2Bridge.Service + + return { + description: DESCRIPTION, + parameters: Parameters, + execute: (params: Schema.Schema.Type, ctx: Tool.Context) => + Effect.gen(function* () { + const speech = yield* prepareSpeech(params) + const key = yield* speechKey(auth) + + const instance = yield* InstanceState.context + const directory = path.join(instance.directory, "artifacts") + const stem = slug(params.filename?.trim() || speech.text) + let filepath = path.join(directory, `${stem}.mp3`) + for (let n = 2; yield* fs.existsSafe(filepath); n += 1) { + filepath = path.join(directory, `${stem}-${n}.mp3`) + } + yield* assertExternalDirectoryEffect(ctx, filepath) + yield* ctx.ask({ + permission: "edit", + patterns: [path.relative(instance.worktree, filepath)], + always: ["*"], + metadata: { filepath, model: speech.model, voice: speech.voice }, + }) + + const spoken = yield* requestSpeech(http, key, speech) + yield* fs.writeWithDirs(filepath, spoken.audio) + yield* events.publish(FileSystem.Event.Edited, { file: filepath }) + yield* events.publish(Watcher.Event.Updated, { file: filepath, event: "add" }) + + const relative = path.relative(instance.worktree, filepath) + return { + title: relative, + output: `Saved ${relative}: ${speech.characters} characters spoken with ${speech.model}${speech.voice ? ` (${speech.voice})` : ""}.`, + metadata: { + filepath, + model: speech.model, + ...(speech.voice ? { voice: speech.voice } : {}), + characters: speech.characters, + ...(spoken.costUsd === undefined ? {} : { costUsd: spoken.costUsd }), + }, + } + }).pipe( + Effect.catchTag("SpeechError", (error) => Effect.die(new Error(error.message))), + Effect.orDie, + ), + } + }), +) diff --git a/packages/redrob/src/tool/speech_generate.txt b/packages/redrob/src/tool/speech_generate.txt new file mode 100644 index 0000000000..6efe5784c8 --- /dev/null +++ b/packages/redrob/src/tool/speech_generate.txt @@ -0,0 +1,10 @@ +- Turns text into spoken audio and saves it as an MP3 in the project's `artifacts/` folder +- Use this when the user asks for a voice-over, narration, a spoken summary, an audio version of a document, or anything else they want to listen to +- The audio is not played back to you; you get the file's path. The user can play it in the conversation +- Billed to the user's Redrob account per character, so speak only what the user asked for + +Usage notes: + - `text` is read aloud as written, up to 4096 characters per call; split longer material into parts and save each as its own file + - Write numbers, abbreviations and names the way they should be spoken + - Leave `model` and `voice` unset unless the user asks for a specific one; the default speaks Korean, English and many other languages + - `filename` is optional and has no extension; a name is derived from the text when it is omitted, and an existing file is never overwritten diff --git a/packages/redrob/test/server/httpapi-exercise/index.ts b/packages/redrob/test/server/httpapi-exercise/index.ts index 4b245ad0c2..682530f7c3 100644 --- a/packages/redrob/test/server/httpapi-exercise/index.ts +++ b/packages/redrob/test/server/httpapi-exercise/index.ts @@ -649,6 +649,17 @@ const scenarios: Scenario[] = [ .json(200, (body) => { check(body === true, "log route should return true") }), + /* No Redrob key in the exercise home, so both media routes refuse up front with 503, spending nothing. */ + http.protected + .post("/redrob/speech", "redrob.speech") + .global() + .at(() => ({ path: "/redrob/speech", body: { text: "route coverage" } })) + .status(503, undefined, "status"), + http.protected + .post("/redrob/transcribe", "redrob.transcribe") + .global() + .at(() => ({ path: "/redrob/transcribe", body: { audio: "UklGRg==", format: "wav" } })) + .status(503, undefined, "status"), http.protected .put("/auth/{providerID}", "auth.set") .global() diff --git a/packages/redrob/test/server/httpapi-speech.test.ts b/packages/redrob/test/server/httpapi-speech.test.ts new file mode 100644 index 0000000000..de7aa1fb2f --- /dev/null +++ b/packages/redrob/test/server/httpapi-speech.test.ts @@ -0,0 +1,184 @@ +import { NodeHttpServer, NodeServices } from "@effect/platform-node" +import { describe, expect } from "bun:test" +import { Config, Context, Effect, Layer } from "effect" +import { + HttpClient, + HttpClientRequest, + HttpRouter, + HttpServer, + HttpServerRequest, + HttpServerResponse, +} from "effect/unstable/http" +import * as Socket from "effect/unstable/socket/Socket" +import * as Http from "node:http" +import { ControlPaths } from "../../src/server/routes/instance/httpapi/groups/control" +import { HttpApiApp } from "../../src/server/routes/instance/httpapi/server" +import { DEFAULT_SPEECH_MODEL, DEFAULT_SPEECH_VOICE, PLAYBACK_SPEECH_MODEL } from "../../src/tool/speech_generate" +import { testEffect } from "../lib/effect" + +/** + * `POST /redrob/speech` through the production route tree, with a stand-in for the Redrob gateway + * behind it. What matters is that the app gets audio while the engine's key only ever goes to the + * gateway, and that every refusal arrives as the endpoint's declared error. + */ + +const servedRoutes: Layer.Layer = HttpRouter.serve( + HttpApiApp.routes, + { disableListenLog: true, disableLogger: true }, +) +const it = testEffect( + servedRoutes.pipe( + Layer.provide(Socket.layerWebSocketConstructorGlobal), + Layer.provideMerge(NodeHttpServer.layerTest), + Layer.provideMerge(NodeServices.layer), + ), +) + +const MP3 = new Uint8Array([0xff, 0xf3, 0x44, 0xc4, 0x01, 0x02]) + +type Seen = { authorization: string | undefined; body: unknown } +type Answer = { status: number; type: string; body: Uint8Array | string } + +/** The engine pointed at the stand-in and holding a Redrob key, restored when the test scope closes. */ +const engineEnv = (gatewayUrl: string) => + Effect.acquireRelease( + Effect.sync(() => { + const saved = { + url: process.env.REDROB_CONSOLE_URL, + auth: process.env.REDROB_AUTH_CONTENT, + key: process.env.REDROB_API_KEY, + } + process.env.REDROB_CONSOLE_URL = gatewayUrl + process.env.REDROB_AUTH_CONTENT = JSON.stringify({ redrob: { type: "api", key: "rk_engine_key" } }) + delete process.env.REDROB_API_KEY + return saved + }), + (saved) => + Effect.sync(() => { + for (const [name, value] of [ + ["REDROB_CONSOLE_URL", saved.url], + ["REDROB_AUTH_CONTENT", saved.auth], + ["REDROB_API_KEY", saved.key], + ] as const) { + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + }), + ) + +/** + * Topology: the engine's routes run on the test's primary server (NodeHttpServer.layerTest), and the + * Redrob gateway stand-in is a second listener built into this test's scope with its own fresh router, + * so its `/audio/speech` cannot collide with anything the engine registers. The engine reaches the + * stand-in through `REDROB_CONSOLE_URL`, the developer seam the gateway URL already honours. + */ +const gateway = Effect.fn("SpeechTest.gateway")(function* (answer: (body: Record) => Answer) { + const seen: Seen[] = [] + const context = yield* Layer.build( + Layer.mergeAll( + Layer.fresh(HttpRouter.layer), + NodeHttpServer.layer(() => Http.createServer(), { host: "127.0.0.1", port: 0 }), + ), + ) + const server = Context.get(context, HttpServer.HttpServer) + const router = Context.get(context, HttpRouter.HttpRouter) + yield* router.add( + "POST", + "/audio/speech", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest + const raw = yield* request.json.pipe(Effect.orElseSucceed(() => ({}))) + const body = typeof raw === "object" && raw !== null ? Object.fromEntries(Object.entries(raw)) : {} + seen.push({ authorization: request.headers.authorization, body }) + const reply = answer(body) + return typeof reply.body === "string" + ? HttpServerResponse.text(reply.body, { status: reply.status, contentType: reply.type }) + : HttpServerResponse.uint8Array(reply.body, { status: reply.status, contentType: reply.type }) + }), + ) + yield* server.serve(router.asHttpEffect()) + if (server.address._tag !== "TcpAddress") throw new Error("expected a TCP listener") + yield* engineEnv(`http://127.0.0.1:${server.address.port}`) + return { seen } +}) + +const speak = (payload: Record) => + HttpClientRequest.post(ControlPaths.speech).pipe( + HttpClientRequest.bodyJson(payload), + Effect.flatMap(HttpClient.execute), + ) + +describe("redrob speech HttpApi", () => { + it.live("answers with the gateway's mp3, sending the engine's key only to the gateway", () => + Effect.gen(function* () { + const { seen } = yield* gateway(() => ({ status: 200, type: "audio/mpeg", body: MP3 })) + const response = yield* speak({ text: "안녕하세요" }) + + expect(response.status).toBe(200) + expect(response.headers["content-type"]).toContain("audio/mpeg") + expect(new Uint8Array(yield* response.arrayBuffer)).toEqual(MP3) + expect(seen).toEqual([ + { + authorization: "Bearer rk_engine_key", + body: { + model: PLAYBACK_SPEECH_MODEL, + input: "안녕하세요", + response_format: "mp3", + voice: DEFAULT_SPEECH_VOICE, + }, + }, + ]) + }), + ) + + it.live("plays with the fast model by default, and a chosen model keeps the voice it was given", () => + Effect.gen(function* () { + const { seen } = yield* gateway(() => ({ status: 200, type: "audio/mpeg", body: MP3 })) + expect((yield* speak({ text: "One" })).status).toBe(200) + expect((yield* speak({ text: "Two", model: DEFAULT_SPEECH_MODEL })).status).toBe(200) + expect((yield* speak({ text: "Three", model: DEFAULT_SPEECH_MODEL, voice: "george" })).status).toBe(200) + expect(seen.map((call) => call.body)).toEqual([ + { model: PLAYBACK_SPEECH_MODEL, input: "One", response_format: "mp3", voice: DEFAULT_SPEECH_VOICE }, + // Another model's voices are its own, so none is assumed for it. + { model: DEFAULT_SPEECH_MODEL, input: "Two", response_format: "mp3" }, + { model: DEFAULT_SPEECH_MODEL, input: "Three", response_format: "mp3", voice: "george" }, + ]) + }), + ) + + it.live("refuses empty or over-long text with a 400, sending nothing", () => + Effect.gen(function* () { + const { seen } = yield* gateway(() => ({ status: 200, type: "audio/mpeg", body: MP3 })) + expect((yield* speak({ text: " " })).status).toBe(400) + expect((yield* speak({ text: "가".repeat(4097) })).status).toBe(400) + expect(seen).toHaveLength(0) + }), + ) + + it.live("without a Redrob key it is a 503, sending nothing", () => + Effect.gen(function* () { + const { seen } = yield* gateway(() => ({ status: 200, type: "audio/mpeg", body: MP3 })) + process.env.REDROB_AUTH_CONTENT = JSON.stringify({}) + const response = yield* speak({ text: "Hello" }) + expect(response.status).toBe(503) + expect(JSON.stringify(yield* response.json)).toContain("Connect Redrob") + expect(seen).toHaveLength(0) + }), + ) + + it.live("a gateway refusal, or an answer that is not audio, is a 502 with the gateway's message", () => + Effect.gen(function* () { + yield* gateway((body) => + body.input === "json" + ? { status: 200, type: "application/json", body: "{}" } + : { status: 402, type: "application/json", body: JSON.stringify({ error: { message: "Out of credit." } }) }, + ) + const refused = yield* speak({ text: "Hello" }) + expect(refused.status).toBe(502) + expect(JSON.stringify(yield* refused.json)).toContain("answered 402: Out of credit.") + const json = yield* speak({ text: "json" }) + expect(json.status).toBe(502) + expect(JSON.stringify(yield* json.json)).toContain("not audio") + }), + ) +}) diff --git a/packages/redrob/test/server/httpapi-transcribe.test.ts b/packages/redrob/test/server/httpapi-transcribe.test.ts new file mode 100644 index 0000000000..2760cf91f0 --- /dev/null +++ b/packages/redrob/test/server/httpapi-transcribe.test.ts @@ -0,0 +1,200 @@ +import { NodeHttpServer, NodeServices } from "@effect/platform-node" +import { describe, expect } from "bun:test" +import { Config, Context, Effect, Layer } from "effect" +import { + HttpClient, + HttpClientRequest, + HttpRouter, + HttpServer, + HttpServerRequest, + HttpServerResponse, +} from "effect/unstable/http" +import * as Socket from "effect/unstable/socket/Socket" +import * as Http from "node:http" +import { ControlPaths } from "../../src/server/routes/instance/httpapi/groups/control" +import { HttpApiApp } from "../../src/server/routes/instance/httpapi/server" +import { DEFAULT_TRANSCRIPTION_MODEL, MAX_TRANSCRIPTION_BYTES } from "../../src/redrob/transcription" +import { testEffect } from "../lib/effect" + +/** + * `POST /redrob/transcribe` through the production route tree, with a stand-in for the Redrob + * gateway behind it. What matters is that the app gets a transcript while the engine's key only ever + * goes to the gateway, that a request the gateway would refuse is refused here before anything is + * sent, and that every refusal arrives as the endpoint's declared error. + */ + +const servedRoutes: Layer.Layer = HttpRouter.serve( + HttpApiApp.routes, + { disableListenLog: true, disableLogger: true }, +) +const it = testEffect( + servedRoutes.pipe( + Layer.provide(Socket.layerWebSocketConstructorGlobal), + Layer.provideMerge(NodeHttpServer.layerTest), + Layer.provideMerge(NodeServices.layer), + ), +) + +/** A few bytes standing in for a webm/opus recording; the gateway stand-in never decodes them. */ +const CLIP = Buffer.from([0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x42, 0x86, 0x81]).toString("base64") + +type Seen = { authorization: string | undefined; body: unknown } +type Answer = { status: number; body: string } + +/** The engine pointed at the stand-in and holding a Redrob key, restored when the test scope closes. */ +const engineEnv = (gatewayUrl: string) => + Effect.acquireRelease( + Effect.sync(() => { + const saved = { + url: process.env.REDROB_CONSOLE_URL, + auth: process.env.REDROB_AUTH_CONTENT, + key: process.env.REDROB_API_KEY, + } + process.env.REDROB_CONSOLE_URL = gatewayUrl + process.env.REDROB_AUTH_CONTENT = JSON.stringify({ redrob: { type: "api", key: "rk_engine_key" } }) + delete process.env.REDROB_API_KEY + return saved + }), + (saved) => + Effect.sync(() => { + for (const [name, value] of [ + ["REDROB_CONSOLE_URL", saved.url], + ["REDROB_AUTH_CONTENT", saved.auth], + ["REDROB_API_KEY", saved.key], + ] as const) { + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + }), + ) + +/** + * Topology: the engine's routes run on the test's primary server (NodeHttpServer.layerTest), and the + * Redrob gateway stand-in is a second listener built into this test's scope with its own fresh router, + * so its `/audio/transcriptions` cannot collide with anything the engine registers. The engine reaches + * the stand-in through `REDROB_CONSOLE_URL`, the developer seam the gateway URL already honours. + */ +const gateway = Effect.fn("TranscribeTest.gateway")(function* (answer: (body: Record) => Answer) { + const seen: Seen[] = [] + const context = yield* Layer.build( + Layer.mergeAll( + Layer.fresh(HttpRouter.layer), + NodeHttpServer.layer(() => Http.createServer(), { host: "127.0.0.1", port: 0 }), + ), + ) + const server = Context.get(context, HttpServer.HttpServer) + const router = Context.get(context, HttpRouter.HttpRouter) + yield* router.add( + "POST", + "/audio/transcriptions", + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest + const raw = yield* request.json.pipe(Effect.orElseSucceed(() => ({}))) + const body = typeof raw === "object" && raw !== null ? Object.fromEntries(Object.entries(raw)) : {} + seen.push({ authorization: request.headers.authorization, body }) + const reply = answer(body) + return HttpServerResponse.text(reply.body, { status: reply.status, contentType: "application/json" }) + }), + ) + yield* server.serve(router.asHttpEffect()) + if (server.address._tag !== "TcpAddress") throw new Error("expected a TCP listener") + yield* engineEnv(`http://127.0.0.1:${server.address.port}`) + return { seen } +}) + +const transcript = (text: string) => ({ + status: 200, + body: JSON.stringify({ text, seconds: 3.2, requestId: "req_1", costUsd: 0.0004 }), +}) + +const transcribe = (payload: Record) => + HttpClientRequest.post(ControlPaths.transcribe).pipe( + HttpClientRequest.bodyJson(payload), + Effect.flatMap(HttpClient.execute), + ) + +describe("redrob transcribe HttpApi", () => { + it.live("answers with the gateway's transcript, sending the engine's key only to the gateway", () => + Effect.gen(function* () { + const { seen } = yield* gateway(() => transcript(" 회의 요약해 줘, and send it to Minji ")) + const response = yield* transcribe({ audio: CLIP, format: "webm", language: "KO" }) + + expect(response.status).toBe(200) + expect(yield* response.json).toEqual({ + text: "회의 요약해 줘, and send it to Minji", + seconds: 3.2, + costUsd: 0.0004, + }) + expect(seen).toEqual([ + { + authorization: "Bearer rk_engine_key", + body: { + model: DEFAULT_TRANSCRIPTION_MODEL, + input_audio: { data: CLIP, format: "webm" }, + response_format: "json", + language: "ko", + }, + }, + ]) + }), + ) + + it.live("leaves the language to the gateway when none is given, and passes a chosen model through", () => + Effect.gen(function* () { + const { seen } = yield* gateway(() => transcript("Hello")) + const response = yield* transcribe({ audio: CLIP, format: "ogg", model: "openai/gpt-transcribe" }) + expect(response.status).toBe(200) + expect(seen[0]?.body).toEqual({ + model: "openai/gpt-transcribe", + input_audio: { data: CLIP, format: "ogg" }, + response_format: "json", + }) + }), + ) + + it.live("refuses audio the gateway would refuse with a 400, sending nothing", () => + Effect.gen(function* () { + const { seen } = yield* gateway(() => transcript("Hello")) + /* The smallest unpadded base64 that decodes to more than the limit. */ + const oversize = "A".repeat(Math.ceil((MAX_TRANSCRIPTION_BYTES + 1) / 3) * 4) + for (const payload of [ + { audio: " ", format: "webm" }, + { audio: `data:audio/webm;base64,${CLIP}`, format: "webm" }, + { audio: "abc", format: "webm" }, + { audio: oversize, format: "webm" }, + { audio: CLIP, format: "webm", language: "korean" }, + { audio: CLIP, format: "opus" }, + ]) { + expect((yield* transcribe(payload)).status).toBe(400) + } + expect(seen).toHaveLength(0) + }), + ) + + it.live("without a Redrob key it is a 503, sending nothing", () => + Effect.gen(function* () { + const { seen } = yield* gateway(() => transcript("Hello")) + process.env.REDROB_AUTH_CONTENT = JSON.stringify({}) + const response = yield* transcribe({ audio: CLIP, format: "webm" }) + expect(response.status).toBe(503) + expect(JSON.stringify(yield* response.json)).toContain("Connect Redrob") + expect(seen).toHaveLength(0) + }), + ) + + it.live("a gateway refusal, or an answer that is not a transcript, is a 502 with the gateway's message", () => + Effect.gen(function* () { + yield* gateway((body) => + body.model === "shapeless" + ? { status: 200, body: JSON.stringify({ transcript: "Hello" }) } + : { status: 402, body: JSON.stringify({ error: { message: "Out of credit." } }) }, + ) + const refused = yield* transcribe({ audio: CLIP, format: "webm" }) + expect(refused.status).toBe(502) + expect(JSON.stringify(yield* refused.json)).toContain("answered 402: Out of credit.") + const shapeless = yield* transcribe({ audio: CLIP, format: "webm", model: "shapeless" }) + expect(shapeless.status).toBe(502) + expect(JSON.stringify(yield* shapeless.json)).toContain("without a transcript") + }), + ) +}) diff --git a/packages/redrob/test/tool/image_generate.test.ts b/packages/redrob/test/tool/image_generate.test.ts new file mode 100644 index 0000000000..2d56ad2d36 --- /dev/null +++ b/packages/redrob/test/tool/image_generate.test.ts @@ -0,0 +1,223 @@ +import { afterEach, beforeEach, describe, expect } from "bun:test" +import { LayerNode } from "@redrob-code/core/effect/layer-node" +import { httpClient } from "@redrob-code/core/effect/app-node-platform" +import { FSUtil } from "@redrob-code/core/fs-util" +import { Cause, Effect, Exit } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import fs from "fs/promises" +import path from "path" +import { Auth } from "@/auth" +import { Tool } from "@/tool/tool" +import { Truncate } from "@/tool/truncate" +import { Agent } from "../../src/agent/agent" +import { EventV2Bridge } from "../../src/event-v2-bridge" +import { SessionID, MessageID } from "../../src/session/schema" +import { DEFAULT_IMAGE_MODEL, ImageGenerateTool, decodeImageDataUrl, slug } from "../../src/tool/image_generate" +import { disposeAllInstances, TestInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" + +/** A real one-pixel PNG, so the bytes written are an image rather than filler. */ +const PIXEL_PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + +const it = testEffect( + LayerNode.compile( + LayerNode.group([httpClient, Auth.node, FSUtil.node, EventV2Bridge.node, Truncate.node, Agent.node]), + [[httpClient, FetchHttpClient.layer]], + ), +) + +type Seen = { authorization: string | null; body: Record } + +/** A stand-in for the Redrob gateway that records what reached it. */ +function gateway(answer: (body: Record) => Response) { + const seen: Seen[] = [] + const server = Bun.serve({ + port: 0, + async fetch(request) { + const body: Record = await request.json() + seen.push({ authorization: request.headers.get("authorization"), body }) + return answer(body) + }, + }) + return { seen, server } +} + +const imageAnswer = () => + Response.json({ + choices: [ + { + message: { + content: "A red square.", + images: [{ type: "image_url", image_url: { url: `data:image/png;base64,${PIXEL_PNG}` } }], + }, + }, + ], + redrob: { costUsd: 0.039 }, + }) + +const asks: unknown[] = [] +const ctx = { + sessionID: SessionID.make("ses_test-image"), + messageID: MessageID.make("msg_test"), + callID: "", + agent: "build", + abort: AbortSignal.any([]), + messages: [], + metadata: () => Effect.void, + ask: (request: unknown) => Effect.sync(() => void asks.push(request)), +} + +const run = Effect.fn("ImageGenerateToolTest.run")(function* (args: Tool.InferParameters) { + const info = yield* ImageGenerateTool + const tool = yield* info.init() + return yield* tool.execute(args, ctx) +}) + +const failure = (exit: Exit.Exit) => (Exit.isFailure(exit) ? Cause.pretty(exit.cause) : "") + +const saved = { + url: process.env.REDROB_CONSOLE_URL, + auth: process.env.REDROB_AUTH_CONTENT, + key: process.env.REDROB_API_KEY, +} + +beforeEach(() => { + asks.length = 0 + process.env.REDROB_AUTH_CONTENT = JSON.stringify({ redrob: { type: "api", key: "rk_test_key" } }) + delete process.env.REDROB_API_KEY +}) + +afterEach(async () => { + for (const [name, value] of [ + ["REDROB_CONSOLE_URL", saved.url], + ["REDROB_AUTH_CONTENT", saved.auth], + ["REDROB_API_KEY", saved.key], + ] as const) { + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + await disposeAllInstances() +}) + +describe("tool.image_generate", () => { + it.instance("asks the gateway for an image on the engine's Redrob key, saves it, and attaches it", () => + Effect.gen(function* () { + const test = yield* TestInstance + const { seen, server } = gateway(imageAnswer) + process.env.REDROB_CONSOLE_URL = server.url.toString() + try { + const result = yield* run({ prompt: "A small red square on white" }) + + expect(seen).toHaveLength(1) + expect(seen[0].authorization).toBe("Bearer rk_test_key") + expect(seen[0].body).toMatchObject({ + model: DEFAULT_IMAGE_MODEL, + modalities: ["image", "text"], + messages: [{ role: "user", content: "A small red square on white" }], + }) + + const filepath = path.join(test.directory, "artifacts", "a-small-red-square-on-white.png") + expect(Buffer.from(yield* Effect.promise(() => fs.readFile(filepath))).toString("base64")).toBe(PIXEL_PNG) + expect(result.title.endsWith(path.join("artifacts", "a-small-red-square-on-white.png"))).toBe(true) + expect(result.output).toContain("A red square.") + expect(result.metadata).toMatchObject({ filepath, model: DEFAULT_IMAGE_MODEL, costUsd: 0.039 }) + expect(result.attachments).toEqual([ + { + type: "file", + mime: "image/png", + filename: "a-small-red-square-on-white.png", + url: `data:image/png;base64,${PIXEL_PNG}`, + }, + ]) + expect(asks).toEqual([expect.objectContaining({ permission: "edit", patterns: [result.title] })]) + } finally { + void server.stop(true) + } + }), + ) + + it.instance("never overwrites an existing file, and uses a named model and file name", () => + Effect.gen(function* () { + const test = yield* TestInstance + const existing = path.join(test.directory, "artifacts", "logo.png") + yield* Effect.promise(() => fs.mkdir(path.dirname(existing), { recursive: true })) + yield* Effect.promise(() => fs.writeFile(existing, "keep me")) + const { seen, server } = gateway(imageAnswer) + process.env.REDROB_CONSOLE_URL = server.url.toString() + try { + const result = yield* run({ prompt: "A logo", filename: "Logo", model: "openai/gpt-5-image-mini" }) + expect(seen[0].body.model).toBe("openai/gpt-5-image-mini") + expect(result.title.endsWith(path.join("artifacts", "logo-2.png"))).toBe(true) + expect(yield* Effect.promise(() => fs.readFile(existing, "utf8"))).toBe("keep me") + } finally { + void server.stop(true) + } + }), + ) + + it.instance("fails with what the model said when it returns no image", () => + Effect.gen(function* () { + const { server } = gateway(() => Response.json({ choices: [{ message: { content: "I cannot draw that." } }] })) + process.env.REDROB_CONSOLE_URL = server.url.toString() + try { + const exit = yield* run({ prompt: "Something" }).pipe(Effect.exit) + expect(failure(exit)).toContain("returned no image. It said: I cannot draw that.") + } finally { + void server.stop(true) + } + }), + ) + + it.instance("relays the gateway's own refusal, such as an empty balance", () => + Effect.gen(function* () { + const { server } = gateway(() => + Response.json( + { error: { message: "This workspace is out of credit.", type: "insufficient_quota" } }, + { status: 402 }, + ), + ) + process.env.REDROB_CONSOLE_URL = server.url.toString() + try { + const exit = yield* run({ prompt: "Something" }).pipe(Effect.exit) + expect(failure(exit)).toContain("answered 402: This workspace is out of credit.") + } finally { + void server.stop(true) + } + }), + ) + + it.instance("without a Redrob key it fails before anything is sent", () => + Effect.gen(function* () { + process.env.REDROB_AUTH_CONTENT = JSON.stringify({}) + const { seen, server } = gateway(imageAnswer) + process.env.REDROB_CONSOLE_URL = server.url.toString() + try { + const exit = yield* run({ prompt: "Something" }).pipe(Effect.exit) + expect(failure(exit)).toContain("Connect Redrob to generate images") + expect(seen).toHaveLength(0) + } finally { + void server.stop(true) + } + }), + ) +}) + +describe("image_generate helpers", () => { + it.effect("derives a safe file name stem", () => + Effect.sync(() => { + expect(slug("A Red Square, on white!")).toBe("a-red-square-on-white") + expect(slug("../../etc/passwd")).toBe("etc-passwd") + expect(slug("로고")).toBe("image") + expect(slug("x".repeat(100))).toHaveLength(48) + }), + ) + + it.effect("accepts only inline images of a type it writes", () => + Effect.sync(() => { + expect(decodeImageDataUrl(`data:image/png;base64,${PIXEL_PNG}`)?.mime).toBe("image/png") + expect(decodeImageDataUrl("https://example.com/a.png")).toBeUndefined() + expect(decodeImageDataUrl("data:image/svg+xml;base64,PHN2Zz4=")).toBeUndefined() + expect(decodeImageDataUrl("data:text/html;base64,PGI+")).toBeUndefined() + }), + ) +}) diff --git a/packages/redrob/test/tool/registry.test.ts b/packages/redrob/test/tool/registry.test.ts index c7c5348aa2..cc61d2704a 100644 --- a/packages/redrob/test/tool/registry.test.ts +++ b/packages/redrob/test/tool/registry.test.ts @@ -150,6 +150,39 @@ describe("tool.registry", () => { }), ) + it.instance("offers image_generate only while a Redrob credential is available", () => + Effect.gen(function* () { + const registry = yield* ToolRegistry.Service + const agents = yield* Agent.Service + const agent = yield* agents.defaultInfo() + const saved = { auth: process.env.REDROB_AUTH_CONTENT, key: process.env.REDROB_API_KEY } + const visible = () => + registry + .tools({ providerID: ProviderV2.ID.redrob, modelID: ModelV2.ID.make("test"), agent }) + .pipe(Effect.map((tools) => tools.map((tool) => tool.id).includes("image_generate"))) + try { + delete process.env.REDROB_API_KEY + process.env.REDROB_AUTH_CONTENT = JSON.stringify({}) + expect(yield* visible()).toBe(false) + + process.env.REDROB_AUTH_CONTENT = JSON.stringify({ redrob: { type: "api", key: "rk_test" } }) + expect(yield* visible()).toBe(true) + + process.env.REDROB_AUTH_CONTENT = JSON.stringify({}) + process.env.REDROB_API_KEY = "rk_env" + expect(yield* visible()).toBe(true) + } finally { + for (const [name, value] of [ + ["REDROB_AUTH_CONTENT", saved.auth], + ["REDROB_API_KEY", saved.key], + ] as const) { + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + } + }), + ) + it.instance("hides task background parameter unless experimental background subagents are enabled", () => Effect.gen(function* () { const registry = yield* ToolRegistry.Service diff --git a/packages/redrob/test/tool/speech_generate.test.ts b/packages/redrob/test/tool/speech_generate.test.ts new file mode 100644 index 0000000000..23843344a9 --- /dev/null +++ b/packages/redrob/test/tool/speech_generate.test.ts @@ -0,0 +1,193 @@ +import { afterEach, beforeEach, describe, expect } from "bun:test" +import { LayerNode } from "@redrob-code/core/effect/layer-node" +import { httpClient } from "@redrob-code/core/effect/app-node-platform" +import { FSUtil } from "@redrob-code/core/fs-util" +import { Cause, Effect, Exit } from "effect" +import { FetchHttpClient } from "effect/unstable/http" +import fs from "fs/promises" +import path from "path" +import { Auth } from "@/auth" +import { Tool } from "@/tool/tool" +import { Truncate } from "@/tool/truncate" +import { Agent } from "../../src/agent/agent" +import { EventV2Bridge } from "../../src/event-v2-bridge" +import { SessionID, MessageID } from "../../src/session/schema" +import { DEFAULT_SPEECH_MODEL, DEFAULT_SPEECH_VOICE, SpeechGenerateTool } from "../../src/tool/speech_generate" +import { disposeAllInstances, TestInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" + +/** The first bytes of an MP3 frame, which is what a real answer starts with. */ +const MP3 = new Uint8Array([0xff, 0xf3, 0x44, 0xc4, 0x00, 0x01, 0x02, 0x03]) + +const it = testEffect( + LayerNode.compile( + LayerNode.group([httpClient, Auth.node, FSUtil.node, EventV2Bridge.node, Truncate.node, Agent.node]), + [[httpClient, FetchHttpClient.layer]], + ), +) + +type Seen = { path: string; authorization: string | null; body: Record } + +/** A stand-in for the gateway's speech route, recording what reached it. */ +function gateway(answer: () => Response) { + const seen: Seen[] = [] + const server = Bun.serve({ + port: 0, + async fetch(request) { + const body: Record = await request.json() + seen.push({ path: new URL(request.url).pathname, authorization: request.headers.get("authorization"), body }) + return answer() + }, + }) + return { seen, server } +} + +const audioAnswer = () => + new Response(MP3, { headers: { "content-type": "audio/mpeg", "x-redrob-cost-usd": "0.000042" } }) + +const asks: unknown[] = [] +const ctx = { + sessionID: SessionID.make("ses_test-speech"), + messageID: MessageID.make("msg_test"), + callID: "", + agent: "build", + abort: AbortSignal.any([]), + messages: [], + metadata: () => Effect.void, + ask: (request: unknown) => Effect.sync(() => void asks.push(request)), +} + +const run = Effect.fn("SpeechGenerateToolTest.run")(function* (args: Tool.InferParameters) { + const info = yield* SpeechGenerateTool + const tool = yield* info.init() + return yield* tool.execute(args, ctx) +}) + +const failure = (exit: Exit.Exit) => (Exit.isFailure(exit) ? Cause.pretty(exit.cause) : "") + +const saved = { + url: process.env.REDROB_CONSOLE_URL, + auth: process.env.REDROB_AUTH_CONTENT, + key: process.env.REDROB_API_KEY, +} + +beforeEach(() => { + asks.length = 0 + process.env.REDROB_AUTH_CONTENT = JSON.stringify({ redrob: { type: "api", key: "rk_test_key" } }) + delete process.env.REDROB_API_KEY +}) + +afterEach(async () => { + for (const [name, value] of [ + ["REDROB_CONSOLE_URL", saved.url], + ["REDROB_AUTH_CONTENT", saved.auth], + ["REDROB_API_KEY", saved.key], + ] as const) { + if (value === undefined) delete process.env[name] + else process.env[name] = value + } + await disposeAllInstances() +}) + +describe("tool.speech_generate", () => { + it.instance("speaks the text through the gateway on the engine's key and saves an mp3, with no attachment", () => + Effect.gen(function* () { + const test = yield* TestInstance + const { seen, server } = gateway(audioAnswer) + process.env.REDROB_CONSOLE_URL = server.url.toString() + try { + const result = yield* run({ text: "안녕하세요, quarterly update." }) + + expect(seen).toHaveLength(1) + expect(seen[0].path).toBe("/audio/speech") + expect(seen[0].authorization).toBe("Bearer rk_test_key") + expect(seen[0].body).toEqual({ + model: DEFAULT_SPEECH_MODEL, + input: "안녕하세요, quarterly update.", + response_format: "mp3", + voice: DEFAULT_SPEECH_VOICE, + }) + + const filepath = path.join(test.directory, "artifacts", "quarterly-update.mp3") + expect(new Uint8Array(yield* Effect.promise(() => fs.readFile(filepath)))).toEqual(MP3) + expect(result.output).toContain("artifacts") + expect(result.output).toContain("quarterly-update.mp3") + // 24 code points: five Hangul syllables, ", ", "quarterly", " ", "update." (as the gateway bills). + expect(result.metadata).toMatchObject({ + filepath, + model: DEFAULT_SPEECH_MODEL, + characters: 24, + costUsd: 0.000042, + }) + expect(result.attachments).toBeUndefined() + expect(asks).toEqual([expect.objectContaining({ permission: "edit" })]) + } finally { + void server.stop(true) + } + }), + ) + + it.instance("a named model gets no default voice, and an existing file is never overwritten", () => + Effect.gen(function* () { + const test = yield* TestInstance + const existing = path.join(test.directory, "artifacts", "intro.mp3") + yield* Effect.promise(() => fs.mkdir(path.dirname(existing), { recursive: true })) + yield* Effect.promise(() => fs.writeFile(existing, "keep me")) + const { seen, server } = gateway(audioAnswer) + process.env.REDROB_CONSOLE_URL = server.url.toString() + try { + const result = yield* run({ text: "Hello", model: "hexgrad/kokoro-82m", filename: "intro" }) + expect(seen[0].body.voice).toBeUndefined() + expect(seen[0].body.model).toBe("hexgrad/kokoro-82m") + expect(result.title.endsWith(path.join("artifacts", "intro-2.mp3"))).toBe(true) + expect(yield* Effect.promise(() => fs.readFile(existing, "utf8"))).toBe("keep me") + } finally { + void server.stop(true) + } + }), + ) + + it.instance("text over the limit, or no key, is refused before anything is sent", () => + Effect.gen(function* () { + const { seen, server } = gateway(audioAnswer) + process.env.REDROB_CONSOLE_URL = server.url.toString() + try { + const long = yield* run({ text: "가".repeat(4097) }).pipe(Effect.exit) + expect(failure(long)).toContain("The text is 4097 characters") + process.env.REDROB_AUTH_CONTENT = JSON.stringify({}) + const keyless = yield* run({ text: "Hello" }).pipe(Effect.exit) + expect(failure(keyless)).toContain("Connect Redrob to generate speech") + expect(seen).toHaveLength(0) + } finally { + void server.stop(true) + } + }), + ) + + it.instance("a gateway refusal is relayed, and an answer that is not audio is not saved", () => + Effect.gen(function* () { + const test = yield* TestInstance + const refusal = gateway(() => + Response.json({ error: { message: "Model `x` has no voice `y`." } }, { status: 400 }), + ) + process.env.REDROB_CONSOLE_URL = refusal.server.url.toString() + try { + const exit = yield* run({ text: "Hello", model: "x", voice: "y" }).pipe(Effect.exit) + expect(failure(exit)).toContain("answered 400: Model `x` has no voice `y`.") + } finally { + void refusal.server.stop(true) + } + + const json = gateway(() => Response.json({ ok: true })) + process.env.REDROB_CONSOLE_URL = json.server.url.toString() + try { + const exit = yield* run({ text: "Hello" }).pipe(Effect.exit) + expect(failure(exit)).toContain("not audio") + const written = yield* Effect.promise(() => fs.readdir(path.join(test.directory, "artifacts")).catch(() => [])) + expect(written).toEqual([]) + } finally { + void json.server.stop(true) + } + }), + ) +}) diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index 5bf5bf79a9..c3c53ef505 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -179,6 +179,10 @@ import type { QuestionReplyErrors, QuestionReplyResponses, QuestionV2Reply, + RedrobSpeechErrors, + RedrobSpeechResponses, + RedrobTranscribeErrors, + RedrobTranscribeResponses, SessionAbortErrors, SessionAbortResponses, SessionChildrenErrors, @@ -531,6 +535,84 @@ export class Auth extends HeyApiClient { } } +export class Redrob extends HeyApiClient { + /** + * Speak text + * + * Turn text into mp3 speech through the Redrob gateway, billed to the connected Redrob account. + */ + public speech( + parameters?: { + text?: string + voice?: string + model?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "body", key: "text" }, + { in: "body", key: "voice" }, + { in: "body", key: "model" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/redrob/speech", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } + + /** + * Transcribe speech + * + * Turn recorded speech into text through the Redrob gateway, billed to the connected Redrob account. + */ + public transcribe( + parameters?: { + audio?: string + format?: "wav" | "mp3" | "flac" | "m4a" | "ogg" | "webm" | "aac" + language?: string + model?: string + }, + options?: Options, + ) { + const params = buildClientParams( + [parameters], + [ + { + args: [ + { in: "body", key: "audio" }, + { in: "body", key: "format" }, + { in: "body", key: "language" }, + { in: "body", key: "model" }, + ], + }, + ], + ) + return (options?.client ?? this.client).post({ + url: "/redrob/transcribe", + ...options, + ...params, + headers: { + "Content-Type": "application/json", + ...options?.headers, + ...params.headers, + }, + }) + } +} + export class App extends HeyApiClient { /** * Write log @@ -7427,6 +7509,11 @@ export class RedrobClient extends HeyApiClient { return (this._auth ??= new Auth({ client: this.client })) } + private _redrob?: Redrob + get redrob(): Redrob { + return (this._redrob ??= new Redrob({ client: this.client })) + } + private _app?: App get app(): App { return (this._app ??= new App({ client: this.client })) diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 96f3a1a7e2..cf3c1f128b 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -174,6 +174,25 @@ export type InvalidRequestError = { field?: string } +export type ServiceUnavailableError = { + _tag: "ServiceUnavailableError" + message: string + service?: string +} + +export type UpstreamError = { + _tag: "UpstreamError" + message: string + service?: string + status?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" +} + +export type TimeoutError = { + _tag: "TimeoutError" + message: string + operation?: string +} + export type MoveSessionError = { name: "MoveSessionError" data: { @@ -2798,12 +2817,6 @@ export type ConflictError = { resource?: string } -export type ServiceUnavailableError = { - _tag: "ServiceUnavailableError" - message: string - service?: string -} - export type MessageNotFoundError = { _tag: "MessageNotFoundError" sessionID: string @@ -7581,6 +7594,111 @@ export type AuthSetResponses = { export type AuthSetResponse = AuthSetResponses[keyof AuthSetResponses] +export type RedrobSpeechData = { + body?: { + /** + * The words to speak, at most 4096 characters + */ + text: string + voice?: string + model?: string + } + path?: never + query?: never + url: "/redrob/speech" +} + +export type RedrobSpeechErrors = { + /** + * InvalidRequestError + */ + 400: InvalidRequestError + /** + * UpstreamError + */ + 502: UpstreamError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError + /** + * TimeoutError + */ + 504: TimeoutError +} + +export type RedrobSpeechError = RedrobSpeechErrors[keyof RedrobSpeechErrors] + +export type RedrobSpeechResponses = { + /** + * Success + */ + 200: Blob | File +} + +export type RedrobSpeechResponse = RedrobSpeechResponses[keyof RedrobSpeechResponses] + +export type RedrobTranscribeData = { + body?: { + /** + * Base64-encoded audio, the raw bytes rather than a data URL, at most 10 MB + */ + audio: string + /** + * The audio's container format + */ + format: "wav" | "mp3" | "flac" | "m4a" | "ogg" | "webm" | "aac" + language?: string + model?: string + } + path?: never + query?: never + url: "/redrob/transcribe" +} + +export type RedrobTranscribeErrors = { + /** + * InvalidRequestError + */ + 400: InvalidRequestError + /** + * UpstreamError + */ + 502: UpstreamError + /** + * ServiceUnavailableError + */ + 503: ServiceUnavailableError + /** + * TimeoutError + */ + 504: TimeoutError +} + +export type RedrobTranscribeError = RedrobTranscribeErrors[keyof RedrobTranscribeErrors] + +export type RedrobTranscribeResponses = { + /** + * The transcript + */ + 200: { + /** + * The transcript, trimmed; empty when nothing was said + */ + text: string + /** + * Seconds of audio billed + */ + seconds?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + /** + * What the request cost, in USD + */ + costUsd?: number | "NaN" | "Infinity" | "-Infinity" | "Infinity" | "-Infinity" | "NaN" + } +} + +export type RedrobTranscribeResponse = RedrobTranscribeResponses[keyof RedrobTranscribeResponses] + export type AppLogData = { body?: { /**