Image and speech tools, and /redrob/speech and /redrob/transcribe routes

packages/redrob/src/redrob/transcription.ts

@@ -0,0 +1,163 @@
+import { Effect, Schema } from "effect"
+import { HttpClient, HttpClientRequest } from "effect/unstable/http"
+import { Auth } from "@/auth"
+import { gatewayMessage, gatewayUrl, redrobKey } from "@/tool/image_generate"
+
+/**
+ * Transcription through the Redrob gateway's `POST /v1/audio/transcriptions`, on the engine's own
+ * Redrob credential, for the reason speech gives: the key is the engine's and stays here. Its caller
+ * is the engine's `POST /redrob/transcribe` route, which is how Redrob Cowork's push-to-talk turns a
+ * recording into text without the app ever holding the key.
+ *
+ * Everything that can be refused is refused before anything is sent, so a bad request costs nothing.
+ */
+
+/** Scribe handles Korean, English and the two mixed in one sentence, which is how users speak. */
+export const DEFAULT_TRANSCRIPTION_MODEL = "elevenlabs/scribe-v2"
+
+/** The formats the gateway accepts, which are the ones its upstream documents. */
+export const TRANSCRIPTION_FORMATS = ["wav", "mp3", "flac", "m4a", "ogg", "webm", "aac"] as const
+export type TranscriptionFormat = (typeof TRANSCRIPTION_FORMATS)[number]
+
+/** The gateway's limit for one request, decoded, checked here so an over-size clip costs nothing. */
+export const MAX_TRANSCRIPTION_BYTES = 10 * 1024 * 1024
+
+const TIMEOUT_MS = 120_000
+const BASE64 = /^[A-Za-z0-9+/]*={0,2}$/
+const LANGUAGE = /^[a-z]{2}$/
+
+/**
+ * Why audio could not be transcribed. `invalid` and `no_key` are found before anything is sent and
+ * cost nothing; `upstream` and `timeout` are the gateway's.
+ */
+export class TranscriptionError extends Schema.TaggedErrorClass<TranscriptionError>()("TranscriptionError", {
+  reason: Schema.Literals(["invalid", "no_key", "upstream", "timeout"]),
+  message: Schema.String,
+  status: Schema.optional(Schema.Number),
+}) {}
+
+export type PreparedTranscription = {
+  audio: string
+  format: TranscriptionFormat
+  model: string
+  language?: string
+  bytes: number
+}
+
+export type Transcript = { text: string; seconds?: number; costUsd?: number }
+
+/** What base64 decodes to, without decoding it: three bytes per four characters, less the padding. */
+function decodedBytes(base64: string): number {
+  const padding = base64.endsWith("==") ? 2 : base64.endsWith("=") ? 1 : 0
+  return (base64.length / 4) * 3 - padding
+}
+
+/** The request as it will be sent, or why it would be refused, with nothing sent either way. */
+export const prepare = Effect.fn("RedrobTranscription.prepare")(function* (input: {
+  audio: string
+  format: TranscriptionFormat
+  model?: string
+  language?: string
+}) {
+  const audio = input.audio.replace(/\s+/g, "")
+  if (!audio) return yield* new TranscriptionError({ reason: "invalid", message: "Give the audio to transcribe." })
+  if (audio.length % 4 !== 0 || !BASE64.test(audio)) {
+    return yield* new TranscriptionError({
+      reason: "invalid",
+      message: "The audio must be base64: the raw bytes, not a data URL.",
+    })
+  }
+  const bytes = decodedBytes(audio)
+  if (bytes > MAX_TRANSCRIPTION_BYTES) {
+    return yield* new TranscriptionError({
+      reason: "invalid",
+      message: `The audio is ${bytes} bytes; transcribe at most ${MAX_TRANSCRIPTION_BYTES} per call.`,
+    })
+  }
+  const language = input.language?.trim().toLowerCase() || undefined
+  if (language && !LANGUAGE.test(language)) {
+    return yield* new TranscriptionError({
+      reason: "invalid",
+      message: `"${input.language}" is not an ISO-639-1 language such as "ko" or "en".`,
+    })
+  }
+  const prepared: PreparedTranscription = {
+    audio,
+    format: input.format,
+    model: input.model?.trim() || DEFAULT_TRANSCRIPTION_MODEL,
+    ...(language ? { language } : {}),
+    bytes,
+  }
+  return prepared
+})
+
+/** The engine's Redrob key, or a `no_key` failure, which is found before anything is sent. */
+export const key = Effect.fn("RedrobTranscription.key")(function* (auth: Auth.Interface) {
+  const found = redrobKey(yield* auth.get("redrob").pipe(Effect.orElseSucceed(() => undefined)))
+  if (!found) {
+    return yield* new TranscriptionError({
+      reason: "no_key",
+      message: "Connect Redrob to transcribe speech: no Redrob API key is available.",
+    })
+  }
+  return found
+})
+
+const GatewayTranscript = Schema.Struct({
+  text: Schema.String,
+  seconds: Schema.optional(Schema.Number),
+  costUsd: Schema.optional(Schema.Number),
+})
+const decodeTranscript = Schema.decodeUnknownEffect(Schema.fromJsonString(GatewayTranscript))
+
+/** One call to the gateway. The answer is checked for being a transcript before anyone reads it. */
+export const request = Effect.fn("RedrobTranscription.request")(function* (
+  http: HttpClient.HttpClient,
+  key: string,
+  transcription: PreparedTranscription,
+) {
+  const outgoing = yield* HttpClientRequest.post(`${gatewayUrl()}/audio/transcriptions`).pipe(
+    HttpClientRequest.bearerToken(key),
+    HttpClientRequest.bodyJson({
+      model: transcription.model,
+      input_audio: { data: transcription.audio, format: transcription.format },
+      response_format: "json",
+      ...(transcription.language ? { language: transcription.language } : {}),
+    }),
+    Effect.orDie,
+  )
+  const response = yield* http.execute(outgoing).pipe(
+    Effect.mapError(
+      () => new TranscriptionError({ reason: "upstream", message: "The Redrob gateway could not be reached." }),
+    ),
+    Effect.timeoutOrElse({
+      duration: TIMEOUT_MS,
+      orElse: () => Effect.fail(new TranscriptionError({ reason: "timeout", message: "Transcription timed out." })),
+    }),
+  )
+  const body = yield* response.text.pipe(
+    Effect.mapError(
+      () => new TranscriptionError({ reason: "upstream", message: "The Redrob gateway's answer was cut off." }),
+    ),
+  )
+  if (response.status < 200 || response.status >= 300) {
+    return yield* new TranscriptionError({
+      reason: "upstream",
+      status: response.status,
+      message: `The Redrob gateway answered ${response.status}: ${gatewayMessage(body)}`,
+    })
+  }
+  const parsed = yield* decodeTranscript(body).pipe(
+    Effect.mapError(
+      () => new TranscriptionError({ reason: "upstream", message: "The Redrob gateway answered without a transcript." }),
+    ),
+  )
+  const transcript: Transcript = {
+    text: parsed.text.trim(),
+    ...(parsed.seconds !== undefined && Number.isFinite(parsed.seconds) ? { seconds: parsed.seconds } : {}),
+    ...(parsed.costUsd !== undefined && Number.isFinite(parsed.costUsd) ? { costUsd: parsed.costUsd } : {}),
+  }
+  return transcript
+})
+
+export * as RedrobTranscription from "./transcription"

packages/redrob/src/server/routes/instance/httpapi/groups/control.ts

@@ -1,9 +1,11 @@
 import { Auth } from "@/auth"
 
 import { Schema } from "effect"
-import { HttpApi, HttpApiEndpoint, HttpApiError, HttpApiGroup, OpenApi } from "effect/unstable/httpapi"
+import { HttpApi, HttpApiEndpoint, HttpApiError, HttpApiGroup, HttpApiSchema, OpenApi } from "effect/unstable/httpapi"
+import { InvalidRequestError, ServiceUnavailableError, TimeoutError, UpstreamError } from "../errors"
 import { described } from "./metadata"
 import { ProviderV2 } from "@redrob-code/core/provider"
+import { TRANSCRIPTION_FORMATS } from "@/redrob/transcription"
 
 const AuthParams = Schema.Struct({
   providerID: ProviderV2.ID,
@@ -28,9 +30,36 @@ export const LogInput = Schema.Struct({
   }),
 })
 
+export const SpeechInput = Schema.Struct({
+  text: Schema.String.annotate({ description: "The words to speak, at most 4096 characters" }),
+  voice: Schema.optional(Schema.String).annotate({ description: "A voice of the chosen model" }),
+  model: Schema.optional(Schema.String).annotate({
+    description: "A speech model id served by the Redrob gateway. Defaults to elevenlabs/eleven-flash-v2.5, for playback.",
+  }),
+})
+
+export const TranscribeInput = Schema.Struct({
+  audio: Schema.String.annotate({ description: "Base64-encoded audio, the raw bytes rather than a data URL, at most 10 MB" }),
+  format: Schema.Literals(TRANSCRIPTION_FORMATS).annotate({ description: "The audio's container format" }),
+  language: Schema.optional(Schema.String).annotate({
+    description: 'ISO-639-1 language, e.g. "ko". Detected when omitted.',
+  }),
+  model: Schema.optional(Schema.String).annotate({
+    description: "A transcription model id served by the Redrob gateway",
+  }),
+})
+
+export const TranscribeOutput = Schema.Struct({
+  text: Schema.String.annotate({ description: "The transcript, trimmed; empty when nothing was said" }),
+  seconds: Schema.optional(Schema.Number).annotate({ description: "Seconds of audio billed" }),
+  costUsd: Schema.optional(Schema.Number).annotate({ description: "What the request cost, in USD" }),
+})
+
 export const ControlPaths = {
   auth: "/auth/:providerID",
   log: "/log",
+  speech: "/redrob/speech",
+  transcribe: "/redrob/transcribe",
 } as const
 
 export const ControlApi = HttpApi.make("control").add(
@@ -59,6 +88,33 @@ export const ControlApi = HttpApi.make("control").add(
           description: "Remove authentication credentials",
         }),
       ),
+      /*
+       * Speech on the engine's own Redrob credential, for a client that has to play audio and must
+       * not hold the key: Redrob Cowork's read-aloud. The key never leaves this process.
+       */
+      HttpApiEndpoint.post("speech", ControlPaths.speech, {
+        payload: SpeechInput,
+        success: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array({ contentType: "audio/mpeg" })),
+        error: [InvalidRequestError, ServiceUnavailableError, UpstreamError, TimeoutError],
+      }).annotateMerge(
+        OpenApi.annotations({
+          identifier: "redrob.speech",
+          summary: "Speak text",
+          description: "Turn text into mp3 speech through the Redrob gateway, billed to the connected Redrob account.",
+        }),
+      ),
+      /* Transcription on the same credential, for Redrob Cowork's push-to-talk, for the same reason. */
+      HttpApiEndpoint.post("transcribe", ControlPaths.transcribe, {
+        payload: TranscribeInput,
+        success: described(TranscribeOutput, "The transcript"),
+        error: [InvalidRequestError, ServiceUnavailableError, UpstreamError, TimeoutError],
+      }).annotateMerge(
+        OpenApi.annotations({
+          identifier: "redrob.transcribe",
+          summary: "Transcribe speech",
+          description: "Turn recorded speech into text through the Redrob gateway, billed to the connected Redrob account.",
+        }),
+      ),
       HttpApiEndpoint.post("log", ControlPaths.log, {
         query: LogQuery,
         payload: LogInput,

packages/redrob/src/server/routes/instance/httpapi/handlers/control.ts

@@ -3,12 +3,49 @@ import { Auth } from "@/auth"
 import { Effect } from "effect"
 import { HttpApiBuilder } from "effect/unstable/httpapi"
 import { RootHttpApi } from "../api"
-import { LogInput } from "../groups/control"
+import { LogInput, SpeechInput, TranscribeInput } from "../groups/control"
+import { RedrobTranscription, type TranscriptionError } from "@/redrob/transcription"
+import { InvalidRequestError, ServiceUnavailableError, TimeoutError, UpstreamError } from "../errors"
+import { PLAYBACK_DEFAULTS, prepareSpeech, requestSpeech, speechKey, type SpeechError } from "@/tool/speech_generate"
+import { HttpClient } from "effect/unstable/http"
 import { ProviderV2 } from "@redrob-code/core/provider"
 
 export const controlHandlers = HttpApiBuilder.group(RootHttpApi, "control", (handlers) =>
   Effect.gen(function* () {
     const auth = yield* Auth.Service
+    const http = yield* HttpClient.HttpClient
+
+    /**
+     * Each gateway failure as the endpoints declare it; no_key is 503 because the engine, not the
+     * caller, lacks it. Speech and transcription fail for the same four reasons.
+     */
+    const gatewayFailure = (operation: string) => (error: SpeechError | TranscriptionError) =>
+      error.reason === "invalid"
+        ? new InvalidRequestError({ message: error.message })
+        : error.reason === "no_key"
+          ? new ServiceUnavailableError({ message: error.message, service: "redrob" })
+          : error.reason === "timeout"
+            ? new TimeoutError({ message: error.message, operation })
+            : new UpstreamError({
+                message: error.message,
+                service: "redrob",
+                ...(error.status ? { status: error.status } : {}),
+              })
+
+    const speech = Effect.fn("ControlHttpApi.speech")(function* (ctx: { payload: typeof SpeechInput.Type }) {
+      // This route is for audio that is played at once, so it defaults to the fast playback model.
+      const prepared = yield* prepareSpeech(ctx.payload, PLAYBACK_DEFAULTS)
+      const key = yield* speechKey(auth)
+      return (yield* requestSpeech(http, key, prepared)).audio
+    }, Effect.mapError(gatewayFailure("speech")))
+
+    const transcribe = Effect.fn("ControlHttpApi.transcribe")(function* (ctx: {
+      payload: typeof TranscribeInput.Type
+    }) {
+      const prepared = yield* RedrobTranscription.prepare(ctx.payload)
+      const key = yield* RedrobTranscription.key(auth)
+      return yield* RedrobTranscription.request(http, key, prepared)
+    }, Effect.mapError(gatewayFailure("transcribe")))
 
     const authSet = Effect.fn("ControlHttpApi.authSet")(function* (ctx: {
       params: { providerID: ProviderV2.ID }
@@ -38,6 +75,11 @@ export const controlHandlers = HttpApiBuilder.group(RootHttpApi, "control", (han
       return true
     })
 
-    return handlers.handle("authSet", authSet).handle("authRemove", authRemove).handle("log", log)
+    return handlers
+      .handle("authSet", authSet)
+      .handle("authRemove", authRemove)
+      .handle("speech", speech)
+      .handle("transcribe", transcribe)
+      .handle("log", log)
   }),
 )