From b1325bdaf8fe7d9212c94b4ff3f45d41353bc6f7 Mon Sep 17 00:00:00 2001 From: iam-truongtrungnghia <149591043+ttncode@users.noreply.github.com> Date: Mon, 14 Sep 2026 03:41:49 +0700 Subject: [PATCH 01/77] docs: add the OpenMedia design spec --- docs/.vitepress/config.ts | 1 + .../specs/2026-09-14-openmedia-design.md | 391 ++++++++++++++++++ 2 files changed, 392 insertions(+) create mode 100644 docs/superpowers/specs/2026-09-14-openmedia-design.md diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 307343f..b194c86 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -6,6 +6,7 @@ export default defineConfig({ head: [['link', { rel: 'icon', href: '/logo.png' }]], // a dead link is a failed build, not a warning ignoreDeadLinks: false, + srcExclude: ['superpowers/**'], themeConfig: { logo: '/logo.png', sidebar: [ diff --git a/docs/superpowers/specs/2026-09-14-openmedia-design.md b/docs/superpowers/specs/2026-09-14-openmedia-design.md new file mode 100644 index 0000000..ee9994d --- /dev/null +++ b/docs/superpowers/specs/2026-09-14-openmedia-design.md @@ -0,0 +1,391 @@ +# OpenMedia design + +Status: approved by the product owner on 2026-09-14 (autonomous delivery mandate). + +## 1. Goal + +OpenMedia is a self-hosted web app that downloads video and audio from almost any +website through yt-dlp and ffmpeg. It is a rebuild of +[averygan/reclip](https://github.com/averygan/reclip) (MIT): the Flask + yt-dlp +backend foundation is kept, patched and extended; the web UI is replaced by a +Next.js app in an Apple Human Interface style; the project is generated by the +`scaffold` toolbox (`--web nextjs --api flask --db none --cache none`). + +### Definition of shippable + +1. `mise run checklist` passes locally and the CI workflow is green on the pull request. +2. `docker compose up` with the released images downloads a real YouTube video as + MP4 and a SoundCloud track as M4A, with live progress, cancel, trim and cleanup. +3. A `v0.1.0` release exists with both images on GHCR and `compose.yaml`, + `example.env`, `install.sh` attached. +4. README (English and Vietnamese), documentation site, LICENSE and NOTICE are complete. +5. No TODO, placeholder or unfinished path anywhere in the repository. + +## 2. Confirmed product decisions + +| Topic | Decision | +| ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Backend | Keep reclip's Flask, yt-dlp and ffmpeg foundation and its endpoint shapes; fix security; extend | +| Features | Smart queue (progress, cancel, concurrency), formats, quality, trim, subtitles, embedded metadata, cookies, optional password, rate limit, history, paste and drag-drop, PWA share target, light and dark theme, accent colors, keyboard shortcuts, Vietnamese and English | +| Deployment | One image set for a laptop or a VPS, safe defaults, password optional | +| Language | Follows the browser; user can override | +| Brand | Logo "viewfinder": frame corners plus three waveform bars; wordmark `OpenMedia` in Geist 600; accent teal | +| UI | Layout: sidebar, queue, inspector. Style: Apple HIG (translucent materials, spring motion, iOS sheets, Dynamic Island style toast) | +| Repository | github.com/ttncode/openmedia, public, MIT with NOTICE crediting reclip | + +## 3. Architecture + +``` +browser ──HTTPS──▶ web (Next.js standalone, :8080) + │ /api/* route handler proxy (same origin) + ▼ + api (Flask + gunicorn, :8080, one worker, threads) + │ subprocess (argument list, never a shell) + ▼ + yt-dlp ─▶ ffmpeg, deno + │ + /data volume: downloads/, cookies.txt, settings.json, secret_key, yt-dlp/ +``` + +- The browser only talks to the web origin. The web app proxies every `/api/*` + request to `API_URL` at runtime, so no API address is baked into the build and + cookies stay first-party. +- The API runs one gunicorn worker with eight threads because the job registry lives + in process memory. Jobs do not survive a restart; files older than the retention + period are removed on start. +- Compose publishes the web port publicly and binds the API port to `127.0.0.1` only. + +### Repository layout (generated by scaffold, then extended) + +| Path | Purpose | +| -------------- | -------------------------------------------------------------- | +| `apps/api` | Flask application, yt-dlp job engine, pytest suite, Dockerfile | +| `apps/web` | Next.js 16 application, vitest suite, Dockerfile | +| `docs` | VitePress documentation site and ADRs | +| `compose.yaml` | Production stack attached to releases | +| `install.sh` | Operator installer attached to releases | + +## 4. Backend design (`apps/api`) + +### 4.1 Modules + +| Module | Responsibility | +| -------------------------------- | ----------------------------------------------------------------------------------- | +| `apps/api/app/__init__.py` | `create_app()`: config, blueprints, proxy fix, error handlers, background services | +| `apps/api/app/config.py` | Typed `Settings` read once from environment variables | +| `apps/api/app/health.py` | `/health/live` and `/health/ready` (ready checks yt-dlp, ffmpeg, writable data dir) | +| `apps/api/app/media.py` | Blueprint with every `/api/*` route; thin, delegates to services | +| `apps/api/app/validation.py` | URL, format id, trim, language and option validation | +| `apps/api/app/network_guard.py` | Resolves hosts and rejects private, loopback, link-local and reserved addresses | +| `apps/api/app/ytdlp.py` | Builds yt-dlp argument lists; runs info and playlist extraction; maps errors | +| `apps/api/app/progress.py` | Parses progress lines and maps multi-stream progress to one percentage | +| `apps/api/app/jobs.py` | `JobManager`: queue, concurrency, worker threads, cancel, file collection | +| `apps/api/app/cleanup.py` | Retention sweeper and startup orphan removal | +| `apps/api/app/storage.py` | Disk usage and storage limit | +| `apps/api/app/cookies.py` | Cookie file upload validation, domain and expiry summary | +| `apps/api/app/settings_store.py` | Persists runtime settings (retention, concurrency) in `settings.json` | +| `apps/api/app/security.py` | Optional password session, cross-site request guard, rate limiter | +| `apps/api/app/errors.py` | `ApiError` with HTTP status, machine code and message | + +### 4.2 Configuration + +| Variable | Default | Meaning | +| --------------------------------- | --------- | ------------------------------------------------------------------- | +| `OPENMEDIA_DATA_DIR` | `/data` | Root for downloads, cookies, settings, secret key, yt-dlp updates | +| `OPENMEDIA_PASSWORD` | empty | When set, every API route except session and health needs a login | +| `OPENMEDIA_SECRET_KEY` | generated | Session signing key; generated once into the data dir when empty | +| `OPENMEDIA_RETENTION_MINUTES` | `60` | Default retention; runtime setting overrides it | +| `OPENMEDIA_MAX_CONCURRENT` | `3` | Default concurrent downloads (1 to 5); runtime setting overrides it | +| `OPENMEDIA_MAX_FILESIZE_MB` | `4096` | Passed to yt-dlp `--max-filesize` | +| `OPENMEDIA_MAX_STORAGE_GB` | `0` | Total download storage limit; `0` means unlimited | +| `OPENMEDIA_MAX_PLAYLIST_ITEMS` | `50` | Upper bound for playlist expansion | +| `OPENMEDIA_RATE_LIMIT_PER_MINUTE` | `30` | Per client, for info, playlist and download requests | +| `OPENMEDIA_STALL_TIMEOUT_SECONDS` | `180` | A download with no output for this long is stopped | +| `OPENMEDIA_ALLOW_PRIVATE_URLS` | `false` | Allows URLs that resolve to private networks | +| `OPENMEDIA_TRUSTED_PROXY_HOPS` | `1` | Number of `X-Forwarded-*` hops trusted (the web proxy) | +| `OPENMEDIA_AUTO_UPDATE_YTDLP` | `true` | Container start installs the newest yt-dlp into the data dir | +| `OPENMEDIA_YTDLP_PROXY` | empty | Optional proxy passed to yt-dlp | + +### 4.3 HTTP API + +All responses are JSON unless stated. Every error has the shape +`{"error": "", "code": ""}`, which keeps +reclip clients working because they read `error`. Field names are snake_case. + +| Method and path | Request | Success response | +| ------------------------------ | ------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| GET /api/session | none | `{auth_required, authenticated, version, limits: {max_filesize_mb, max_playlist_items}}` | +| POST /api/session | `{password}` | 204, sets the session cookie; 401 `invalid_password` | +| DELETE /api/session | none | 204 | +| POST /api/info | `{url}` | `{id, title, thumbnail, duration, uploader, platform, webpage_url, formats: [{id, label, height, ext, filesize}], subtitle_languages: [..], has_chapters}` | +| POST /api/playlist | `{url, limit?}` | `{title, count, urls: [..]}` | +| POST /api/download | see 4.4 | 202 `{job_id, job}` | +| GET /api/jobs | none | `{jobs: [Job]}` newest first | +| GET /api/status/{job_id} | none | `Job` (includes reclip's `status`, `error`, `filename`) | +| DELETE /api/jobs/{job_id} | none | 204; cancels an active job or removes a finished one and its files | +| GET /api/file/{job_id} | none | Media file as attachment, range requests supported | +| GET /api/file/{job_id}/{index} | none | Any file of the job (subtitle files) | +| GET /api/settings | none | `{retention_minutes, max_concurrent}` | +| PUT /api/settings | same shape | Saved settings; retention in 15, 60, 360, 1440; concurrency 1 to 5 | +| GET /api/storage | none | `{used_bytes, limit_bytes, free_bytes}` (`limit_bytes` null when unlimited) | +| GET /api/cookies | none | `{present, domains: [..], expires_at, uploaded_at}` | +| PUT /api/cookies | multipart field `file`, at most 1 MB, Netscape cookie format | Same as GET | +| DELETE /api/cookies | none | 204 | + +`Job` object: + +```json +{ + "job_id": "3f9c2a7b1e", + "url": "https://www.youtube.com/watch?v=...", + "title": "...", + "status": "queued | downloading | processing | done | error | cancelled", + "progress": 63.4, + "speed_bps": 4400000, + "eta_seconds": 7, + "downloaded_bytes": 120000000, + "total_bytes": 190000000, + "queue_position": 0, + "options": { "kind": "video", "container": "mp4", "quality_height": 1080, "format_id": "137", "audio_format": null, "audio_quality": null, "trim": { "start": 0, "end": 1122 }, "subtitles": { "languages": ["vi"], "mode": "embed" }, "embed_metadata": true }, + "filename": "Pho bo Ha Noi.mp4", + "files": [{ "index": 0, "name": "Pho bo Ha Noi.mp4", "kind": "media", "size_bytes": 412000000 }], + "error": null, + "error_code": null, + "created_at": "2026-09-14T08:00:00Z", + "finished_at": null, + "expires_at": null +} +``` + +Error codes: `invalid_url`, `unsupported_url`, `private_network`, `invalid_option`, +`not_found`, `file_not_ready`, `rate_limited`, `auth_required`, `invalid_password`, +`cross_site_request`, `storage_full`, `too_large`, `bot_check`, `private_video`, +`geo_blocked`, `unavailable`, `timeout`, `extractor_error`, `invalid_cookies`. + +### 4.4 Download request + +```json +{ + "url": "https://...", + "title": "optional, used for the file name", + "format": "video | audio", + "format_id": "optional yt-dlp format id from /api/info", + "container": "mp4 | mkv", + "quality_height": 1080, + "audio_format": "mp3 | m4a | opus | flac | wav", + "audio_quality": "320k | best", + "trim": { "start": 0, "end": 120 }, + "subtitles": { "languages": ["vi", "en"], "mode": "embed | srt" }, + "embed_metadata": true +} +``` + +Only `url` is required; reclip's `{url, format, format_id, title}` request remains valid. + +### 4.5 yt-dlp invocation + +The command is always an argument list run with `sys.executable -m yt_dlp`, with +`PYTHONPATH` pointing at `/data/yt-dlp` when an updated copy exists. The URL is the +last argument, after `--`. + +Common arguments: `--no-playlist --newline --no-colors --no-warnings --restrict-filenames +--progress-template "download:OMPROGRESS %(progress.downloaded_bytes)s %(progress.total_bytes)s %(progress.total_bytes_estimate)s %(progress.speed)s %(progress.eta)s" +--max-filesize {limit}M -P {job_dir} -o "media.%(ext)s"`, plus `--cookies {job copy}` +when cookies exist and `--proxy` when configured. + +| Option | Arguments | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Video MP4 | `-f "{format_id}+bestaudio[ext=m4a]/{format_id}+bestaudio/best"` or `-f "bv*[height<={h}]+ba/b[height<={h}]/b"`; `-S "vcodec:h264,acodec:aac"`; `--merge-output-format mp4` | +| Video MKV | Same selection without `-S`; `--merge-output-format mkv` | +| Audio | `-f "ba/b" -x --audio-format {fmt}`; `--audio-quality 320K` for `320k`, `0` for `best` | +| Trim | `--download-sections "*{start}-{end}" --force-keyframes-at-cuts` | +| Subtitles embed | `--write-subs --write-auto-subs --sub-langs "{langs}" --embed-subs` | +| Subtitles srt | `--write-subs --write-auto-subs --sub-langs "{langs}" --convert-subs srt` | +| Metadata | `--embed-metadata --embed-chapters`, and `--embed-thumbnail` except for WAV | + +Info extraction uses `-J --no-playlist` (first JSON document) with a 60 second +timeout. Playlist expansion uses `--flat-playlist -J --playlist-end {limit}`. + +Error mapping from the last stderr line: "Sign in to confirm" becomes `bot_check`, +"Private video" `private_video`, "not available in your country" `geo_blocked`, +"Video unavailable" `unavailable`, "Unsupported URL" `unsupported_url`, +"File is larger than max-filesize" `too_large`; anything else `extractor_error`. + +### 4.6 Job lifecycle + +``` +queued ──slot free──▶ downloading ──post-processing line──▶ processing ──exit 0──▶ done + │ │ │ + └──cancel──▶ cancelled ◀┴──cancel─────────────────────────────┘ + └──non-zero exit, stall, too large──▶ error +``` + +- A dispatcher starts queued jobs while running jobs are fewer than `max_concurrent`; + changing the setting takes effect immediately. +- Progress: the first stream maps to 0 to 90 percent, a second stream to 90 to 99, + post-processing to 99, done to 100. Progress never decreases. +- Cancel terminates the process group, removes the job directory and marks `cancelled`. +- Stall: no output line for `OPENMEDIA_STALL_TIMEOUT_SECONDS` stops the job with `timeout`. +- On success the media file is the largest non-subtitle file; the download name is the + sanitized title (100 characters, reclip's rule) plus extension; subtitle files are + listed after it. +- Retention: the sweeper runs every 60 seconds and removes finished, failed and + cancelled jobs older than the retention period, including their files. +- Storage: a download request is refused with 507 `storage_full` when the limit is reached. + +### 4.7 Security + +- URL validation: `http` or `https`, at most 2048 characters, no whitespace or control + characters, host present. `format_id` matches `^[A-Za-z0-9_.+-]{1,64}$`; subtitle + languages match `^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})?$` (at most five); trim values are + non-negative with `start < end`. +- Network guard: every resolved address of the URL host must be public unless + `OPENMEDIA_ALLOW_PRIVATE_URLS=true`. yt-dlp may still follow redirects; the + documentation says so and recommends `OPENMEDIA_YTDLP_PROXY` for hardened setups. +- Cross-site guard: state-changing requests are rejected with 403 `cross_site_request` + when `Sec-Fetch-Site` is `cross-site` or `same-site`, or when `Origin` is present + and does not match the forwarded host. +- Password: `OPENMEDIA_PASSWORD` compared with `hmac.compare_digest`; Flask signed + session cookie, `HttpOnly`, `SameSite=Lax`, `Secure` when the forwarded protocol is + https; login attempts limited to 5 per minute per client. +- Rate limit: token bucket per client address (after `ProxyFix`), 429 `rate_limited` + with `Retry-After`. +- Cookies file: stored with mode 0600, never logged, copied per job so yt-dlp writes do + not race; content must start with the Netscape header or contain tab-separated rows. + +### 4.8 Container + +- Build stage: uv sync as generated by the adapter. +- Runtime: `python:3.13-slim` with `ffmpeg` from apt and the `deno` binary copied from + the pinned `denoland/deno:bin` image; `yt-dlp[default]` and `yt-dlp-ejs` come from + `uv.lock`. +- `apps/api/docker-entrypoint.sh` creates the data directories, optionally installs the + newest `yt-dlp` and `yt-dlp-ejs` into `/data/yt-dlp` (failure keeps the locked version), + then runs gunicorn: `--workers 1 --threads 8 --timeout 120 --bind 0.0.0.0:8080`. +- Non-root user `app` (uid 10001) owns `/data`; compose mounts the named volume + `openmedia-data` there. + +## 5. Web design (`apps/web`) + +### 5.1 Stack + +Next.js 16 App Router with React 19, TypeScript strict, CSS Modules plus one global +token sheet, `@phosphor-icons/react` (regular weight), the next font loader for Inter and +JetBrains Mono as fallbacks behind the Apple system font stack. No state library: +one reducer and context. Tailwind stays installed by the generator but components +use CSS Modules, because the design relies on layered tokens, springs and materials +that read more clearly as CSS. + +### 5.2 Modules + +| Path | Responsibility | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------- | +| `apps/web/src/app/layout.tsx` | Fonts, metadata, theme bootstrap script, providers | +| `apps/web/src/app/page.tsx` | Renders the client application shell | +| `apps/web/src/app/api/[...path]/route.ts` | Runtime proxy to `API_URL` for every method, streaming bodies, forwarding cookies and `X-Forwarded-*` | +| `apps/web/src/app/manifest.ts` | PWA manifest with share target | +| `apps/web/src/app/globals.css` | Design tokens (colors, accents, radii, springs), base elements | +| `apps/web/src/lib/api` | Typed client and response types mirroring section 4.3 | +| `apps/web/src/lib/links.ts` | Link parsing, platform detection, playlist detection | +| `apps/web/src/lib/format.ts` | Sizes, clocks, speeds, remaining time, Vietnamese and English number formats | +| `apps/web/src/lib/i18n` | Dictionaries (`vi`, `en`), provider, `useT()` | +| `apps/web/src/lib/preferences.ts` | Theme, accent, language, default format, history and ready items in localStorage | +| `apps/web/src/state` | Reducer, actions, context, job polling hook | +| `apps/web/src/components` | UI components grouped by area: shell, importer, queue, inspector, history, settings, overlays, controls, auth | + +### 5.3 Client state + +- Queue items are either `ready` (client side, created from `/api/info`, persisted in + localStorage) or linked to a server job by `job_id`. +- Polling: `GET /api/jobs` every second while any job is queued, downloading or + processing, every ten seconds otherwise, paused while the tab is hidden. +- When a job becomes `done` it is added to history (localStorage, newest first, at most + 200 entries). "Download again" re-runs `/api/info` for the stored URL. +- Saving a file is a plain link to `/api/file/{job_id}` with the `download` attribute. + +### 5.4 Screens and components (mapped from the approved prototype) + +| Area | Behavior | +| ------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Sidebar (1024 px and up) | Glass panel, brand lockup, filters: queue, downloading, done, needs attention, history, settings; install hint; storage bar | +| Toolbar | Large title that collapses into an inline title on scroll; theme toggle; shortcuts; sidebar toggle on tablets | +| Importer | Auto-growing field, Paste, Get info; platform chips; playlist choice (this video or whole playlist); Enter fetches, Shift+Enter new line; drag and drop anywhere; paste anywhere fetches | +| Queue | Inset grouped list; rows show thumbnail, title, state line; trailing control per state: Download capsule, progress ring with stop, waiting ring, Save capsule, Fix capsule; skeleton while fetching | +| Inspector (desktop column, phone sheet) | Artwork, title, platform; for ready items: kind and container segmented controls, quality list with sizes, trim editor with yellow frame handles and time fields, subtitles, embed switch, sticky footer with estimate and primary action; status card for other states | +| History | List with Download again, Clear history with confirmation alert | +| Settings (modal on desktop, sheet on phone) | Cookies upload and removal, retention, concurrency stepper, default format, theme, accent (7 colors, teal default), language, password status, storage | +| Login | Shown when `auth_required` and not authenticated; password field, error shake | +| Feedback | Dynamic Island style toast, alert dialog, keyboard shortcuts HUD, drop overlay | +| Phone (below 768 px) | Floating tab bar (queue, history, settings), inspector and settings as draggable sheets with the page scaling behind | + +### 5.5 Visual system + +- Tokens come from the prototype: light and dark palettes, accent-l and accent-d and + accent-text-l and accent-text-d per accent, radii (panel 18, group 12, artwork 14, thumb 7, + capsule), spring easings generated as CSS `linear()` with `cubic-bezier` fallback. +- Default accent teal: fill `#12939c`, light text `#0b7178`, dark fill `#3fbac2`, + dark text `#5cc9d0`. Primary buttons use the light text tone so white labels pass + WCAG AA in both themes. +- The logo mark uses the label color for the frame and the accent for the waveform, so + it follows the chosen accent. +- Motion: springs for segmented thumbs, sheets, modals and toasts; arrival blur for new + rows; view transition for theme changes; everything disabled under + `prefers-reduced-motion`; glass falls back to solid under + `prefers-reduced-transparency` or without `backdrop-filter` support. + +### 5.6 Internationalization + +Dictionaries hold every visible string. The initial language is `vi` when +`navigator.language` starts with `vi`, otherwise `en`; the choice persists. The `lang` +attribute follows the selection. Server error codes map to localized messages with a +suggested action (for `bot_check`: open cookie settings). + +### 5.7 PWA + +`manifest.ts` declares name, short name, teal theme color, standalone display, icons +(SVG plus PNG 192 and 512 rendered with `ImageResponse`) and +`share_target` `{action: "/", method: "GET", params: {url: "url", text: "text", title: "title"}}`. +On load the app reads `url` or the first link in `text`, fills the importer and fetches. +An Apple touch icon is rendered the same way. + +### 5.8 Accessibility and responsiveness + +Real buttons, labels and inputs; `aria-live` for progress and toasts; radiogroup +keyboard handling for segmented controls; slider semantics and arrow, Shift+arrow, +Home, End for trim handles; focus trapped in dialogs and restored on close; 44 px touch +targets on phones; layouts verified at 360, 390, 768, 1024, 1440 and 1920 px. + +## 6. Testing + +| Root | Tooling | Coverage | +| ---------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `apps/api` | pytest, ruff, mypy strict | validation, network guard (resolver stubbed), command builder, progress parser, job manager with a fake process runner (queue, concurrency change, cancel, stall, success file selection), cleanup, cookies parsing, settings store, security (password, cross-site guard, rate limit), every route through the Flask test client | +| `apps/web` | vitest with jsdom and Testing Library, eslint, tsc | link parsing, formatting, reducer transitions, API client error mapping, proxy header forwarding, segmented and trim keyboard behavior, importer submit flow | +| `docs` | VitePress build, path and ADR checks | every page builds, no dead links | +| Stack | Manual verification recorded in the pull request | `docker compose` build and run, real downloads, cancel, trim, cleanup, phone and desktop screenshots | + +## 7. Documentation and repository + +- `README.md` follows the escrcpy README structure: logo, badges, one-line pitch, + language link, features, preview image, installation (Docker Compose, installer + script, from source), documentation links, for developers, get help, acknowledgments + (reclip, yt-dlp, ffmpeg, Deno, Next.js, Flask, Phosphor), disclaimer, license. + `README.vi.md` is the Vietnamese version. +- Documentation site pages: getting started, configuration, usage, deployment, + troubleshooting, security; the logo replaces the scaffold placeholder. +- ADRs: keep and patch the reclip backend; same-origin web proxy; in-memory job + registry with one worker; client-side history; Apple style design system. +- `LICENSE` (MIT) and `NOTICE` crediting reclip's MIT license. +- Screenshots in `docs/public/screenshots` are captured from the running stack. + +## 8. Delivery + +1. Generate with scaffold, publish the public repository (done). +2. Work on branch feat-openmedia, commits in Conventional Commit format. +3. Open a pull request, wait for CI, merge with squash. +4. Merge the Release Please pull request to cut `v0.1.0`; confirm images and assets. + +## 9. Out of scope + +User accounts, databases, channel subscriptions, browser extensions, transcription, +GIF export, SponsorBlock, a hosted public instance. From d067951fe506fe0f3ad3506b1cad43635e6cff46 Mon Sep 17 00:00:00 2001 From: iam-truongtrungnghia <149591043+ttncode@users.noreply.github.com> Date: Mon, 14 Sep 2026 04:19:09 +0700 Subject: [PATCH 02/77] docs: add the OpenMedia implementation plan --- .../superpowers/plans/2026-09-14-openmedia.md | 8584 +++++++++++++++++ .../specs/2026-09-14-openmedia-design.md | 2 +- 2 files changed, 8585 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/plans/2026-09-14-openmedia.md diff --git a/docs/superpowers/plans/2026-09-14-openmedia.md b/docs/superpowers/plans/2026-09-14-openmedia.md new file mode 100644 index 0000000..3e54d1b --- /dev/null +++ b/docs/superpowers/plans/2026-09-14-openmedia.md @@ -0,0 +1,8584 @@ +# OpenMedia Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Turn the scaffold-generated project into a shippable OpenMedia release: a secured, extended Flask + yt-dlp API and an Apple-style Next.js web app, documented and released. + +**Architecture:** `apps/api` keeps reclip's Flask foundation and endpoint shapes and adds a job engine (queue, progress, cancel, retention), security (network guard, cross-site guard, rate limit, optional password) and cookie handling. `apps/web` serves the UI and proxies every `/api/*` request to the API at runtime. Compose runs both images with a data volume. + +**Tech Stack:** Python 3.13, Flask 3, gunicorn, yt-dlp 2026.8.19 (`default`, `deno`, `curl-cffi` extras), ffmpeg, uv, pytest, ruff, mypy strict; Next.js 16.3.5, React 19.2.8, TypeScript strict, CSS Modules, `@phosphor-icons/react`, vitest, Testing Library; VitePress; Docker. + +**Spec:** `docs/superpowers/specs/2026-09-14-openmedia-design.md` + +## Global Constraints + +- Zero code comments in Python, TypeScript, CSS, shell and YAML you write ("clean code needs no comments"); intent lives in names. Generated files keep their existing comments. +- Python: type hints everywhere, `mypy --strict app tests` clean, `ruff check` and `ruff format --check` clean, functions at most about 20 lines, no bare `except`. +- TypeScript: no `any`, explicit return types on exported functions, `const` by default, `===` only, no `console.log`. +- Every commit uses Conventional Commits (`feat:`, `fix:`, `test:`, `docs:`, `build:`, `ci:`, `chore:`), enforced by commitlint. +- API JSON is snake_case; every error is `{"error": "", "code": ""}`. +- reclip compatibility: `POST /api/info`, `POST /api/playlist`, `POST /api/download {url, format, format_id, title}`, `GET /api/status/{id}` (`status`, `error`, `filename`), `GET /api/file/{id}` keep working. +- UI copy: Vietnamese and English dictionaries, no em dash or en dash characters in visible text, no emoji. +- Accent default teal: fill `#12939c`, light text `#0b7178`, dark fill `#3fbac2`, dark text `#5cc9d0`. +- Brand: logo frame path `M9 22 V9 H22 M42 9 H55 V22 M55 42 V55 H42 M22 55 H9 V42`, wave path `M23 27 V37 M32 20 V44 M41 25 V39`, viewBox `0 0 64 64`, stroke 6, round caps and joins. +- Per-root verification is `mise run //apps/api:ci-unit`, `mise run //apps/web:ci-unit`, `mise run //docs:ci-unit`, all run from the project root. +- Reference material outside the repository (read-only): + - Approved prototype: `/tmp/claude-1000/-home-ttndev-workspace-playground-openmedia/f044fd99-123a-4570-90dc-70fb559509d9/scratchpad/designs/apple/` (tokens.css, base.css, layout.css, components.css, overlays.css, body.html, data.js, format.js, render.js, inspector.js, overlays.js, actions.js, app.js) + - Logo assets: `/tmp/claude-1000/-home-ttndev-workspace-playground-openmedia/f044fd99-123a-4570-90dc-70fb559509d9/scratchpad/logo/assets/` + - reclip source: `/tmp/claude-1000/-home-ttndev-workspace-playground-openmedia/f044fd99-123a-4570-90dc-70fb559509d9/scratchpad/reclip/` + +## File Map + +### API (`apps/api`) + +| File | Task | Responsibility | +| ------------------------------------------------------------------------------- | ---- | ------------------------------------------------------------- | +| `apps/api/pyproject.toml` | 1 | Adds `yt-dlp[default,deno,curl-cffi]==2026.8.19` | +| `apps/api/app/config.py` | 1 | `Settings`, `load_settings()` | +| `apps/api/app/errors.py` | 1 | `ApiError`, `register_error_handlers()` | +| `apps/api/app/health.py` | 1 | Liveness and readiness (yt-dlp, ffmpeg, data dir) | +| `apps/api/app/validation.py` | 2 | URL and download option validation | +| `apps/api/app/network_guard.py` | 2 | Public address enforcement | +| `apps/api/app/progress.py` | 3 | Progress line parsing and tracking | +| `apps/api/app/ytdlp.py` | 3 | Command builders, error mapping, info and playlist extraction | +| `apps/api/app/settings_store.py` | 4 | Runtime settings persisted to JSON | +| `apps/api/app/storage.py` | 4 | Disk usage and storage limit | +| `apps/api/app/jobs.py` | 4 | `JobManager` and job model | +| `apps/api/app/cleanup.py` | 4 | Retention sweeper and orphan removal | +| `apps/api/app/cookies.py` | 5 | Cookie file validation and summary | +| `apps/api/app/security.py` | 5 | Secret key, password session, cross-site guard, rate limiter | +| `apps/api/app/services.py` | 6 | `Services` container and `build_services()` | +| `apps/api/app/media.py` | 6 | All `/api/*` routes | +| `apps/api/app/__init__.py` | 6 | `create_app()` | +| `apps/api/tests/*` | 1-6 | pytest suites named after modules | +| `apps/api/Dockerfile`, `apps/api/docker-entrypoint.sh`, `apps/api/.env.example` | 7 | Container image and runtime | +| `compose.yaml`, `compose.dev.yaml`, `example.env`, `renovate.json` | 7 | Stack wiring, yt-dlp update policy | + +### Web (`apps/web`) + +| File | Task | Responsibility | +| -------------------------------------------------------------------------------------------------------------------------------------------- | ---- | ------------------------------------------------------------------------------------------------------- | +| `apps/web/package.json` | 8 | Dependencies and test tooling | +| `apps/web/vitest.config.ts`, `apps/web/src/test/setup.ts` | 8 | Test environment | +| `apps/web/src/app/globals.css` | 8 | Tokens and base styles ported from the prototype | +| `apps/web/src/app/layout.tsx` | 8 | Fonts, metadata, theme bootstrap, providers | +| `apps/web/src/app/api/[...path]/route.ts` | 8 | Runtime API proxy | +| `apps/web/src/lib/api/types.ts`, `apps/web/src/lib/api/client.ts` | 8 | API types and client | +| `apps/web/src/lib/links.ts`, `apps/web/src/lib/format.ts` | 8 | Link parsing, formatting | +| `apps/web/src/lib/i18n/` | 8 | Dictionaries and provider | +| `apps/web/src/lib/preferences.ts` | 9 | localStorage persistence | +| `apps/web/src/state/` | 9 | Reducer, context, polling, commands | +| `apps/web/src/components/controls/` | 10 | Icon, Capsule, IconButton, Segmented, Switch, Stepper, BrandMark | +| `apps/web/src/components/overlays/` | 10 | Sheet, Island, AlertDialog, ShortcutsHud, DropOverlay | +| `apps/web/src/components/shell/` | 13 | AppShell, Sidebar, Toolbar, TabBar | +| `apps/web/src/components/importer/` | 13 | Importer with platform chips and playlist choice | +| `apps/web/src/components/queue/` | 11 | QueueView, QueueRow, ProgressRing, RowSkeleton | +| `apps/web/src/components/inspector/` | 11 | Inspector, Artwork, OptionsPanel, QualityList, TrimEditor, SubtitleOptions, StatusCard, InspectorFooter | +| `apps/web/src/components/history/`, `settings/`, `auth/` | 12 | History view, settings sheet, login screen | +| `apps/web/src/app/manifest.ts`, `apps/web/src/app/icon.svg`, `apps/web/src/app/apple-icon.tsx`, `apps/web/src/app/pwa-icon/[size]/route.tsx` | 12 | PWA | + +### Repository and docs + +| File | Task | Responsibility | +| --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | ----------------------------------- | +| `LICENSE`, `NOTICE`, `README.md`, `README.vi.md` | 14 | Licensing and landing documentation | +| `docs/index.md`, `docs/getting-started.md`, `docs/configuration.md`, `docs/usage.md`, `docs/deployment.md`, `docs/troubleshooting.md`, `docs/security.md` | 14 | Documentation site | +| `docs/decisions/0001-*.md` to `docs/decisions/0005-*.md` | 14 | ADRs | +| `docs/public/logo.svg`, `docs/public/screenshots/` | 14, 15 | Brand and screenshots | + +--- + +### Task 1: API foundation (dependencies, settings, errors, readiness) + +**Files:** + +- Modify: `apps/api/pyproject.toml` (via `uv add`), `apps/api/uv.lock` +- Create: `apps/api/app/config.py`, `apps/api/app/errors.py`, `apps/api/tests/conftest.py`, `apps/api/tests/test_config.py` +- Modify: `apps/api/app/health.py`, `apps/api/tests/test_health.py` + +**Interfaces:** + +- Produces: `Settings` (frozen dataclass) with fields `data_dir: Path, password: str, secret_key: str, retention_minutes: int, max_concurrent: int, max_filesize_mb: int, max_storage_gb: int, max_playlist_items: int, rate_limit_per_minute: int, stall_timeout_seconds: float, allow_private_urls: bool, trusted_proxy_hops: int, ytdlp_proxy: str` and properties `downloads_dir, cookies_file, settings_file, secret_key_file, ytdlp_dir`; `load_settings() -> Settings`; `ApiError(status: int, code: str, message: str, headers: Mapping[str, str] | None = None)`; `register_error_handlers(app: Flask) -> None`; pytest fixture `settings(tmp_path) -> Settings`. + +- [ ] **Step 1: Add the download engine dependency** + +Run from `apps/api`: + +```bash +mise exec -- uv add "yt-dlp[default,deno,curl-cffi]==2026.8.19" +``` + +Expected: `pyproject.toml` lists the dependency and `uv.lock` updates. + +- [ ] **Step 2: Write the failing settings tests** + +`apps/api/tests/conftest.py`: + +```python +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest + +from app.config import Settings + + +def make_settings(data_dir: Path, **overrides: Any) -> Settings: + base = Settings( + data_dir=data_dir, + password="", + secret_key="test-secret-key", + retention_minutes=60, + max_concurrent=3, + max_filesize_mb=4096, + max_storage_gb=0, + max_playlist_items=50, + rate_limit_per_minute=30, + stall_timeout_seconds=180, + allow_private_urls=False, + trusted_proxy_hops=1, + ytdlp_proxy="", + ) + return replace(base, **overrides) + + +@pytest.fixture +def settings(tmp_path: Path) -> Settings: + return make_settings(tmp_path) +``` + +`apps/api/tests/test_config.py`: + +```python +from pathlib import Path + +import pytest + +from app.config import load_settings + + +def test_defaults_point_at_data_volume(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ("OPENMEDIA_DATA_DIR", "OPENMEDIA_MAX_CONCURRENT", "OPENMEDIA_ALLOW_PRIVATE_URLS"): + monkeypatch.delenv(name, raising=False) + settings = load_settings() + assert settings.data_dir == Path("/data") + assert settings.max_concurrent == 3 + assert settings.allow_private_urls is False + assert settings.downloads_dir == Path("/data/downloads") + + +def test_environment_overrides(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setenv("OPENMEDIA_DATA_DIR", str(tmp_path)) + monkeypatch.setenv("OPENMEDIA_MAX_CONCURRENT", "5") + monkeypatch.setenv("OPENMEDIA_ALLOW_PRIVATE_URLS", "true") + settings = load_settings() + assert settings.data_dir == tmp_path + assert settings.max_concurrent == 5 + assert settings.allow_private_urls is True + + +def test_out_of_range_value_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("OPENMEDIA_MAX_CONCURRENT", "9") + with pytest.raises(ValueError, match="OPENMEDIA_MAX_CONCURRENT"): + load_settings() +``` + +- [ ] **Step 3: Run the tests to see them fail** + +Run from `apps/api`: `mise exec -- uv run pytest -q tests/test_config.py` +Expected: FAIL with `ModuleNotFoundError: No module named 'app.config'`. + +- [ ] **Step 4: Implement settings and errors** + +`apps/api/app/config.py`: + +```python +import os +from dataclasses import dataclass +from pathlib import Path + +TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) + + +def _text(name: str, default: str) -> str: + return os.environ.get(name, default).strip() + + +def _flag(name: str, default: bool) -> bool: + raw = os.environ.get(name, "").strip().lower() + return default if not raw else raw in TRUE_VALUES + + +def _integer(name: str, default: int, minimum: int, maximum: int) -> int: + raw = os.environ.get(name, "").strip() + if not raw: + return default + try: + value = int(raw) + except ValueError as error: + raise ValueError(f"{name} must be an integer, got {raw!r}") from error + if not minimum <= value <= maximum: + raise ValueError(f"{name} must be between {minimum} and {maximum}, got {value}") + return value + + +@dataclass(frozen=True) +class Settings: + data_dir: Path + password: str + secret_key: str + retention_minutes: int + max_concurrent: int + max_filesize_mb: int + max_storage_gb: int + max_playlist_items: int + rate_limit_per_minute: int + stall_timeout_seconds: float + allow_private_urls: bool + trusted_proxy_hops: int + ytdlp_proxy: str + + @property + def downloads_dir(self) -> Path: + return self.data_dir / "downloads" + + @property + def cookies_file(self) -> Path: + return self.data_dir / "cookies.txt" + + @property + def settings_file(self) -> Path: + return self.data_dir / "settings.json" + + @property + def secret_key_file(self) -> Path: + return self.data_dir / "secret_key" + + @property + def ytdlp_dir(self) -> Path: + return self.data_dir / "yt-dlp" + + +def load_settings() -> Settings: + return Settings( + data_dir=Path(_text("OPENMEDIA_DATA_DIR", "/data")), + password=os.environ.get("OPENMEDIA_PASSWORD", ""), + secret_key=_text("OPENMEDIA_SECRET_KEY", ""), + retention_minutes=_integer("OPENMEDIA_RETENTION_MINUTES", 60, 1, 10080), + max_concurrent=_integer("OPENMEDIA_MAX_CONCURRENT", 3, 1, 5), + max_filesize_mb=_integer("OPENMEDIA_MAX_FILESIZE_MB", 4096, 1, 1048576), + max_storage_gb=_integer("OPENMEDIA_MAX_STORAGE_GB", 0, 0, 1048576), + max_playlist_items=_integer("OPENMEDIA_MAX_PLAYLIST_ITEMS", 50, 1, 500), + rate_limit_per_minute=_integer("OPENMEDIA_RATE_LIMIT_PER_MINUTE", 30, 1, 10000), + stall_timeout_seconds=_integer("OPENMEDIA_STALL_TIMEOUT_SECONDS", 180, 10, 3600), + allow_private_urls=_flag("OPENMEDIA_ALLOW_PRIVATE_URLS", False), + trusted_proxy_hops=_integer("OPENMEDIA_TRUSTED_PROXY_HOPS", 1, 0, 5), + ytdlp_proxy=_text("OPENMEDIA_YTDLP_PROXY", ""), + ) +``` + +`apps/api/app/errors.py`: + +```python +from collections.abc import Mapping + +from flask import Flask, Response, jsonify +from werkzeug.exceptions import HTTPException + + +class ApiError(Exception): + def __init__( + self, + status: int, + code: str, + message: str, + headers: Mapping[str, str] | None = None, + ) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = dict(headers or {}) + + +def error_response(status: int, code: str, message: str) -> tuple[Response, int]: + return jsonify(error=message, code=code), status + + +def _api_error(error: ApiError) -> tuple[Response, int, dict[str, str]]: + response, status = error_response(error.status, error.code, error.message) + return response, status, error.headers + + +def _http_error(error: HTTPException) -> tuple[Response, int]: + status = error.code or 500 + code = (error.name or "error").lower().replace(" ", "_") + return error_response(status, code, error.description or error.name) + + +def register_error_handlers(app: Flask) -> None: + app.register_error_handler(ApiError, _api_error) + app.register_error_handler(HTTPException, _http_error) +``` + +- [ ] **Step 5: Replace readiness with runtime checks** + +`apps/api/app/health.py`: + +```python +import importlib.util +import os +import shutil +from pathlib import Path + +from flask import Blueprint, Response, current_app, jsonify + +health = Blueprint("health", __name__) + +Reply = Response | tuple[Response, int] + + +def missing_dependencies(data_dir: Path) -> list[str]: + checks = { + "yt-dlp": importlib.util.find_spec("yt_dlp") is not None, + "ffmpeg": shutil.which("ffmpeg") is not None, + "data directory": data_dir.is_dir() and os.access(data_dir, os.W_OK), + } + return [name for name, passed in checks.items() if not passed] + + +@health.get("/health/live") +def live() -> Reply: + return jsonify(status="ok") + + +@health.get("/health/ready") +def ready() -> Reply: + data_dir = Path(current_app.config["OPENMEDIA_DATA_DIR"]) + missing = missing_dependencies(data_dir) + if missing: + return jsonify(status="unavailable", reason=f"missing: {', '.join(missing)}"), 503 + return jsonify(status="ok") +``` + +`apps/api/tests/test_health.py`: + +```python +from pathlib import Path + +import pytest +from flask import Flask + +from app.health import health, missing_dependencies + + +def make_app(data_dir: Path) -> Flask: + app = Flask(__name__) + app.config["OPENMEDIA_DATA_DIR"] = str(data_dir) + app.register_blueprint(health) + return app + + +def test_live_reports_ok(tmp_path: Path) -> None: + response = make_app(tmp_path).test_client().get("/health/live") + assert response.status_code == 200 + assert response.get_json() == {"status": "ok"} + + +def test_ready_names_missing_dependencies(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("app.health.shutil.which", lambda name: None) + assert missing_dependencies(tmp_path) == ["ffmpeg"] + response = make_app(tmp_path).test_client().get("/health/ready") + assert response.status_code == 503 + assert "ffmpeg" in response.get_json()["reason"] + + +def test_ready_passes_when_everything_is_present(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("app.health.shutil.which", lambda name: "/usr/bin/ffmpeg") + assert make_app(tmp_path).test_client().get("/health/ready").status_code == 200 +``` + +`apps/api/app/__init__.py` stays as generated in this task (it still registers `health`); it sets `app.config["OPENMEDIA_DATA_DIR"] = "/data"` so the readiness route works until Task 6 replaces the factory: + +```python +from flask import Flask + +from .health import health + + +def create_app() -> Flask: + app = Flask(__name__) + app.config["OPENMEDIA_DATA_DIR"] = "/data" + app.register_blueprint(health) + return app +``` + +- [ ] **Step 6: Run the API checks** + +Run from the project root: `mise run //apps/api:ci-unit` +Expected: format, lint, `mypy --strict` and every test pass. + +- [ ] **Step 7: Commit** + +```bash +git add apps/api +git commit -m "feat(api): add settings, error model and runtime readiness checks" +``` + +### Task 2: Input validation and network guard + +**Files:** + +- Create: `apps/api/app/validation.py`, `apps/api/app/network_guard.py`, `apps/api/tests/test_validation.py`, `apps/api/tests/test_network_guard.py` + +**Interfaces:** + +- Consumes: `ApiError` (Task 1). +- Produces: + - `validate_url(value: object) -> str` + - `Trim(start: float, end: float)`, `SubtitleOptions(languages: tuple[str, ...], mode: str)`, `DownloadOptions(kind: str, container: str, quality_height: int | None, format_id: str | None, audio_format: str | None, audio_quality: str | None, trim: Trim | None, subtitles: SubtitleOptions | None, embed_metadata: bool)` with `DownloadOptions.to_json() -> dict[str, object]` + - `parse_download_options(payload: Mapping[str, object]) -> DownloadOptions` + - `resolve_host(host: str) -> list[str]`, `ensure_public_url(url: str, resolver: Callable[[str], list[str]] = resolve_host) -> None` + +- [ ] **Step 1: Write the failing validation tests** + +`apps/api/tests/test_validation.py`: + +```python +import pytest + +from app.errors import ApiError +from app.validation import DownloadOptions, Trim, parse_download_options, validate_url + + +@pytest.mark.parametrize( + "value", + ["--exec=touch /tmp/pwned", "file:///etc/passwd", "ftp://example.com/a", "https://", "https://exa mple.com", 42, None, "https://example.com/" + "a" * 2100], +) +def test_rejects_unsafe_or_malformed_urls(value: object) -> None: + with pytest.raises(ApiError) as caught: + validate_url(value) + assert caught.value.code == "invalid_url" + + +def test_accepts_and_trims_http_urls() -> None: + assert validate_url(" https://www.youtube.com/watch?v=abc ") == "https://www.youtube.com/watch?v=abc" + + +def test_reclip_request_maps_to_video_defaults() -> None: + options = parse_download_options({"url": "https://x.com/a", "format": "video", "format_id": "137"}) + assert options == DownloadOptions( + kind="video", container="mp4", quality_height=None, format_id="137", audio_format=None, + audio_quality=None, trim=None, subtitles=None, embed_metadata=True, + ) + + +def test_reclip_audio_request_defaults_to_mp3() -> None: + options = parse_download_options({"format": "audio"}) + assert (options.kind, options.audio_format, options.audio_quality) == ("audio", "mp3", "best") + + +def test_full_request_is_parsed() -> None: + options = parse_download_options({ + "format": "video", "container": "mkv", "quality_height": 720, + "trim": {"start": 5, "end": 65.5}, + "subtitles": {"languages": ["vi", "en-US"], "mode": "srt"}, + "embed_metadata": False, + }) + assert options.container == "mkv" + assert options.quality_height == 720 + assert options.trim == Trim(start=5.0, end=65.5) + assert options.subtitles is not None and options.subtitles.languages == ("vi", "en-US") + assert options.embed_metadata is False + + +@pytest.mark.parametrize( + "payload", + [ + {"format": "gif"}, + {"format_id": "137; rm -rf /"}, + {"container": "avi"}, + {"quality_height": 999}, + {"format": "audio", "audio_format": "aac"}, + {"trim": {"start": 10, "end": 5}}, + {"trim": {"start": -1, "end": 5}}, + {"subtitles": {"languages": ["vi", "en", "fr", "de", "ja", "ko"], "mode": "embed"}}, + {"subtitles": {"languages": ["../x"], "mode": "embed"}}, + {"subtitles": {"languages": ["vi"], "mode": "burn"}}, + {"embed_metadata": "yes"}, + ], +) +def test_invalid_options_are_rejected(payload: dict[str, object]) -> None: + with pytest.raises(ApiError) as caught: + parse_download_options(payload) + assert caught.value.code == "invalid_option" + + +def test_options_serialize_for_the_job_payload() -> None: + options = parse_download_options({"format": "audio", "audio_format": "flac"}) + assert options.to_json()["audio_format"] == "flac" + assert options.to_json()["trim"] is None +``` + +- [ ] **Step 2: Write the failing network guard tests** + +`apps/api/tests/test_network_guard.py`: + +```python +from collections.abc import Callable + +import pytest + +from app.errors import ApiError +from app.network_guard import ensure_public_url + + +def resolver_for(*addresses: str) -> Callable[[str], list[str]]: + return lambda host: list(addresses) + + +@pytest.mark.parametrize("address", ["127.0.0.1", "10.1.2.3", "192.168.1.20", "169.254.169.254", "::1", "fd00::1", "::ffff:10.0.0.1", "0.0.0.0"]) +def test_private_addresses_are_blocked(address: str) -> None: + with pytest.raises(ApiError) as caught: + ensure_public_url("https://internal.example/x", resolver_for(address)) + assert caught.value.code == "private_network" + + +def test_any_private_address_blocks_the_host() -> None: + with pytest.raises(ApiError): + ensure_public_url("https://mixed.example", resolver_for("142.250.1.1", "10.0.0.5")) + + +def test_public_addresses_pass() -> None: + ensure_public_url("https://www.youtube.com/watch?v=a", resolver_for("142.250.190.14", "2607:f8b0:4005:80b::200e")) + + +def test_unresolvable_host_is_invalid() -> None: + def failing(host: str) -> list[str]: + raise OSError("no such host") + + with pytest.raises(ApiError) as caught: + ensure_public_url("https://nope.invalid", failing) + assert caught.value.code == "invalid_url" +``` + +- [ ] **Step 3: Run both suites to see them fail** + +Run from `apps/api`: `mise exec -- uv run pytest -q tests/test_validation.py tests/test_network_guard.py` +Expected: FAIL with `ModuleNotFoundError`. + +- [ ] **Step 4: Implement validation** + +`apps/api/app/validation.py`: + +```python +import re +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from urllib.parse import urlsplit + +from .errors import ApiError + +MAX_URL_LENGTH = 2048 +FORMAT_ID_PATTERN = re.compile(r"^[A-Za-z0-9_.+-]{1,64}$") +LANGUAGE_PATTERN = re.compile(r"^[A-Za-z]{2,3}(-[A-Za-z0-9]{2,8})?$") +UNSAFE_URL_CHARACTERS = re.compile(r"[\s\x00-\x1f\x7f]") +MAX_SUBTITLE_LANGUAGES = 5 +KINDS = ("video", "audio") +CONTAINERS = ("mp4", "mkv") +AUDIO_FORMATS = ("mp3", "m4a", "opus", "flac", "wav") +AUDIO_QUALITIES = ("320k", "best") +SUBTITLE_MODES = ("embed", "srt") +QUALITY_HEIGHTS = (2160, 1440, 1080, 720, 480, 360) + + +@dataclass(frozen=True) +class Trim: + start: float + end: float + + +@dataclass(frozen=True) +class SubtitleOptions: + languages: tuple[str, ...] + mode: str + + +@dataclass(frozen=True) +class DownloadOptions: + kind: str + container: str + quality_height: int | None + format_id: str | None + audio_format: str | None + audio_quality: str | None + trim: Trim | None + subtitles: SubtitleOptions | None + embed_metadata: bool + + def to_json(self) -> dict[str, object]: + data = asdict(self) + if self.subtitles is not None: + data["subtitles"] = {"languages": list(self.subtitles.languages), "mode": self.subtitles.mode} + return data + + +def invalid_option(message: str) -> ApiError: + return ApiError(400, "invalid_option", message) + + +def validate_url(value: object) -> str: + if not isinstance(value, str): + raise ApiError(400, "invalid_url", "Provide a link that starts with http:// or https://.") + url = value.strip() + parts = urlsplit(url) + is_valid = ( + len(url) <= MAX_URL_LENGTH + and parts.scheme in ("http", "https") + and bool(parts.hostname) + and not UNSAFE_URL_CHARACTERS.search(url) + ) + if not is_valid: + raise ApiError(400, "invalid_url", "Provide a link that starts with http:// or https://.") + return url + + +def _choice(payload: Mapping[str, object], key: str, choices: tuple[str, ...], default: str) -> str: + value = payload.get(key) or default + if value not in choices: + raise invalid_option(f"{key} must be one of {', '.join(choices)}.") + return str(value) + + +def _format_id(payload: Mapping[str, object]) -> str | None: + value = payload.get("format_id") + if value in (None, ""): + return None + if not isinstance(value, str) or not FORMAT_ID_PATTERN.fullmatch(value): + raise invalid_option("format_id is not a valid format identifier.") + return value + + +def _quality_height(payload: Mapping[str, object]) -> int | None: + value = payload.get("quality_height") + if value is None: + return None + if not isinstance(value, int) or isinstance(value, bool) or value not in QUALITY_HEIGHTS: + raise invalid_option("quality_height must be one of 2160, 1440, 1080, 720, 480, 360.") + return value + + +def _number(value: object, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise invalid_option(f"trim.{name} must be a number of seconds.") + return float(value) + + +def _trim(payload: Mapping[str, object]) -> Trim | None: + value = payload.get("trim") + if value is None: + return None + if not isinstance(value, Mapping): + raise invalid_option("trim must be an object with start and end.") + start, end = _number(value.get("start"), "start"), _number(value.get("end"), "end") + if start < 0 or start >= end: + raise invalid_option("trim.start must be at least 0 and before trim.end.") + return Trim(start=start, end=end) + + +def _subtitles(payload: Mapping[str, object]) -> SubtitleOptions | None: + value = payload.get("subtitles") + if value is None: + return None + if not isinstance(value, Mapping) or not isinstance(value.get("languages"), list): + raise invalid_option("subtitles must contain a languages list.") + languages = tuple(value["languages"]) + valid = 0 < len(languages) <= MAX_SUBTITLE_LANGUAGES and all( + isinstance(language, str) and LANGUAGE_PATTERN.fullmatch(language) for language in languages + ) + if not valid: + raise invalid_option("subtitles.languages must hold one to five language codes.") + return SubtitleOptions(languages=languages, mode=_choice(value, "mode", SUBTITLE_MODES, "embed")) + + +def _embed_metadata(payload: Mapping[str, object]) -> bool: + value = payload.get("embed_metadata", True) + if not isinstance(value, bool): + raise invalid_option("embed_metadata must be true or false.") + return value + + +def parse_download_options(payload: Mapping[str, object]) -> DownloadOptions: + kind = _choice(payload, "format", KINDS, "video") + is_audio = kind == "audio" + return DownloadOptions( + kind=kind, + container=_choice(payload, "container", CONTAINERS, "mp4"), + quality_height=None if is_audio else _quality_height(payload), + format_id=None if is_audio else _format_id(payload), + audio_format=_choice(payload, "audio_format", AUDIO_FORMATS, "mp3") if is_audio else None, + audio_quality=_choice(payload, "audio_quality", AUDIO_QUALITIES, "best") if is_audio else None, + trim=_trim(payload), + subtitles=None if is_audio else _subtitles(payload), + embed_metadata=_embed_metadata(payload), + ) +``` + +When `format` is `video`, an invalid `audio_format` in the payload is ignored because audio options do not apply; the parametrized case `{"format": "audio", "audio_format": "aac"}` covers audio validation. + +- [ ] **Step 5: Implement the network guard** + +`apps/api/app/network_guard.py`: + +```python +import ipaddress +import socket +from collections.abc import Callable +from urllib.parse import urlsplit + +from .errors import ApiError + +Resolver = Callable[[str], list[str]] + + +def resolve_host(host: str) -> list[str]: + return sorted({str(info[4][0]) for info in socket.getaddrinfo(host, None)}) + + +def _is_public(address: str) -> bool: + ip = ipaddress.ip_address(address.split("%", 1)[0]) + mapped = ip.ipv4_mapped if isinstance(ip, ipaddress.IPv6Address) else None + return (mapped or ip).is_global + + +def ensure_public_url(url: str, resolver: Resolver = resolve_host) -> None: + host = urlsplit(url).hostname + if not host: + raise ApiError(400, "invalid_url", "The link has no host name.") + try: + addresses = resolver(host) + except OSError as error: + raise ApiError(400, "invalid_url", f"Could not resolve {host}.") from error + if not addresses or not all(_is_public(address) for address in addresses): + raise ApiError(400, "private_network", "Links to private or local network addresses are not allowed.") +``` + +- [ ] **Step 6: Run the API checks** + +Run from the project root: `mise run //apps/api:ci-unit` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add apps/api +git commit -m "feat(api): validate download input and block private network targets" +``` + +### Task 3: yt-dlp command builder, progress parser and extraction + +**Files:** + +- Create: `apps/api/app/progress.py`, `apps/api/app/ytdlp.py`, `apps/api/tests/test_progress.py`, `apps/api/tests/test_ytdlp.py` + +**Interfaces:** + +- Consumes: `Settings`, `ApiError`, `DownloadOptions`, `Trim`, `SubtitleOptions`. +- Produces: + - `progress.PROGRESS_MARKER = "OMPROGRESS"`, `ProgressSample(downloaded_bytes: int | None, total_bytes: int | None, speed_bps: float | None, eta_seconds: int | None)`, `parse_progress_line(line: str) -> ProgressSample | None`, `is_postprocessing_line(line: str) -> bool`, `ProgressTracker` with `percent: float`, `record(sample) -> float`, `record_processing() -> float` + - `ytdlp.CompletedRun(returncode: int, stdout: str, stderr: str)`, `Runner = Callable[[Sequence[str], float, Mapping[str, str]], CompletedRun]`, `run_command`, `CookieCopier = Callable[[Path], Path | None]`, `ytdlp_environment(settings) -> dict[str, str]`, `DownloadRequest(url: str, options: DownloadOptions, job_dir: Path, max_filesize_mb: int, cookies_file: Path | None, proxy: str)`, `build_download_command(request) -> list[str]`, `build_info_command(url, cookies_file, proxy) -> list[str]`, `build_playlist_command(url, limit, cookies_file, proxy) -> list[str]`, `error_from_output(output: str) -> ApiError`, `summarize_info(info: Mapping[str, Any]) -> dict[str, object]`, `YtDlpClient(settings, copy_cookies: CookieCopier, runner: Runner = run_command)` with `fetch_info(url) -> dict[str, object]` and `fetch_playlist(url, limit) -> dict[str, object]` + +- [ ] **Step 1: Write the failing progress tests** + +`apps/api/tests/test_progress.py`: + +```python +from app.progress import ProgressSample, ProgressTracker, is_postprocessing_line, parse_progress_line + + +def test_parses_a_full_progress_line() -> None: + sample = parse_progress_line("OMPROGRESS 1048576 4194304 NA 524288.5 6") + assert sample == ProgressSample(downloaded_bytes=1048576, total_bytes=4194304, speed_bps=524288.5, eta_seconds=6) + + +def test_falls_back_to_the_size_estimate() -> None: + sample = parse_progress_line("OMPROGRESS 100 NA 400.0 NA NA") + assert sample is not None + assert (sample.total_bytes, sample.speed_bps, sample.eta_seconds) == (400, None, None) + + +def test_ignores_other_output() -> None: + assert parse_progress_line("[youtube] abc: Downloading webpage") is None + assert parse_progress_line("OMPROGRESS 1 2") is None + + +def test_detects_postprocessing_lines() -> None: + assert is_postprocessing_line('[Merger] Merging formats into "media.mp4"') + assert is_postprocessing_line("[ExtractAudio] Destination: media.mp3") + assert not is_postprocessing_line("[download] Destination: media.f137.mp4") + + +def test_two_streams_map_into_one_rising_percentage() -> None: + tracker = ProgressTracker() + assert tracker.record(ProgressSample(50, 100, None, None)) == 45.0 + assert tracker.record(ProgressSample(100, 100, None, None)) == 90.0 + assert tracker.record(ProgressSample(10, 20, None, None)) == 94.5 + assert tracker.record_processing() == 99.0 + assert tracker.record(ProgressSample(20, 20, None, None)) == 99.0 + + +def test_unknown_total_keeps_the_percentage() -> None: + tracker = ProgressTracker() + assert tracker.record(ProgressSample(500, None, 10.0, None)) == 0.0 +``` + +- [ ] **Step 2: Write the failing yt-dlp tests** + +`apps/api/tests/test_ytdlp.py`: + +```python +import json +from collections.abc import Mapping, Sequence +from pathlib import Path + +import pytest + +from app.config import Settings +from app.errors import ApiError +from app.validation import DownloadOptions, SubtitleOptions, Trim, parse_download_options +from app.ytdlp import ( + CompletedRun, + DownloadRequest, + YtDlpClient, + build_download_command, + build_info_command, + error_from_output, + summarize_info, +) + +URL = "https://www.youtube.com/watch?v=abc" + + +def request_for(options: DownloadOptions, cookies: Path | None = None) -> DownloadRequest: + return DownloadRequest(URL, options, Path("/data/downloads/job1"), 4096, cookies, "") + + +def value_after(command: list[str], flag: str) -> str: + return command[command.index(flag) + 1] + + +def test_url_is_always_the_final_argument_after_the_separator() -> None: + command = build_download_command(request_for(parse_download_options({}))) + assert command[-2:] == ["--", URL] + assert value_after(command, "-P") == "/data/downloads/job1" + assert value_after(command, "-o") == "media.%(ext)s" + assert value_after(command, "--max-filesize") == "4096M" + assert "--newline" in command and "--no-playlist" in command + + +def test_mp4_prefers_compatible_codecs_for_a_chosen_format() -> None: + command = build_download_command(request_for(parse_download_options({"format_id": "137"}))) + assert value_after(command, "-f") == "137+bestaudio[ext=m4a]/137+bestaudio/best" + assert value_after(command, "-S") == "vcodec:h264,acodec:aac" + assert value_after(command, "--merge-output-format") == "mp4" + + +def test_mkv_with_height_cap_skips_codec_sorting() -> None: + command = build_download_command(request_for(parse_download_options({"container": "mkv", "quality_height": 720}))) + assert value_after(command, "-f") == "bv*[height<=720]+ba/b[height<=720]/b" + assert "-S" not in command + assert value_after(command, "--merge-output-format") == "mkv" + + +def test_audio_extraction_arguments() -> None: + command = build_download_command(request_for(parse_download_options({"format": "audio", "audio_format": "m4a", "audio_quality": "320k"}))) + assert value_after(command, "-f") == "ba/b" + assert "-x" in command + assert value_after(command, "--audio-format") == "m4a" + assert value_after(command, "--audio-quality") == "320K" + + +def test_trim_subtitles_metadata_and_cookies() -> None: + options = DownloadOptions("video", "mp4", None, None, None, None, Trim(5, 65.5), SubtitleOptions(("vi", "en"), "srt"), True) + command = build_download_command(request_for(options, Path("/tmp/job/.cookies.txt"))) + assert value_after(command, "--download-sections") == "*5-65.5" + assert "--force-keyframes-at-cuts" in command + assert value_after(command, "--sub-langs") == "vi,en" + assert value_after(command, "--convert-subs") == "srt" + assert "--embed-subs" not in command + assert {"--embed-metadata", "--embed-chapters", "--embed-thumbnail"} <= set(command) + assert value_after(command, "--cookies") == "/tmp/job/.cookies.txt" + + +def test_wav_skips_thumbnail_embedding() -> None: + command = build_download_command(request_for(parse_download_options({"format": "audio", "audio_format": "wav"}))) + assert "--embed-thumbnail" not in command + assert "--embed-metadata" in command + + +def test_info_command_ends_with_separator_and_url() -> None: + assert build_info_command(URL, None, "socks5://proxy:1080")[-4:] == ["--proxy", "socks5://proxy:1080", "--", URL] + + +@pytest.mark.parametrize( + ("line", "code"), + [ + ("ERROR: [youtube] abc: Sign in to confirm you're not a bot", "bot_check"), + ("ERROR: [youtube] abc: Private video. Sign in", "private_video"), + ("ERROR: The uploader has not made this video available in your country", "geo_blocked"), + ("ERROR: [youtube] abc: Video unavailable", "unavailable"), + ("ERROR: Unsupported URL: https://example.com", "unsupported_url"), + ("ERROR: File is larger than max-filesize (5000 bytes > 10 bytes). Aborting.", "too_large"), + ("ERROR: something else broke", "extractor_error"), + ], +) +def test_error_mapping(line: str, code: str) -> None: + error = error_from_output(f"[info] noise\n{line}\n") + assert error.code == code + + +def test_summarize_keeps_best_format_per_height() -> None: + info = { + "id": "abc", "title": "Pho", "thumbnail": "https://i.ytimg.com/a.jpg", "duration": 1122, + "uploader": "Bep", "extractor_key": "Youtube", "webpage_url": URL, "chapters": [{"title": "Intro"}], + "subtitles": {"vi": [], "en": []}, + "formats": [ + {"format_id": "136", "height": 720, "vcodec": "avc1", "tbr": 900, "ext": "mp4", "filesize": 236}, + {"format_id": "247", "height": 720, "vcodec": "vp9", "tbr": 1200, "ext": "webm", "filesize_approx": 250}, + {"format_id": "137", "height": 1080, "vcodec": "avc1", "tbr": 2000, "ext": "mp4", "filesize": 412}, + {"format_id": "140", "height": None, "vcodec": "none", "ext": "m4a"}, + ], + } + summary = summarize_info(info) + formats = summary["formats"] + assert isinstance(formats, list) + assert [entry["id"] for entry in formats] == ["137", "247"] + assert summary["subtitle_languages"] == ["en", "vi"] + assert summary["has_chapters"] is True + assert summary["platform"] == "Youtube" + + +class RecordingRunner: + def __init__(self, result: CompletedRun) -> None: + self.result = result + self.commands: list[list[str]] = [] + + def __call__(self, command: Sequence[str], timeout: float, env: Mapping[str, str]) -> CompletedRun: + self.commands.append(list(command)) + return self.result + + +def no_cookies(directory: Path) -> Path | None: + return None + + +def test_fetch_info_summarizes_output(settings: Settings) -> None: + runner = RecordingRunner(CompletedRun(0, json.dumps({"title": "Pho", "formats": []}), "")) + info = YtDlpClient(settings, no_cookies, runner).fetch_info(URL) + assert info["title"] == "Pho" + assert runner.commands[0][-2:] == ["--", URL] + + +def test_fetch_info_raises_mapped_error(settings: Settings) -> None: + runner = RecordingRunner(CompletedRun(1, "", "ERROR: [youtube] abc: Sign in to confirm you're not a bot")) + with pytest.raises(ApiError) as caught: + YtDlpClient(settings, no_cookies, runner).fetch_info(URL) + assert caught.value.code == "bot_check" + + +def test_fetch_playlist_limits_entries(settings: Settings) -> None: + document = {"title": "Mix", "entries": [{"url": f"https://www.youtube.com/watch?v={n}"} for n in range(5)]} + runner = RecordingRunner(CompletedRun(0, json.dumps(document), "")) + playlist = YtDlpClient(settings, no_cookies, runner).fetch_playlist(URL, 3) + assert playlist == {"title": "Mix", "count": 3, "urls": [f"https://www.youtube.com/watch?v={n}" for n in range(3)]} + assert runner.commands[0][runner.commands[0].index("--playlist-end") + 1] == "3" +``` + +- [ ] **Step 3: Run both suites to see them fail** + +Run from `apps/api`: `mise exec -- uv run pytest -q tests/test_progress.py tests/test_ytdlp.py` +Expected: FAIL with `ModuleNotFoundError`. + +- [ ] **Step 4: Implement the progress parser** + +`apps/api/app/progress.py`: + +```python +from dataclasses import dataclass + +PROGRESS_MARKER = "OMPROGRESS" +PROGRESS_FIELD_COUNT = 6 +POSTPROCESSOR_TAGS = ( + "[Merger]", + "[ExtractAudio]", + "[EmbedSubtitle]", + "[Metadata]", + "[EmbedThumbnail]", + "[FixupM3u8]", + "[FixupM4a]", + "[VideoConvertor]", + "[VideoRemuxer]", + "[SubtitlesConvertor]", + "[ThumbnailsConvertor]", + "[ModifyChapters]", +) +STREAM_RANGES = ((0.0, 90.0), (90.0, 99.0)) +PROCESSING_PERCENT = 99.0 + + +@dataclass(frozen=True) +class ProgressSample: + downloaded_bytes: int | None + total_bytes: int | None + speed_bps: float | None + eta_seconds: int | None + + +def _number(token: str) -> float | None: + try: + value = float(token) + except ValueError: + return None + return value if value >= 0 else None + + +def _whole(value: float | None) -> int | None: + return None if value is None else int(value) + + +def parse_progress_line(line: str) -> ProgressSample | None: + parts = line.split() + if len(parts) != PROGRESS_FIELD_COUNT or parts[0] != PROGRESS_MARKER: + return None + downloaded, total, estimate, speed, eta = (_number(token) for token in parts[1:]) + return ProgressSample( + downloaded_bytes=_whole(downloaded), + total_bytes=_whole(total if total is not None else estimate), + speed_bps=speed, + eta_seconds=_whole(eta), + ) + + +def is_postprocessing_line(line: str) -> bool: + return line.lstrip().startswith(POSTPROCESSOR_TAGS) + + +class ProgressTracker: + def __init__(self) -> None: + self.percent = 0.0 + self._stream = 0 + self._last_downloaded = -1 + + def record(self, sample: ProgressSample) -> float: + downloaded = sample.downloaded_bytes or 0 + if downloaded < self._last_downloaded and self._stream < len(STREAM_RANGES) - 1: + self._stream += 1 + self._last_downloaded = downloaded + if sample.total_bytes: + low, high = STREAM_RANGES[self._stream] + fraction = min(downloaded / sample.total_bytes, 1.0) + self.percent = max(self.percent, low + (high - low) * fraction) + return self.percent + + def record_processing(self) -> float: + self.percent = max(self.percent, PROCESSING_PERCENT) + return self.percent +``` + +- [ ] **Step 5: Implement the yt-dlp module** + +`apps/api/app/ytdlp.py`: + +```python +import json +import os +import subprocess +import sys +import tempfile +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .config import Settings +from .errors import ApiError +from .progress import PROGRESS_MARKER +from .validation import DownloadOptions + +PROGRESS_TEMPLATE = ( + f"download:{PROGRESS_MARKER} %(progress.downloaded_bytes)s %(progress.total_bytes)s " + "%(progress.total_bytes_estimate)s %(progress.speed)s %(progress.eta)s" +) +MEDIA_OUTPUT_TEMPLATE = "media.%(ext)s" +INFO_TIMEOUT_SECONDS = 60.0 +PLAYLIST_TIMEOUT_SECONDS = 90.0 +MAX_ERROR_MESSAGE_LENGTH = 300 +ERROR_PATTERNS = ( + ("sign in to confirm", "bot_check", "The site asked to confirm you are not a bot. Add cookies and try again."), + ("private video", "private_video", "This video is private."), + ("available in your country", "geo_blocked", "This video is not available in the server's region."), + ("video unavailable", "unavailable", "This video is unavailable."), + ("unsupported url", "unsupported_url", "This link is not supported."), + ("larger than max-filesize", "too_large", "The file is larger than the configured size limit."), +) + + +@dataclass(frozen=True) +class CompletedRun: + returncode: int + stdout: str + stderr: str + + +Runner = Callable[[Sequence[str], float, Mapping[str, str]], CompletedRun] +CookieCopier = Callable[[Path], Path | None] + + +@dataclass(frozen=True) +class DownloadRequest: + url: str + options: DownloadOptions + job_dir: Path + max_filesize_mb: int + cookies_file: Path | None + proxy: str + + +def run_command(command: Sequence[str], timeout: float, env: Mapping[str, str]) -> CompletedRun: + try: + result = subprocess.run(list(command), capture_output=True, text=True, timeout=timeout, env=dict(env), check=False) + except subprocess.TimeoutExpired as error: + raise ApiError(504, "timeout", "The site took too long to respond. Try again.") from error + return CompletedRun(result.returncode, result.stdout, result.stderr) + + +def base_command() -> list[str]: + return [sys.executable, "-m", "yt_dlp"] + + +def ytdlp_environment(settings: Settings) -> dict[str, str]: + environment = dict(os.environ) + if (settings.ytdlp_dir / "yt_dlp").is_dir(): + paths = [str(settings.ytdlp_dir), environment.get("PYTHONPATH", "")] + environment["PYTHONPATH"] = os.pathsep.join(path for path in paths if path) + return environment + + +def _network_arguments(cookies_file: Path | None, proxy: str) -> list[str]: + cookies = ["--cookies", str(cookies_file)] if cookies_file is not None else [] + return [*cookies, *(["--proxy", proxy] if proxy else [])] + + +def _seconds(value: float) -> str: + return f"{value:g}" + + +def _video_arguments(options: DownloadOptions) -> list[str]: + if options.format_id: + selector = f"{options.format_id}+bestaudio[ext=m4a]/{options.format_id}+bestaudio/best" + elif options.quality_height: + height = options.quality_height + selector = f"bv*[height<={height}]+ba/b[height<={height}]/b" + else: + selector = "bv*+ba/b" + sorting = ["-S", "vcodec:h264,acodec:aac"] if options.container == "mp4" else [] + return ["-f", selector, *sorting, "--merge-output-format", options.container] + + +def _audio_arguments(options: DownloadOptions) -> list[str]: + quality = "320K" if options.audio_quality == "320k" else "0" + return ["-f", "ba/b", "-x", "--audio-format", options.audio_format or "mp3", "--audio-quality", quality] + + +def _trim_arguments(options: DownloadOptions) -> list[str]: + if options.trim is None: + return [] + section = f"*{_seconds(options.trim.start)}-{_seconds(options.trim.end)}" + return ["--download-sections", section, "--force-keyframes-at-cuts"] + + +def _subtitle_arguments(options: DownloadOptions) -> list[str]: + if options.subtitles is None: + return [] + delivery = ["--embed-subs"] if options.subtitles.mode == "embed" else ["--convert-subs", "srt"] + return ["--write-subs", "--write-auto-subs", "--sub-langs", ",".join(options.subtitles.languages), *delivery] + + +def _metadata_arguments(options: DownloadOptions) -> list[str]: + if not options.embed_metadata: + return [] + thumbnail = [] if options.audio_format == "wav" else ["--embed-thumbnail"] + return ["--embed-metadata", "--embed-chapters", *thumbnail] + + +def build_download_command(request: DownloadRequest) -> list[str]: + options = request.options + selection = _audio_arguments(options) if options.kind == "audio" else _video_arguments(options) + return [ + *base_command(), + "--no-playlist", "--newline", "--no-colors", "--no-warnings", "--progress", + "--progress-template", PROGRESS_TEMPLATE, + "--max-filesize", f"{request.max_filesize_mb}M", + "-P", str(request.job_dir), "-o", MEDIA_OUTPUT_TEMPLATE, + *selection, *_trim_arguments(options), *_subtitle_arguments(options), *_metadata_arguments(options), + *_network_arguments(request.cookies_file, request.proxy), + "--", request.url, + ] + + +def build_info_command(url: str, cookies_file: Path | None, proxy: str) -> list[str]: + return [*base_command(), "-J", "--no-playlist", "--no-warnings", *_network_arguments(cookies_file, proxy), "--", url] + + +def build_playlist_command(url: str, limit: int, cookies_file: Path | None, proxy: str) -> list[str]: + return [ + *base_command(), "-J", "--flat-playlist", "--playlist-end", str(limit), "--no-warnings", + *_network_arguments(cookies_file, proxy), "--", url, + ] + + +def _last_line(output: str) -> str: + lines = [line.strip() for line in output.splitlines() if line.strip()] + return lines[-1] if lines else "yt-dlp failed without output" + + +def error_from_output(output: str) -> ApiError: + line = _last_line(output) + lowered = line.lower() + for fragment, code, message in ERROR_PATTERNS: + if fragment in lowered: + return ApiError(400, code, message) + detail = line.removeprefix("ERROR:").strip()[:MAX_ERROR_MESSAGE_LENGTH] + return ApiError(400, "extractor_error", detail) + + +def first_json_document(stdout: str) -> dict[str, Any]: + for candidate in (stdout, *stdout.splitlines()): + try: + document = json.loads(candidate) + except json.JSONDecodeError: + continue + if isinstance(document, dict): + return document + raise ApiError(502, "extractor_error", "yt-dlp returned no data.") + + +def _best_formats_by_height(formats: Sequence[Mapping[str, Any]]) -> list[dict[str, object]]: + best: dict[int, Mapping[str, Any]] = {} + for entry in formats: + height = entry.get("height") + if not isinstance(height, int) or entry.get("vcodec", "none") == "none": + continue + if height not in best or (entry.get("tbr") or 0) > (best[height].get("tbr") or 0): + best[height] = entry + return [ + {"id": str(entry["format_id"]), "label": f"{height}p", "height": height, "ext": entry.get("ext"), + "filesize": entry.get("filesize") or entry.get("filesize_approx")} + for height, entry in sorted(best.items(), reverse=True) + ] + + +def summarize_info(info: Mapping[str, Any]) -> dict[str, object]: + return { + "id": info.get("id"), + "title": info.get("title") or "", + "thumbnail": info.get("thumbnail") or "", + "duration": info.get("duration"), + "uploader": info.get("uploader") or info.get("channel") or "", + "platform": info.get("extractor_key") or "", + "webpage_url": info.get("webpage_url") or "", + "formats": _best_formats_by_height(info.get("formats") or []), + "subtitle_languages": sorted((info.get("subtitles") or {}).keys()), + "has_chapters": bool(info.get("chapters")), + } + + +class YtDlpClient: + def __init__(self, settings: Settings, copy_cookies: CookieCopier, runner: Runner = run_command) -> None: + self._settings = settings + self._copy_cookies = copy_cookies + self._runner = runner + + def _run(self, build: Callable[[Path | None], list[str]], timeout: float) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="openmedia-") as workdir: + command = build(self._copy_cookies(Path(workdir))) + result = self._runner(command, timeout, ytdlp_environment(self._settings)) + if result.returncode != 0: + raise error_from_output(result.stderr) + return first_json_document(result.stdout) + + def fetch_info(self, url: str) -> dict[str, object]: + proxy = self._settings.ytdlp_proxy + document = self._run(lambda cookies: build_info_command(url, cookies, proxy), INFO_TIMEOUT_SECONDS) + return summarize_info(document) + + def fetch_playlist(self, url: str, limit: int) -> dict[str, object]: + proxy = self._settings.ytdlp_proxy + document = self._run(lambda cookies: build_playlist_command(url, limit, cookies, proxy), PLAYLIST_TIMEOUT_SECONDS) + entries = document.get("entries") or [] + urls = [str(entry.get("url") or entry.get("webpage_url")) for entry in entries if entry.get("url") or entry.get("webpage_url")] + return {"title": document.get("title") or "", "count": len(urls[:limit]), "urls": urls[:limit]} +``` + +- [ ] **Step 6: Run the API checks** + +Run from the project root: `mise run //apps/api:ci-unit` +Expected: PASS (ruff format may reflow long lines; run `mise run //apps/api:format-fix` first if `format` fails). + +- [ ] **Step 7: Commit** + +```bash +git add apps/api +git commit -m "feat(api): build yt-dlp commands, parse progress and map extraction errors" +``` + +### Task 4: Job engine, runtime settings, storage and retention + +**Files:** + +- Create: `apps/api/app/settings_store.py`, `apps/api/app/storage.py`, `apps/api/app/jobs.py`, `apps/api/app/cleanup.py` +- Test: `apps/api/tests/test_settings_store.py`, `apps/api/tests/test_storage.py`, `apps/api/tests/test_jobs.py`, `apps/api/tests/test_cleanup.py` + +**Interfaces:** + +- Consumes: `Settings`, `ApiError`, `DownloadOptions`, `parse_download_options`, `ProgressTracker`, `parse_progress_line`, `is_postprocessing_line`, `DownloadRequest`, `build_download_command`, `error_from_output`, `ytdlp_environment`, `CookieCopier`. +- Produces: + - `RETENTION_CHOICES = (15, 60, 360, 1440)`, `RuntimeSettings(retention_minutes: int, max_concurrent: int)` with `to_json()`, `parse_runtime_settings(payload, fallback) -> RuntimeSettings`, `SettingsStore(path: Path, defaults: RuntimeSettings)` with `current()` and `update(payload)` + - `StorageUsage(used_bytes: int, limit_bytes: int | None, free_bytes: int)` with `to_json()`, `storage_usage(downloads_dir: Path, max_storage_gb: int) -> StorageUsage`, `ensure_capacity(usage: StorageUsage) -> None` + - `JobStatus` (`queued`, `downloading`, `processing`, `done`, `error`, `cancelled`), `Job`, `JobFile`, `ProcessHandle` protocol (`output_lines() -> Iterator[str]`, `wait() -> int`, `terminate() -> None`), `ProcessFactory`, `start_subprocess`, `JobRuntime(settings, store, copy_cookies, process_factory=start_subprocess, now=utc_now)`, `JobManager(runtime)` with `submit(url, title, options) -> Job`, `get(job_id) -> Job`, `list_jobs() -> list[Job]`, `to_json(job) -> dict[str, object]`, `dispatch() -> None`, `cancel_or_remove(job_id) -> None`, `remove_finished_before(cutoff: datetime) -> int`, `known_job_ids() -> set[str]`, `wait_until_idle(timeout: float) -> bool`, `utc_now() -> datetime` + - `remove_orphan_directories(downloads_dir, known_job_ids) -> int`, `RetentionSweeper(manager, store, now=utc_now)` with `sweep_once() -> int`, `start()`, `stop()` + +- [ ] **Step 1: Write the failing settings store and storage tests** + +`apps/api/tests/test_settings_store.py`: + +```python +import json +from pathlib import Path + +import pytest + +from app.errors import ApiError +from app.settings_store import RuntimeSettings, SettingsStore + +DEFAULTS = RuntimeSettings(retention_minutes=60, max_concurrent=3) + + +def test_missing_file_uses_defaults(tmp_path: Path) -> None: + assert SettingsStore(tmp_path / "settings.json", DEFAULTS).current() == DEFAULTS + + +def test_update_persists_and_reloads(tmp_path: Path) -> None: + path = tmp_path / "settings.json" + SettingsStore(path, DEFAULTS).update({"retention_minutes": 360, "max_concurrent": 5}) + assert json.loads(path.read_text()) == {"retention_minutes": 360, "max_concurrent": 5} + assert SettingsStore(path, DEFAULTS).current() == RuntimeSettings(360, 5) + + +def test_partial_update_keeps_other_values(tmp_path: Path) -> None: + store = SettingsStore(tmp_path / "settings.json", DEFAULTS) + assert store.update({"max_concurrent": 1}) == RuntimeSettings(60, 1) + + +@pytest.mark.parametrize("payload", [{"retention_minutes": 30}, {"max_concurrent": 0}, {"max_concurrent": 6}, {"max_concurrent": True}, {"retention_minutes": "60"}]) +def test_invalid_updates_are_rejected(tmp_path: Path, payload: dict[str, object]) -> None: + with pytest.raises(ApiError) as caught: + SettingsStore(tmp_path / "settings.json", DEFAULTS).update(payload) + assert caught.value.code == "invalid_option" + + +def test_corrupt_file_falls_back_to_defaults(tmp_path: Path) -> None: + path = tmp_path / "settings.json" + path.write_text("{not json") + assert SettingsStore(path, DEFAULTS).current() == DEFAULTS +``` + +`apps/api/tests/test_storage.py`: + +```python +from pathlib import Path + +import pytest + +from app.errors import ApiError +from app.storage import StorageUsage, ensure_capacity, storage_usage + + +def test_usage_counts_files_recursively(tmp_path: Path) -> None: + (tmp_path / "job1").mkdir() + (tmp_path / "job1" / "media.mp4").write_bytes(b"x" * 1500) + (tmp_path / "loose.bin").write_bytes(b"x" * 500) + usage = storage_usage(tmp_path, 0) + assert usage.used_bytes == 2000 + assert usage.limit_bytes is None + assert usage.free_bytes > 0 + + +def test_limit_is_reported_in_bytes(tmp_path: Path) -> None: + assert storage_usage(tmp_path, 2).limit_bytes == 2 * 1024**3 + + +def test_missing_directory_counts_as_empty(tmp_path: Path) -> None: + assert storage_usage(tmp_path / "absent", 0).used_bytes == 0 + + +def test_full_storage_is_refused() -> None: + with pytest.raises(ApiError) as caught: + ensure_capacity(StorageUsage(used_bytes=10, limit_bytes=10, free_bytes=100)) + assert (caught.value.status, caught.value.code) == (507, "storage_full") + ensure_capacity(StorageUsage(used_bytes=9, limit_bytes=10, free_bytes=100)) + ensure_capacity(StorageUsage(used_bytes=10**12, limit_bytes=None, free_bytes=100)) +``` + +- [ ] **Step 2: Implement the settings store and storage** + +`apps/api/app/settings_store.py`: + +```python +import json +import threading +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from pathlib import Path + +from .errors import ApiError + +RETENTION_CHOICES = (15, 60, 360, 1440) +MIN_CONCURRENT = 1 +MAX_CONCURRENT = 5 + + +@dataclass(frozen=True) +class RuntimeSettings: + retention_minutes: int + max_concurrent: int + + def to_json(self) -> dict[str, int]: + return asdict(self) + + +def _integer_or_none(value: object) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def parse_runtime_settings(payload: Mapping[str, object], fallback: RuntimeSettings) -> RuntimeSettings: + retention = _integer_or_none(payload.get("retention_minutes", fallback.retention_minutes)) + concurrency = _integer_or_none(payload.get("max_concurrent", fallback.max_concurrent)) + if retention is None or (retention != fallback.retention_minutes and retention not in RETENTION_CHOICES): + raise ApiError(400, "invalid_option", "retention_minutes must be 15, 60, 360 or 1440.") + if concurrency is None or not MIN_CONCURRENT <= concurrency <= MAX_CONCURRENT: + raise ApiError(400, "invalid_option", "max_concurrent must be between 1 and 5.") + return RuntimeSettings(retention_minutes=retention, max_concurrent=concurrency) + + +class SettingsStore: + def __init__(self, path: Path, defaults: RuntimeSettings) -> None: + self._path = path + self._lock = threading.Lock() + self._current = self._load(defaults) + + def _load(self, defaults: RuntimeSettings) -> RuntimeSettings: + try: + raw = json.loads(self._path.read_text(encoding="utf-8")) + return parse_runtime_settings(raw, defaults) if isinstance(raw, dict) else defaults + except (OSError, json.JSONDecodeError, ApiError): + return defaults + + def current(self) -> RuntimeSettings: + with self._lock: + return self._current + + def update(self, payload: Mapping[str, object]) -> RuntimeSettings: + updated = parse_runtime_settings(payload, self.current()) + self._path.parent.mkdir(parents=True, exist_ok=True) + temporary = self._path.with_suffix(".tmp") + temporary.write_text(json.dumps(updated.to_json()), encoding="utf-8") + temporary.replace(self._path) + with self._lock: + self._current = updated + return updated +``` + +Because `parse_runtime_settings` compares against the fallback, a retention value equal to the current one is accepted even when it came from the environment (for example 120); any other change must be one of the four choices. The test `{"retention_minutes": 30}` uses the default fallback of 60, so it is rejected. + +`apps/api/app/storage.py`: + +```python +import shutil +from dataclasses import asdict, dataclass +from pathlib import Path + +from .errors import ApiError + +BYTES_PER_GIGABYTE = 1024**3 + + +@dataclass(frozen=True) +class StorageUsage: + used_bytes: int + limit_bytes: int | None + free_bytes: int + + def to_json(self) -> dict[str, int | None]: + return asdict(self) + + +def directory_size(path: Path) -> int: + if not path.is_dir(): + return 0 + return sum(entry.stat().st_size for entry in path.rglob("*") if entry.is_file()) + + +def _free_bytes(path: Path) -> int: + existing = path if path.exists() else path.parent + return shutil.disk_usage(existing).free + + +def storage_usage(downloads_dir: Path, max_storage_gb: int) -> StorageUsage: + limit = max_storage_gb * BYTES_PER_GIGABYTE if max_storage_gb > 0 else None + return StorageUsage(used_bytes=directory_size(downloads_dir), limit_bytes=limit, free_bytes=_free_bytes(downloads_dir)) + + +def ensure_capacity(usage: StorageUsage) -> None: + if usage.limit_bytes is not None and usage.used_bytes >= usage.limit_bytes: + raise ApiError(507, "storage_full", "Server storage is full. Remove finished downloads or raise the limit.") +``` + +- [ ] **Step 3: Run these suites** + +Run from `apps/api`: `mise exec -- uv run pytest -q tests/test_settings_store.py tests/test_storage.py` +Expected: PASS. + +- [ ] **Step 4: Write the failing job engine tests** + +`apps/api/tests/test_jobs.py`: + +```python +import threading +import time +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest + +from app.config import Settings +from app.errors import ApiError +from app.jobs import JobManager, JobRuntime, JobStatus +from app.settings_store import RuntimeSettings, SettingsStore +from app.validation import parse_download_options + +URL = "https://www.youtube.com/watch?v=abc" + + +class ScriptedProcess: + def __init__(self, command: Sequence[str], script: "ProcessScript") -> None: + self.job_dir = Path(command[list(command).index("-P") + 1]) + self.script = script + self.terminated = threading.Event() + + def output_lines(self) -> Iterator[str]: + yield from self.script.lines + while self.script.hold and not (self.script.release.is_set() or self.terminated.is_set()): + time.sleep(0.01) + if not self.terminated.is_set(): + for name, size in self.script.files.items(): + (self.job_dir / name).write_bytes(b"x" * size) + + def wait(self) -> int: + return -15 if self.terminated.is_set() else self.script.returncode + + def terminate(self) -> None: + self.terminated.set() + + +class ProcessScript: + def __init__(self, lines: list[str], files: dict[str, int], returncode: int = 0, hold: bool = False) -> None: + self.lines = lines + self.files = files + self.returncode = returncode + self.hold = hold + self.release = threading.Event() + self.processes: list[ScriptedProcess] = [] + + def __call__(self, command: Sequence[str], env: Mapping[str, str]) -> ScriptedProcess: + process = ScriptedProcess(command, self) + self.processes.append(process) + return process + + +def no_cookies(directory: Path) -> Path | None: + return None + + +def make_manager(settings: Settings, script: ProcessScript, concurrency: int = 3) -> JobManager: + store = SettingsStore(settings.settings_file, RuntimeSettings(60, concurrency)) + return JobManager(JobRuntime(settings=settings, store=store, copy_cookies=no_cookies, process_factory=script)) + + +def test_successful_download_collects_named_files(settings: Settings) -> None: + script = ProcessScript(["OMPROGRESS 50 100 NA 1000 5", '[Merger] Merging formats into "media.mp4"'], {"media.mp4": 30, "media.vi.srt": 5}) + manager = make_manager(settings, script) + job = manager.submit(URL, 'Phở: bò/Hà Nội', parse_download_options({"subtitles": {"languages": ["vi"], "mode": "srt"}})) + assert manager.wait_until_idle(5) + assert job.status is JobStatus.DONE + assert job.progress == 100.0 + assert [(f.name, f.kind, f.size_bytes) for f in job.files] == [("Phở bòHà Nội.mp4", "media", 30), ("Phở bòHà Nội.vi.srt", "subtitle", 5)] + payload = manager.to_json(job) + assert payload["status"] == "done" + assert payload["filename"] == "Phở bòHà Nội.mp4" + assert payload["expires_at"] is not None + + +def test_failure_maps_the_last_error_line(settings: Settings) -> None: + script = ProcessScript(["ERROR: [youtube] abc: Sign in to confirm you're not a bot"], {}, returncode=1) + manager = make_manager(settings, script) + job = manager.submit(URL, "x", parse_download_options({})) + assert manager.wait_until_idle(5) + assert (job.status, job.error_code) == (JobStatus.ERROR, "bot_check") + + +def test_missing_output_file_is_an_error(settings: Settings) -> None: + manager = make_manager(settings, ProcessScript([], {})) + job = manager.submit(URL, "x", parse_download_options({})) + assert manager.wait_until_idle(5) + assert job.error_code == "extractor_error" + + +def test_concurrency_limit_queues_and_releases(settings: Settings) -> None: + script = ProcessScript([], {"media.mp4": 1}, hold=True) + manager = make_manager(settings, script, concurrency=1) + first = manager.submit(URL, "first", parse_download_options({})) + second = manager.submit(URL, "second", parse_download_options({})) + time.sleep(0.05) + assert first.status is JobStatus.DOWNLOADING + assert second.status is JobStatus.QUEUED + assert manager.to_json(second)["queue_position"] == 1 + script.release.set() + assert manager.wait_until_idle(5) + assert (first.status, second.status) == (JobStatus.DONE, JobStatus.DONE) + + +def test_raising_concurrency_starts_queued_jobs(settings: Settings) -> None: + script = ProcessScript([], {"media.mp4": 1}, hold=True) + store = SettingsStore(settings.settings_file, RuntimeSettings(60, 1)) + manager = JobManager(JobRuntime(settings=settings, store=store, copy_cookies=no_cookies, process_factory=script)) + manager.submit(URL, "a", parse_download_options({})) + queued = manager.submit(URL, "b", parse_download_options({})) + store.update({"max_concurrent": 2}) + manager.dispatch() + assert queued.status is JobStatus.DOWNLOADING + script.release.set() + assert manager.wait_until_idle(5) + + +def test_cancelling_a_running_job_stops_the_process_and_removes_files(settings: Settings) -> None: + script = ProcessScript(["OMPROGRESS 10 100 NA NA NA"], {"media.mp4": 1}, hold=True) + manager = make_manager(settings, script) + job = manager.submit(URL, "x", parse_download_options({})) + time.sleep(0.05) + manager.cancel_or_remove(job.job_id) + assert manager.wait_until_idle(5) + assert job.status is JobStatus.CANCELLED + assert script.processes[0].terminated.is_set() + assert not (settings.downloads_dir / job.job_id).exists() + + +def test_cancelling_a_queued_job(settings: Settings) -> None: + script = ProcessScript([], {"media.mp4": 1}, hold=True) + manager = make_manager(settings, script, concurrency=1) + manager.submit(URL, "running", parse_download_options({})) + queued = manager.submit(URL, "queued", parse_download_options({})) + manager.cancel_or_remove(queued.job_id) + assert queued.status is JobStatus.CANCELLED + script.release.set() + assert manager.wait_until_idle(5) + assert len(script.processes) == 1 + + +def test_removing_a_finished_job_deletes_it(settings: Settings) -> None: + manager = make_manager(settings, ProcessScript([], {"media.mp4": 1})) + job = manager.submit(URL, "x", parse_download_options({})) + assert manager.wait_until_idle(5) + manager.cancel_or_remove(job.job_id) + with pytest.raises(ApiError): + manager.get(job.job_id) + assert not (settings.downloads_dir / job.job_id).exists() + + +def test_stalled_download_times_out(settings: Settings) -> None: + script = ProcessScript([], {}, hold=True) + manager = make_manager(replace(settings, stall_timeout_seconds=0.3), script) + job = manager.submit(URL, "x", parse_download_options({})) + assert manager.wait_until_idle(5) + assert (job.status, job.error_code) == (JobStatus.ERROR, "timeout") + + +def test_remove_finished_before_cutoff(settings: Settings) -> None: + manager = make_manager(settings, ProcessScript([], {"media.mp4": 1})) + job = manager.submit(URL, "x", parse_download_options({})) + assert manager.wait_until_idle(5) + assert manager.remove_finished_before(datetime.now(UTC) - timedelta(minutes=5)) == 0 + assert manager.remove_finished_before(datetime.now(UTC) + timedelta(seconds=1)) == 1 + assert manager.known_job_ids() == set() + + +def test_unknown_job_is_not_found(settings: Settings) -> None: + with pytest.raises(ApiError) as caught: + make_manager(settings, ProcessScript([], {})).get("missing") + assert caught.value.code == "not_found" +``` + +`apps/api/tests/test_cleanup.py`: + +```python +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from app.cleanup import RetentionSweeper, remove_orphan_directories +from app.config import Settings +from app.jobs import JobManager, JobRuntime +from app.settings_store import RuntimeSettings, SettingsStore +from app.validation import parse_download_options + +from .test_jobs import ProcessScript, no_cookies + + +def test_orphan_directories_are_removed(tmp_path: Path) -> None: + (tmp_path / "keep").mkdir() + (tmp_path / "orphan").mkdir() + (tmp_path / "orphan" / "media.mp4").write_bytes(b"x") + assert remove_orphan_directories(tmp_path, {"keep"}) == 1 + assert sorted(path.name for path in tmp_path.iterdir()) == ["keep"] + + +def test_sweeper_uses_the_current_retention(settings: Settings) -> None: + store = SettingsStore(settings.settings_file, RuntimeSettings(15, 3)) + manager = JobManager(JobRuntime(settings=settings, store=store, copy_cookies=no_cookies, process_factory=ProcessScript([], {"media.mp4": 1}))) + manager.submit("https://www.youtube.com/watch?v=a", "x", parse_download_options({})) + assert manager.wait_until_idle(5) + later = datetime.now(UTC) + timedelta(minutes=16) + assert RetentionSweeper(manager, store, now=lambda: later).sweep_once() == 1 +``` + +Add an empty `apps/api/tests/__init__.py` so `from .test_jobs import ...` resolves as a package import. + +- [ ] **Step 5: Run the job suites to see them fail** + +Run from `apps/api`: `mise exec -- uv run pytest -q tests/test_jobs.py tests/test_cleanup.py` +Expected: FAIL with `ModuleNotFoundError: No module named 'app.jobs'`. + +- [ ] **Step 6: Implement the job engine** + +`apps/api/app/jobs.py`: + +```python +import os +import secrets +import shutil +import signal +import subprocess +import threading +import time +from collections import deque +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from pathlib import Path +from typing import Protocol + +from .config import Settings +from .errors import ApiError +from .progress import ProgressTracker, is_postprocessing_line, parse_progress_line +from .settings_store import SettingsStore +from .validation import DownloadOptions +from .ytdlp import CookieCopier, DownloadRequest, build_download_command, error_from_output, ytdlp_environment + +OUTPUT_TAIL_LINES = 40 +MAX_TITLE_LENGTH = 100 +TITLE_UNSAFE_CHARACTERS = frozenset('\\/:*?"<>|') +SUBTITLE_SUFFIXES = frozenset({".srt", ".vtt", ".ass", ".lrc"}) +PARTIAL_SUFFIXES = frozenset({".part", ".ytdl", ".temp"}) +WATCHDOG_INTERVAL_SECONDS = 1.0 +TERMINATED_EXIT_CODE = -15 + + +class JobStatus(StrEnum): + QUEUED = "queued" + DOWNLOADING = "downloading" + PROCESSING = "processing" + DONE = "done" + ERROR = "error" + CANCELLED = "cancelled" + + +ACTIVE_STATUSES = frozenset({JobStatus.QUEUED, JobStatus.DOWNLOADING, JobStatus.PROCESSING}) + + +class ProcessHandle(Protocol): + def output_lines(self) -> Iterator[str]: ... + + def wait(self) -> int: ... + + def terminate(self) -> None: ... + + +ProcessFactory = Callable[[Sequence[str], Mapping[str, str]], ProcessHandle] + + +class SubprocessHandle: + def __init__(self, command: Sequence[str], env: Mapping[str, str]) -> None: + self._process = subprocess.Popen( + list(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1, + env=dict(env), start_new_session=True, + ) + + def output_lines(self) -> Iterator[str]: + stream = self._process.stdout + return iter(stream.readline, "") if stream is not None else iter(()) + + def wait(self) -> int: + return self._process.wait() + + def terminate(self) -> None: + try: + os.killpg(self._process.pid, signal.SIGTERM) + except ProcessLookupError: + return + + +def start_subprocess(command: Sequence[str], env: Mapping[str, str]) -> ProcessHandle: + return SubprocessHandle(command, env) + + +def utc_now() -> datetime: + return datetime.now(UTC) + + +def isoformat(moment: datetime | None) -> str | None: + return None if moment is None else moment.isoformat().replace("+00:00", "Z") + + +@dataclass +class JobFile: + index: int + name: str + kind: str + size_bytes: int + path: Path + + def to_json(self) -> dict[str, object]: + return {"index": self.index, "name": self.name, "kind": self.kind, "size_bytes": self.size_bytes} + + +@dataclass +class Job: + job_id: str + url: str + title: str + options: DownloadOptions + created_at: datetime + status: JobStatus = JobStatus.QUEUED + progress: float = 0.0 + speed_bps: float | None = None + eta_seconds: int | None = None + downloaded_bytes: int | None = None + total_bytes: int | None = None + files: list[JobFile] = field(default_factory=list) + error: str | None = None + error_code: str | None = None + finished_at: datetime | None = None + + @property + def filename(self) -> str | None: + return self.files[0].name if self.files else None + + @property + def is_active(self) -> bool: + return self.status in ACTIVE_STATUSES + + +@dataclass(frozen=True) +class JobRuntime: + settings: Settings + store: SettingsStore + copy_cookies: CookieCopier + process_factory: ProcessFactory = start_subprocess + now: Callable[[], datetime] = utc_now + + +@dataclass(frozen=True) +class Outcome: + returncode: int + output: str + stalled: bool + cookies_file: Path | None + + +def safe_title(title: str, fallback: str) -> str: + cleaned = "".join(character for character in title if character not in TITLE_UNSAFE_CHARACTERS) + return cleaned.strip()[:MAX_TITLE_LENGTH].strip() or fallback + + +def collect_files(job_dir: Path, title: str, job_id: str) -> list[JobFile]: + candidates = [ + path for path in sorted(job_dir.iterdir()) + if path.is_file() and not path.name.startswith(".") and path.suffix not in PARTIAL_SUFFIXES + ] + media = [path for path in candidates if path.suffix not in SUBTITLE_SUFFIXES] + if not media: + return [] + stem = safe_title(title, f"openmedia-{job_id}") + primary = max(media, key=lambda path: path.stat().st_size) + subtitles = [path for path in candidates if path.suffix in SUBTITLE_SUFFIXES] + named = [(primary, f"{stem}{primary.suffix}", "media")] + named += [(path, f"{stem}.{path.name.split('.', 1)[1]}", "subtitle") for path in subtitles] + return [JobFile(index, name, kind, path.stat().st_size, path) for index, (path, name, kind) in enumerate(named)] + + +class StallWatchdog: + def __init__(self, handle: ProcessHandle, timeout_seconds: float) -> None: + self._handle = handle + self._timeout = timeout_seconds + self._last_activity = time.monotonic() + self._stopped = threading.Event() + self.fired = False + self._thread = threading.Thread(target=self._watch, daemon=True) + + def start(self) -> None: + self._thread.start() + + def touch(self) -> None: + self._last_activity = time.monotonic() + + def stop(self) -> None: + self._stopped.set() + + def _watch(self) -> None: + interval = min(WATCHDOG_INTERVAL_SECONDS, self._timeout / 4) + while not self._stopped.wait(interval): + if time.monotonic() - self._last_activity > self._timeout: + self.fired = True + self._handle.terminate() + return + + +class JobManager: + def __init__(self, runtime: JobRuntime) -> None: + self._runtime = runtime + self._jobs: dict[str, Job] = {} + self._handles: dict[str, ProcessHandle] = {} + self._running: set[str] = set() + self._cancelled: set[str] = set() + self._threads: list[threading.Thread] = [] + self._lock = threading.RLock() + + def submit(self, url: str, title: str, options: DownloadOptions) -> Job: + job = Job(job_id=secrets.token_hex(5), url=url, title=title, options=options, created_at=self._runtime.now()) + with self._lock: + self._jobs[job.job_id] = job + self.dispatch() + return job + + def get(self, job_id: str) -> Job: + with self._lock: + job = self._jobs.get(job_id) + if job is None: + raise ApiError(404, "not_found", "Job not found.") + return job + + def list_jobs(self) -> list[Job]: + with self._lock: + return sorted(self._jobs.values(), key=lambda job: job.created_at, reverse=True) + + def known_job_ids(self) -> set[str]: + with self._lock: + return set(self._jobs) + + def _queued_in_order(self) -> list[Job]: + return [job for job in sorted(self._jobs.values(), key=lambda job: job.created_at) if job.status is JobStatus.QUEUED] + + def queue_position(self, job: Job) -> int: + with self._lock: + queued = self._queued_in_order() + return queued.index(job) + 1 if job in queued else 0 + + def _expires_at(self, job: Job) -> datetime | None: + if job.status is not JobStatus.DONE or job.finished_at is None: + return None + return job.finished_at + timedelta(minutes=self._runtime.store.current().retention_minutes) + + def to_json(self, job: Job) -> dict[str, object]: + with self._lock: + return { + "job_id": job.job_id, "url": job.url, "title": job.title, "status": job.status.value, + "progress": job.progress, "speed_bps": job.speed_bps, "eta_seconds": job.eta_seconds, + "downloaded_bytes": job.downloaded_bytes, "total_bytes": job.total_bytes, + "queue_position": self.queue_position(job), "options": job.options.to_json(), + "filename": job.filename, "files": [entry.to_json() for entry in job.files], + "error": job.error, "error_code": job.error_code, "created_at": isoformat(job.created_at), + "finished_at": isoformat(job.finished_at), "expires_at": isoformat(self._expires_at(job)), + } + + def dispatch(self) -> None: + with self._lock: + open_slots = self._runtime.store.current().max_concurrent - len(self._running) + for job in self._queued_in_order()[: max(open_slots, 0)]: + job.status = JobStatus.DOWNLOADING + self._running.add(job.job_id) + thread = threading.Thread(target=self._run, args=(job,), name=f"job-{job.job_id}", daemon=True) + self._threads.append(thread) + thread.start() + + def cancel_or_remove(self, job_id: str) -> None: + job = self.get(job_id) + with self._lock: + was_active = job.is_active + is_running = job_id in self._running + if was_active: + self._mark_cancelled(job) + else: + del self._jobs[job_id] + handle = self._handles.get(job_id) + if handle is not None: + handle.terminate() + if not is_running: + self._remove_directory(job) + self.dispatch() + + def remove_finished_before(self, cutoff: datetime) -> int: + with self._lock: + expired = [job for job in self._jobs.values() if not job.is_active and job.finished_at is not None and job.finished_at < cutoff] + for job in expired: + del self._jobs[job.job_id] + for job in expired: + self._remove_directory(job) + return len(expired) + + def wait_until_idle(self, timeout: float) -> bool: + deadline = time.monotonic() + timeout + for thread in list(self._threads): + thread.join(max(deadline - time.monotonic(), 0)) + return not any(thread.is_alive() for thread in self._threads) + + def _job_dir(self, job: Job) -> Path: + return self._runtime.settings.downloads_dir / job.job_id + + def _remove_directory(self, job: Job) -> None: + shutil.rmtree(self._job_dir(job), ignore_errors=True) + + def _mark_cancelled(self, job: Job) -> None: + self._cancelled.add(job.job_id) + job.status = JobStatus.CANCELLED + job.finished_at = self._runtime.now() + job.speed_bps = None + job.eta_seconds = None + + def _run(self, job: Job) -> None: + job_dir = self._job_dir(job) + job_dir.mkdir(parents=True, exist_ok=True) + try: + outcome = self._execute(job, job_dir) + except OSError as error: + outcome = Outcome(returncode=1, output=f"ERROR: {error}", stalled=False, cookies_file=None) + self._finish(job, job_dir, outcome) + self.dispatch() + + def _execute(self, job: Job, job_dir: Path) -> Outcome: + settings = self._runtime.settings + cookies_file = self._runtime.copy_cookies(job_dir) + request = DownloadRequest(job.url, job.options, job_dir, settings.max_filesize_mb, cookies_file, settings.ytdlp_proxy) + handle = self._runtime.process_factory(build_download_command(request), ytdlp_environment(settings)) + with self._lock: + self._handles[job.job_id] = handle + watchdog = StallWatchdog(handle, settings.stall_timeout_seconds) + watchdog.start() + tail: deque[str] = deque(maxlen=OUTPUT_TAIL_LINES) + tracker = ProgressTracker() + for line in handle.output_lines(): + watchdog.touch() + tail.append(line.rstrip()) + self._apply_line(job, tracker, line) + returncode = handle.wait() + watchdog.stop() + return Outcome(returncode, "\n".join(tail), watchdog.fired, cookies_file) + + def _apply_line(self, job: Job, tracker: ProgressTracker, line: str) -> None: + sample = parse_progress_line(line) + with self._lock: + if job.job_id in self._cancelled: + return + if sample is not None: + job.progress = round(tracker.record(sample), 1) + job.downloaded_bytes, job.total_bytes = sample.downloaded_bytes, sample.total_bytes + job.speed_bps, job.eta_seconds = sample.speed_bps, sample.eta_seconds + elif is_postprocessing_line(line): + job.status = JobStatus.PROCESSING + job.progress = tracker.record_processing() + job.speed_bps, job.eta_seconds = None, None + + def _finish(self, job: Job, job_dir: Path, outcome: Outcome) -> None: + if outcome.cookies_file is not None: + outcome.cookies_file.unlink(missing_ok=True) + with self._lock: + self._handles.pop(job.job_id, None) + self._running.discard(job.job_id) + cancelled = job.job_id in self._cancelled + if not cancelled: + self._record_outcome(job, job_dir, outcome) + if cancelled: + self._remove_directory(job) + + def _record_outcome(self, job: Job, job_dir: Path, outcome: Outcome) -> None: + job.finished_at = self._runtime.now() + job.speed_bps, job.eta_seconds = None, None + if outcome.stalled: + return self._fail(job, "timeout", "The download stalled and was stopped.") + if outcome.returncode != 0: + error = error_from_output(outcome.output) + return self._fail(job, error.code, error.message) + files = collect_files(job_dir, job.title, job.job_id) + if not files: + return self._fail(job, "extractor_error", "The download finished but no file was found.") + job.files, job.status, job.progress = files, JobStatus.DONE, 100.0 + return None + + def _fail(self, job: Job, code: str, message: str) -> None: + job.status, job.error_code, job.error = JobStatus.ERROR, code, message +``` + +- [ ] **Step 7: Implement retention cleanup** + +`apps/api/app/cleanup.py`: + +```python +import shutil +import threading +from collections.abc import Callable +from datetime import datetime, timedelta +from pathlib import Path + +from .jobs import JobManager, utc_now +from .settings_store import SettingsStore + +SWEEP_INTERVAL_SECONDS = 60.0 + + +def remove_orphan_directories(downloads_dir: Path, known_job_ids: set[str]) -> int: + if not downloads_dir.is_dir(): + return 0 + orphans = [path for path in downloads_dir.iterdir() if path.is_dir() and path.name not in known_job_ids] + for path in orphans: + shutil.rmtree(path, ignore_errors=True) + return len(orphans) + + +class RetentionSweeper: + def __init__(self, manager: JobManager, store: SettingsStore, now: Callable[[], datetime] = utc_now) -> None: + self._manager = manager + self._store = store + self._now = now + self._stopped = threading.Event() + self._thread = threading.Thread(target=self._loop, name="retention-sweeper", daemon=True) + + def sweep_once(self) -> int: + cutoff = self._now() - timedelta(minutes=self._store.current().retention_minutes) + return self._manager.remove_finished_before(cutoff) + + def start(self) -> None: + self._thread.start() + + def stop(self) -> None: + self._stopped.set() + + def _loop(self) -> None: + while not self._stopped.wait(SWEEP_INTERVAL_SECONDS): + self.sweep_once() +``` + +- [ ] **Step 8: Run the API checks** + +Run from the project root: `mise run //apps/api:ci-unit` +Expected: PASS. If `mypy --strict` flags `return self._fail(...)` in `_record_outcome`, change those lines to call `self._fail(...)` followed by `return`. + +- [ ] **Step 9: Commit** + +```bash +git add apps/api +git commit -m "feat(api): add the download job engine with retention and storage limits" +``` + +### Task 5: Cookies and security primitives + +**Files:** + +- Create: `apps/api/app/cookies.py`, `apps/api/app/security.py` +- Test: `apps/api/tests/test_cookies.py`, `apps/api/tests/test_security.py` + +**Interfaces:** + +- Consumes: `Settings`, `ApiError`. +- Produces: + - `COOKIE_COPY_NAME = ".cookies.txt"`, `MAX_COOKIE_BYTES = 1048576`, `CookieRow(domain: str, expires: int)`, `CookieSummary(present: bool, domains: tuple[str, ...], expires_at: datetime | None, uploaded_at: datetime | None)` with `to_json()`, `parse_cookie_rows(text) -> list[CookieRow]`, `validate_cookie_file(raw: bytes) -> str`, `CookieStore(path)` with `summary()`, `save(raw: bytes) -> CookieSummary`, `delete()`, `copy_into(directory: Path) -> Path | None` + - `load_or_create_secret_key(settings) -> str`, `RateLimiter(per_minute: int, clock=time.monotonic)` with `retry_after(key) -> float | None` and `enforce(key) -> None`, `client_address() -> str`, `ensure_same_origin_request() -> None`, `is_authenticated(settings) -> bool`, `ensure_authenticated(settings) -> None`, `password_matches(settings, candidate: object) -> bool`, `sign_in() -> None`, `sign_out() -> None`, `ForwardedProtoSessionInterface` + +- [ ] **Step 1: Write the failing cookie tests** + +`apps/api/tests/test_cookies.py`: + +```python +import stat +from pathlib import Path + +import pytest + +from app.cookies import CookieStore, parse_cookie_rows, validate_cookie_file +from app.errors import ApiError + +COOKIES = ( + "# Netscape HTTP Cookie File\n" + ".youtube.com\tTRUE\t/\tTRUE\t1893456000\tSID\tabc\n" + "#HttpOnly_.youtube.com\tTRUE\t/\tTRUE\t1861920000\tHSID\tdef\n" + "accounts.google.com\tFALSE\t/\tTRUE\t0\tLSID\tghi\n" +) + + +def test_rows_include_http_only_entries() -> None: + rows = parse_cookie_rows(COOKIES) + assert [(row.domain, row.expires) for row in rows] == [("youtube.com", 1893456000), ("youtube.com", 1861920000), ("accounts.google.com", 0)] + + +@pytest.mark.parametrize("raw", [b"hello world", b"\xff\xfe", b"x" * (1024 * 1024 + 1)]) +def test_invalid_files_are_rejected(raw: bytes) -> None: + with pytest.raises(ApiError) as caught: + validate_cookie_file(raw) + assert caught.value.code == "invalid_cookies" + + +def test_store_saves_privately_and_summarizes(tmp_path: Path) -> None: + store = CookieStore(tmp_path / "cookies.txt") + assert store.summary().present is False + summary = store.save(COOKIES.encode()) + assert summary.present is True + assert summary.domains == ("accounts.google.com", "youtube.com") + assert summary.expires_at is not None and summary.expires_at.year == 2030 + assert stat.S_IMODE((tmp_path / "cookies.txt").stat().st_mode) == 0o600 + assert summary.to_json()["expires_at"] == "2030-01-01T00:00:00Z" + + +def test_copy_into_and_delete(tmp_path: Path) -> None: + store = CookieStore(tmp_path / "cookies.txt") + assert store.copy_into(tmp_path) is None + store.save(COOKIES.encode()) + job_dir = tmp_path / "job" + job_dir.mkdir() + copied = store.copy_into(job_dir) + assert copied == job_dir / ".cookies.txt" + assert copied.read_text() == COOKIES + store.delete() + assert store.summary().present is False +``` + +- [ ] **Step 2: Write the failing security tests** + +`apps/api/tests/test_security.py`: + +```python +from dataclasses import replace + +import pytest +from flask import Flask + +from app.config import Settings +from app.errors import ApiError +from app.security import ( + RateLimiter, + ensure_authenticated, + ensure_same_origin_request, + is_authenticated, + load_or_create_secret_key, + password_matches, + sign_in, +) + + +class FakeClock: + def __init__(self) -> None: + self.now = 0.0 + + def __call__(self) -> float: + return self.now + + +def test_rate_limiter_refills_over_time() -> None: + clock = FakeClock() + limiter = RateLimiter(2, clock) + assert limiter.retry_after("a") is None + assert limiter.retry_after("a") is None + wait = limiter.retry_after("a") + assert wait is not None and 29 <= wait <= 30 + assert limiter.retry_after("b") is None + clock.now = 30.0 + assert limiter.retry_after("a") is None + + +def test_enforce_sets_retry_after_header() -> None: + limiter = RateLimiter(1, FakeClock()) + limiter.enforce("a") + with pytest.raises(ApiError) as caught: + limiter.enforce("a") + assert caught.value.status == 429 + assert caught.value.headers["Retry-After"] == "60" + + +@pytest.mark.parametrize( + ("headers", "allowed"), + [ + ({}, True), + ({"Sec-Fetch-Site": "same-origin"}, True), + ({"Sec-Fetch-Site": "cross-site"}, False), + ({"Sec-Fetch-Site": "same-site"}, False), + ({"Origin": "http://localhost"}, True), + ({"Origin": "https://evil.example"}, False), + ], +) +def test_cross_site_guard(headers: dict[str, str], allowed: bool) -> None: + app = Flask(__name__) + with app.test_request_context("/api/download", method="POST", headers=headers, base_url="http://localhost"): + if allowed: + ensure_same_origin_request() + else: + with pytest.raises(ApiError) as caught: + ensure_same_origin_request() + assert caught.value.code == "cross_site_request" + + +def test_safe_methods_skip_the_guard() -> None: + app = Flask(__name__) + with app.test_request_context("/api/jobs", method="GET", headers={"Sec-Fetch-Site": "cross-site"}): + ensure_same_origin_request() + + +def test_password_session(settings: Settings) -> None: + protected = replace(settings, password="correct horse") + app = Flask(__name__) + app.secret_key = "test" + with app.test_request_context("/api/jobs"): + assert is_authenticated(settings) is True + assert is_authenticated(protected) is False + with pytest.raises(ApiError) as caught: + ensure_authenticated(protected) + assert caught.value.code == "auth_required" + assert password_matches(protected, "wrong") is False + assert password_matches(protected, 42) is False + assert password_matches(protected, "correct horse") is True + sign_in() + assert is_authenticated(protected) is True + + +def test_secret_key_is_generated_once(settings: Settings) -> None: + generated = replace(settings, secret_key="") + first = load_or_create_secret_key(generated) + assert len(first) == 64 + assert load_or_create_secret_key(generated) == first + assert load_or_create_secret_key(settings) == "test-secret-key" +``` + +- [ ] **Step 3: Run both suites to see them fail** + +Run from `apps/api`: `mise exec -- uv run pytest -q tests/test_cookies.py tests/test_security.py` +Expected: FAIL with `ModuleNotFoundError`. + +- [ ] **Step 4: Implement cookies** + +`apps/api/app/cookies.py`: + +```python +import os +import shutil +import threading +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +from .errors import ApiError + +COOKIE_COPY_NAME = ".cookies.txt" +MAX_COOKIE_BYTES = 1024 * 1024 +HTTP_ONLY_PREFIX = "#HttpOnly_" +COOKIE_FIELD_COUNT = 7 +EXPIRY_FIELD = 4 +PRIVATE_FILE_MODE = 0o600 + + +@dataclass(frozen=True) +class CookieRow: + domain: str + expires: int + + +def _iso(moment: datetime | None) -> str | None: + return None if moment is None else moment.isoformat().replace("+00:00", "Z") + + +@dataclass(frozen=True) +class CookieSummary: + present: bool + domains: tuple[str, ...] + expires_at: datetime | None + uploaded_at: datetime | None + + def to_json(self) -> dict[str, object]: + return { + "present": self.present, + "domains": list(self.domains), + "expires_at": _iso(self.expires_at), + "uploaded_at": _iso(self.uploaded_at), + } + + +EMPTY_SUMMARY = CookieSummary(present=False, domains=(), expires_at=None, uploaded_at=None) + + +def parse_cookie_rows(text: str) -> list[CookieRow]: + rows = [] + for raw_line in text.splitlines(): + line = raw_line.removeprefix(HTTP_ONLY_PREFIX) + if not line.strip() or line.startswith("#"): + continue + fields = line.split("\t") + if len(fields) == COOKIE_FIELD_COUNT and fields[EXPIRY_FIELD].isdigit(): + rows.append(CookieRow(domain=fields[0].lstrip("."), expires=int(fields[EXPIRY_FIELD]))) + return rows + + +def validate_cookie_file(raw: bytes) -> str: + if len(raw) > MAX_COOKIE_BYTES: + raise ApiError(413, "invalid_cookies", "The cookie file must be 1 MB or smaller.") + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + raise ApiError(400, "invalid_cookies", "The cookie file must be UTF-8 text.") from error + if not parse_cookie_rows(text): + raise ApiError(400, "invalid_cookies", "This is not a cookies.txt file in Netscape format.") + return text + + +def summarize_cookies(text: str, uploaded_at: datetime) -> CookieSummary: + rows = parse_cookie_rows(text) + expiries = [row.expires for row in rows if row.expires > 0] + expires_at = datetime.fromtimestamp(max(expiries), UTC) if expiries else None + return CookieSummary(True, tuple(sorted({row.domain for row in rows})), expires_at, uploaded_at) + + +class CookieStore: + def __init__(self, path: Path) -> None: + self._path = path + self._lock = threading.Lock() + + def summary(self) -> CookieSummary: + with self._lock: + if not self._path.is_file(): + return EMPTY_SUMMARY + text = self._path.read_text(encoding="utf-8") + uploaded_at = datetime.fromtimestamp(self._path.stat().st_mtime, UTC) + return summarize_cookies(text, uploaded_at) + + def save(self, raw: bytes) -> CookieSummary: + text = validate_cookie_file(raw) + with self._lock: + self._path.parent.mkdir(parents=True, exist_ok=True) + temporary = self._path.with_suffix(".tmp") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, PRIVATE_FILE_MODE) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(text) + temporary.replace(self._path) + return self.summary() + + def delete(self) -> None: + with self._lock: + self._path.unlink(missing_ok=True) + + def copy_into(self, directory: Path) -> Path | None: + with self._lock: + if not self._path.is_file(): + return None + target = directory / COOKIE_COPY_NAME + shutil.copyfile(self._path, target) + target.chmod(PRIVATE_FILE_MODE) + return target +``` + +The test expects `expires_at` for year 2030 from the latest expiry `1893456000` (2030-01-01T00:00:00Z); the summary reports the latest expiry because long-lived login cookies decide whether the file still works. + +- [ ] **Step 5: Implement security** + +`apps/api/app/security.py`: + +```python +import hmac +import math +import os +import secrets +import threading +import time +from collections.abc import Callable +from urllib.parse import urlsplit + +from flask import Flask, request, session +from flask.sessions import SecureCookieSessionInterface + +from .config import Settings +from .errors import ApiError + +AUTHENTICATED_SESSION_KEY = "openmedia_authenticated" +SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"}) +CROSS_SITE_FETCH_VALUES = frozenset({"cross-site", "same-site"}) +SECONDS_PER_MINUTE = 60.0 +SECRET_KEY_BYTES = 32 +PRIVATE_FILE_MODE = 0o600 + + +def load_or_create_secret_key(settings: Settings) -> str: + if settings.secret_key: + return settings.secret_key + path = settings.secret_key_file + if path.is_file(): + return path.read_text(encoding="utf-8").strip() + path.parent.mkdir(parents=True, exist_ok=True) + key = secrets.token_hex(SECRET_KEY_BYTES) + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, PRIVATE_FILE_MODE) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(key) + return key + + +class RateLimiter: + def __init__(self, per_minute: int, clock: Callable[[], float] = time.monotonic) -> None: + self._capacity = float(per_minute) + self._refill_per_second = per_minute / SECONDS_PER_MINUTE + self._clock = clock + self._buckets: dict[str, tuple[float, float]] = {} + self._lock = threading.Lock() + + def retry_after(self, key: str) -> float | None: + with self._lock: + now = self._clock() + tokens, updated = self._buckets.get(key, (self._capacity, now)) + tokens = min(self._capacity, tokens + (now - updated) * self._refill_per_second) + if tokens < 1: + self._buckets[key] = (tokens, now) + return (1 - tokens) / self._refill_per_second + self._buckets[key] = (tokens - 1, now) + return None + + def enforce(self, key: str) -> None: + wait = self.retry_after(key) + if wait is not None: + headers = {"Retry-After": str(math.ceil(wait))} + raise ApiError(429, "rate_limited", "Too many requests. Try again shortly.", headers) + + +def client_address() -> str: + return request.remote_addr or "unknown" + + +def _cross_site_error() -> ApiError: + return ApiError(403, "cross_site_request", "Requests from other websites are not allowed.") + + +def ensure_same_origin_request() -> None: + if request.method in SAFE_METHODS: + return + if request.headers.get("Sec-Fetch-Site", "").lower() in CROSS_SITE_FETCH_VALUES: + raise _cross_site_error() + origin = request.headers.get("Origin") + if origin and urlsplit(origin).netloc != request.host: + raise _cross_site_error() + + +def is_authenticated(settings: Settings) -> bool: + return not settings.password or session.get(AUTHENTICATED_SESSION_KEY) is True + + +def ensure_authenticated(settings: Settings) -> None: + if not is_authenticated(settings): + raise ApiError(401, "auth_required", "Sign in to continue.") + + +def password_matches(settings: Settings, candidate: object) -> bool: + if not settings.password or not isinstance(candidate, str): + return False + return hmac.compare_digest(candidate.encode(), settings.password.encode()) + + +def sign_in() -> None: + session.clear() + session[AUTHENTICATED_SESSION_KEY] = True + session.permanent = True + + +def sign_out() -> None: + session.clear() + + +class ForwardedProtoSessionInterface(SecureCookieSessionInterface): + def get_cookie_secure(self, app: Flask) -> bool: + return request.is_secure +``` + +- [ ] **Step 6: Run the API checks** + +Run from the project root: `mise run //apps/api:ci-unit` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add apps/api +git commit -m "feat(api): add cookie storage, cross-site guard, rate limiting and optional password" +``` + +### Task 6: HTTP routes and application factory + +**Files:** + +- Create: `apps/api/app/services.py`, `apps/api/app/media.py`, `apps/api/tests/test_routes.py` +- Modify: `apps/api/app/__init__.py` + +**Interfaces:** + +- Consumes: everything from Tasks 1 to 5. +- Produces: `Services(settings, store, cookies, ytdlp, jobs, request_limiter, login_limiter)`, `EXTENSION_KEY = "openmedia"`, `build_services(settings, process_factory=start_subprocess, runner=run_command) -> Services`, `current_services() -> Services`, blueprint `media`, `create_app(settings: Settings | None = None, services: Services | None = None) -> Flask`. The gunicorn entry point stays `app:create_app()`. + +- [ ] **Step 1: Write the failing route tests** + +`apps/api/tests/test_routes.py`: + +```python +import io +import json +from collections.abc import Iterator +from dataclasses import replace + +import pytest +from flask import Flask +from flask.testing import FlaskClient + +from app import create_app +from app.config import Settings +from app.services import Services, build_services +from app.storage import StorageUsage +from app.ytdlp import CompletedRun + +from .test_cookies import COOKIES +from .test_jobs import ProcessScript +from .test_ytdlp import RecordingRunner + +URL = "https://www.youtube.com/watch?v=abc" +INFO = {"id": "abc", "title": "Pho", "duration": 1122, "formats": [{"format_id": "137", "height": 1080, "vcodec": "avc1", "tbr": 1}]} + + +def build(settings: Settings, script: ProcessScript | None = None, output: dict[str, object] | None = None) -> tuple[Flask, Services]: + runner = RecordingRunner(CompletedRun(0, json.dumps(output or INFO), "")) + services = build_services(settings, process_factory=script or ProcessScript([], {"media.mp4": 12}), runner=runner) + return create_app(settings, services), services + + +@pytest.fixture +def open_settings(settings: Settings) -> Settings: + return replace(settings, allow_private_urls=True) + + +@pytest.fixture +def client(open_settings: Settings) -> Iterator[FlaskClient]: + app, _ = build(open_settings) + yield app.test_client() + + +def test_session_without_password(client: FlaskClient) -> None: + body = client.get("/api/session").get_json() + assert body == {"auth_required": False, "authenticated": True, "limits": {"max_filesize_mb": 4096, "max_playlist_items": 50}} + + +def test_password_protects_the_api(open_settings: Settings) -> None: + app, _ = build(replace(open_settings, password="hunter2")) + client = app.test_client() + assert client.get("/api/jobs").get_json()["code"] == "auth_required" + assert client.post("/api/session", json={"password": "nope"}).status_code == 401 + assert client.post("/api/session", json={"password": "hunter2"}).status_code == 204 + assert client.get("/api/jobs").status_code == 200 + assert client.delete("/api/session").status_code == 204 + assert client.get("/api/jobs").status_code == 401 + + +def test_cross_site_post_is_rejected(client: FlaskClient) -> None: + response = client.post("/api/info", json={"url": URL}, headers={"Sec-Fetch-Site": "cross-site"}) + assert (response.status_code, response.get_json()["code"]) == (403, "cross_site_request") + + +def test_info_returns_the_summary(client: FlaskClient) -> None: + body = client.post("/api/info", json={"url": URL}).get_json() + assert body["title"] == "Pho" + assert body["formats"][0] == {"id": "137", "label": "1080p", "height": 1080, "ext": None, "filesize": None} + + +def test_reclip_injection_payload_is_rejected(client: FlaskClient) -> None: + response = client.post("/api/info", json={"url": "--exec=touch /tmp/pwned"}) + assert (response.status_code, response.get_json()["code"]) == (400, "invalid_url") + assert "error" in response.get_json() + + +def test_private_network_is_blocked_by_default(settings: Settings) -> None: + app, _ = build(settings) + response = app.test_client().post("/api/info", json={"url": "http://127.0.0.1:8080/admin"}) + assert response.get_json()["code"] == "private_network" + + +def test_playlist_is_limited(open_settings: Settings) -> None: + document = {"title": "Mix", "entries": [{"url": f"{URL}{n}"} for n in range(80)]} + app, _ = build(open_settings, output=document) + body = app.test_client().post("/api/playlist", json={"url": URL}).get_json() + assert body["count"] == 50 + + +def test_reclip_download_flow(open_settings: Settings) -> None: + app, services = build(open_settings) + client = app.test_client() + response = client.post("/api/download", json={"url": URL, "format": "video", "format_id": "137", "title": "Pho bo"}) + assert response.status_code == 202 + job_id = response.get_json()["job_id"] + assert services.jobs.wait_until_idle(5) + status = client.get(f"/api/status/{job_id}").get_json() + assert (status["status"], status["error"], status["filename"]) == ("done", None, "Pho bo.mp4") + file_response = client.get(f"/api/file/{job_id}") + assert file_response.status_code == 200 + assert file_response.data == b"x" * 12 + assert "Pho%20bo.mp4" in file_response.headers["Content-Disposition"] or "Pho bo.mp4" in file_response.headers["Content-Disposition"] + assert client.get(f"/api/file/{job_id}/5").get_json()["code"] == "not_found" + + +def test_file_not_ready_while_downloading(open_settings: Settings) -> None: + script = ProcessScript([], {"media.mp4": 1}, hold=True) + app, services = build(open_settings, script=script) + client = app.test_client() + job_id = client.post("/api/download", json={"url": URL}).get_json()["job_id"] + assert client.get(f"/api/file/{job_id}").get_json()["code"] == "file_not_ready" + assert client.delete(f"/api/jobs/{job_id}").status_code == 204 + script.release.set() + assert services.jobs.wait_until_idle(5) + assert client.get(f"/api/status/{job_id}").get_json()["status"] == "cancelled" + + +def test_jobs_list_and_remove(client: FlaskClient) -> None: + job_id = client.post("/api/download", json={"url": URL, "format": "audio"}).get_json()["job_id"] + jobs = client.get("/api/jobs").get_json()["jobs"] + assert [job["job_id"] for job in jobs] == [job_id] + + +def test_storage_full_refuses_downloads(open_settings: Settings, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("app.media.storage_usage", lambda directory, limit: StorageUsage(10, 10, 0)) + app, _ = build(open_settings) + response = app.test_client().post("/api/download", json={"url": URL}) + assert (response.status_code, response.get_json()["code"]) == (507, "storage_full") + + +def test_settings_round_trip(client: FlaskClient) -> None: + assert client.get("/api/settings").get_json() == {"retention_minutes": 60, "max_concurrent": 3} + assert client.put("/api/settings", json={"max_concurrent": 5}).get_json()["max_concurrent"] == 5 + assert client.put("/api/settings", json={"retention_minutes": 7}).status_code == 400 + + +def test_storage_reports_usage(client: FlaskClient) -> None: + body = client.get("/api/storage").get_json() + assert set(body) == {"used_bytes", "limit_bytes", "free_bytes"} + + +def test_cookie_upload_and_removal(client: FlaskClient) -> None: + upload = client.put("/api/cookies", data={"file": (io.BytesIO(COOKIES.encode()), "cookies.txt")}, content_type="multipart/form-data") + assert upload.status_code == 200 + assert upload.get_json()["domains"] == ["accounts.google.com", "youtube.com"] + bad = client.put("/api/cookies", data={"file": (io.BytesIO(b"nope"), "cookies.txt")}, content_type="multipart/form-data") + assert bad.get_json()["code"] == "invalid_cookies" + assert client.put("/api/cookies", data={}, content_type="multipart/form-data").get_json()["code"] == "invalid_cookies" + assert client.delete("/api/cookies").status_code == 204 + assert client.get("/api/cookies").get_json()["present"] is False + + +def test_rate_limit(open_settings: Settings) -> None: + app, _ = build(replace(open_settings, rate_limit_per_minute=2)) + client = app.test_client() + client.post("/api/info", json={"url": URL}) + client.post("/api/info", json={"url": URL}) + limited = client.post("/api/info", json={"url": URL}) + assert limited.status_code == 429 + assert "Retry-After" in limited.headers + + +def test_unknown_route_is_json(client: FlaskClient) -> None: + response = client.get("/api/nope") + assert response.status_code == 404 + assert response.get_json()["code"] == "not_found" +``` + +- [ ] **Step 2: Run the suite to see it fail** + +Run from `apps/api`: `mise exec -- uv run pytest -q tests/test_routes.py` +Expected: FAIL with `ModuleNotFoundError: No module named 'app.services'`. + +- [ ] **Step 3: Implement the services container** + +`apps/api/app/services.py`: + +```python +from dataclasses import dataclass +from typing import cast + +from flask import current_app + +from .config import Settings +from .cookies import CookieStore +from .jobs import JobManager, JobRuntime, ProcessFactory, start_subprocess +from .security import RateLimiter +from .settings_store import RuntimeSettings, SettingsStore +from .ytdlp import Runner, YtDlpClient, run_command + +EXTENSION_KEY = "openmedia" +LOGIN_ATTEMPTS_PER_MINUTE = 5 + + +@dataclass(frozen=True) +class Services: + settings: Settings + store: SettingsStore + cookies: CookieStore + ytdlp: YtDlpClient + jobs: JobManager + request_limiter: RateLimiter + login_limiter: RateLimiter + + +def build_services( + settings: Settings, + process_factory: ProcessFactory = start_subprocess, + runner: Runner = run_command, +) -> Services: + settings.downloads_dir.mkdir(parents=True, exist_ok=True) + store = SettingsStore(settings.settings_file, RuntimeSettings(settings.retention_minutes, settings.max_concurrent)) + cookies = CookieStore(settings.cookies_file) + runtime = JobRuntime(settings=settings, store=store, copy_cookies=cookies.copy_into, process_factory=process_factory) + return Services( + settings=settings, + store=store, + cookies=cookies, + ytdlp=YtDlpClient(settings, cookies.copy_into, runner), + jobs=JobManager(runtime), + request_limiter=RateLimiter(settings.rate_limit_per_minute), + login_limiter=RateLimiter(LOGIN_ATTEMPTS_PER_MINUTE), + ) + + +def current_services() -> Services: + return cast(Services, current_app.extensions[EXTENSION_KEY]) +``` + +- [ ] **Step 4: Implement the routes** + +`apps/api/app/media.py`: + +```python +from collections.abc import Mapping + +from flask import Blueprint, Response, jsonify, request, send_file + +from .errors import ApiError +from .network_guard import ensure_public_url +from .security import ( + client_address, + ensure_authenticated, + ensure_same_origin_request, + is_authenticated, + password_matches, + sign_in, + sign_out, +) +from .services import current_services +from .storage import ensure_capacity, storage_usage +from .validation import parse_download_options, validate_url + +media = Blueprint("media", __name__, url_prefix="/api") + +PUBLIC_ENDPOINTS = frozenset({"media.session_status", "media.create_session", "media.delete_session"}) +MAX_TITLE_LENGTH = 300 +NO_CONTENT = ("", 204) + + +@media.before_request +def guard_request() -> None: + ensure_same_origin_request() + if request.endpoint not in PUBLIC_ENDPOINTS: + ensure_authenticated(current_services().settings) + + +def json_payload() -> Mapping[str, object]: + payload = request.get_json(silent=True, force=True) + if not isinstance(payload, dict): + raise ApiError(400, "invalid_option", "Send a JSON object.") + return payload + + +def checked_url(payload: Mapping[str, object]) -> str: + url = validate_url(payload.get("url")) + if not current_services().settings.allow_private_urls: + ensure_public_url(url) + return url + + +def enforce_request_limit() -> None: + current_services().request_limiter.enforce(client_address()) + + +@media.get("/session") +def session_status() -> Response: + settings = current_services().settings + limits = {"max_filesize_mb": settings.max_filesize_mb, "max_playlist_items": settings.max_playlist_items} + return jsonify(auth_required=bool(settings.password), authenticated=is_authenticated(settings), limits=limits) + + +@media.post("/session") +def create_session() -> tuple[str, int]: + services = current_services() + services.login_limiter.enforce(client_address()) + if services.settings.password and not password_matches(services.settings, json_payload().get("password")): + raise ApiError(401, "invalid_password", "The password is not correct.") + sign_in() + return NO_CONTENT + + +@media.delete("/session") +def delete_session() -> tuple[str, int]: + sign_out() + return NO_CONTENT + + +@media.post("/info") +def get_info() -> Response: + enforce_request_limit() + url = checked_url(json_payload()) + return jsonify(current_services().ytdlp.fetch_info(url)) + + +@media.post("/playlist") +def get_playlist() -> Response: + enforce_request_limit() + payload = json_payload() + maximum = current_services().settings.max_playlist_items + requested = payload.get("limit") + limit = requested if isinstance(requested, int) and not isinstance(requested, bool) and 0 < requested < maximum else maximum + return jsonify(current_services().ytdlp.fetch_playlist(checked_url(payload), limit)) + + +@media.post("/download") +def start_download() -> tuple[Response, int]: + enforce_request_limit() + services = current_services() + payload = json_payload() + url = checked_url(payload) + options = parse_download_options(payload) + ensure_capacity(storage_usage(services.settings.downloads_dir, services.settings.max_storage_gb)) + title = str(payload.get("title") or "")[:MAX_TITLE_LENGTH] + job = services.jobs.submit(url, title, options) + return jsonify(job_id=job.job_id, job=services.jobs.to_json(job)), 202 + + +@media.get("/jobs") +def list_jobs() -> Response: + jobs = current_services().jobs + return jsonify(jobs=[jobs.to_json(job) for job in jobs.list_jobs()]) + + +@media.get("/status/") +def job_status(job_id: str) -> Response: + jobs = current_services().jobs + return jsonify(jobs.to_json(jobs.get(job_id))) + + +@media.delete("/jobs/") +def delete_job(job_id: str) -> tuple[str, int]: + current_services().jobs.cancel_or_remove(job_id) + return NO_CONTENT + + +@media.get("/file/", defaults={"index": 0}) +@media.get("/file//") +def download_file(job_id: str, index: int) -> Response: + job = current_services().jobs.get(job_id) + if job.status.value != "done": + raise ApiError(404, "file_not_ready", "The file is not ready yet.") + if index >= len(job.files): + raise ApiError(404, "not_found", "File not found.") + entry = job.files[index] + return send_file(entry.path, as_attachment=True, download_name=entry.name, conditional=True) + + +@media.get("/settings") +def get_settings() -> Response: + return jsonify(current_services().store.current().to_json()) + + +@media.put("/settings") +def update_settings() -> Response: + services = current_services() + updated = services.store.update(json_payload()) + services.jobs.dispatch() + return jsonify(updated.to_json()) + + +@media.get("/storage") +def get_storage() -> Response: + settings = current_services().settings + return jsonify(storage_usage(settings.downloads_dir, settings.max_storage_gb).to_json()) + + +@media.get("/cookies") +def get_cookies() -> Response: + return jsonify(current_services().cookies.summary().to_json()) + + +@media.put("/cookies") +def upload_cookies() -> Response: + upload = request.files.get("file") + if upload is None: + raise ApiError(400, "invalid_cookies", "Choose a cookies.txt file to upload.") + raw = upload.stream.read(1024 * 1024 + 1) + return jsonify(current_services().cookies.save(raw).to_json()) + + +@media.delete("/cookies") +def delete_cookies() -> tuple[str, int]: + current_services().cookies.delete() + return NO_CONTENT +``` + +- [ ] **Step 5: Implement the application factory** + +`apps/api/app/__init__.py`: + +```python +from datetime import timedelta + +from flask import Flask +from werkzeug.middleware.proxy_fix import ProxyFix + +from .cleanup import RetentionSweeper, remove_orphan_directories +from .config import Settings, load_settings +from .errors import register_error_handlers +from .health import health +from .media import media +from .security import ForwardedProtoSessionInterface, load_or_create_secret_key +from .services import EXTENSION_KEY, Services, build_services + +MAX_REQUEST_BYTES = 2 * 1024 * 1024 +SESSION_LIFETIME = timedelta(days=30) + + +def _configure(app: Flask, settings: Settings) -> None: + settings.data_dir.mkdir(parents=True, exist_ok=True) + app.config.update( + SECRET_KEY=load_or_create_secret_key(settings), + SESSION_COOKIE_NAME="openmedia_session", + SESSION_COOKIE_HTTPONLY=True, + SESSION_COOKIE_SAMESITE="Lax", + PERMANENT_SESSION_LIFETIME=SESSION_LIFETIME, + MAX_CONTENT_LENGTH=MAX_REQUEST_BYTES, + OPENMEDIA_DATA_DIR=str(settings.data_dir), + ) + app.session_interface = ForwardedProtoSessionInterface() + hops = settings.trusted_proxy_hops + setattr(app, "wsgi_app", ProxyFix(app.wsgi_app, x_for=hops, x_proto=hops, x_host=hops)) + + +def _start_services(settings: Settings) -> Services: + services = build_services(settings) + remove_orphan_directories(settings.downloads_dir, services.jobs.known_job_ids()) + RetentionSweeper(services.jobs, services.store).start() + return services + + +def create_app(settings: Settings | None = None, services: Services | None = None) -> Flask: + resolved = settings or load_settings() + app = Flask(__name__) + _configure(app, resolved) + register_error_handlers(app) + app.register_blueprint(health) + app.register_blueprint(media) + app.extensions[EXTENSION_KEY] = services or _start_services(resolved) + return app +``` + +Update `apps/api/tests/test_health.py` only if `create_app` imports break it; its tests build their own Flask app and keep passing. + +- [ ] **Step 6: Run the API checks** + +Run from the project root: `mise run //apps/api:ci-unit` +Expected: PASS. `test_unknown_route_is_json` relies on the `HTTPException` handler mapping "Not Found" to `not_found`. + +- [ ] **Step 7: Smoke-test against real yt-dlp** + +Run from `apps/api`: + +```bash +OPENMEDIA_DATA_DIR=$(mktemp -d) mise exec -- uv run flask --app "app:create_app()" run --port 8095 & +sleep 4 +curl -s -X POST localhost:8095/api/info -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=jNQXAC9IVRw"}' | head -c 300 +kill %1 +``` + +Expected: JSON with `"title": "Me at the zoo"`. If the network is unavailable, record that in the report and continue. + +- [ ] **Step 8: Commit** + +```bash +git add apps/api +git commit -m "feat(api): expose the media API with reclip-compatible routes" +``` + +### Task 7: API container, compose stack and update policy + +**Files:** + +- Modify: `apps/api/Dockerfile`, `apps/api/.env.example`, `apps/api/mise.toml`, `apps/api/.dockerignore`, `compose.yaml`, `example.env`, `renovate.json`, `.gitignore` +- Create: `apps/api/docker-entrypoint.sh` + +**Interfaces:** + +- Consumes: `create_app()` (Task 6), `OPENMEDIA_*` variables (spec 4.2). +- Produces: image `ghcr.io/ttncode/openmedia-api` that serves port 8080 as user `app` (uid 10001) with `/data` as a volume; compose services `web` and `api` with `API_URL=http://api:8080` for `web`; mise task `//apps/api:dev`. + +- [ ] **Step 1: Rewrite the runtime stage of the Dockerfile** + +Keep the generated `deps` stage and its comment lines untouched. Replace everything from `FROM python:3.13-slim@sha256:... AS runtime` to the end with (keep the same pinned digest line the generator wrote): + +```dockerfile +FROM python:3.13-slim@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285 AS runtime +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ffmpeg ca-certificates \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=deps /usr/local/bin/uv /usr/local/bin/uv +COPY --from=deps /app/.venv ./.venv +COPY . . +ENV PATH="/app/.venv/bin:${PATH}" \ + OPENMEDIA_DATA_DIR=/data \ + UV_CACHE_DIR=/data/.cache/uv \ + PYTHONUNBUFFERED=1 +RUN useradd --create-home --uid 10001 app \ + && mkdir -p /data/downloads \ + && chown -R app:app /data \ + && chmod 0755 /app/docker-entrypoint.sh +USER app +VOLUME ["/data"] +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=3s \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/live')" || exit 1 +ENTRYPOINT ["/app/docker-entrypoint.sh"] +CMD ["gunicorn", "--workers", "1", "--threads", "8", "--timeout", "120", "--bind", "0.0.0.0:8080", "--access-logfile", "-", "app:create_app()"] +``` + +The `# @SERVICE_SETUP@` anchor was already removed by scaffold for `--db none`; do not re-add it. If `COPY --from=deps /usr/local/bin/uv` fails because the uv image stores the binary elsewhere, find it with `docker run --rm --entrypoint sh ghcr.io/astral-sh/uv:0.12.13-python3.13-trixie-slim -c 'command -v uv'` and use that path. + +- [ ] **Step 2: Write the entrypoint** + +`apps/api/docker-entrypoint.sh`: + +```sh +#!/bin/sh +set -eu + +data_dir="${OPENMEDIA_DATA_DIR:-/data}" +mkdir -p "$data_dir/downloads" + +if [ "${OPENMEDIA_AUTO_UPDATE_YTDLP:-true}" = "true" ]; then + echo "openmedia: updating yt-dlp in $data_dir/yt-dlp" + if ! uv pip install --quiet --python /app/.venv/bin/python --target "$data_dir/yt-dlp" --upgrade "yt-dlp[default]"; then + echo "openmedia: yt-dlp update failed, using the bundled version" + fi +fi + +exec "$@" +``` + +Make it executable in git: `git update-index --chmod=+x apps/api/docker-entrypoint.sh` after adding. + +- [ ] **Step 3: Development environment and ignores** + +`apps/api/.env.example` (replace the generated content): + +```dotenv +FLASK_DEBUG=0 +OPENMEDIA_DATA_DIR=./data +OPENMEDIA_ALLOW_PRIVATE_URLS=false +OPENMEDIA_AUTO_UPDATE_YTDLP=false +``` + +Append to `apps/api/mise.toml`: + +```toml +[tasks.dev] +env = { OPENMEDIA_DATA_DIR = "./data", OPENMEDIA_TRUSTED_PROXY_HOPS = "1" } +run = "uv run flask --app 'app:create_app()' run --port 8081 --debug" +``` + +Append `data/` to `apps/api/.dockerignore` and `apps/api/data/` to the root `.gitignore`. + +- [ ] **Step 4: Wire the compose stack** + +`compose.yaml` services become (keep the header comment block): + +```yaml +name: app +services: + web: + image: ghcr.io/ttncode/openmedia-web:${IMAGE_TAG:-latest} + env_file: + - path: .env + required: false + environment: + API_URL: http://api:8080 + restart: always + ports: + - "${WEB_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + api: + image: ghcr.io/ttncode/openmedia-api:${IMAGE_TAG:-latest} + env_file: + - path: .env + required: false + restart: always + ports: + - "127.0.0.1:${API_PORT:-8081}:8080" + volumes: + - openmedia-data:/data +volumes: + openmedia-data: +``` + +Append to `example.env`: + +```dotenv + +OPENMEDIA_PASSWORD= +OPENMEDIA_RETENTION_MINUTES=60 +OPENMEDIA_MAX_CONCURRENT=3 +OPENMEDIA_MAX_FILESIZE_MB=4096 +OPENMEDIA_MAX_STORAGE_GB=0 +OPENMEDIA_MAX_PLAYLIST_ITEMS=50 +OPENMEDIA_RATE_LIMIT_PER_MINUTE=30 +OPENMEDIA_STALL_TIMEOUT_SECONDS=180 +OPENMEDIA_ALLOW_PRIVATE_URLS=false +OPENMEDIA_AUTO_UPDATE_YTDLP=true +OPENMEDIA_YTDLP_PROXY= +``` + +- [ ] **Step 5: Let yt-dlp updates through quickly** + +`renovate.json`: + +```json +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:recommended", "helpers:pinGitHubActionDigests", "docker:pinDigests"], + "minimumReleaseAge": "3 days", + "packageRules": [ + { + "matchPackageNames": ["yt-dlp", "yt-dlp-ejs"], + "minimumReleaseAge": "0 days", + "groupName": "yt-dlp" + } + ] +} +``` + +- [ ] **Step 6: Build and run the image** + +```bash +docker build -t openmedia-api:local apps/api +docker volume create openmedia-verify +docker run -d --name openmedia-api-verify -p 127.0.0.1:18081:8080 -v openmedia-verify:/data -e OPENMEDIA_AUTO_UPDATE_YTDLP=false openmedia-api:local +sleep 8 +curl -fsS localhost:18081/health/ready +curl -fsS -X POST localhost:18081/api/info -H 'content-type: application/json' -d '{"url":"https://www.youtube.com/watch?v=jNQXAC9IVRw"}' | head -c 200 +docker exec openmedia-api-verify sh -c 'command -v ffmpeg deno && id -u' +docker rm -f openmedia-api-verify && docker volume rm openmedia-verify +``` + +Expected: `{"status":"ok"}`, JSON containing `Me at the zoo`, paths for ffmpeg and deno, uid `10001`. Then run `docker compose config --quiet` from the project root; expected: no output. + +- [ ] **Step 7: Run the API checks and commit** + +Run from the project root: `mise run //apps/api:ci-unit` +Expected: PASS. + +```bash +git add apps/api compose.yaml example.env renovate.json .gitignore +git update-index --chmod=+x apps/api/docker-entrypoint.sh +git commit -m "build(api): ship ffmpeg, deno and a data volume in the API image" +``` + +### Task 8: Web foundation (tooling, tokens, i18n, API client, proxy) + +**Files:** + +- Modify: `apps/web/package.json`, `apps/web/pnpm-lock.yaml`, `apps/web/src/app/layout.tsx`, `apps/web/src/app/page.tsx`, `apps/web/src/app/globals.css`, `apps/web/.env.example`, `apps/web/mise.toml` +- Delete: `apps/web/public/next.svg`, `apps/web/public/vercel.svg`, `apps/web/public/file.svg`, `apps/web/public/globe.svg`, `apps/web/public/window.svg`, `apps/web/src/app/favicon.ico` +- Create: `apps/web/vitest.config.ts`, `apps/web/src/test/setup.ts`, `apps/web/src/app/api/[...path]/route.ts`, `apps/web/src/app/api/[...path]/proxy.ts`, `apps/web/src/app/api/[...path]/proxy.test.ts`, `apps/web/src/lib/api/types.ts`, `apps/web/src/lib/api/client.ts`, `apps/web/src/lib/api/client.test.ts`, `apps/web/src/lib/links.ts`, `apps/web/src/lib/links.test.ts`, `apps/web/src/lib/format.ts`, `apps/web/src/lib/format.test.ts`, `apps/web/src/lib/i18n/en.ts`, `apps/web/src/lib/i18n/vi.ts`, `apps/web/src/lib/i18n/I18nProvider.tsx`, `apps/web/src/lib/i18n/i18n.test.ts`, `apps/web/src/lib/theme.ts` + +**Interfaces:** + +- Produces: + - `types.ts`: `JobStatus`, `DownloadKind = "video" | "audio"`, `Container = "mp4" | "mkv"`, `AudioFormat = "mp3" | "m4a" | "opus" | "flac" | "wav"`, `AudioQuality = "320k" | "best"`, `SubtitleMode = "embed" | "srt"`, `MediaFormat`, `MediaInfo`, `PlaylistInfo`, `DownloadRequest`, `JobOptions`, `JobFile`, `Job`, `SessionInfo`, `RuntimeSettings`, `StorageUsage`, `CookieSummary`, `ApiErrorBody` + - `client.ts`: `ApiRequestError(status, code, message, retryAfterSeconds)`, `api` object with `session()`, `signIn(password)`, `signOut()`, `info(url)`, `playlist(url)`, `download(request)`, `jobs()`, `removeJob(jobId)`, `settings()`, `updateSettings(patch)`, `storage()`, `cookies()`, `uploadCookies(file)`, `removeCookies()`, and `fileUrl(jobId, index?)` + - `links.ts`: `PlatformId`, `parseLinks(text) -> string[]`, `detectPlatform(url) -> PlatformId`, `detectPlatforms(urls) -> PlatformId[]`, `hasPlaylist(url) -> boolean`, `linkFromShare({ url, text }) -> string | null` + - `format.ts`: `Locale = "vi" | "en"`, `formatBytes(bytes, locale)`, `formatSpeed(bytesPerSecond, locale)`, `formatClock(seconds)`, `parseClock(text) -> number | null`, `splitDuration(seconds) -> { hours, minutes, seconds }` + - `i18n`: `Messages` type (from `en`), `en`, `vi`, `LanguagePreference = "auto" | Locale`, `resolveLocale(preference, navigatorLanguage) -> Locale`, `I18nProvider({ locale, children })`, `useI18n() -> { locale, t: Messages }` + - `theme.ts`: `ThemePreference = "system" | "light" | "dark"`, `AccentId = "teal" | "blue" | "purple" | "pink" | "orange" | "green" | "graphite"`, `ACCENTS`, `applyTheme(theme)`, `applyAccent(accent)`, `THEME_BOOTSTRAP_SCRIPT` + - `proxy.ts`: `buildUpstreamUrl(requestUrl, path, apiBaseUrl) -> URL`, `forwardedRequestHeaders(request) -> Headers`, `proxyToApi(request, path, apiBaseUrl, fetchImpl?) -> Promise` + +- [ ] **Step 1: Install dependencies** + +Run from `apps/web`: + +```bash +mise exec -- pnpm add @phosphor-icons/react +mise exec -- pnpm add -D jsdom @testing-library/react @testing-library/user-event @testing-library/jest-dom @vitejs/plugin-react +``` + +Expected: `package.json` and `pnpm-lock.yaml` update. If pnpm refuses a postinstall build, add the package to `allowBuilds` in `apps/web/pnpm-workspace.yaml` only when it is required to run. + +- [ ] **Step 2: Configure vitest** + +`apps/web/vitest.config.ts`: + +```ts +import react from "@vitejs/plugin-react"; +import { fileURLToPath } from "node:url"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [react()], + resolve: { + alias: { "@": fileURLToPath(new URL("./src", import.meta.url)) }, + }, + test: { + restoreMocks: true, + css: { modules: { classNameStrategy: "non-scoped" } }, + projects: [ + { + extends: true, + test: { + name: "dom", + environment: "jsdom", + include: ["src/**/*.test.{ts,tsx}"], + exclude: ["src/app/api/**"], + setupFiles: ["./src/test/setup.ts"], + }, + }, + { + extends: true, + test: { name: "node", environment: "node", include: ["src/app/api/**/*.test.ts"] }, + }, + ], + }, +}); +``` + +`apps/web/src/test/setup.ts`: + +```ts +import "@testing-library/jest-dom/vitest"; +import { cleanup } from "@testing-library/react"; +import { afterEach } from "vitest"; + +afterEach(() => { + cleanup(); + window.localStorage.clear(); +}); +``` + +- [ ] **Step 3: Write the failing library tests** + +`apps/web/src/lib/links.test.ts`: + +```ts +import { describe, expect, it } from "vitest"; +import { detectPlatform, detectPlatforms, hasPlaylist, linkFromShare, parseLinks } from "./links"; + +describe("links", () => { + it("parses links separated by spaces, commas and newlines without duplicates", () => { + const text = "https://youtu.be/a, https://www.tiktok.com/@x/video/1\nhttps://youtu.be/a not-a-link ftp://x.y"; + expect(parseLinks(text)).toEqual(["https://youtu.be/a", "https://www.tiktok.com/@x/video/1"]); + }); + + it("detects platforms by host", () => { + expect(detectPlatform("https://m.youtube.com/watch?v=1")).toBe("youtube"); + expect(detectPlatform("https://x.com/a/status/1")).toBe("x"); + expect(detectPlatform("https://soundcloud.com/a/b")).toBe("soundcloud"); + expect(detectPlatform("https://example.org/v")).toBe("other"); + expect(detectPlatforms(["https://youtu.be/a", "https://youtube.com/b", "https://vimeo.com/1"])).toEqual(["youtube", "vimeo"]); + }); + + it("recognizes playlist parameters", () => { + expect(hasPlaylist("https://www.youtube.com/watch?v=a&list=PL1")).toBe(true); + expect(hasPlaylist("https://www.youtube.com/watch?v=a")).toBe(false); + }); + + it("extracts a link from share target parameters", () => { + expect(linkFromShare({ url: null, text: "Look https://youtu.be/a nice" })).toBe("https://youtu.be/a"); + expect(linkFromShare({ url: "https://vimeo.com/1", text: null })).toBe("https://vimeo.com/1"); + expect(linkFromShare({ url: null, text: "no link" })).toBeNull(); + }); +}); +``` + +`apps/web/src/lib/format.test.ts`: + +```ts +import { describe, expect, it } from "vitest"; +import { formatBytes, formatClock, formatSpeed, parseClock, splitDuration } from "./format"; + +describe("format", () => { + it("formats sizes with locale decimal separators", () => { + expect(formatBytes(1_600_000_000, "vi")).toBe("1,6 GB"); + expect(formatBytes(412_000_000, "en")).toBe("412 MB"); + expect(formatBytes(57_300_000, "vi")).toBe("57,3 MB"); + expect(formatBytes(800, "en")).toBe("0.1 MB"); + }); + + it("formats speeds", () => { + expect(formatSpeed(4_200_000, "vi")).toBe("4,2 MB/s"); + }); + + it("formats and parses clocks", () => { + expect(formatClock(1122)).toBe("18:42"); + expect(formatClock(3735)).toBe("1:02:15"); + expect(parseClock("1:02:15")).toBe(3735); + expect(parseClock("18:42")).toBe(1122); + expect(parseClock("90")).toBe(90); + expect(parseClock("1:xx")).toBeNull(); + }); + + it("splits durations", () => { + expect(splitDuration(3735)).toEqual({ hours: 1, minutes: 2, seconds: 15 }); + }); +}); +``` + +`apps/web/src/lib/api/client.test.ts`: + +```ts +import { describe, expect, it, vi } from "vitest"; +import { api, ApiRequestError } from "./client"; + +function jsonResponse(body: unknown, init: ResponseInit = {}): Response { + return new Response(JSON.stringify(body), { headers: { "content-type": "application/json" }, ...init }); +} + +describe("api client", () => { + it("posts JSON and parses the response", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue(jsonResponse({ title: "Pho", formats: [] })); + const info = await api.info("https://youtu.be/a"); + expect(info.title).toBe("Pho"); + const [url, init] = fetchMock.mock.calls[0]; + expect(url).toBe("/api/info"); + expect(init?.method).toBe("POST"); + expect(new Headers(init?.headers).get("content-type")).toBe("application/json"); + }); + + it("raises typed errors with retry hints", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue(jsonResponse({ error: "Too many requests.", code: "rate_limited" }, { status: 429, headers: { "retry-after": "12", "content-type": "application/json" } })); + await expect(api.jobs()).rejects.toMatchObject({ status: 429, code: "rate_limited", retryAfterSeconds: 12 }); + }); + + it("maps network failures to api_unreachable", async () => { + vi.spyOn(globalThis, "fetch").mockRejectedValue(new TypeError("fetch failed")); + const error = await api.session().catch((caught: unknown) => caught); + expect(error).toBeInstanceOf(ApiRequestError); + expect((error as ApiRequestError).code).toBe("api_unreachable"); + }); + + it("resolves empty responses for deletes", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response(null, { status: 204 })); + await expect(api.removeJob("abc")).resolves.toBeUndefined(); + }); + + it("builds file urls", () => { + expect(api.fileUrl("abc")).toBe("/api/file/abc"); + expect(api.fileUrl("abc", 1)).toBe("/api/file/abc/1"); + }); +}); +``` + +`apps/web/src/lib/i18n/i18n.test.ts`: + +```ts +import { describe, expect, it } from "vitest"; +import { en } from "./en"; +import { resolveLocale } from "./I18nProvider"; +import { vi as vietnamese } from "./vi"; + +function keysOf(value: object, prefix = ""): string[] { + return Object.entries(value).flatMap(([key, child]) => (typeof child === "object" && child !== null ? keysOf(child, `${prefix}${key}.`) : [`${prefix}${key}`])); +} + +describe("i18n", () => { + it("resolves the browser language", () => { + expect(resolveLocale("auto", "vi-VN")).toBe("vi"); + expect(resolveLocale("auto", "en-US")).toBe("en"); + expect(resolveLocale("auto", "fr-FR")).toBe("en"); + expect(resolveLocale("vi", "en-US")).toBe("vi"); + }); + + it("keeps both dictionaries in sync", () => { + expect(keysOf(vietnamese).sort()).toEqual(keysOf(en).sort()); + }); + + it("contains no dash characters reserved by the style guide", () => { + const strings = JSON.stringify([en, vietnamese]); + expect(strings).not.toMatch(/[–—]/); + }); +}); +``` + +`apps/web/src/app/api/[...path]/proxy.test.ts`: + +```ts +import { describe, expect, it, vi } from "vitest"; +import { buildUpstreamUrl, forwardedRequestHeaders, proxyToApi } from "./proxy"; + +describe("api proxy", () => { + it("maps the path and query onto the API base url", () => { + const url = buildUpstreamUrl("http://localhost:8080/api/status/abc?x=1", ["status", "abc"], "http://api:8080"); + expect(url.toString()).toBe("http://api:8080/api/status/abc?x=1"); + }); + + it("forwards host and protocol and drops hop-by-hop headers", () => { + const request = new Request("http://media.local:8080/api/download", { + method: "POST", + headers: { host: "media.local:8080", connection: "keep-alive", cookie: "openmedia_session=1", origin: "http://media.local:8080" }, + }); + const headers = forwardedRequestHeaders(request); + expect(headers.get("x-forwarded-host")).toBe("media.local:8080"); + expect(headers.get("x-forwarded-proto")).toBe("http"); + expect(headers.get("cookie")).toBe("openmedia_session=1"); + expect(headers.get("connection")).toBeNull(); + expect(headers.get("host")).toBeNull(); + }); + + it("streams the upstream response and keeps every set-cookie", async () => { + const upstreamHeaders = new Headers({ "content-type": "application/json" }); + upstreamHeaders.append("set-cookie", "a=1; Path=/"); + upstreamHeaders.append("set-cookie", "b=2; Path=/"); + const fetchImpl = vi.fn().mockResolvedValue(new Response('{"ok":true}', { status: 201, headers: upstreamHeaders })); + const response = await proxyToApi(new Request("http://localhost/api/session", { method: "POST", body: "{}" }), ["session"], "http://api:8080", fetchImpl); + expect(response.status).toBe(201); + expect(response.headers.getSetCookie()).toEqual(["a=1; Path=/", "b=2; Path=/"]); + expect(await response.text()).toBe('{"ok":true}'); + expect(fetchImpl.mock.calls[0][1].method).toBe("POST"); + }); + + it("answers 502 when the API is down", async () => { + const fetchImpl = vi.fn().mockRejectedValue(new TypeError("connect ECONNREFUSED")); + const response = await proxyToApi(new Request("http://localhost/api/jobs"), ["jobs"], "http://api:8080", fetchImpl); + expect(response.status).toBe(502); + expect(await response.json()).toEqual({ error: "The OpenMedia API is not reachable.", code: "api_unreachable" }); + }); +}); +``` + +The proxy test runs in the `node` project defined in `vitest.config.ts`, so it needs no per-file environment marker. + +- [ ] **Step 4: Run the tests to see them fail** + +Run from `apps/web`: `mise exec -- pnpm exec vitest --run` +Expected: FAIL, modules not found. + +- [ ] **Step 5: Implement the libraries** + +`apps/web/src/lib/links.ts`: + +```ts +export type PlatformId = "youtube" | "tiktok" | "instagram" | "soundcloud" | "x" | "facebook" | "vimeo" | "other"; + +const PLATFORM_HOSTS: ReadonlyArray = [ + ["youtube", /(^|\.)(youtube\.com|youtu\.be)$/], + ["tiktok", /(^|\.)tiktok\.com$/], + ["instagram", /(^|\.)instagram\.com$/], + ["soundcloud", /(^|\.)soundcloud\.com$/], + ["x", /(^|\.)(x\.com|twitter\.com)$/], + ["facebook", /(^|\.)(facebook\.com|fb\.watch)$/], + ["vimeo", /(^|\.)vimeo\.com$/], +]; + +const LINK_PATTERN = /^https?:\/\/[^\s/$.?#].\S*$/i; + +function hostOf(url: string): string { + try { + return new URL(url).hostname.replace(/^www\./, ""); + } catch { + return ""; + } +} + +export function parseLinks(text: string): string[] { + const tokens = text.split(/[\s,]+/).filter((token) => LINK_PATTERN.test(token)); + return [...new Set(tokens)]; +} + +export function detectPlatform(url: string): PlatformId { + const host = hostOf(url); + return PLATFORM_HOSTS.find(([, pattern]) => pattern.test(host))?.[0] ?? "other"; +} + +export function detectPlatforms(urls: readonly string[]): PlatformId[] { + return [...new Set(urls.map(detectPlatform))]; +} + +export function hasPlaylist(url: string): boolean { + try { + return new URL(url).searchParams.has("list"); + } catch { + return false; + } +} + +export function linkFromShare({ url, text }: { url: string | null; text: string | null }): string | null { + return parseLinks([url ?? "", text ?? ""].join(" "))[0] ?? null; +} +``` + +`apps/web/src/lib/format.ts`: + +```ts +export type Locale = "vi" | "en"; + +const BYTES_PER_MEGABYTE = 1_000_000; +const MEGABYTES_PER_GIGABYTE = 1000; +const MINIMUM_MEGABYTES = 0.1; +const LOCALE_TAGS: Record = { vi: "vi-VN", en: "en-US" }; + +function decimal(value: number, locale: Locale): string { + return new Intl.NumberFormat(LOCALE_TAGS[locale], { maximumFractionDigits: 1 }).format(value); +} + +export function formatBytes(bytes: number, locale: Locale): string { + const megabytes = Math.max(bytes / BYTES_PER_MEGABYTE, MINIMUM_MEGABYTES); + return megabytes >= MEGABYTES_PER_GIGABYTE ? `${decimal(megabytes / MEGABYTES_PER_GIGABYTE, locale)} GB` : `${decimal(megabytes, locale)} MB`; +} + +export function formatSpeed(bytesPerSecond: number, locale: Locale): string { + return `${formatBytes(bytesPerSecond, locale)}/s`; +} + +export function splitDuration(totalSeconds: number): { hours: number; minutes: number; seconds: number } { + const whole = Math.max(0, Math.round(totalSeconds)); + return { hours: Math.floor(whole / 3600), minutes: Math.floor((whole % 3600) / 60), seconds: whole % 60 }; +} + +const pad = (value: number): string => String(value).padStart(2, "0"); + +export function formatClock(totalSeconds: number): string { + const { hours, minutes, seconds } = splitDuration(totalSeconds); + return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds)}` : `${minutes}:${pad(seconds)}`; +} + +export function parseClock(text: string): number | null { + const parts = text.trim().split(":"); + const valid = parts.length >= 1 && parts.length <= 3 && parts.every((part) => /^\d+$/.test(part)); + return valid ? parts.reduce((total, part) => total * 60 + Number(part), 0) : null; +} +``` + +`apps/web/src/lib/api/types.ts`: + +```ts +export type JobStatus = "queued" | "downloading" | "processing" | "done" | "error" | "cancelled"; +export type DownloadKind = "video" | "audio"; +export type Container = "mp4" | "mkv"; +export type AudioFormat = "mp3" | "m4a" | "opus" | "flac" | "wav"; +export type AudioQuality = "320k" | "best"; +export type SubtitleMode = "embed" | "srt"; + +export interface MediaFormat { + readonly id: string; + readonly label: string; + readonly height: number; + readonly ext: string | null; + readonly filesize: number | null; +} + +export interface MediaInfo { + readonly id: string | null; + readonly title: string; + readonly thumbnail: string; + readonly duration: number | null; + readonly uploader: string; + readonly platform: string; + readonly webpage_url: string; + readonly formats: readonly MediaFormat[]; + readonly subtitle_languages: readonly string[]; + readonly has_chapters: boolean; +} + +export interface PlaylistInfo { + readonly title: string; + readonly count: number; + readonly urls: readonly string[]; +} + +export interface TrimRange { + readonly start: number; + readonly end: number; +} + +export interface SubtitleSelection { + readonly languages: readonly string[]; + readonly mode: SubtitleMode; +} + +export interface DownloadRequest { + readonly url: string; + readonly title: string; + readonly format: DownloadKind; + readonly format_id?: string; + readonly container?: Container; + readonly quality_height?: number; + readonly audio_format?: AudioFormat; + readonly audio_quality?: AudioQuality; + readonly trim?: TrimRange; + readonly subtitles?: SubtitleSelection; + readonly embed_metadata: boolean; +} + +export interface JobOptions { + readonly kind: DownloadKind; + readonly container: Container; + readonly quality_height: number | null; + readonly format_id: string | null; + readonly audio_format: AudioFormat | null; + readonly audio_quality: AudioQuality | null; + readonly trim: TrimRange | null; + readonly subtitles: SubtitleSelection | null; + readonly embed_metadata: boolean; +} + +export interface JobFile { + readonly index: number; + readonly name: string; + readonly kind: "media" | "subtitle"; + readonly size_bytes: number; +} + +export interface Job { + readonly job_id: string; + readonly url: string; + readonly title: string; + readonly status: JobStatus; + readonly progress: number; + readonly speed_bps: number | null; + readonly eta_seconds: number | null; + readonly downloaded_bytes: number | null; + readonly total_bytes: number | null; + readonly queue_position: number; + readonly options: JobOptions; + readonly filename: string | null; + readonly files: readonly JobFile[]; + readonly error: string | null; + readonly error_code: string | null; + readonly created_at: string; + readonly finished_at: string | null; + readonly expires_at: string | null; +} + +export interface SessionInfo { + readonly auth_required: boolean; + readonly authenticated: boolean; + readonly limits: { readonly max_filesize_mb: number; readonly max_playlist_items: number }; +} + +export interface RuntimeSettings { + readonly retention_minutes: number; + readonly max_concurrent: number; +} + +export interface StorageUsage { + readonly used_bytes: number; + readonly limit_bytes: number | null; + readonly free_bytes: number; +} + +export interface CookieSummary { + readonly present: boolean; + readonly domains: readonly string[]; + readonly expires_at: string | null; + readonly uploaded_at: string | null; +} + +export interface ApiErrorBody { + readonly error: string; + readonly code: string; +} +``` + +`apps/web/src/lib/api/client.ts`: + +```ts +import type { CookieSummary, DownloadRequest, Job, MediaInfo, PlaylistInfo, RuntimeSettings, SessionInfo, StorageUsage } from "./types"; + +export class ApiRequestError extends Error { + constructor( + readonly status: number, + readonly code: string, + message: string, + readonly retryAfterSeconds: number | null, + ) { + super(message); + this.name = "ApiRequestError"; + } +} + +const API_PREFIX = "/api"; +const UNREACHABLE_STATUS = 0; + +function jsonInit(method: string, body?: unknown): RequestInit { + return body === undefined ? { method } : { method, body: JSON.stringify(body), headers: { "content-type": "application/json" } }; +} + +async function send(path: string, init: RequestInit = {}): Promise { + try { + return await fetch(`${API_PREFIX}${path}`, { credentials: "same-origin", cache: "no-store", ...init }); + } catch { + throw new ApiRequestError(UNREACHABLE_STATUS, "api_unreachable", "The OpenMedia API is not reachable.", null); + } +} + +async function errorFrom(response: Response): Promise { + const body: unknown = await response.json().catch(() => null); + const record = typeof body === "object" && body !== null ? (body as Record) : {}; + const retryAfter = Number(response.headers.get("retry-after")); + return new ApiRequestError(response.status, typeof record.code === "string" ? record.code : "unknown_error", typeof record.error === "string" ? record.error : response.statusText, Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : null); +} + +async function requestJson(path: string, init?: RequestInit): Promise { + const response = await send(path, init); + if (!response.ok) throw await errorFrom(response); + return (await response.json()) as T; +} + +async function requestVoid(path: string, init?: RequestInit): Promise { + const response = await send(path, init); + if (!response.ok) throw await errorFrom(response); +} + +export const api = { + session: (): Promise => requestJson("/session"), + signIn: (password: string): Promise => requestVoid("/session", jsonInit("POST", { password })), + signOut: (): Promise => requestVoid("/session", { method: "DELETE" }), + info: (url: string): Promise => requestJson("/info", jsonInit("POST", { url })), + playlist: (url: string): Promise => requestJson("/playlist", jsonInit("POST", { url })), + download: (request: DownloadRequest): Promise<{ job_id: string; job: Job }> => requestJson("/download", jsonInit("POST", request)), + jobs: async (): Promise => (await requestJson<{ jobs: Job[] }>("/jobs")).jobs, + removeJob: (jobId: string): Promise => requestVoid(`/jobs/${encodeURIComponent(jobId)}`, { method: "DELETE" }), + settings: (): Promise => requestJson("/settings"), + updateSettings: (patch: Partial): Promise => requestJson("/settings", jsonInit("PUT", patch)), + storage: (): Promise => requestJson("/storage"), + cookies: (): Promise => requestJson("/cookies"), + uploadCookies: (file: File): Promise => { + const body = new FormData(); + body.append("file", file); + return requestJson("/cookies", { method: "PUT", body }); + }, + removeCookies: (): Promise => requestVoid("/cookies", { method: "DELETE" }), + fileUrl: (jobId: string, index?: number): string => `${API_PREFIX}/file/${encodeURIComponent(jobId)}${index === undefined ? "" : `/${index}`}`, +}; +``` + +`apps/web/src/app/api/[...path]/proxy.ts`: + +```ts +const HOP_BY_HOP_HEADERS = new Set(["connection", "keep-alive", "proxy-connection", "transfer-encoding", "upgrade", "te", "trailer", "host", "content-length"]); +const METHODS_WITHOUT_BODY = new Set(["GET", "HEAD"]); + +export function buildUpstreamUrl(requestUrl: string, path: readonly string[], apiBaseUrl: string): URL { + const upstream = new URL(`/api/${path.map(encodeURIComponent).join("/")}`, apiBaseUrl); + upstream.search = new URL(requestUrl).search; + return upstream; +} + +export function forwardedRequestHeaders(request: Request): Headers { + const incoming = new URL(request.url); + const headers = new Headers(); + request.headers.forEach((value, key) => { + if (!HOP_BY_HOP_HEADERS.has(key)) headers.set(key, value); + }); + headers.set("x-forwarded-host", request.headers.get("host") ?? incoming.host); + headers.set("x-forwarded-proto", request.headers.get("x-forwarded-proto") ?? incoming.protocol.replace(":", "")); + return headers; +} + +function responseHeaders(upstream: Response): Headers { + const headers = new Headers(); + upstream.headers.forEach((value, key) => { + if (!HOP_BY_HOP_HEADERS.has(key) && key !== "set-cookie") headers.set(key, value); + }); + upstream.headers.getSetCookie().forEach((cookie) => headers.append("set-cookie", cookie)); + return headers; +} + +export async function proxyToApi(request: Request, path: readonly string[], apiBaseUrl: string, fetchImpl: typeof fetch = fetch): Promise { + const body = METHODS_WITHOUT_BODY.has(request.method) ? undefined : await request.arrayBuffer(); + try { + const upstream = await fetchImpl(buildUpstreamUrl(request.url, path, apiBaseUrl), { + method: request.method, + headers: forwardedRequestHeaders(request), + body, + redirect: "manual", + cache: "no-store", + }); + return new Response(upstream.body, { status: upstream.status, statusText: upstream.statusText, headers: responseHeaders(upstream) }); + } catch { + return Response.json({ error: "The OpenMedia API is not reachable.", code: "api_unreachable" }, { status: 502 }); + } +} +``` + +`apps/web/src/app/api/[...path]/route.ts`: + +```ts +import { proxyToApi } from "./proxy"; + +export const dynamic = "force-dynamic"; + +const DEFAULT_API_URL = "http://localhost:8081"; + +type ProxyContext = { params: Promise<{ path: string[] }> }; + +async function handle(request: Request, context: ProxyContext): Promise { + const { path } = await context.params; + return proxyToApi(request, path, process.env.API_URL ?? DEFAULT_API_URL); +} + +export const GET = handle; +export const HEAD = handle; +export const POST = handle; +export const PUT = handle; +export const PATCH = handle; +export const DELETE = handle; +``` + +The existing `apps/web/src/app/api/health/live/route.ts` keeps answering the container health check because a static segment wins over the catch-all. + +`apps/web/src/lib/theme.ts`: + +```ts +export type ThemePreference = "system" | "light" | "dark"; +export type AccentId = "teal" | "blue" | "purple" | "pink" | "orange" | "green" | "graphite"; + +export const DEFAULT_ACCENT: AccentId = "teal"; + +export const ACCENTS: ReadonlyArray<{ readonly id: AccentId; readonly swatch: string }> = [ + { id: "teal", swatch: "#12939c" }, + { id: "blue", swatch: "#007aff" }, + { id: "purple", swatch: "#af52de" }, + { id: "pink", swatch: "#ff2d55" }, + { id: "orange", swatch: "#ff9500" }, + { id: "green", swatch: "#34c759" }, + { id: "graphite", swatch: "#8e8e93" }, +]; + +export const PREFERENCES_STORAGE_KEY = "openmedia.preferences"; + +export function applyTheme(theme: ThemePreference): void { + const root = document.documentElement; + if (theme === "system") root.removeAttribute("data-theme"); + else root.dataset.theme = theme; +} + +export function applyAccent(accent: AccentId): void { + document.documentElement.dataset.accent = accent; +} + +export const THEME_BOOTSTRAP_SCRIPT = `try{var p=JSON.parse(localStorage.getItem("${PREFERENCES_STORAGE_KEY}")||"{}");if(p.theme==="light"||p.theme==="dark"){document.documentElement.dataset.theme=p.theme}document.documentElement.dataset.accent=p.accent||"${DEFAULT_ACCENT}"}catch(e){document.documentElement.dataset.accent="${DEFAULT_ACCENT}"}`; +``` + +`apps/web/src/lib/i18n/I18nProvider.tsx`: + +```tsx +"use client"; + +import { createContext, useContext, useMemo, type ReactNode } from "react"; +import type { Locale } from "../format"; +import { en, type Messages } from "./en"; +import { vi } from "./vi"; + +export type LanguagePreference = "auto" | Locale; + +const DICTIONARIES: Record = { en, vi }; + +export function resolveLocale(preference: LanguagePreference, navigatorLanguage: string): Locale { + if (preference !== "auto") return preference; + return navigatorLanguage.toLowerCase().startsWith("vi") ? "vi" : "en"; +} + +interface I18nValue { + readonly locale: Locale; + readonly t: Messages; +} + +const I18nContext = createContext({ locale: "en", t: en }); + +export function I18nProvider({ locale, children }: { locale: Locale; children: ReactNode }): ReactNode { + const value = useMemo(() => ({ locale, t: DICTIONARIES[locale] }), [locale]); + return {children}; +} + +export function useI18n(): I18nValue { + return useContext(I18nContext); +} +``` + +The language preference itself lives in the preferences store (Task 9); the provider only receives the resolved locale. + +`apps/web/src/lib/i18n/en.ts`: + +```ts +export const en = { + app: { name: "OpenMedia", sampleNote: "Sample data" }, + nav: { queue: "Queue", downloading: "Downloading", done: "Done", attention: "Needs attention", history: "History", settings: "Settings", other: "More", downloads: "Downloads", showSidebar: "Show sidebar", toggleTheme: "Switch light or dark", shortcuts: "Keyboard shortcuts" }, + importer: { + label: "Links to download", + placeholder: "Paste a YouTube, TikTok or SoundCloud link...", + paste: "Paste", + fetch: "Get info", + hint: "Enter gets info, Shift+Enter adds a line, or drop links anywhere", + pasteFallback: "Press Cmd+V or Ctrl+V to paste", + noLinks: "No links recognized", + playlistPrompt: "This link belongs to a playlist.", + playlistSingle: "Only this video", + playlistAll: (count: number): string => `Whole playlist (up to ${count} videos)`, + dropTitle: "Drop links to add them to the queue", + installHint: "Install OpenMedia on your home screen to share links straight from other apps.", + dismissInstallHint: "Hide install tip", + }, + queue: { + summary: (total: number, active: number, done: number): string => `${total} items, ${active} downloading, ${done} done`, + startAll: (count: number): string => `Download all (${count})`, + concurrency: (count: number): string => `Up to ${count} downloads at once`, + empty: "Nothing here yet. Paste a link to start.", + download: "Download", + save: "Save", + fix: "Fix", + retry: "Try again", + cancel: (title: string): string => `Cancel ${title}`, + removeQueued: "Remove from queue", + remove: "Remove", + fetching: "Getting info", + queued: (position: number): string => `Waiting, position ${position}`, + processing: "Finishing up", + doneLine: (format: string, size: string, expiry: string): string => `${format}, ${size}. ${expiry}`, + downloadingLine: (percent: number, speed: string, remaining: string): string => `${percent}% · ${speed}, ${remaining}`, + cancelled: "Cancelled", + listLabel: "Download queue", + }, + time: { + secondsLeft: (seconds: number): string => `${seconds} s left`, + minutesLeft: (minutes: number, seconds: number): string => (seconds === 0 ? `${minutes} min left` : `${minutes} min ${seconds} s left`), + expiresIn: (minutes: number): string => (minutes >= 60 ? `Deleted in ${Math.round(minutes / 60)} h` : `Deleted in ${minutes} min`), + retention: { 15: "15 minutes", 60: "1 hour", 360: "6 hours", 1440: "24 hours" }, + }, + inspector: { + title: "Details", + done: "Done", + empty: "Select an item to see its details.", + kind: "Type", + video: "Video", + audio: "Audio", + format: "Format", + quality: "Quality", + qualityBest: "Best available", + audioOriginal: "Original", + audioLossless: "Original, lossless", + trim: "Trim", + trimStart: "Start", + trimEnd: "End", + trimLength: (length: string): string => `Length ${length}`, + trimHelp: "Drag the yellow handles, or select a handle and use the arrow keys.", + trimStartHandle: "Start point", + trimEndHandle: "End point", + subtitles: "Subtitles", + subtitlesOff: "Off", + subtitlesVietnamese: "Vietnamese", + subtitlesEnglish: "English", + subtitleEmbed: "Embed in video", + subtitleFile: "Separate .srt file", + embedMetadata: "Embed cover and details", + estimate: "Estimated", + selection: "Selected part", + downloadAll: "Download", + downloadSelection: "Download selection", + downloadingTitle: (label: string): string => `Downloading ${label}`, + keepsRunning: "You can close this page; the server keeps downloading.", + queuedTitle: (position: number): string => `Waiting, position ${position}`, + queuedHelp: "Starts when a download slot is free.", + doneTitle: "Download complete", + saveToDevice: "Save to device", + subtitleFileName: (name: string): string => `Save ${name}`, + errorTitle: "Could not download", + addCookies: "Add cookies to continue", + cookiesReady: "Cookies for this site are loaded", + processingTitle: "Finishing up", + processingHelp: "Merging streams and embedding details.", + artworkAlt: (title: string): string => `Cover of ${title}`, + }, + history: { + title: "History", + note: "Stored in this browser", + clear: "Clear history", + clearTitle: "Clear history?", + clearMessage: "The list in this browser will be removed. Files on the server are not affected.", + clearConfirm: "Clear", + cancel: "Cancel", + again: "Download again", + empty: "History is empty.", + }, + settings: { + title: "Settings", + done: "Done", + cookies: "Cookies", + cookiesNone: "No cookies yet", + cookiesLoaded: (domains: string, days: number): string => `Loaded for ${domains}, expires in ${days} days`, + cookiesChoose: "Choose cookies.txt", + cookiesRemove: "Remove", + cookiesNote: "Used for age-restricted videos or when YouTube asks to confirm you are not a bot.", + downloads: "Downloads", + retention: "Keep files on the server", + concurrency: "Simultaneous downloads", + decrease: "Fewer downloads", + increase: "More downloads", + defaultFormat: "Default", + defaultFormats: { "video-mp4-1080": "Video MP4 1080p", "video-mp4-720": "Video MP4 720p", "audio-m4a": "Audio M4A", "audio-mp3": "Audio MP3 320 kbps" }, + appearance: "Appearance", + theme: "Theme", + themeSystem: "System", + themeLight: "Light", + themeDark: "Dark", + accent: "Accent color", + accents: { teal: "Teal", blue: "Blue", purple: "Purple", pink: "Pink", orange: "Orange", green: "Green", graphite: "Graphite" }, + language: "Language", + languages: { auto: "Automatic", vi: "Tiếng Việt", en: "English" }, + access: "Access", + passwordOn: "Password protection is on", + passwordOff: "Password protection is off", + passwordNote: "Set OPENMEDIA_PASSWORD on the server to turn it on.", + signOut: "Sign out", + storage: "Storage", + storageUsed: (used: string, total: string): string => `${used} used of ${total}`, + storageUsedUnlimited: (used: string, free: string): string => `${used} used, ${free} free`, + }, + shortcuts: { title: "Keyboard shortcuts", focus: "Enter a link", pasteFetch: "Paste and get info", close: "Close panel", show: "Show shortcuts", dismiss: "Close" }, + auth: { title: "Sign in to OpenMedia", password: "Password", submit: "Sign in", wrong: "The password is not correct." }, + island: { + fetched: "Info ready", + playlistAdded: (count: number): string => `Added ${count} videos from the playlist`, + downloaded: (title: string): string => `Downloaded: ${title}`, + cancelled: "Download cancelled", + removedFromQueue: "Removed from queue", + addedAgain: "Added back to the queue", + cookiesLoaded: "Cookies loaded", + cookiesRemoved: "Cookies removed", + settingsSaved: "Settings saved", + }, + errors: { + invalid_url: "That does not look like a link. Paste an address that starts with http:// or https://.", + unsupported_url: "This site is not supported.", + private_network: "Links to private or local network addresses are blocked on this server.", + invalid_option: "One of the download options is not valid.", + not_found: "That item no longer exists on the server.", + file_not_ready: "The file is not ready yet.", + rate_limited: "Too many requests. Wait a moment and try again.", + auth_required: "Sign in to continue.", + invalid_password: "The password is not correct.", + cross_site_request: "The request was blocked because it came from another website.", + storage_full: "Server storage is full. Remove finished downloads first.", + too_large: "The file is larger than this server allows.", + bot_check: "The site asked to confirm you are not a bot. Add cookies in Settings.", + private_video: "This video is private.", + geo_blocked: "This video is not available in the server's region.", + unavailable: "This video is unavailable.", + timeout: "The site took too long to respond. Try again.", + extractor_error: "The site could not be read. Try again later.", + invalid_cookies: "That file is not a cookies.txt file in Netscape format.", + api_unreachable: "The OpenMedia server is not reachable.", + unknown_error: "Something went wrong. Try again.", + }, +}; + +export type Messages = typeof en; +``` + +`apps/web/src/lib/i18n/vi.ts` exports `export const vi: Messages = { ... }` with the same keys in natural Vietnamese (use the prototype copy for wording, for example "Hàng đợi", "Dán liên kết YouTube, TikTok, SoundCloud...", "Lấy thông tin", "Tải tất cả (n)", "Tối đa n lượt tải cùng lúc", "Kéo hai tay nắm vàng, hoặc chọn một tay nắm rồi dùng phím mũi tên.", "Dùng cho video giới hạn tuổi hoặc khi YouTube yêu cầu xác minh.", "Đặt biến OPENMEDIA_PASSWORD để bật."). Every function keeps the English parameter list; numbers are formatted by the caller. No en or em dash characters. + +- [ ] **Step 6: Port the design tokens and base styles** + +Replace `apps/web/src/app/globals.css` with the concatenation of the prototype's `tokens.css` and `base.css`, with these changes: + +- Keep `@import "tailwindcss";` as the first line. +- Change the bare `:root` accent block to the teal defaults (`--accent-l: #12939c; --accent-d: #3fbac2; --accent-text-l: #0b7178; --accent-text-d: #5cc9d0;`), add a `:root[data-accent="blue"]` rule with the prototype's blue values, and delete the teal rule that duplicates the defaults. +- Set `--font-text` to `-apple-system, BlinkMacSystemFont, "SF Pro Text", var(--font-inter), "Segoe UI Variable Text", system-ui, sans-serif`, `--font-display` to the same with `"SF Pro Display"`, `--font-mono` to `ui-monospace, "SF Mono", var(--font-geist-mono), Menlo, Consolas, monospace`, and add `--font-brand: var(--font-geist-sans), var(--font-text)`. +- Remove the `body { overflow: hidden; }` line from `base.css` only for widths below 768 px (phones scroll the document); keep it for 768 px and up. +- Delete the `.brand-mark`, `.capsule`, `.icon-button`, `kbd`, `.sample-tag`, `.glass` rules from the global file; those move into component modules in Task 10. Keep resets, tokens, keyframes, `:focus-visible`, `.visually-hidden` and the reduced-motion block. + +- [ ] **Step 7: Replace the layout and page** + +`apps/web/src/app/layout.tsx`: + +```tsx +import type { Metadata, Viewport } from "next"; +import { Geist, Geist_Mono, Inter } from "next/font/google"; +import type { ReactNode } from "react"; +import { THEME_BOOTSTRAP_SCRIPT } from "@/lib/theme"; +import "./globals.css"; + +const geistSans = Geist({ variable: "--font-geist-sans", subsets: ["latin", "latin-ext"] }); +const geistMono = Geist_Mono({ variable: "--font-geist-mono", subsets: ["latin", "latin-ext"] }); +const inter = Inter({ variable: "--font-inter", subsets: ["latin", "latin-ext", "vietnamese"] }); + +export const metadata: Metadata = { + title: "OpenMedia", + description: "Download videos from almost any website. Lightweight, self-hosted media downloader with a clean web UI.", + applicationName: "OpenMedia", + appleWebApp: { capable: true, title: "OpenMedia", statusBarStyle: "black-translucent" }, +}; + +export const viewport: Viewport = { + width: "device-width", + initialScale: 1, + viewportFit: "cover", + themeColor: [ + { media: "(prefers-color-scheme: light)", color: "#f5f5f7" }, + { media: "(prefers-color-scheme: dark)", color: "#1e1e20" }, + ], +}; + +export default function RootLayout({ children }: { children: ReactNode }): ReactNode { + return ( + + +