From 4ceece08c974d12a389d06309acb1ccca8b7562b Mon Sep 17 00:00:00 2001 From: Nika Siradze Date: Thu, 1 Oct 2026 03:58:54 +0400 Subject: [PATCH] Edit a full multicam episode: sync every camera and mic, cut to the speaker podcli could only start from one finished video. A podcast recorded on three cameras and two recorders had to be synced and cut somewhere else first. Point podcli at the folder of one episode's recordings. It guesses who each file belongs to from its name, syncs everything by audio, and cuts cameras to whoever is talking. Crosstalk and long monologues go to the wide shot. The result renders as an MP4 with one WAV per person, or exports as a Premiere XML or FCPXML timeline that points at the original files. Sync cross-correlates onset envelopes over the full length of both files, then refines with GCC-PHAT at several checkpoints and fits a line through them. The line gives the offset and the clock drift between devices, which the render and both timelines correct. A file that shares no sound with the reference gets a second try against every file that did sync, so split camera parts line up without a continuous mic. Audio that starts late inside a video file is anchored to container time, and offsets set by hand survive a re-sync. Who is talking comes from each person's mic, normalized to that mic's own speech level so bleed and a hot preamp don't decide the cut. Shows with one shared mic fall back to speaker detection. The same edit runs from the studio (Multicam edit), the CLI (podcli multicam ) and the manage_multicam MCP tool. All three share one saved edit per set of files, so re-running reuses the sync and the cut, and an unchanged render is skipped. A job holds its edit until it finishes, so a second job or an edit made meanwhile is refused instead of lost. --- CLAUDE.md | 8 +- README.md | 14 +- backend/cli.py | 270 ++++ backend/main.py | 69 + .../services/integrations/_shared/fcpxml.py | 7 +- .../integrations/_shared/multicam_xml.py | 272 ++++ backend/services/multicam.py | 1195 +++++++++++++++++ backend/services/multicam_signal.py | 464 +++++++ cli/main.go | 3 +- src/models/index.ts | 2 +- src/server.ts | 89 ++ src/ui/client/Layout.tsx | 3 + src/ui/client/icons.tsx | 3 +- src/ui/client/main.tsx | 2 + src/ui/client/multicam-cut.tsx | 139 ++ src/ui/client/multicam-deliver.tsx | 65 + src/ui/client/multicam-look.tsx | 43 + src/ui/client/multicam-page.tsx | 246 ++++ src/ui/client/multicam-person-chip.tsx | 31 + src/ui/client/multicam-sources.tsx | 176 +++ src/ui/client/multicam-start.tsx | 125 ++ src/ui/client/multicam-sync.tsx | 127 ++ src/ui/client/multicam-types.ts | 139 ++ src/ui/public/css/styles.css | 7 + src/ui/web-server.ts | 159 ++- tests/test_multicam.py | 449 +++++++ tests/test_multicam_signal.py | 220 +++ 27 files changed, 4316 insertions(+), 11 deletions(-) create mode 100644 backend/services/integrations/_shared/multicam_xml.py create mode 100644 backend/services/multicam.py create mode 100644 backend/services/multicam_signal.py create mode 100644 src/ui/client/multicam-cut.tsx create mode 100644 src/ui/client/multicam-deliver.tsx create mode 100644 src/ui/client/multicam-look.tsx create mode 100644 src/ui/client/multicam-page.tsx create mode 100644 src/ui/client/multicam-person-chip.tsx create mode 100644 src/ui/client/multicam-sources.tsx create mode 100644 src/ui/client/multicam-start.tsx create mode 100644 src/ui/client/multicam-sync.tsx create mode 100644 src/ui/client/multicam-types.ts create mode 100644 tests/test_multicam.py create mode 100644 tests/test_multicam_signal.py diff --git a/CLAUDE.md b/CLAUDE.md index 5b9ea426..0ffafcd9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -31,7 +31,7 @@ Both share the same knowledge base at `.podcli/knowledge/`. ## MCP tools (podcli engine) -All 26 tools registered by the MCP server. +All 27 tools registered by the MCP server. **Transcription and input** @@ -57,6 +57,12 @@ All 26 tools registered by the MCP server. | `manage_reel` | Build a highlights reel: detect once, edit moments, rebuild without re-detecting | | `analyze_energy` | Analyze audio energy levels to find high-energy moments | +**Full-episode editing** + +| Tool | What it does | +|------|-------------| +| `manage_multicam` | Map camera and mic files to people, sync them by audio, auto-cut to the speaker, render or export to Premiere / FCPXML | + **Content and configuration** | Tool | What it does | diff --git a/README.md b/README.md index 2e27d7f8..79b22aff 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Runs on macOS (Apple Silicon), Linux (x64 and arm64), and Windows (x64). Intel M ```bash podcli # interactive menu, opens the web studio podcli process episode.mp4 # transcribe, pick moments, render clips +podcli multicam ~/ep12 # sync every camera and mic, cut to the speaker, render the episode ``` Clips land in `podcli-clips/` in the directory you ran it from, so each show keeps its own renders. Everything else (knowledge, presets, assets, clip history, cache) lives in one managed folder that follows you between directories. Set `PODCLI_OUTPUT` to render somewhere fixed instead. @@ -70,6 +71,13 @@ Clips land in `podcli-clips/` in the directory you ran it from, so each show kee - Logos, intros, outros, and background music from a reusable asset library - Loudness-normalized audio and hardware encoding on VideoToolbox, NVENC, and VAAPI, with a CPU fallback +**Editing the full episode** + +- Multicam edit: point at one folder of camera and mic files, confirm who each file belongs to, and podcli syncs them by audio and corrects clock drift between devices +- Automatic camera cuts that follow whoever is talking, with wide shots for crosstalk and long monologues +- Render the episode as an MP4 with one WAV per person, or export a Premiere XML or FCPXML timeline that points at your original files +- Same edit from the studio (Multicam edit), the CLI (`podcli multicam `), or an agent (`manage_multicam`). Re-running reuses the sync and the cut, so only the step you changed runs again + **Finding the moments** - Whisper transcription with speaker diarization, or bring your own transcript as `.txt`, `.srt`, or `.vtt` @@ -79,7 +87,7 @@ Clips land in `podcli-clips/` in the directory you ran it from, so each show kee **The studio at `localhost:3847`** -- Library, episode workspace, per-clip detail, highlights, thumbnails, content, analytics, assets, knowledge, config, integrations, and MCP setup +- Library, episode workspace, multicam edit, per-clip detail, highlights, thumbnails, content, analytics, assets, knowledge, config, integrations, and MCP setup - `⌘K` command palette across pages, clips, and assets - Titles, descriptions, tags, and hashtags, with any section regenerated on your own guidance - Thumbnail studio for 16:9 and 9:16, with frame and text options @@ -87,7 +95,7 @@ Clips land in `podcli-clips/` in the directory you ran it from, so each show kee **Shipping it** -- 26 MCP tools, so an agent can transcribe, score, render, and publish through conversation +- 27 MCP tools, so an agent can transcribe, score, render, and publish through conversation - YouTube publishing plus performance analytics to see which clips landed - DaVinci Resolve export as FCPXML when you want to finish by hand - Presets, clip history with duplicate detection, and a transcript cache @@ -98,7 +106,7 @@ If you are weighing podcli against the cloud clippers, this is the difference: - Runs locally. Transcription and rendering happen on your machine by default, so episodes stay there. Only the optional cloud engine (AssemblyAI) and publishing to YouTube send anything out. - Free and open source under AGPL-3.0. Exports are unlimited, full quality, and watermark-free. -- Agent-native. 26 MCP tools let Claude Code or Codex drive the whole flow, transcription through publishing. +- Agent-native. 27 MCP tools let Claude Code or Codex drive the whole flow, transcription through publishing. - A knowledge base keeps titles, captions, and descriptions in your show's voice, and stops the engine from resuggesting moments you already published. - DaVinci Resolve handoff. Export any clip as FCPXML when you want to finish the edit yourself. diff --git a/backend/cli.py b/backend/cli.py index 54c62f56..f3f74009 100644 --- a/backend/cli.py +++ b/backend/cli.py @@ -629,6 +629,240 @@ def _show(session): "edit N [secs] | build | delete ") +def _fmt_time(seconds: float) -> str: + seconds = int(seconds) + h, rem = divmod(seconds, 3600) + return f"{h}:{rem // 60:02d}:{rem % 60:02d}" if h else f"{rem // 60}:{rem % 60:02d}" + + +def _multicam_table(session) -> None: + names = {p.id: p.name for p in session.people} + names.update({"wide": "Everyone (wide)", "": "Shared room mic"}) + print(f"\n {session.name} · people: {', '.join(p.name for p in session.people)} · session {session.session_id}") + print(f" {'#':>2} {'File':<28} {'Role':<7} {'Who':<22} {'Length':>8} {'Format':<10} Sync") + for i, s in enumerate(session.sources, 1): + if s.role == "ignore": + who = "" + elif s.channel_people: + who = " / ".join(names.get(p, p) for p in s.channel_people) + else: + who = names.get(s.person, s.person) + size = f"{s.width}x{s.height}" if s.kind == "video" else f"{s.audio_channels}ch" + sync = {"reference": "reference", "ok": "synced", "rough": "rough sync", "manual": "set by hand", + "failed": "not synced"}.get(s.sync.get("status", ""), "") + if s.synced: + drift = s.sync.get("drift_ppm") or 0 + sync = f"{s.offset:+.3f}s {sync}" + (f", drift {drift:+.0f} ppm" if abs(drift) >= 5 else "") + flag = " (guessed)" if s.guessed and s.role != "ignore" else "" + name = os.path.basename(s.path) + print(f" {i:>2} {name[:28]:<28} {s.role:<7} {who[:22]:<22} {_fmt_time(s.duration):>8} {size:<10} {sync}{flag}") + + +def _multicam_progress(label: str): + """One self-overwriting status line on a terminal, plain lines otherwise.""" + tty = sys.stdout.isatty() + last = [""] + + def report(percent: int, message: str) -> None: + line = f" {label} {percent:>3}% {message}"[:110] + if tty: + print("\r" + line.ljust(len(last[0])), end="", flush=True) + last[0] = line + elif message != last[0]: + print(line) + last[0] = message + + def done() -> None: + if tty and last[0]: + print() + + return report, done + + +def _apply_multicam_fixes(session, fixes: list[str]): + """--set FILE=ROLE[:PERSON[,PERSON]] edits, matched by 1-based number or filename.""" + from services.multicam import update_mapping + + people = {p.name.lower(): p.id for p in session.people} | {p.id: p.id for p in session.people} + people.update({"wide": "wide", "room": "", "shared": ""}) + edits = [] + for fix in fixes: + target, _, value = fix.partition("=") + role, _, who = value.partition(":") + role = {"cam": "camera", "mic": "mic", "camera": "camera", "ignore": "ignore", "skip": "ignore"}.get(role.strip().lower()) + if not target or not role: + raise ValueError(f"Can't read --set {fix!r}. Use FILE=camera:NAME, FILE=mic:NAME, or FILE=ignore") + target = target.strip() + if target.isdigit() and 1 <= int(target) <= len(session.sources): + source = session.sources[int(target) - 1] + else: + matches = [s for s in session.sources if os.path.basename(s.path).lower().startswith(target.lower())] + if len(matches) != 1: + raise ValueError(f"--set {fix!r} matches {len(matches)} files. Use the file's number from the table.") + source = matches[0] + edit = {"id": source.id, "role": role} + names = [n.strip().lower() for n in who.split(",") if n.strip()] + unknown = [n for n in names if n not in people] + if unknown: + raise ValueError(f"Unknown person {unknown[0]!r}. People: {', '.join(p.name for p in session.people)}") + if role == "mic" and len(names) == 2: + edit["channel_people"] = [people[n] for n in names] + elif names: + edit["person"] = people[names[0]] + edit["channel_people"] = [] + edits.append(edit) + return update_mapping(session, {"sources": edits}) if edits else session + + +def _review_multicam_interactively(session): + """Let the user fix guessed roles before the slow steps. Returns None to quit.""" + import questionary + from services.multicam import rename_people, update_mapping + + while True: + _multicam_table(session) + choice = questionary.select( + "Does this look right?", + choices=[ + questionary.Choice("Yes, sync and cut", value="go"), + questionary.Choice("Fix a file", value="fix"), + questionary.Choice("Rename people", value="people"), + questionary.Choice("Quit", value="quit"), + ], + ).ask() + if choice in (None, "quit"): + return None + if choice == "go": + # Confirming the table counts as mapping every file, so the next run doesn't ask again. + return update_mapping(session, {"sources": [{"id": s.id, "role": s.role} for s in session.sources]}) + if choice == "people": + raw = questionary.text("People, comma separated:", default=", ".join(p.name for p in session.people)).ask() + if raw: + session = rename_people(session, [n.strip() for n in raw.split(",") if n.strip()]) + continue + source = questionary.select( + "Which file?", + choices=[questionary.Choice(os.path.basename(s.path), value=s) for s in session.sources], + ).ask() + if source is None: + continue + roles = ["camera", "mic", "ignore"] if source.kind == "video" else ["mic", "ignore"] + role = questionary.select("It is a", choices=roles, default=source.role if source.role in roles else roles[0]).ask() + if role is None: + continue + edit = {"id": source.id, "role": role} + if role != "ignore": + options = [questionary.Choice(p.name, value=p.id) for p in session.people] + options.append(questionary.Choice("Everyone (wide)", value="wide") if role == "camera" + else questionary.Choice("Shared room mic", value="")) + who = questionary.select("Who does it show or record?" if role == "camera" else "Whose voice?", choices=options).ask() + if who is None: + continue + edit.update({"person": who, "channel_people": []}) + session = update_mapping(session, {"sources": [edit]}) + + +def _open_multicam_session(target: str, people): + from services import multicam as mc + + if not os.path.isdir(target): + session = mc.MulticamSession.load(target) + return mc.rename_people(session, people) if people else session + report, done = _multicam_progress("Reading files") + session = mc.new_session(folder=target, people=people, progress_callback=report) + done() + return session + + +def cmd_multicam(args): + """Edit a full multicam episode: map every file, sync, auto-cut, then render or export.""" + from services import multicam as mc + + target = _clean_path(args.target) if args.target else "" + if target == "list": + sessions = mc.list_sessions() + if not sessions: + print(" No multicam edits yet. Start one with: podcli multicam ") + for s in sessions: + state = "rendered" if s["video"] else f"{s['shots']} shots" if s["shots"] else "synced" if s["synced"] else "mapped" + print(f" {s['session_id']} {s['name'][:32]:<32} {s['sources']} files {state}") + return + if not target: + print(" Usage: podcli multicam [options]\n" + " Example: podcli multicam ~/Podcasts/ep12 --people \"Nika, Ana\"") + sys.exit(2) + + try: + if args.delete: + session = (mc.find_session(mc.collect_files(target)) if os.path.isdir(target) + else mc.MulticamSession.load(target)) + if not session: + raise ValueError(f"No multicam edit for {target}") + mc.delete_session(session.session_id) + print(f" ✓ Deleted multicam edit {session.session_id}. Your recordings are untouched.") + return + + people = [n.strip() for n in (args.people or "").split(",") if n.strip()] or None + session = _apply_multicam_fixes(_open_multicam_session(target, people), args.set or []) + + guessed = any(s.guessed and s.role != "ignore" for s in session.sources) + if guessed and sys.stdin.isatty() and sys.stdout.isatty() and not args.yes: + session = _review_multicam_interactively(session) + if session is None: + return + else: + _multicam_table(session) + if guessed: + print(" Using the guessed roles above. Fix any with --set FILE=ROLE:NAME.") + + if args.resync or mc.needs_sync(session): + report, done = _multicam_progress("Syncing") + session = mc.sync_session(session, force=args.resync, progress_callback=report) + done() + _multicam_table(session) + failed = [s for s in session.sources if s.role != "ignore" and not s.synced] + if failed: + names = ", ".join(os.path.basename(s.path) for s in failed) + print(f"\n ✗ Couldn't sync {names}. Ignore it with --set {os.path.basename(failed[0].path)}=ignore, " + "or set its offset in the studio (podcli ui).") + sys.exit(1) + + cut_edits = { + **({"range_start": args.start} if args.start is not None else {}), + **({"range_end": args.end} if args.end is not None else {}), + **({"cut_settings": {k: v for k, v in (("min_shot", args.min_shot), ("max_shot", args.max_shot)) if v is not None}} + if args.min_shot is not None or args.max_shot is not None else {}), + **({"look": args.look} if args.look else {}), + } + if cut_edits: + session = mc.update_mapping(session, cut_edits) + if not session.cuts: + report, done = _multicam_progress("Cutting") + session = mc.plan_session(session, progress_callback=report) + done() + stats = mc.cut_stats(session) + names = {s.id: os.path.basename(s.path) for s in session.sources} + share = ", ".join(f"{names[k]} {v:.0%}" for k, v in sorted(stats["share"].items(), key=lambda kv: -kv[1])) + print(f"\n {stats['shots']} shots over {_fmt_time(stats['duration'])}, average {stats['average_shot']:.1f}s ({share})") + + for fmt in (["premiere", "fcpxml"] if args.export == "all" else [args.export] if args.export else []): + print(f" ✓ {mc.export_xml(session, fmt)}") + if not args.no_render: + report, done = _multicam_progress("Rendering") + outputs = mc.render_session(session, stems=not args.no_stems, progress_callback=report) + done() + for path in [outputs["video"], *(outputs.get("stems") or [])]: + print(f" ✓ {path}") + print(f"\n Change anything and re-run: podcli multicam {session.session_id} [options]") + video = session.outputs.get("video") + if video and os.path.exists(video): + print(f" Make clips from it: podcli process \"{video}\"") + except (ValueError, OSError, RuntimeError, ImportError, TypeError) as e: + sys.stdout.flush() + print(f" ✗ {e}", file=sys.stderr) + sys.exit(1) + + def cmd_process(args): """Full auto pipeline: transcribe → suggest → export.""" from services.clip_generator import generate_clip @@ -4554,6 +4788,40 @@ def main(): rbd = reel_sub.add_parser("build", help="Rebuild the reel (re-cuts only changed moments)") rbd.add_argument("session") + # ── multicam (full-episode edit from several cameras and mics) ── + mc_p = sub.add_parser( + "multicam", + help="Edit a multicam episode: sync cameras and mics, auto-cut to the speaker, render or export", + description="Point at one episode's folder. podcli guesses who each file belongs to, syncs everything by audio, " + "cuts to whoever is talking, and renders an MP4. Re-run with the session id to change anything; " + "finished steps are reused.", + epilog="Examples:\n" + " podcli multicam ~/Podcasts/ep12 --people \"Nika, Ana\"\n" + " podcli multicam ~/Podcasts/ep12 --set cam_b.mp4=camera:Ana --set zoom_lr.wav=mic:Nika,Ana -y\n" + " podcli multicam 3f9c2a1b7e40 --export premiere --no-render\n" + " podcli multicam list", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + mc_p.add_argument("target", nargs="?", help="Episode folder, a session id to resume, or 'list'") + mc_p.add_argument("--people", help="Comma-separated speaker names (default: Host, Guest)") + mc_p.add_argument("--set", action="append", metavar="FILE=ROLE[:NAME]", + help="Fix a file's role: camera:NAME, camera:wide, mic:NAME, mic:NAME,NAME (stereo L/R), mic:room, or ignore. " + "FILE is its number in the table or the start of its name. Repeatable.") + mc_p.add_argument("--start", type=float, help="Episode start on the synced timeline, in seconds") + mc_p.add_argument("--end", type=float, help="Episode end on the synced timeline, in seconds") + mc_p.add_argument("--min-shot", type=float, dest="min_shot", help="Shortest shot in seconds (default 2)") + mc_p.add_argument("--max-shot", type=float, dest="max_shot", + help="Cut to the wide shot after this many seconds on one speaker; 0 never (default 30)") + mc_p.add_argument("--look", choices=["none", "natural", "warm", "contrast"], help="Color look for the render") + mc_p.add_argument("--export", choices=["premiere", "fcpxml", "all"], + help="Also write an editor timeline that points at the original files") + mc_p.add_argument("--no-render", action="store_true", dest="no_render", help="Skip the MP4 render (fast, export only)") + mc_p.add_argument("--no-stems", action="store_true", dest="no_stems", help="Skip the per-person WAV files") + mc_p.add_argument("--resync", action="store_true", + help="Sync every file again, including offsets you set by hand") + mc_p.add_argument("-y", "--yes", action="store_true", help="Don't stop to review guessed roles") + mc_p.add_argument("--delete", action="store_true", help="Delete this multicam edit (source files are untouched)") + # ── studio ── studio = sub.add_parser("studio", help="Cut a fragment + add Remotion intro/outro (follow-us) bookends") studio.add_argument("video", nargs="?", default=None, help="Path to the source video (omit only with --save-brand)") @@ -4892,6 +5160,8 @@ def main(): cmd_studio(args) elif args.command == "reel": cmd_reel(args) + elif args.command == "multicam": + cmd_multicam(args) elif args.command == "templates": cmd_templates(args) elif args.command == "thumbnails": diff --git a/backend/main.py b/backend/main.py index da7e2377..49d96a73 100644 --- a/backend/main.py +++ b/backend/main.py @@ -963,6 +963,74 @@ def handle_render_silence_removed(task_id: str, params: dict): emit_result(task_id, "error", error=str(e)) +MULTICAM_MAP_KEYS = ("people", "sources", "range_start", "range_end", "cut_settings", "speaker_map", "look") + + +def handle_manage_multicam(task_id: str, params: dict): + """Multicam podcast editing: map sources, sync, plan cuts, render, export. + + Actions: new (folder or files, people), list, show, map, sync, plan, cut + (swap one shot's camera), previews, render, export (premiere|fcpxml), delete. + sync, plan and render apply any mapping fields sent with them first. + """ + from services import multicam as mc + + action = params.get("action", "show") + + def progress(stage): + return lambda p, m: emit_progress(task_id, stage, p, m) + + try: + if action == "list": + emit_result(task_id, "success", data={"sessions": mc.list_sessions()}) + return + if action == "new": + session = mc.new_session( + folder=str(params.get("folder") or ""), + files=params.get("files"), + people=params.get("people"), + name=str(params.get("name") or ""), + progress_callback=progress("scanning"), + ) + emit_result(task_id, "success", data=mc.payload(session)) + return + + session_id = params.get("session_id") + if not isinstance(session_id, str) or not session_id: + raise ValueError("session_id is required. Get one from action 'new' or 'list'.") + if action == "delete": + mc.delete_session(session_id) + emit_result(task_id, "success", data={"deleted": True, "session_id": session_id}) + return + + session = mc.MulticamSession.load(session_id) + if action in ("map", "sync", "plan", "render") and any(k in params for k in MULTICAM_MAP_KEYS): + session = mc.update_mapping(session, params) + data: dict = {} + if action in ("show", "map"): + pass + elif action == "sync": + session = mc.sync_session(session, force=bool(params.get("force")), progress_callback=progress("syncing")) + elif action == "plan": + session = mc.plan_session(session, progress_callback=progress("planning")) + elif action == "cut": + index, source_id = params.get("index"), params.get("source_id") + if not isinstance(index, int) or not isinstance(source_id, str): + raise ValueError("cut needs index (0-based shot number) and source_id (a camera's id)") + session = mc.set_cut(session, index, source_id) + elif action == "previews": + data["previews"] = mc.previews(session, looks=bool(params.get("looks")), at=params.get("at")) + elif action == "render": + mc.render_session(session, stems=params.get("stems", True), progress_callback=progress("rendering")) + elif action == "export": + data["export_path"] = mc.export_xml(session, params.get("format", "premiere")) + else: + raise ValueError(f"Unknown multicam action {action!r}") + emit_result(task_id, "success", data={**mc.payload(session), **data}) + except (IndexError, ValueError, OSError, RuntimeError, ImportError, TypeError, AttributeError) as e: + emit_result(task_id, "error", error=str(e)) + + def handle_run_integration_tool(task_id: str, params: dict): from services.integrations import IntegrationRegistry, IntegrationsManager @@ -1022,6 +1090,7 @@ def handle_run_integration_tool(task_id: str, params: dict): "manage_config": handle_manage_config, "analyze_silence": handle_analyze_silence, "render_silence_removed": handle_render_silence_removed, + "manage_multicam": handle_manage_multicam, } diff --git a/backend/services/integrations/_shared/fcpxml.py b/backend/services/integrations/_shared/fcpxml.py index 8da54ca5..dbc6a170 100644 --- a/backend/services/integrations/_shared/fcpxml.py +++ b/backend/services/integrations/_shared/fcpxml.py @@ -1,7 +1,6 @@ """FCPXML 1.10 primitives shared by FCPXML-consuming editors (Resolve, FCP, Premiere).""" from __future__ import annotations -import urllib.parse import xml.etree.ElementTree as ET from fractions import Fraction from pathlib import Path @@ -45,7 +44,8 @@ def rational_time(frames: int, fps: float) -> str: def file_uri(p: Path) -> str: - return "file://" + urllib.parse.quote(str(p.resolve())) + # as_uri handles Windows drive letters and backslashes (file:///C:/...). + return p.resolve().as_uri() def make_format(format_id: str, fps: float, width: int, height: int) -> ET.Element: @@ -71,11 +71,12 @@ def make_asset( has_video: bool = True, has_audio: bool = False, audio_channels: int = 0, + start: str = "0s", ) -> ET.Element: attrs = { "id": asset_id, "name": name, - "start": "0s", + "start": start, "duration": rational_time(frames, fps), "hasVideo": "1" if has_video else "0", "format": format_id, diff --git a/backend/services/integrations/_shared/multicam_xml.py b/backend/services/integrations/_shared/multicam_xml.py new file mode 100644 index 00000000..229df2ae --- /dev/null +++ b/backend/services/integrations/_shared/multicam_xml.py @@ -0,0 +1,272 @@ +"""Editor timelines for a planned multicam session. + +write_xmeml emits FCP7 XML (xmeml v4), which Premiere Pro imports natively and +DaVinci Resolve also reads. write_fcpxml emits FCPXML 1.10 for Final Cut Pro and +Resolve. Both reference the original camera and mic files, so nothing is +re-encoded: V1 carries the camera cuts, and each person's mic sits on its own +audio track. + +Clock drift can't be expressed as a clip property in either format. When a mic +drifts, its track is split at every camera cut and each piece gets its own +in-point, which keeps it within a millisecond or two of picture. +""" +from __future__ import annotations + +import os +import xml.etree.ElementTree as ET +from fractions import Fraction +from pathlib import Path + +from . import fcpxml as fx + + +def _pathurl(path: str) -> str: + return "file://localhost" + Path(path).resolve().as_uri()[len("file://"):] + + +def _rate(parent: ET.Element, fps: float) -> None: + rate = ET.SubElement(parent, "rate") + ET.SubElement(rate, "timebase").text = str(int(round(fps))) + ET.SubElement(rate, "ntsc").text = "TRUE" if abs(fps - round(fps)) > 0.001 else "FALSE" + + +def _text(parent: ET.Element, tag: str, value) -> ET.Element: + el = ET.SubElement(parent, tag) + el.text = str(value) + return el + + +def _audio_pieces(session, source, start: float, end: float, fps: float) -> list[tuple[int, int, float]]: + """(timeline_frame_in, timeline_frame_out, source_seconds_at_in) for one mic across the episode.""" + lo = max(start, source.timeline_start()) + hi = min(end, source.timeline_end()) + if hi <= lo: + return [] + bounds = [lo, hi] + if abs(source.speed - 1.0) > 1e-6: + bounds = [lo] + [c["start"] for c in session.cuts if lo < c["start"] < hi] + [hi] + pieces = [] + for a, b in zip(bounds, bounds[1:]): + f0 = round((a - start) * fps) + f1 = round((b - start) * fps) + if f1 > f0: + pieces.append((f0, f1, source.source_time(start + f0 / fps))) + return pieces + + +def _label(session, source, channel: int, person: str) -> str: + names = {p.id: p.name for p in session.people} + base = names.get(person, "Room") + return f"{base} ({os.path.basename(source.path)}{'' if channel < 0 else f' ch{channel + 1}'})" + + +def _mic_tracks(session, *, split_stereo: bool = False) -> list[tuple[object, int, str]]: + """(source, channel, person) per audio track; channel -1 is the whole file. + + xmeml addresses one channel per track, so a stereo room mic becomes an L + and an R track there instead of losing its right side. + """ + feeds = [(s, ch, pid) for s, ch, pid in session.person_mics() if s.synced] + if not feeds: + rooms = [s for s in session.sources if s.role == "mic" and s.synced] + room = max(rooms, key=lambda s: s.duration) if rooms else session.source(session.reference_id) + feeds = [(room, -1, "")] + if not split_stereo: + return feeds + out = [] + for s, ch, pid in feeds: + out += [(s, 0, pid), (s, 1, pid)] if ch < 0 and s.audio_channels >= 2 else [(s, max(ch, 0), pid)] + return out + + +def write_xmeml(session, out_path: Path, *, width: int, height: int, fps: float) -> None: + from services.multicam import shot_frames + + start = session.cuts[0]["start"] + end = session.cuts[-1]["end"] + frames = shot_frames(session, fps) + shots = [s for s in frames if s[0]] + total = max(f0 + n for _, f0, n in frames) + + root = ET.Element("xmeml", {"version": "4"}) + seq = ET.SubElement(root, "sequence", {"id": "sequence-1"}) + _text(seq, "name", session.name) + _text(seq, "duration", total) + _rate(seq, fps) + media = ET.SubElement(seq, "media") + video = ET.SubElement(media, "video") + fmt = ET.SubElement(video, "format") + chars = ET.SubElement(fmt, "samplecharacteristics") + _rate(chars, fps) + _text(chars, "width", width) + _text(chars, "height", height) + _text(chars, "pixelaspectratio", "square") + vtrack = ET.SubElement(video, "track") + + declared: set[str] = set() + counter = [0] + + def file_el(parent: ET.Element, source) -> None: + fid = f"file-{source.id}" + if fid in declared: + ET.SubElement(parent, "file", {"id": fid}) + return + declared.add(fid) + f = ET.SubElement(parent, "file", {"id": fid}) + _text(f, "name", os.path.basename(source.path)) + _text(f, "pathurl", _pathurl(source.path)) + _rate(f, source.fps or fps) + _text(f, "duration", round(source.duration * (source.fps or fps))) + fm = ET.SubElement(f, "media") + if source.kind == "video": + v = ET.SubElement(fm, "video") + sc = ET.SubElement(v, "samplecharacteristics") + _rate(sc, source.fps or fps) + _text(sc, "width", source.width) + _text(sc, "height", source.height) + if source.has_audio: + a = ET.SubElement(fm, "audio") + _text(a, "channelcount", max(1, source.audio_channels)) + + def clipitem(track: ET.Element, source, f0: int, f1: int, src_seconds: float, + *, name: str, audio_channel: int | None = None) -> None: + counter[0] += 1 + item = ET.SubElement(track, "clipitem", {"id": f"clipitem-{counter[0]}"}) + _text(item, "name", name) + _text(item, "enabled", "TRUE") + _text(item, "duration", round(source.duration * fps)) + _rate(item, fps) + _text(item, "start", f0) + _text(item, "end", f1) + src_in = max(0, round(src_seconds * fps)) + _text(item, "in", src_in) + _text(item, "out", src_in + (f1 - f0)) + file_el(item, source) + if audio_channel is not None: + st = ET.SubElement(item, "sourcetrack") + _text(st, "mediatype", "audio") + _text(st, "trackindex", max(1, audio_channel + 1)) + + for source_id, f0, n in shots: + cam = session.source(source_id) + clipitem(vtrack, cam, f0, f0 + n, cam.source_time(start + f0 / fps), name=os.path.basename(cam.path)) + + audio = ET.SubElement(media, "audio") + _text(audio, "numOutputChannels", 2) + for source, channel, person in _mic_tracks(session, split_stereo=True): + track = ET.SubElement(audio, "track") + label = _label(session, source, channel, person) + for f0, f1, src_seconds in _audio_pieces(session, source, start, end, fps): + clipitem(track, source, f0, f1, src_seconds, name=label, audio_channel=channel) + _text(track, "enabled", "TRUE") + + ET.indent(ET.ElementTree(root), space=" ") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text( + '\n\n' + + ET.tostring(root, encoding="unicode") + "\n", + encoding="utf-8", + ) + + +def write_fcpxml(session, out_path: Path, *, width: int, height: int, fps: float) -> None: + from services.multicam import shot_frames + + start = session.cuts[0]["start"] + end = session.cuts[-1]["end"] + frames = shot_frames(session, fps) + total = max(f0 + n for _, f0, n in frames) + + def t(frame_count: int) -> str: + return fx.rational_time(frame_count, fps) + + def src_t(source, seconds: float) -> str: + # Asset times start at the file's own timecode. Video lands on its frame + # grid; audio stays sample-accurate so two mics never sit a frame apart. + seconds = max(0.0, seconds) + source.timecode + if source.kind == "audio": + return fx.seconds_to_time(Fraction(round(seconds * 48000), 48000)) + rate = source.fps or fps + return fx.rational_time(round(seconds * rate), rate) + + fmt_id = "r1" + resources = [fx.make_format(fmt_id, fps, width, height)] + asset_ids: dict[str, str] = {} + formats: dict[str, str] = {} + used = {s for s, _, _ in frames if s} | {s.id for s, _, _ in _mic_tracks(session)} + for i, source in enumerate(s for s in session.sources if s.id in used): + aid = f"a{i + 1}" + asset_ids[source.id] = aid + if source.kind == "video": + key = f"{source.width}x{source.height}@{round(source.fps or fps, 3)}" + if key not in formats: + formats[key] = f"r{len(formats) + 2}" + resources.append(fx.make_format(formats[key], source.fps or fps, source.width, source.height)) + resources.append(fx.make_asset( + asset_id=aid, + name=os.path.basename(source.path), + media_path=Path(source.path), + frames=round(source.duration * (source.fps or fps)), + fps=source.fps or fps, + format_id=formats[key], + has_video=True, + has_audio=source.has_audio, + audio_channels=source.audio_channels, + start=src_t(source, 0.0), + )) + else: + asset = ET.Element("asset", { + "id": aid, + "name": os.path.basename(source.path), + "start": src_t(source, 0.0), + "duration": fx.seconds_to_time(Fraction(round(source.duration * 48000), 48000)), + "hasVideo": "0", + "hasAudio": "1", + "audioSources": "1", + "audioChannels": str(max(1, source.audio_channels)), + "audioRate": "48000", + }) + ET.SubElement(asset, "media-rep", {"kind": "original-media", "src": fx.file_uri(Path(source.path))}) + resources.append(asset) + + library = ET.Element("library") + event = ET.SubElement(library, "event", {"name": session.name}) + project = ET.SubElement(event, "project", {"name": f"{session.name} multicam"}) + seq = ET.SubElement(project, "sequence", { + "format": fmt_id, "tcStart": "0s", "tcFormat": fx.tc_format(fps), "duration": t(total), + }) + spine = ET.SubElement(seq, "spine") + # A full-length gap anchors everything: picture on lane 1, mics on negative + # lanes, so every clip is positioned against the same zero. + gap = ET.SubElement(spine, "gap", {"name": "Episode", "offset": "0s", "start": "0s", "duration": t(total)}) + + for source_id, f0, n in frames: + if not source_id: + continue + cam = session.source(source_id) + ET.SubElement(gap, "asset-clip", { + "ref": asset_ids[source_id], + "lane": "1", + "offset": t(f0), + "start": src_t(cam, cam.source_time(start + f0 / fps)), + "duration": t(n), + "name": os.path.basename(cam.path), + "srcEnable": "video", + }) + + for lane, (source, channel, person) in enumerate(_mic_tracks(session), start=1): + for f0, f1, src_seconds in _audio_pieces(session, source, start, end, fps): + clip = ET.SubElement(gap, "asset-clip", { + "ref": asset_ids[source.id], + "lane": str(-lane), + "offset": t(f0), + "start": src_t(source, src_seconds), + "duration": t(f1 - f0), + "name": _label(session, source, channel, person), + "audioRole": "dialogue", + "srcEnable": "audio", + }) + if channel >= 0: + ET.SubElement(clip, "audio-channel-source", {"srcCh": str(channel + 1), "role": "dialogue"}) + + fx.write_fcpxml(out_path, resources, library) diff --git a/backend/services/multicam.py b/backend/services/multicam.py new file mode 100644 index 00000000..52256d93 --- /dev/null +++ b/backend/services/multicam.py @@ -0,0 +1,1195 @@ +"""Multicam podcast editing sessions: map sources, sync, cut, and deliver. + +A session holds every camera and microphone file for one recording, who each +one belongs to, and how each sits on a shared timeline. The flow is: + + new (scan + guess roles) -> map (user fixes roles) -> sync -> plan -> render / export + +Sessions persist as JSON under .podcli/multicam so the UI, MCP, and CLI all edit +the same state. Heavy intermediates (8 kHz sync audio, previews) live under the +working dir and are safe to delete. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import tempfile +import threading +import uuid +import wave +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Callable, Optional + +import numpy as np + +from config.paths import paths +from services import multicam_signal as sig +from services.media_probe import get_video_info +from utils.proc import run as proc_run + +ProgressCallback = Optional[Callable[[int, str], None]] + +SYNC_RATE = 8000 +MEDIA_EXTS = {".mp4", ".mov", ".mkv", ".m4v", ".avi", ".mts", ".mxf", ".webm", + ".wav", ".mp3", ".m4a", ".aac", ".flac", ".aif", ".aiff"} + +LOOKS = { + "none": "", + "natural": "eq=contrast=1.04:saturation=1.06", + "warm": "colorbalance=rs=0.05:bs=-0.05:rm=0.03:bm=-0.03,eq=contrast=1.03:saturation=1.08", + "contrast": "curves=preset=medium_contrast,eq=saturation=1.1", +} + +DEFAULT_CUT = {"min_shot": 2.0, "max_shot": 30.0, "wide_insert": 4.0} + + +def _emit(callback: ProgressCallback, percent: float, message: str) -> None: + if callback: + callback(max(0, min(100, int(percent))), message) + + +def _sessions_dir() -> Path: + d = Path(os.path.dirname(paths["packed"])) / "multicam" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _work_dir(session_id: str) -> Path: + d = Path(paths["working"]) / "multicam" / session_id + d.mkdir(parents=True, exist_ok=True) + return d + + +@dataclass +class Person: + id: str + name: str + + +@dataclass +class Source: + id: str + path: str + kind: str # "video" | "audio" + duration: float + has_audio: bool + audio_channels: int = 0 + width: int = 0 + height: int = 0 + fps: float = 0.0 + timecode: float = 0.0 # embedded start timecode in seconds; editors address media from here + role: str = "ignore" # "camera" | "mic" | "ignore" + # camera: a person id or "wide"; mic: a person id, or "" for a shared room mic + person: str = "" + # mic only: one person id per channel when a recorder puts two people on L/R + channel_people: list[str] = field(default_factory=list) + guessed: bool = True + offset: Optional[float] = None # timeline seconds where source time 0 sits + speed: float = 1.0 # timeline seconds per source second (clock drift) + sync: dict = field(default_factory=dict) + + @property + def synced(self) -> bool: + return self.offset is not None + + def timeline_start(self) -> float: + return float(self.offset or 0.0) + + def timeline_end(self) -> float: + return self.timeline_start() + self.duration * self.speed + + def source_time(self, timeline_seconds: float) -> float: + return (timeline_seconds - self.timeline_start()) / self.speed + + +@dataclass +class MulticamSession: + session_id: str + name: str + folder: str = "" + people: list[Person] = field(default_factory=list) + sources: list[Source] = field(default_factory=list) + reference_id: str = "" + range_start: Optional[float] = None + range_end: Optional[float] = None + look: str = "none" + cut_settings: dict = field(default_factory=lambda: dict(DEFAULT_CUT)) + cuts: list[dict] = field(default_factory=list) + speaker_map: dict = field(default_factory=dict) + activity_key: str = "" + outputs: dict = field(default_factory=dict) + skipped: list[str] = field(default_factory=list) + + def path(self) -> Path: + return _sessions_dir() / f"{self.session_id}.json" + + def save(self) -> None: + tmp = self.path().with_suffix(".json.tmp") + tmp.write_text(json.dumps(asdict(self), indent=2), encoding="utf-8") + os.replace(tmp, self.path()) + + @classmethod + def load(cls, session_id: str) -> "MulticamSession": + if not re.fullmatch(r"[a-f0-9]{6,32}", session_id or ""): + raise ValueError(f"invalid session id {session_id!r}") + p = _sessions_dir() / f"{session_id}.json" + if not p.exists(): + raise FileNotFoundError(f"No multicam session {session_id}") + data = json.loads(p.read_text(encoding="utf-8")) + data["people"] = [Person(**x) for x in data.get("people", [])] + data["sources"] = [Source(**x) for x in data.get("sources", [])] + return cls(**data) + + def source(self, source_id: str) -> Source: + for s in self.sources: + if s.id == source_id: + return s + raise ValueError(f"No file with id {source_id!r} in this edit") + + def person_ids(self) -> list[str]: + return [p.id for p in self.people] + + def cameras(self) -> list[Source]: + return [s for s in self.sources if s.role == "camera" and s.kind == "video"] + + def person_mics(self) -> list[tuple[Source, int, str]]: + """(source, channel or -1 for a mono mixdown, person) for each personal mic feed.""" + feeds = [] + people = set(self.person_ids()) + for s in self.sources: + if s.role != "mic": + continue + if s.channel_people: + for ch, pid in enumerate(s.channel_people[: max(1, s.audio_channels)]): + if pid in people: + feeds.append((s, ch, pid)) + elif s.person in people: + feeds.append((s, -1, s.person)) + return feeds + + def timeline_duration(self) -> float: + synced = [s for s in self.sources if s.role != "ignore" and s.synced] + return max((s.timeline_end() for s in synced), default=0.0) + + +# --------------------------------------------------------------------------- +# Scanning and role guessing +# --------------------------------------------------------------------------- + +MAX_SOURCES = 40 +MAX_SCANNED_ENTRIES = 20_000 + + +def scan_folder(folder: str) -> list[str]: + """Media files under one episode folder, stopping early on folders that are clearly too big.""" + root = Path(folder).expanduser() + if not root.is_dir(): + raise FileNotFoundError(f"Folder not found: {folder}") + found: list[str] = [] + seen = 0 + for dirpath, dirnames, filenames in os.walk(root, onerror=lambda _e: None): + # Skip hidden folders and podcli's own renders if someone points at the output folder. + dirnames[:] = sorted(d for d in dirnames if not d.startswith(".") and not d.endswith("_multicam_podcli")) + for name in sorted(filenames): + seen += 1 + if seen > MAX_SCANNED_ENTRIES or len(found) > MAX_SOURCES: + raise ValueError( + f"{folder} holds too many files for one episode. Pick the folder with just this recording's " + f"cameras and mics ({MAX_SOURCES} media files at most)." + ) + if name.startswith(".") or Path(name).suffix.lower() not in MEDIA_EXTS or "_silence_removed_podcli" in name: + continue + found.append(str((Path(dirpath) / name).resolve())) + return found + + +def _timecode_seconds(info: dict, fps: float, sample_rate: int) -> float: + """Embedded start timecode (pro cameras) or BWF time reference (field recorders), in seconds.""" + tags = [info.get("format", {}).get("tags") or {}] + [s.get("tags") or {} for s in info.get("streams", [])] + for t in tags: + tc = t.get("timecode") or t.get("TIMECODE") + if tc and fps > 0: + parts = re.split(r"[:;.]", tc) + if len(parts) == 4 and all(p.isdigit() for p in parts): + h, m, sec, frames = (int(p) for p in parts) + return h * 3600 + m * 60 + sec + frames / round(fps) + for t in tags: + ref = t.get("time_reference") + if ref and str(ref).isdigit() and sample_rate > 0: + return int(ref) / sample_rate + return 0.0 + + +def probe_source(path: str) -> Source: + info = get_video_info(path) + streams = info.get("streams", []) + video = next( + (s for s in streams if s.get("codec_type") == "video" + and not (s.get("disposition") or {}).get("attached_pic")), + None, + ) + audio = next((s for s in streams if s.get("codec_type") == "audio"), None) + durations = [float(info.get("format", {}).get("duration") or 0)] + durations += [float(s.get("duration") or 0) for s in streams] + duration = max(durations) + if duration <= 0: + raise RuntimeError(f"Could not read the duration of {os.path.basename(path)}") + fps = 0.0 + if video: + num, _, den = str(video.get("avg_frame_rate") or video.get("r_frame_rate") or "0/1").partition("/") + try: + fps = float(num) / float(den or 1) if float(den or 1) else 0.0 + except ValueError: + fps = 0.0 + if not 1 <= fps <= 240: + fps = 30.0 + sample_rate = int(audio.get("sample_rate") or 0) if audio else 0 + return Source( + id=hashlib.sha1(os.path.abspath(path).encode()).hexdigest()[:10], + path=os.path.abspath(path), + kind="video" if video else "audio", + duration=round(duration, 3), + has_audio=audio is not None, + audio_channels=int(audio.get("channels") or 0) if audio else 0, + width=int(video.get("width") or 0) if video else 0, + height=int(video.get("height") or 0) if video else 0, + fps=round(fps, 3), + timecode=round(_timecode_seconds(info, fps, sample_rate), 6), + ) + + +_WIDE = re.compile(r"(^|[^a-z])(wide|master|main|both|all|group|ws|two.?shot|2.?shot|overview)([^a-z]|$)") +_MIX = re.compile(r"(^|[^a-z])(mix|lr|stereo|master.?mix|program)([^a-z]|$)") +_TRACK = re.compile(r"(?:tr|track|ch|channel|in|input|mic)[ _-]?0*(\d{1,2})(?!\d)") + + +def _tokens(path: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", Path(path).stem.lower()).strip() + + +def guess_roles(session: MulticamSession) -> None: + """Fill in role/person for sources the user hasn't mapped yet. + + Filenames that mention a person or 'wide' win. Otherwise cameras and mic + tracks go to people in filename order, which matches how most recorders + number their files. Everything guessed stays flagged so the UI can ask. + """ + names = {p.id: _tokens(p.name) for p in session.people} + people = session.person_ids() + + def named(stem: str) -> str: + for pid, name in names.items(): + if name and re.search(rf"(^|\s){re.escape(name)}(\s|$)", stem): + return pid + if re.search(rf"(^|\s){re.escape(pid)}(\s|$)", stem): + return pid + return "" + + videos = [s for s in session.sources if s.kind == "video" and s.guessed] + audios = [s for s in session.sources if s.kind == "audio" and s.guessed] + + unnamed_cams = [] + for s in videos: + stem = _tokens(s.path) + s.role = "camera" + if _WIDE.search(stem): + s.person = "wide" + elif named(stem): + s.person = named(stem) + else: + unnamed_cams.append(s) + if len(session.cameras()) == 1 and unnamed_cams: + unnamed_cams[0].person = "wide" + else: + taken = {s.person for s in videos if s.person} + free = [p for p in people if p not in taken] + for s in unnamed_cams: + s.person = free.pop(0) if free else "wide" + + unnamed_mics = [] + for s in audios: + stem = _tokens(s.path) + if not s.has_audio: + s.role = "ignore" + continue + s.role = "mic" + who = named(stem) + if who: + s.person = who + elif _MIX.search(stem): + s.person = "" + elif s.audio_channels == 2 and len(people) >= 2 and len(audios) == 1: + # A single stereo recorder file usually carries one person per side. + s.channel_people = people[:2] + else: + unnamed_mics.append(s) + + def track_no(s: Source) -> int: + m = _TRACK.search(Path(s.path).stem.lower()) + return int(m.group(1)) if m else 999 + + unnamed_mics.sort(key=lambda s: (track_no(s), s.path)) + taken = {s.person for s in audios if s.person} | {p for s in audios for p in s.channel_people} + free = [p for p in people if p not in taken] + for s in unnamed_mics: + s.person = free.pop(0) if free else "" + + +def collect_files(folder: str = "", files: Optional[list] = None) -> list[str]: + if files is not None and not (isinstance(files, list) and all(isinstance(f, str) for f in files)): + raise ValueError("files must be a list of file paths") + found = list(files or []) + if folder: + found += scan_folder(folder) + found = list(dict.fromkeys(os.path.abspath(p) for p in found)) + if not found: + raise ValueError("No video or audio files found. Pick the folder that holds one episode's recordings.") + if len(found) > MAX_SOURCES: + raise ValueError(f"Found {len(found)} media files. Point at one episode's folder ({MAX_SOURCES} files at most).") + return found + + +def find_session(found: list[str]) -> Optional[MulticamSession]: + """The saved edit for exactly these files, including ones that couldn't be read.""" + wanted = set(found) + for p in sorted(_sessions_dir().glob("*.json"), key=lambda x: x.stat().st_mtime, reverse=True): + try: + session = MulticamSession.load(p.stem) + except (ValueError, OSError, TypeError, json.JSONDecodeError): + continue + if {s.path for s in session.sources} | set(session.skipped) == wanted: + return session + return None + + +def rename_people(session: MulticamSession, names: list[str]) -> MulticamSession: + """Rename people in order, keeping ids so every file mapped to them stays mapped.""" + ids = [p.id for p in session.people] + return update_mapping(session, {"people": [ + {"id": ids[i], "name": n} if i < len(ids) else {"name": n} for i, n in enumerate(names) + ]}) + + +def new_session( + *, + folder: str = "", + files: Optional[list] = None, + people: Optional[list] = None, + name: str = "", + progress_callback: ProgressCallback = None, +) -> MulticamSession: + """Scan and probe one episode's files, or reopen the saved edit for the same files.""" + found = collect_files(folder, files) + raw_people = [p.get("name", "") if isinstance(p, dict) else str(p or "") for p in (people or [])] + person_names = [n.strip() for n in raw_people if n.strip()] + + existing = find_session(found) + if existing: + if person_names and person_names != [p.name for p in existing.people]: + existing = rename_people(existing, person_names) + _emit(progress_callback, 100, f"Reopened the edit for these {len(found)} files") + return existing + + sources, skipped = [], [] + for i, p in enumerate(found): + _emit(progress_callback, 5 + 90 * i / len(found), f"Reading {os.path.basename(p)}") + try: + sources.append(probe_source(p)) + except Exception as e: # unreadable sidecar files shouldn't block the rest + skipped.append(p) + _emit(progress_callback, 5 + 90 * i / len(found), f"Skipped {os.path.basename(p)}: {e}") + if not sources: + raise ValueError("None of the files could be read as audio or video.") + + person_names = person_names or ["Host", "Guest"] + session = MulticamSession( + session_id=uuid.uuid4().hex[:12], + name=name or (Path(folder).name if folder else Path(found[0]).stem), + folder=os.path.abspath(folder) if folder else "", + people=[Person(id=_person_id(n, i), name=n) for i, n in enumerate(person_names)], + sources=sources, + skipped=skipped, + ) + guess_roles(session) + session.save() + _emit(progress_callback, 100, f"Found {len(sources)} sources") + return session + + +def needs_sync(session: MulticamSession) -> bool: + return any(s.role != "ignore" and not s.synced for s in session.sources) + + +def _person_id(name: str, index: int) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-") + return slug or f"person-{index + 1}" + + +def update_mapping(session: MulticamSession, params: dict) -> MulticamSession: + """Apply user edits: people, per-source role/person/channels/offset, range, look, cut settings. + + Any edit that changes who is on which camera or mic, or where a file sits, + drops the planned cut so the next cut reflects it. + """ + before = _cut_inputs(session) + if "people" in params: + people = [] + for i, entry in enumerate(params["people"] or []): + name = str(entry.get("name") if isinstance(entry, dict) else entry).strip() + if not name: + continue + pid = (entry.get("id") if isinstance(entry, dict) else None) or _person_id(name, i) + if pid == "wide" or pid in {p.id for p in people}: + pid = f"{pid}-{i + 1}" + people.append(Person(id=pid, name=name)) + if not people: + raise ValueError("Add at least one person") + session.people = people + valid = set(session.person_ids()) + for s in session.sources: + if s.role == "camera" and s.person not in valid | {"wide"}: + s.person = "wide" + if s.role == "mic" and s.person and s.person not in valid: + s.person = "" + s.channel_people = [p if p in valid else "" for p in s.channel_people] + + valid = set(session.person_ids()) + edits = params.get("sources") or [] + if not isinstance(edits, list) or not all(isinstance(e, dict) for e in edits): + raise ValueError("sources must be a list of {id, role, person, ...} objects") + for edit in edits: + s = session.source(str(edit.get("id"))) + if "role" in edit: + role = edit["role"] + if role not in {"camera", "mic", "ignore"}: + raise ValueError(f"Unknown role {role!r}") + if role == "camera" and s.kind != "video": + raise ValueError(f"{os.path.basename(s.path)} has no video, so it can't be a camera") + if role == "mic" and not s.has_audio: + raise ValueError(f"{os.path.basename(s.path)} has no audio, so it can't be a mic") + s.role = role + if "person" in edit: + person = edit["person"] or "" + allowed = valid | ({"wide"} if s.role == "camera" else {""}) + if person not in allowed: + raise ValueError(f"Unknown person {person!r}") + s.person = person + if "channel_people" in edit: + chans = [c if c in valid else "" for c in (edit["channel_people"] or [])] + s.channel_people = chans[: max(0, s.audio_channels)] if any(chans) else [] + if "offset" in edit and edit["offset"] is not None: + s.offset = float(edit["offset"]) + s.sync = {**s.sync, "status": "manual"} + if "nudge" in edit: + if s.offset is None: + raise ValueError(f"{os.path.basename(s.path)} isn't synced yet. Set its offset first.") + s.offset = float(s.offset) + float(edit["nudge"]) + s.sync = {**s.sync, "status": "manual"} + s.guessed = False + + if "range_start" in params: + session.range_start = None if params["range_start"] is None else max(0.0, float(params["range_start"])) + if "range_end" in params: + session.range_end = None if params["range_end"] is None else float(params["range_end"]) + if "look" in params: + if params["look"] not in LOOKS: + raise ValueError(f"Unknown look {params['look']!r}. Use: {', '.join(LOOKS)}") + session.look = params["look"] + if "cut_settings" in params: + merged = {**session.cut_settings, **(params["cut_settings"] or {})} + session.cut_settings = { + "min_shot": max(0.5, min(10.0, float(merged["min_shot"]))), + "max_shot": max(0.0, min(600.0, float(merged["max_shot"]))), + "wide_insert": max(1.0, min(15.0, float(merged["wide_insert"]))), + } + if "speaker_map" in params: + session.speaker_map = {str(k): v for k, v in (params["speaker_map"] or {}).items() if v in valid} + if _cut_inputs(session) != before: + session.cuts = [] + session.save() + return session + + +def _cut_inputs(session: MulticamSession) -> str: + return json.dumps([ + [(s.id, s.role, s.person, s.channel_people, s.offset) for s in session.sources], + session.person_ids(), session.range_start, session.range_end, session.cut_settings, session.speaker_map, + ], default=str) + + +# --------------------------------------------------------------------------- +# Audio extraction +# --------------------------------------------------------------------------- + +def _extract(source: Source, out: Path, channel: int = -1, rate: int = SYNC_RATE) -> Path: + if out.exists() and out.stat().st_mtime >= os.path.getmtime(source.path): + return out + tmp = out.with_suffix(".tmp.wav") + pick = "pan=mono|c0=c0" if source.audio_channels < 2 else "pan=mono|c0=0.5*c0+0.5*c1" + if channel >= 0: + pick = f"pan=mono|c0=c{channel}" + # Anchoring the first sample at container time zero keeps sync in the same + # clock the render and editors use; audio often starts 20-100 ms after video. + proc_run([ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", + "-i", source.path, "-vn", "-map", "0:a:0", + "-af", f"aresample=async=1:first_pts=0,{pick}", + "-ar", str(rate), "-acodec", "pcm_s16le", str(tmp), + ], timeout=3600, check=True) + os.replace(tmp, out) + return out + + +def _read_wav(p: Path) -> np.ndarray: + with wave.open(str(p), "rb") as w: + return np.frombuffer(w.readframes(w.getnframes()), dtype=np.int16) + + +# --------------------------------------------------------------------------- +# Sync +# --------------------------------------------------------------------------- + +FINE_WINDOW = 10.0 +FINE_SEARCH = 1.0 + + +def _fine_points(ref: np.ndarray, src: np.ndarray, lag: float) -> list[tuple[float, float, float]]: + """GCC-PHAT checkpoints across the overlap, as (source_s, timeline_s, weight).""" + src_len = len(src) / SYNC_RATE + ref_len = len(ref) / SYNC_RATE + lo = max(0.0, -lag) + FINE_SEARCH + hi = min(src_len, ref_len - lag) - FINE_WINDOW - FINE_SEARCH + if hi <= lo: + return [] + span = hi - lo + # One checkpoint per 5 minutes, but at least three when there's room, so + # drift is measurable even on short recordings. + count = int(max(1, min(9, span // 300 + 1), min(3, span // (FINE_WINDOW * 1.5)))) + centers = [lo + (hi - lo) * (i + 0.5) / count for i in range(count)] + pts = [] + for a in centers: + s0 = int(a * SYNC_RATE) + s_win = src[s0:s0 + int(FINE_WINDOW * SYNC_RATE)].astype(np.float32) + r0 = int((a + lag - FINE_SEARCH) * SYNC_RATE) + r_win = ref[r0:r0 + int((FINE_WINDOW + 2 * FINE_SEARCH) * SYNC_RATE)].astype(np.float32) + if r0 < 0 or len(r_win) < int((FINE_WINDOW + 2 * FINE_SEARCH) * SYNC_RATE) or np.abs(s_win).max() < 50: + continue + shift, prominence = sig.gcc_phat(r_win, s_win, SYNC_RATE, FINE_SEARCH) + if prominence < 8: + continue + # A drifting clock smears the match across the window, so the measured + # shift belongs to the window's middle, not its start. + mid = a + FINE_WINDOW / 2 + pts.append((mid, mid + lag + shift, prominence)) + return pts + + +def _match(ref: np.ndarray, ref_env: np.ndarray, src: np.ndarray) -> tuple[Optional[sig.ClockFit], dict]: + """Where src sits on ref's clock, plus the report shown next to the file.""" + match = sig.coarse_lag(ref_env, sig.onset_envelope(src, SYNC_RATE)) + if match is None or not match.ok: + return None, { + "status": "failed", + "score": round(match.score, 1) if match else 0, + "message": "No shared sound found. Check the file belongs to this episode, or set the offset by hand.", + } + fit = sig.fit_clock(_fine_points(ref, src, match.lag_seconds)) + if fit is None: + return sig.ClockFit(match.lag_seconds, 1.0, 10.0, 0), { + "status": "rough", "score": round(match.score, 1), + "message": "Synced to within 10 ms; fine alignment found no clear speech.", + } + return fit, { + "status": "ok", + "score": round(match.score, 1), + "checkpoints": fit.checkpoints, + "residual_ms": round(fit.residual_ms, 1), + "drift_ppm": round((fit.speed - 1.0) * 1e6, 1), + } + + +def sync_session( + session: MulticamSession, + *, + force: bool = False, + progress_callback: ProgressCallback = None, +) -> MulticamSession: + """Place every mapped source on one timeline by matching its audio. + + Offsets set by hand survive a re-sync unless force is set. A file that + shares no sound with the reference gets a second try against every file + that did sync, which covers cameras split into parts with no continuous mic. + """ + candidates = [s for s in session.sources if s.role != "ignore" and s.has_audio] + if not candidates: + raise ValueError("Map at least one camera or mic that has audio before syncing") + # The longest recording is the one most likely to overlap every other file; + # a mic wins a tie because recorders rarely stop mid-episode. + reference = max(candidates, key=lambda s: (s.duration, s.role == "mic")) + manual = [ + s for s in session.sources + if s.role != "ignore" and s.synced and s.sync.get("status") == "manual" and not force and s is not reference + ] + # Manual offsets were measured against the previous timeline zero; re-express + # them against this reference before its offset resets to zero. + anchor = reference.timeline_start() if reference.synced else 0.0 + for s in manual: + s.offset = float(s.offset) - anchor + + session.reference_id = reference.id + work = _work_dir(session.session_id) + + def wav(s: Source) -> np.ndarray: + return _read_wav(_extract(s, work / f"{s.id}.sync.wav")) + + _emit(progress_callback, 2, f"Reading reference audio from {os.path.basename(reference.path)}") + ref = wav(reference) + ref_env = sig.onset_envelope(ref, SYNC_RATE) + reference.offset, reference.speed = 0.0, 1.0 + reference.sync = {"status": "reference"} + + others = [s for s in candidates if s is not reference and s not in manual] + for i, s in enumerate(others): + _emit(progress_callback, 10 + 75 * i / len(others), f"Syncing {os.path.basename(s.path)}") + fit, s.sync = _match(ref, ref_env, wav(s)) + s.offset, s.speed = (fit.offset, fit.speed) if fit else (None, 1.0) + del ref + + failed = [s for s in others if not s.synced] + anchors = [s for s in others if s.synced] + for i, s in enumerate(failed): + _emit(progress_callback, 85 + 10 * i / len(failed), f"Retrying {os.path.basename(s.path)} against the other files") + src = wav(s) + for a in anchors: + a_audio = wav(a) + fit, report = _match(a_audio, sig.onset_envelope(a_audio, SYNC_RATE), src) + if fit: + s.offset, s.speed = a.timeline_start() + a.speed * fit.offset, a.speed * fit.speed + s.sync = {**report, "via": a.id} + anchors.append(s) + break + + for s in session.sources: + if s.role != "ignore" and not s.has_audio and s not in manual: + s.offset = None + s.sync = {"status": "failed", "message": "This camera recorded no audio. Set its offset by hand."} + + synced = [s for s in session.sources if s.synced and s.role != "ignore"] + shift = min(s.timeline_start() for s in synced) + for s in synced: + s.offset = round(float(s.offset) - shift, 4) + session.range_start = session.range_end = None + session.cuts = [] + session.activity_key = "" + session.save() + _emit(progress_callback, 100, "Sources synced") + return session + + +# --------------------------------------------------------------------------- +# Who speaks when +# --------------------------------------------------------------------------- + +def _activity_key(session: MulticamSession) -> str: + feeds = [(s.id, ch, pid, s.offset, s.speed) for s, ch, pid in session.person_mics()] + blob = json.dumps([feeds, session.speaker_map, session.person_ids()], sort_keys=True, default=str) + return hashlib.sha1(blob.encode()).hexdigest()[:16] + + +def _timeline_levels(session: MulticamSession, s: Source, channel: int, length: int) -> np.ndarray: + work = _work_dir(session.session_id) + samples = _read_wav(_extract(s, work / f"{s.id}.{'sync' if channel < 0 else f'ch{channel}'}.wav", channel=channel)) + own = sig.level_db(samples, SYNC_RATE) + t = np.arange(length) * sig.FRAME_SECONDS + idx = (t - s.timeline_start()) / s.speed / sig.FRAME_SECONDS + inside = (idx >= 0) & (idx < len(own) - 1) + out = np.full(length, -100.0, dtype=np.float32) + out[inside] = own[idx[inside].astype(np.int64)] + return out + + +def _diarized_labels(session: MulticamSession, length: int, progress_callback: ProgressCallback) -> np.ndarray: + from services.speaker_detection import run_diarization + + reference = session.source(session.reference_id) + wav = _extract(reference, _work_dir(session.session_id) / f"{reference.id}.diarize.wav", rate=16000) + turns = run_diarization( + str(wav), + num_speakers=len(session.people) or None, + progress_callback=lambda p, m: _emit(progress_callback, 10 + p * 0.7, m), + ) + order = list(dict.fromkeys(t["speaker"] for t in turns)) + people = session.person_ids() + for i, label in enumerate(order): + if label not in session.speaker_map and i < len(people): + session.speaker_map[label] = people[i] + segments = [ + { + "start": reference.timeline_start() + t["start"] * reference.speed, + "end": reference.timeline_start() + t["end"] * reference.speed, + "person": session.speaker_map.get(t["speaker"]), + } + for t in turns + ] + return sig.segments_to_frames(segments, people, length) + + +def speaker_labels(session: MulticamSession, progress_callback: ProgressCallback = None) -> np.ndarray: + """Per-10ms timeline labels (person index, SILENT, BOTH), cached per mapping.""" + length = int(np.ceil(session.timeline_duration() / sig.FRAME_SECONDS)) + cache = _work_dir(session.session_id) / "activity.npy" + key = _activity_key(session) + if session.activity_key == key and cache.exists(): + labels = np.load(cache) + if len(labels) == length: + return labels + + feeds = [(s, ch, pid) for s, ch, pid in session.person_mics() if s.synced] + if feeds: + people = session.person_ids() + per_person: dict[str, np.ndarray] = {} + for i, (s, ch, pid) in enumerate(feeds): + _emit(progress_callback, 10 + 70 * i / len(feeds), f"Listening to {os.path.basename(s.path)}") + level = _timeline_levels(session, s, ch, length) + per_person[pid] = np.maximum(per_person[pid], level) if pid in per_person else level + speaking = [p for p in people if p in per_person] + raw = sig.speaker_frames([per_person[p] for p in speaking]) + index = np.array([people.index(p) for p in speaking], dtype=np.int32) + labels = np.where(raw >= 0, index[np.clip(raw, 0, None)], raw).astype(np.int32) + else: + _emit(progress_callback, 10, "No personal mics mapped; detecting speakers from the shared audio") + labels = _diarized_labels(session, length, progress_callback) + + np.save(cache, labels) + session.activity_key = _activity_key(session) + session.save() + return labels + + +# --------------------------------------------------------------------------- +# Cut plan +# --------------------------------------------------------------------------- + +def _camera_spans(session: MulticamSession) -> list[sig.Camera]: + return [ + sig.Camera(s.id, s.person or "wide", s.timeline_start(), s.timeline_end()) + for s in session.cameras() if s.synced + ] + + +def plan_session(session: MulticamSession, progress_callback: ProgressCallback = None) -> MulticamSession: + cams = _camera_spans(session) + if not cams: + raise ValueError("Sync at least one camera before planning cuts") + labels = speaker_labels(session, progress_callback) + _emit(progress_callback, 85, "Planning camera cuts") + + cover_start = min(c.start for c in cams) + cover_end = max(c.end for c in cams) + if session.range_start is None or session.range_end is None: + bounds = sig.speech_bounds(labels) + a, b = (bounds[0] - 1.0, bounds[1] + 1.0) if bounds else (cover_start, cover_end) + session.range_start = round(max(cover_start, a), 2) if session.range_start is None else session.range_start + session.range_end = round(min(cover_end, b), 2) if session.range_end is None else session.range_end + start = max(cover_start, float(session.range_start)) + end = min(cover_end, float(session.range_end)) + if end - start < 1.0: + raise ValueError("The episode range doesn't overlap any camera. Widen the start and end.") + + session.cuts = sig.plan_cuts( + labels, session.person_ids(), cams, + range_start=start, range_end=end, **session.cut_settings, + ) + if not session.cuts: + raise ValueError("No camera covers the episode range. Check the camera mapping and the start and end.") + session.save() + _emit(progress_callback, 100, f"{len(session.cuts)} shots planned") + return session + + +def cut_stats(session: MulticamSession) -> dict: + if not session.cuts: + return {} + total = sum(c["end"] - c["start"] for c in session.cuts) + per: dict[str, float] = {} + for c in session.cuts: + per[c["source_id"]] = per.get(c["source_id"], 0.0) + c["end"] - c["start"] + return { + "shots": len(session.cuts), + "duration": round(total, 2), + "average_shot": round(total / len(session.cuts), 2), + "share": {k: round(v / total, 3) for k, v in per.items()}, + } + + +def set_cut(session: MulticamSession, index: int, source_id: str) -> MulticamSession: + """Swap one shot to another camera; neighbours on the same camera merge.""" + if not 0 <= index < len(session.cuts): + raise IndexError(f"No shot {index + 1}") + cam = session.source(source_id) + shot = session.cuts[index] + if cam.role != "camera" or not cam.synced: + raise ValueError("Pick a synced camera") + if cam.timeline_start() > shot["start"] + 1e-3 or cam.timeline_end() < shot["end"] - 1e-3: + raise ValueError(f"{os.path.basename(cam.path)} wasn't recording for this whole shot") + shot["source_id"] = source_id + merged: list[dict] = [] + for c in session.cuts: + if merged and merged[-1]["source_id"] == c["source_id"]: + merged[-1]["end"] = c["end"] + else: + merged.append(dict(c)) + session.cuts = merged + session.save() + return session + + +# --------------------------------------------------------------------------- +# Previews +# --------------------------------------------------------------------------- + +def _frame(source: Source, at: float, out: Path, look: str = "none", width: int = 480) -> Path: + vf = ",".join(x for x in [LOOKS.get(look, ""), f"scale={width}:-2"] if x) + proc_run([ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", + "-ss", f"{max(0.0, at):.3f}", "-i", source.path, "-frames:v", "1", + "-vf", vf, "-q:v", "4", str(out), + ], timeout=60, check=True) + return out + + +def previews(session: MulticamSession, *, looks: bool = False, at: Optional[float] = None) -> dict: + """One still per camera, and optionally one still per look from the first camera.""" + work = _work_dir(session.session_id) + frames: dict = {"cameras": {}, "looks": {}} + for s in session.sources: + if s.kind != "video": + continue + t = s.duration * 0.3 if at is None or not s.synced else s.source_time(at) + t = min(max(0.0, t), max(0.0, s.duration - 0.5)) + out = work / f"frame-{s.id}-{int(t * 10)}.jpg" + if not out.exists(): + _frame(s, t, out) + frames["cameras"][s.id] = str(out) + if looks: + cams = session.cameras() or [s for s in session.sources if s.kind == "video"] + if cams: + cam = next((c for c in cams if c.person == "wide"), cams[0]) + t = cam.duration * 0.3 + for name in LOOKS: + out = work / f"look-{cam.id}-{name}.jpg" + if not out.exists(): + _frame(cam, t, out, look=name, width=640) + frames["looks"][name] = str(out) + return frames + + +# --------------------------------------------------------------------------- +# Render +# --------------------------------------------------------------------------- + +def _output_format(session: MulticamSession) -> tuple[int, int, float]: + cams = [s for s in session.cameras() if s.synced] + width = max(s.width for s in cams) + height = max(s.height for s in cams) + rates = [_standard_fps(s.fps) for s in cams] + return width - width % 2, height - height % 2, max(set(rates), key=rates.count) + + +STANDARD_FPS = (24000 / 1001, 24.0, 25.0, 30000 / 1001, 30.0, 50.0, 60000 / 1001, 60.0) + + +def _standard_fps(fps: float) -> float: + """Snap phone rates like 29.92 to the broadcast rate editors expect.""" + nearest = min(STANDARD_FPS, key=lambda r: abs(r - fps)) + return nearest if abs(nearest - fps) / nearest < 0.01 else fps + + +def _output_dir(session: MulticamSession) -> Path: + slug = re.sub(r"[^A-Za-z0-9._-]+", "-", session.name).strip("-") or "episode" + d = Path(paths["output"]) / f"{slug}-{session.session_id[:6]}_multicam_podcli" + d.mkdir(parents=True, exist_ok=True) + return d + + +def _render_shot(cam: Optional[Source], out: Path, tl_start: float, frames: int, + width: int, height: int, fps: float, look: str) -> None: + """Encode exactly `frames` frames starting at timeline second tl_start. + + cam None renders black, which keeps picture and sound aligned across a + stretch no camera covered. + """ + common = [ + "-frames:v", str(frames), "-an", + "-c:v", "libx264", "-crf", "18", "-preset", "fast", "-pix_fmt", "yuv420p", + "-video_track_timescale", "90000", str(out), + ] + if cam is None: + proc_run([ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", + "-f", "lavfi", "-i", f"color=c=black:s={width}x{height}:r={fps:.6f}", *common, + ], timeout=600, check=True) + return + vf = [ + f"scale={width}:{height}:force_original_aspect_ratio=decrease", + f"pad={width}:{height}:(ow-iw)/2:(oh-ih)/2", + "setsar=1", + f"fps={fps:.6f}", + # A camera that stops a few frames early would shorten the shot and slip + # every later shot against the audio; holding its last frame prevents that. + "tpad=stop_mode=clone:stop=-1", + ] + if LOOKS.get(look): + vf.append(LOOKS[look]) + proc_run([ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", + "-ss", f"{max(0.0, cam.source_time(tl_start)):.4f}", "-i", cam.path, + "-map", "0:v:0", "-vf", ",".join(vf), *common, + ], timeout=3600, check=True) + + +def _audio_inputs(session: MulticamSession) -> list[tuple[Source, int]]: + feeds = [(s, ch) for s, ch, _ in session.person_mics() if s.synced] + if feeds: + return feeds + rooms = [s for s in session.sources if s.role == "mic" and s.synced] + if rooms: + return [(max(rooms, key=lambda s: s.duration), -1)] + return [(session.source(session.reference_id), -1)] + + +def _aligned_input(s: Source, channel: int, start: float, duration: float) -> tuple[list[str], list[str]]: + """ffmpeg input args and mono filter steps that place a source on [start, start + duration].""" + src_start = s.source_time(start) + steps = ["aresample=async=1:first_pts=0"] + if channel >= 0: + steps.append(f"pan=mono|c0=c{channel}") + elif s.audio_channels >= 2: + steps.append("pan=mono|c0=0.5*c0+0.5*c1") + else: + steps.append("pan=mono|c0=c0") + if abs(s.speed - 1.0) > 1e-7: + steps.append(f"atempo={1.0 / s.speed:.8f}") + if src_start < 0: + steps.append(f"adelay={int(round(-src_start * 1000))}") + steps.append(f"apad,atrim=0:{duration:.4f}") + return ["-ss", f"{max(0.0, src_start):.4f}", "-i", s.path], steps + + +def _render_audio(session: MulticamSession, out: Path, start: float, duration: float) -> None: + args: list[str] = [] + chains: list[str] = [] + for i, (s, ch) in enumerate(_audio_inputs(session)): + inp, steps = _aligned_input(s, ch, start, duration) + args += inp + chains.append(f"[{i}:a:0]{','.join(steps + ['dynaudnorm=f=250:g=15:p=0.9'])}[a{i}]") + n = len(chains) + mix = "".join(f"[a{i}]" for i in range(n)) + f"amix=inputs={n}:normalize=0," if n > 1 else "[a0]" + graph = ";".join(chains) + f";{mix}loudnorm=I=-16:TP=-1.5:LRA=11,aformat=channel_layouts=stereo[out]" + proc_run([ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", *args, + "-filter_complex", graph, "-map", "[out]", "-ar", "48000", + "-c:a", "aac", "-b:a", "192k", str(out), + ], timeout=7200, check=True) + + +def _render_stems(session: MulticamSession, out_dir: Path, start: float, duration: float) -> list[str]: + """One WAV per person; a person recorded across several files (recorder splits) gets them mixed in.""" + stems = [] + for pid in session.person_ids(): + feeds = [(s, ch) for s, ch, who in session.person_mics() if who == pid and s.synced] + if not feeds: + continue + args, chains = [], [] + for i, (s, ch) in enumerate(feeds): + inp, steps = _aligned_input(s, ch, start, duration) + args += inp + chains.append(f"[{i}:a:0]{','.join(steps)}[a{i}]") + mix = "".join(f"[a{i}]" for i in range(len(feeds))) + graph = ";".join(chains) + (f";{mix}amix=inputs={len(feeds)}:normalize=0[out]" if len(feeds) > 1 else ";[a0]anull[out]") + # Person ids are unique slugs, so two people never share a file name. + out = out_dir / f"{pid}.wav" + proc_run([ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", *args, + "-filter_complex", graph, "-map", "[out]", "-ar", "48000", "-c:a", "pcm_s24le", str(out), + ], timeout=3600, check=True) + stems.append(str(out)) + return stems + + +def shot_frames(session: MulticamSession, fps: float) -> list[tuple[Optional[str], int, int]]: + """(source_id or None for a gap, first frame, frame count) on one grid from the first cut. + + One grid over the whole episode keeps shot lengths from accumulating + rounding drift against the audio. + """ + start = session.cuts[0]["start"] + out: list[tuple[Optional[str], int, int]] = [] + cursor = 0 + for shot in session.cuts: + f0 = round((shot["start"] - start) * fps) + f1 = round((shot["end"] - start) * fps) + if f0 > cursor: + out.append((None, cursor, f0 - cursor)) + if f1 > max(f0, cursor): + out.append((shot["source_id"], max(f0, cursor), f1 - max(f0, cursor))) + cursor = f1 + return out + + +def _render_key(session: MulticamSession, stems: bool) -> str: + """Everything the MP4 depends on, so an unchanged edit isn't rendered twice.""" + used = {c["source_id"] for c in session.cuts} | {s.id for s, _ in _audio_inputs(session)} + files = [(s.id, s.offset, s.speed, s.channel_people, s.person, os.path.getmtime(s.path)) + for s in session.sources if s.id in used and os.path.exists(s.path)] + blob = json.dumps([session.cuts, session.look, stems, files], sort_keys=True, default=str) + return hashlib.sha1(blob.encode()).hexdigest()[:16] + + +def render_session( + session: MulticamSession, + *, + stems: bool = True, + progress_callback: ProgressCallback = None, +) -> dict: + if not session.cuts: + raise ValueError("Plan the cuts before rendering") + key = _render_key(session, stems) + video = session.outputs.get("video") + if session.outputs.get("render_key") == key and video and os.path.exists(video): + _emit(progress_callback, 100, "Already rendered with these settings") + return session.outputs + width, height, fps = _output_format(session) + start = session.cuts[0]["start"] + out_dir = _output_dir(session) + work = Path(tempfile.mkdtemp(prefix="podcli_multicam_", dir=paths["working"] if os.path.isdir(paths["working"]) else None)) + try: + jobs = shot_frames(session, fps) + total_frames = sum(n for _, _, n in jobs) + duration = total_frames / fps + done = [0, 0] + lock = threading.Lock() + + def run(item): + i, (source_id, f0, n) = item + chunk = work / f"shot-{i:05d}.mp4" + cam = session.source(source_id) if source_id else None + _render_shot(cam, chunk, start + f0 / fps, n, width, height, fps, session.look) + with lock: + done[0] += n + done[1] += 1 + _emit(progress_callback, 3 + 80 * done[0] / total_frames, f"Cut {done[1]} of {len(jobs)} shots") + return chunk + + workers = max(1, min(4, (os.cpu_count() or 2) // 2)) + with ThreadPoolExecutor(max_workers=workers) as pool: + chunks = list(pool.map(run, enumerate(jobs))) + + _emit(progress_callback, 84, "Mixing microphones") + audio = work / "audio.m4a" + _render_audio(session, audio, start, duration) + + _emit(progress_callback, 92, "Joining shots") + listing = work / "shots.txt" + listing.write_text("".join(f"file '{c.as_posix()}'\n" for c in chunks), encoding="utf-8") + partial = work / "episode.mp4" + proc_run([ + "ffmpeg", "-y", "-hide_banner", "-loglevel", "error", + "-f", "concat", "-safe", "0", "-i", str(listing), "-i", str(audio), + "-map", "0:v:0", "-map", "1:a:0", "-c", "copy", "-shortest", + "-movflags", "+faststart", str(partial), + ], timeout=3600, check=True) + video = out_dir / "episode.mp4" + # shutil.move falls back to copy+delete when work and output sit on different volumes. + shutil.move(str(partial), str(video)) + + stem_paths = [] + if stems: + _emit(progress_callback, 96, "Writing separate mic tracks") + stem_paths = _render_stems(session, out_dir, start, duration) + + session.outputs = { + **session.outputs, + "video": str(video), + "stems": stem_paths, + "duration": round(duration, 3), + "render_key": key, + } + session.save() + _emit(progress_callback, 100, "Episode ready") + return session.outputs + finally: + shutil.rmtree(work, ignore_errors=True) + + +def export_xml(session: MulticamSession, fmt: str) -> str: + from services.integrations._shared import multicam_xml + + if not session.cuts: + raise ValueError("Plan the cuts before exporting") + if fmt not in {"premiere", "fcpxml"}: + raise ValueError("Export format must be 'premiere' or 'fcpxml'") + width, height, fps = _output_format(session) + out_dir = _output_dir(session) + if fmt == "premiere": + out = out_dir / "episode-premiere.xml" + multicam_xml.write_xmeml(session, out, width=width, height=height, fps=fps) + else: + out = out_dir / "episode.fcpxml" + multicam_xml.write_fcpxml(session, out, width=width, height=height, fps=fps) + session.outputs = {**session.outputs, fmt: str(out)} + session.save() + return str(out) + + +# --------------------------------------------------------------------------- +# Listing and payloads +# --------------------------------------------------------------------------- + +def list_sessions() -> list[dict]: + items = [] + for p in sorted(_sessions_dir().glob("*.json"), key=lambda x: x.stat().st_mtime, reverse=True): + try: + s = MulticamSession.load(p.stem) + except Exception: + continue + items.append({ + "session_id": s.session_id, + "name": s.name, + "sources": len(s.sources), + "cameras": len(s.cameras()), + "synced": all(x.synced for x in s.sources if x.role != "ignore"), + "shots": len(s.cuts), + "video": s.outputs.get("video") if s.outputs.get("video") and os.path.exists(s.outputs["video"]) else None, + }) + return items + + +def delete_session(session_id: str) -> bool: + s = MulticamSession.load(session_id) + s.path().unlink(missing_ok=True) + shutil.rmtree(Path(paths["working"]) / "multicam" / session_id, ignore_errors=True) + return True + + +def payload(session: MulticamSession) -> dict: + data = asdict(session) + data["timeline_duration"] = round(session.timeline_duration(), 3) + data["stats"] = cut_stats(session) + data["looks"] = list(LOOKS) + data["has_person_mics"] = bool(session.person_mics()) + data["outputs"] = { + k: v for k, v in session.outputs.items() + if not isinstance(v, str) or not os.path.isabs(v) or os.path.exists(v) + } + for s in data["sources"]: + s["name"] = os.path.basename(s["path"]) + s["timeline_start"] = s["offset"] + s["timeline_end"] = None if s["offset"] is None else round(s["offset"] + s["duration"] * s["speed"], 3) + return data diff --git a/backend/services/multicam_signal.py b/backend/services/multicam_signal.py new file mode 100644 index 00000000..ca82400b --- /dev/null +++ b/backend/services/multicam_signal.py @@ -0,0 +1,464 @@ +"""Signal math behind multicam editing: sync, drift, who-speaks-when, and the cut plan. + +Everything here works on numpy arrays so it can be tested with synthetic audio. +The orchestration (ffmpeg extraction, sessions, rendering) lives in multicam.py. + +Sync runs in two passes. A coarse pass cross-correlates 10 ms onset envelopes +over the full length of both files, which tolerates different mics, gain and EQ. +A fine pass runs GCC-PHAT on raw audio at several checkpoints; fitting a line +through those checkpoints yields both the offset and the clock drift between +devices (cameras and recorders commonly drift a frame every 10-20 minutes). +""" + +from __future__ import annotations + +import heapq +from dataclasses import dataclass +from typing import Optional + +import numpy as np + +FRAME_SECONDS = 0.01 + + +def onset_envelope(samples: np.ndarray, sample_rate: int) -> np.ndarray: + """Half-wave rectified log-energy rise per 10 ms frame, z-scored.""" + hop = int(round(sample_rate * FRAME_SECONDS)) + frames = len(samples) // hop + if frames < 2: + return np.zeros(0, dtype=np.float32) + x = samples[: frames * hop].astype(np.float32).reshape(frames, hop) + log_energy = np.log10(np.mean(x * x, axis=1) + 1e-9) + onset = np.maximum(0.0, np.diff(log_energy, prepend=log_energy[0])) + std = float(onset.std()) + if std < 1e-9: + return np.zeros(frames, dtype=np.float32) + return ((onset - onset.mean()) / std).astype(np.float32) + + +@dataclass +class CoarseMatch: + lag_seconds: float + score: float + peak_ratio: float + + @property + def ok(self) -> bool: + return self.score >= 6.0 and self.peak_ratio <= 0.85 + + +def coarse_lag( + ref_env: np.ndarray, + src_env: np.ndarray, + *, + min_overlap_seconds: float = 20.0, +) -> Optional[CoarseMatch]: + """Lag (seconds) such that src time 0 sits at that time on the reference. + + score is the peak's robust z-score over all valid lags; peak_ratio is the + runner-up peak (at least 1 s away) divided by the best one. + """ + n, m = len(ref_env), len(src_env) + if n == 0 or m == 0: + return None + size = 1 << int(np.ceil(np.log2(n + m))) + corr = np.fft.irfft(np.fft.rfft(ref_env, size) * np.conj(np.fft.rfft(src_env, size)), size) + lags = np.concatenate([np.arange(0, n), np.arange(-(m - 1), 0)]) + values = np.concatenate([corr[:n], corr[size - (m - 1):]]) if m > 1 else corr[:n] + overlap = np.minimum(n, lags + m) - np.maximum(0, lags) + min_overlap = min(int(min_overlap_seconds / FRAME_SECONDS), int(0.5 * min(n, m))) + valid = overlap >= max(1, min_overlap) + if not valid.any(): + return None + lags, values = lags[valid], values[valid] + + best = int(np.argmax(values)) + peak = float(values[best]) + median = float(np.median(values)) + mad = float(np.median(np.abs(values - median))) * 1.4826 + 1e-9 + guard = int(1.0 / FRAME_SECONDS) + away = np.abs(lags - lags[best]) > guard + runner_up = float(values[away].max()) if away.any() else median + ratio = (runner_up - median) / (peak - median) if peak > median else 1.0 + return CoarseMatch( + lag_seconds=float(lags[best]) * FRAME_SECONDS, + score=(peak - median) / mad, + peak_ratio=max(0.0, float(ratio)), + ) + + +def gcc_phat(ref: np.ndarray, src: np.ndarray, sample_rate: int, max_shift_seconds: float) -> tuple[float, float]: + """Shift (seconds) of src inside ref, and the peak's prominence. + + ref must be the src window padded by max_shift on both sides; a return of 0 + means src lines up with the center of ref. + """ + size = 1 << int(np.ceil(np.log2(len(ref) + len(src)))) + cross = np.fft.rfft(ref.astype(np.float64), size) * np.conj(np.fft.rfft(src.astype(np.float64), size)) + cross /= np.abs(cross) + 1e-12 + corr = np.fft.irfft(cross, size) + pad = int(round(max_shift_seconds * sample_rate)) + window = corr[: 2 * pad + 1] + best = int(np.argmax(window)) + peak = float(window[best]) + spread = float(np.std(window)) + 1e-12 + shift = best + # Parabolic interpolation gets sub-sample precision. + if 0 < best < len(window) - 1: + a, b, c = window[best - 1], window[best], window[best + 1] + denom = a - 2 * b + c + if abs(denom) > 1e-12: + shift = best + 0.5 * (a - c) / denom + return (shift - pad) / sample_rate, peak / spread + + +@dataclass +class ClockFit: + offset: float + speed: float + residual_ms: float + checkpoints: int + + +def fit_clock(points: list[tuple[float, float, float]], *, max_residual: float = 0.02) -> Optional[ClockFit]: + """Fit timeline = offset + speed * source from (source_s, timeline_s, weight) points. + + Checkpoints further than max_residual seconds from a Theil-Sen line are + dropped before the weighted least-squares fit. + A single point yields speed 1. Implausible speeds (over 1000 ppm) fall back + to the median offset at speed 1, since real clock drift is far smaller. + """ + if not points: + return None + pts = np.array(points, dtype=np.float64) + + def solve(p: np.ndarray) -> tuple[float, float]: + if len(p) < 2 or np.ptp(p[:, 0]) < 1.0: + return float(np.median(p[:, 1] - p[:, 0])), 1.0 + w = p[:, 2] + slope, intercept = np.polyfit(p[:, 0], p[:, 1], 1, w=w) + return float(intercept), float(slope) + + # Theil-Sen seeds the outlier test; least squares alone lets one bad + # checkpoint drag the line far enough that good points look bad too. + slopes = [ + (pts[j, 1] - pts[i, 1]) / (pts[j, 0] - pts[i, 0]) + for i in range(len(pts)) for j in range(i + 1, len(pts)) + if pts[j, 0] - pts[i, 0] >= 1.0 + ] + speed = float(np.median(slopes)) if slopes else 1.0 + offset = float(np.median(pts[:, 1] - speed * pts[:, 0])) + keep = np.abs(pts[:, 1] - (offset + speed * pts[:, 0])) <= max_residual + if keep.any(): + pts = pts[keep] + offset, speed = solve(pts) + residual = np.abs(pts[:, 1] - (offset + speed * pts[:, 0])) + if abs(speed - 1.0) > 1e-3: + offset, speed = float(np.median(pts[:, 1] - pts[:, 0])), 1.0 + residual = np.abs(pts[:, 1] - (offset + pts[:, 0])) + return ClockFit( + offset=offset, + speed=speed, + residual_ms=float(residual.max()) * 1000.0, + checkpoints=len(pts), + ) + + +def level_db(samples: np.ndarray, sample_rate: int) -> np.ndarray: + """RMS level in dBFS per 10 ms frame for int16 or float samples.""" + hop = int(round(sample_rate * FRAME_SECONDS)) + frames = len(samples) // hop + if frames == 0: + return np.zeros(0, dtype=np.float32) + x = samples[: frames * hop].astype(np.float32) + if samples.dtype == np.int16: + x /= 32768.0 + power = np.mean(x.reshape(frames, hop) ** 2, axis=1) + return (10.0 * np.log10(power + 1e-10)).astype(np.float32) + + +def _smooth_power_db(db: np.ndarray, frames: int) -> np.ndarray: + if frames <= 1 or len(db) == 0: + return db + power = 10.0 ** (db / 10.0) + kernel = np.ones(frames, dtype=np.float32) / frames + return (10.0 * np.log10(np.convolve(power, kernel, mode="same") + 1e-10)).astype(np.float32) + + +SILENT = -1 +BOTH = -2 + + +def speaker_frames( + levels: list[np.ndarray], + *, + smooth_seconds: float = 0.3, + speech_window_db: float = 20.0, + floor_margin_db: float = 10.0, + both_margin_db: float = 4.0, +) -> np.ndarray: + """Per-frame dominant speaker index from one timeline-aligned level track per person. + + Each track is normalized to its own typical speech level (95th percentile) + so a hot guest mic doesn't win every frame. Bleed from the other person + lands far below that level and loses the argmax. Returns SILENT when nobody + clears their noise floor and BOTH when two people are within both_margin_db. + """ + if not levels: + return np.zeros(0, dtype=np.int32) + length = min(len(level) for level in levels) + frames = int(round(smooth_seconds / FRAME_SECONDS)) + stacked = np.stack([_smooth_power_db(level[:length], frames) for level in levels]) + floors = np.percentile(stacked, 20, axis=1, keepdims=True) + speech = np.percentile(stacked, 95, axis=1, keepdims=True) + normalized = stacked - speech + active = (normalized > -speech_window_db) & (stacked > floors + floor_margin_db) + scores = np.where(active, normalized, -np.inf) + + order = np.argsort(-scores, axis=0) + best = order[0] + labels = best.astype(np.int32) + best_score = np.take_along_axis(scores, order[:1], axis=0)[0] + labels[~np.isfinite(best_score)] = SILENT + if len(levels) > 1: + second_score = np.take_along_axis(scores, order[1:2], axis=0)[0] + gap = np.where(np.isfinite(second_score), best_score - np.nan_to_num(second_score, neginf=0.0), np.inf) + both = gap < both_margin_db + labels[both] = BOTH + return labels + + +def segments_to_frames(segments: list[dict], people: list[str], length: int) -> np.ndarray: + """Rasterize diarized {start, end, person} turns into the speaker_frames label format.""" + labels = np.full(length, SILENT, dtype=np.int32) + index = {person: i for i, person in enumerate(people)} + for seg in segments: + who = index.get(seg.get("person")) + if who is None: + continue + a = max(0, int(float(seg["start"]) / FRAME_SECONDS)) + b = min(length, int(float(seg["end"]) / FRAME_SECONDS)) + if b <= a: + continue + current = labels[a:b] + labels[a:b] = np.where((current != SILENT) & (current != who), BOTH, who) + return labels + + +def _runs(values: np.ndarray) -> list[list[int]]: + """Run-length encode into [value, start_frame, end_frame] lists.""" + if len(values) == 0: + return [] + change = np.flatnonzero(np.diff(values)) + 1 + starts = np.concatenate([[0], change]) + ends = np.concatenate([change, [len(values)]]) + return [[int(values[s]), int(s), int(e)] for s, e in zip(starts, ends)] + + +@dataclass +class Camera: + source_id: str + person: str + start: float + end: float + + def covers(self, a: float, b: float) -> bool: + return self.start <= a + 1e-6 and self.end >= b - 1e-6 + + +def plan_cuts( + labels: np.ndarray, + people: list[str], + cameras: list[Camera], + *, + range_start: float, + range_end: float, + min_shot: float = 2.0, + max_shot: float = 30.0, + wide_insert: float = 4.0, +) -> list[dict]: + """Turn per-frame speaker labels into [{start, end, source_id}] on the timeline. + + Silence holds the previous shot; crosstalk goes wide. Shots shorter than + min_shot fold into a neighbour whose camera also covers them, and a single + shot longer than max_shot gets a wide_insert-second wide shot in the middle + when a wide camera exists. Split camera files (part 1, part 2) are handled + because every decision checks which camera actually covers the moment. + """ + if not cameras or range_end <= range_start: + return [] + wide = [c for c in cameras if c.person == "wide"] + by_person = {p: [c for c in cameras if c.person == p] for p in people} + + def pick(target: str, a: float, b: float) -> Optional[Camera]: + pools = [] + if target == "wide": + pools = [wide] + else: + pools = [by_person.get(target, []), wide] + pools.append(cameras) + for pool in pools: + for cam in pool: + if cam.covers(a, b): + return cam + return None + + first = int(range_start / FRAME_SECONDS) + last = int(np.ceil(range_end / FRAME_SECONDS)) + window = np.full(last - first, SILENT, dtype=np.int32) + avail = labels[max(0, first):min(len(labels), last)] + lead = max(0, -first) + window[lead:lead + len(avail)] = avail + + targets: list[str] = [] + previous = "wide" if wide else (people[0] if people else "wide") + for value in window: + if value == SILENT: + targets.append(previous) + continue + current = "wide" if value == BOTH else people[value] + targets.append(current) + previous = current + codes = {name: i for i, name in enumerate(dict.fromkeys(targets))} + names = list(codes) + shot_runs = _runs(np.array([codes[t] for t in targets], dtype=np.int32)) + + # Camera coverage can change inside a run (a camera file ends); split there. + edges = sorted({c.start for c in cameras} | {c.end for c in cameras}) + shots: list[dict] = [] + for code, a, b in shot_runs: + start = range_start + a * FRAME_SECONDS + end = min(range_end, range_start + b * FRAME_SECONDS) + cuts = [start] + [e for e in edges if start < e < end] + [end] + for s, e in zip(cuts, cuts[1:]): + cam = pick(names[code], s, e) + if cam is None: + continue + if shots and shots[-1]["source_id"] == cam.source_id and abs(shots[-1]["end"] - s) < 1e-6: + shots[-1]["end"] = e + else: + shots.append({"start": s, "end": e, "source_id": cam.source_id}) + + by_id = {c.source_id: c for c in cameras} + shots = _fold_short_shots(shots, min_shot, by_id) + + if wide and max_shot > 0 and wide_insert > 0: + broken: list[dict] = [] + for shot in shots: + length = shot["end"] - shot["start"] + is_wide = by_id[shot["source_id"]].person == "wide" + if is_wide or length <= max_shot: + broken.append(shot) + continue + # Spacing inserts at least a shortest shot apart keeps them from + # overlapping when the wide cutaway is longer than max_shot. + pieces = int(length // max(max_shot, wide_insert + min_shot)) + step = length / (pieces + 1) + cursor = shot["start"] + for k in range(1, pieces + 1): + mid = shot["start"] + step * k + a, b = mid - wide_insert / 2, mid + wide_insert / 2 + if a - cursor < min_shot or shot["end"] - b < min_shot: + continue + cam = pick("wide", a, b) + if cam is None or cam.person != "wide": + continue + broken.append({**shot, "start": cursor, "end": a}) + broken.append({"start": a, "end": b, "source_id": cam.source_id}) + cursor = b + broken.append({**shot, "start": cursor, "end": shot["end"]}) + shots = broken + + return [ + {"start": round(s["start"], 3), "end": round(s["end"], 3), "source_id": s["source_id"]} + for s in shots + if s["end"] - s["start"] > 1e-3 + ] + + +def _merge_neighbours(shots: list[dict]) -> list[dict]: + merged: list[dict] = [] + for s in shots: + if merged and merged[-1]["source_id"] == s["source_id"] and abs(merged[-1]["end"] - s["start"]) < 1e-6: + merged[-1]["end"] = s["end"] + else: + merged.append(s) + return merged + + +def _fold_short_shots(shots: list[dict], min_shot: float, by_id: dict[str, Camera]) -> list[dict]: + """Fold shots under min_shot into their longer neighbour whose camera covers them. + + Always folds the globally shortest shot next, so flickery crosstalk + consolidates into one wide shot before the long shots around it grow into + it. A heap over a linked list keeps that O(n log n) on 3-hour episodes. + """ + shots = _merge_neighbours(shots) + n = len(shots) + prev = list(range(-1, n - 1)) + nxt = list(range(1, n + 1)) + nxt[-1:] = [-1] if n else [] + alive = [True] * n + version = [0] * n + heap: list[tuple[float, int, int]] = [] + + def length(i: int) -> float: + return shots[i]["end"] - shots[i]["start"] + + def push(i: int) -> None: + if 0 <= i and alive[i] and length(i) < min_shot: + heapq.heappush(heap, (length(i), i, version[i])) + + def unlink(i: int) -> None: + alive[i] = False + if prev[i] >= 0: + nxt[prev[i]] = nxt[i] + if nxt[i] >= 0: + prev[nxt[i]] = prev[i] + + for i in range(n): + push(i) + while heap: + _, i, v = heapq.heappop(heap) + if not alive[i] or v != version[i]: + continue + shot = shots[i] + candidates = [ + j for j in (prev[i], nxt[i]) + if j >= 0 + and abs((shots[j]["end"] if j == prev[i] else shots[j]["start"]) - (shot["start"] if j == prev[i] else shot["end"])) < 1e-6 + and by_id[shots[j]["source_id"]].covers(shot["start"], shot["end"]) + ] + if not candidates: + continue + j = max(candidates, key=length) + if j == prev[i]: + shots[j]["end"] = shot["end"] + else: + shots[j]["start"] = shot["start"] + unlink(i) + # The fold can leave j touching a shot on the same camera; join them. + for k in (prev[j], nxt[j]): + if k >= 0 and shots[k]["source_id"] == shots[j]["source_id"]: + shots[j]["start"] = min(shots[j]["start"], shots[k]["start"]) + shots[j]["end"] = max(shots[j]["end"], shots[k]["end"]) + unlink(k) + # j grew, so its old heap entry is stale; its neighbours may now have a + # camera to fold into that they lacked before. + version[j] += 1 + for k in (j, prev[j], nxt[j]): + push(k) + + out, i = [], next((k for k in range(n) if alive[k] and prev[k] == -1), -1) + while i >= 0: + out.append(shots[i]) + i = nxt[i] + return out + + +def speech_bounds(labels: np.ndarray) -> Optional[tuple[float, float]]: + """First and last frame where anyone speaks, in seconds.""" + spoken = np.flatnonzero(labels != SILENT) + if len(spoken) == 0: + return None + return float(spoken[0]) * FRAME_SECONDS, float(spoken[-1] + 1) * FRAME_SECONDS diff --git a/cli/main.go b/cli/main.go index c81ab87a..e402263c 100644 --- a/cli/main.go +++ b/cli/main.go @@ -83,7 +83,7 @@ func wantsRuntime(args []string) bool { return true } switch args[0] { - case "process", "transcribe", "studio", "auto", "ui", "webui": + case "process", "transcribe", "studio", "multicam", "auto", "ui", "webui": return true } return false @@ -768,6 +768,7 @@ Engine commands (routed to the processing backend): process