diff --git a/CLAUDE.md b/CLAUDE.md index 3b76e2197..4c4ca2052 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -30,11 +30,13 @@ that change how you work: - **Projects are not created by Flow Kit any more.** Make one in the Flow UI and pin its uuid as `FLOW_PROJECT_ID`, or pass `flow_project_id` to `POST /api/projects`. -- **Four capabilities are unported** because their payloads were never captured: - 4K upscale, r2v, start+end-frame chaining, and Omni Flash. They fail with +- **Three capabilities are unported**, all on the Veo path, because their + payloads were never captured: **video** upscale (not image export, which + works), Veo r2v, and Veo start+end-frame chaining. They fail with `UNSUPPORTED_ON_BATCH_API` rather than silently producing the wrong thing. - `FLOW_ALLOW_DEGRADED=1` drops chaining and r2v to plain i2v; upscale has no - fallback. To restore one properly, see `docs/CAPTURE.md`. + Omni covers frame, first+last and reference modes — use + `model_family=omni_flash`. `FLOW_ALLOW_DEGRADED=1` drops Veo chaining and r2v + to plain i2v; video upscale has no fallback. See `docs/CAPTURE.md`. - **A poll saying "Media not found." is not a failure.** Finished jobs report it. ## Skills diff --git a/README.md b/README.md index 04be57229..31b866c7e 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,15 @@ > from an older Flow Kit: reload the extension (v0.3.2+) and pin > `FLOW_PROJECT_ID`; Flow Kit can no longer create the project for you. > -> Four capabilities are still unported because their payloads were never -> captured off the new UI — **video upscale**, **reference-to-video (r2v)**, -> **start+end-frame chaining**, and **Omni Flash frame/reference**. They fail -> loudly with `UNSUPPORTED_ON_BATCH_API` instead of quietly producing the wrong -> thing. `FLOW_ALLOW_DEGRADED=1` drops chaining and r2v to plain i2v; video -> upscale has no fallback. To restore one properly see [`docs/CAPTURE.md`](docs/CAPTURE.md). +> Every Omni 1.1 Flash video mode is ported and live-verified: text-to-video, +> first frame, first+last frame, and references. Three capabilities remain +> unported on the **Veo** path because their payloads were never captured off +> the new UI — **video upscale**, **Veo reference-to-video**, and **Veo +> start+end-frame chaining**. They fail loudly with `UNSUPPORTED_ON_BATCH_API` +> instead of quietly producing the wrong thing. For the latter two, Omni covers +> the same shot with `model_family=omni_flash`, or `FLOW_ALLOW_DEGRADED=1` drops +> them to plain i2v; video upscale has no fallback. To restore one properly see +> [`docs/CAPTURE.md`](docs/CAPTURE.md). --- @@ -276,10 +279,10 @@ Three capabilities have no captured payload, so they fail with | Capability | Status | Workaround | |---|---|---| | 4K/1080p upscale (`/fk-pipeline` last step) | unported | none — keep the 1080p render | -| Reference-to-video (r2v) | unported | `FLOW_ALLOW_DEGRADED=1` → i2v off the first reference | -| Start+end-frame chaining (`/fk-gen-chain-videos`) | unported | `FLOW_ALLOW_DEGRADED=1` → i2v off the start frame | +| Veo reference-to-video (r2v) | unported | Omni r2v (`model_family=omni_flash`), or `FLOW_ALLOW_DEGRADED=1` → i2v off the first reference | +| Veo start+end-frame chaining (`/fk-gen-chain-videos`) | unported | Omni first+last (`model_family=omni_flash`), or `FLOW_ALLOW_DEGRADED=1` → i2v off the start frame | | Omni Flash text-to-video | ported | `POST /api/flow/generate-video-omni-text` (4/6/8/10s) | -| Omni Flash frame/reference modes | unported | use Veo or text-to-video until their batch payloads are captured | +| Omni Flash frame / first+last / reference modes | ported | `eb1hJf`, `nprQif`, `MZZa6b` — `POST /api/flow/generate-video` with `model_family=omni_flash` | Restoring one starts with a capture, not a guess: [`docs/CAPTURE.md`](docs/CAPTURE.md). diff --git a/agent/api/flow.py b/agent/api/flow.py index 0d94b780d..7aed5ef3f 100644 --- a/agent/api/flow.py +++ b/agent/api/flow.py @@ -39,6 +39,7 @@ class GenerateVideoRequest(BaseModel): # Backward compatible: legacy requests remain Veo unless explicitly set. model_family: Literal["veo", "omni_flash"] = "veo" duration_s: int = 8 + resolution: Literal["360p", "720p"] = "720p" class GenerateVideoRefsRequest(BaseModel): @@ -52,6 +53,7 @@ class GenerateVideoRefsRequest(BaseModel): # explicitly opt into Omni Flash. model_family: Literal["veo", "omni_flash"] = "veo" duration_s: int = 8 + resolution: Literal["360p", "720p"] = "720p" class GenerateOmniFlashVideoRequest(BaseModel): @@ -60,6 +62,7 @@ class GenerateOmniFlashVideoRequest(BaseModel): project_id: str scene_id: str = "" duration_s: int = 8 + resolution: Literal["360p", "720p"] = "720p" aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT" user_paygate_tier: str = "PAYGATE_TIER_ONE" @@ -175,8 +178,8 @@ async def generate_video(body: GenerateVideoRequest): request uses Omni First frame. When ``end_image_media_id`` is also present, it uses Omni First+Last frames. - Omni responses include ``flowkitPolling.workflows`` and must use workflow - media polling rather than legacy operation polling. + On the migrated batch transport, Omni frame-conditioned responses return + ``flowkitPolling.mode=batch_operation`` and are polled through ``/check-status``. """ client = get_flow_client() if not client.connected: @@ -190,6 +193,7 @@ async def generate_video(body: GenerateVideoRequest): project_id=body.project_id, scene_id=body.scene_id, duration_s=body.duration_s, + resolution=body.resolution, aspect_ratio=body.aspect_ratio, user_paygate_tier=body.user_paygate_tier, ) @@ -204,7 +208,7 @@ async def generate_video(body: GenerateVideoRequest): raise HTTPException(400, str(exc)) from exc else: result = await client.generate_video( - **body.model_dump(exclude={"model_family", "duration_s"}, exclude_none=True) + **body.model_dump(exclude={"model_family", "duration_s", "resolution"}, exclude_none=True) ) if result.get("error") or (isinstance(result.get("status"), int) and result["status"] >= 400): @@ -218,8 +222,8 @@ async def generate_video_refs(body: GenerateVideoRefsRequest): Existing requests default to ``model_family=veo``. Set ``model_family=omni_flash`` and ``duration_s`` to 4/6/8/10 to use Omni. - Omni responses include ``flowkitPolling.workflows``; poll those workflows, - not the operation-looking handles in the raw Flow response. + Migrated Omni Ingredients/R2V returns ``flowkitPolling.mode=batch_operation``; + poll its operations through ``/check-status``. """ client = get_flow_client() if not client.connected: @@ -233,6 +237,7 @@ async def generate_video_refs(body: GenerateVideoRefsRequest): project_id=body.project_id, scene_id=body.scene_id, duration_s=body.duration_s, + resolution=body.resolution, aspect_ratio=body.aspect_ratio, user_paygate_tier=body.user_paygate_tier, ) @@ -240,7 +245,7 @@ async def generate_video_refs(body: GenerateVideoRefsRequest): raise HTTPException(400, str(exc)) from exc else: result = await client.generate_video_from_references( - **body.model_dump(exclude={"model_family", "duration_s"}) + **body.model_dump(exclude={"model_family", "duration_s", "resolution"}) ) if result.get("error") or (isinstance(result.get("status"), int) and result["status"] >= 400): diff --git a/agent/services/flow_batch.py b/agent/services/flow_batch.py index f81c1e44e..e2f41c00e 100644 --- a/agent/services/flow_batch.py +++ b/agent/services/flow_batch.py @@ -37,6 +37,8 @@ RPC_GEN_IMAGE = "ogiZ0b" RPC_GEN_VIDEO = "eb1hJf" RPC_GEN_VIDEO_TEXT = "YhhmEf" +RPC_GEN_VIDEO_FIRST_LAST = "nprQif" +RPC_GEN_VIDEO_REFERENCES = "MZZa6b" RPC_OPERATION = "jwpduf" RPC_PROJECT_MEDIA = "Zzl0ze" RPC_MEDIA = "as29s" @@ -411,6 +413,101 @@ def video_request(prompt: str, project_id: str, source_media_id: str, return build_envelope(RPC_GEN_VIDEO, inner) +def omni_first_frame_request(prompt: str, project_id: str, source_media_id: str, + *, duration_s: int = 8, resolution: str = "720p", + aspect: Any = VIDEO_ASPECT_LANDSCAPE, + crop: Optional[list] = None) -> str: + """Build current Omni first-frame I2V (RPC ``eb1hJf``). + + Live-captured from Flow on 2026-09-14. 720p uses ``abra_i2v_s``; + 360p appends ``_360p`` and carries the UI's low-resolution option slot. + """ + if duration_s not in (4, 6, 8, 10): + raise ValueError("Omni duration must be 4, 6, 8 or 10 seconds") + res = str(resolution).strip().lower() + if res not in {"360p", "720p"}: + raise ValueError("Omni resolution must be 360p or 720p") + model = f"abra_i2v_{duration_s}s" + ("_360p" if res == "360p" else "") + request = [ + [None, None, [[[prompt]]]], + model, + resolve_video_aspect(aspect), + None, + [None, source_media_id, None, None, None, + FULL_FRAME_CROP if crop is None else crop], + [None, None, None, None, _client_uuid(), _client_uuid()], + ] + if res == "360p": + request.extend([None, None, None, [4]]) + return build_envelope(RPC_GEN_VIDEO, [ + [request], + _context(project_id), + [_client_uuid(), 2], + ]) + + +def omni_first_last_request(prompt: str, project_id: str, + start_media_id: str, end_media_id: str, + *, duration_s: int = 8, resolution: str = "720p", + aspect: Any = VIDEO_ASPECT_LANDSCAPE, + start_crop: Optional[list] = None, + end_crop: Optional[list] = None) -> str: + """Build Omni First+Last frames submit (RPC ``nprQif``).""" + if duration_s not in (4, 6, 8, 10): + raise ValueError("Omni duration must be 4, 6, 8 or 10 seconds") + res = str(resolution).strip().lower() + if res not in {"360p", "720p"}: + raise ValueError("Omni resolution must be 360p or 720p") + model = f"omni_flash_i2v_{duration_s}s_first_last" + ("_360p" if res == "360p" else "") + request = [ + [None, None, [[[prompt]]]], + model, + resolve_video_aspect(aspect), + None, + [None, start_media_id, None, None, None, + FULL_FRAME_CROP if start_crop is None else start_crop], + [None, end_media_id, None, None, None, + FULL_FRAME_CROP if end_crop is None else end_crop], + [None, None, None, None, _client_uuid(), _client_uuid()], + ] + return build_envelope(RPC_GEN_VIDEO_FIRST_LAST, [ + [request], + _context(project_id), + [_client_uuid(), 2], + ]) + + +def omni_reference_video_request(prompt: str, project_id: str, + reference_media_ids: list[str], + *, duration_s: int = 8, resolution: str = "720p", + aspect: Any = VIDEO_ASPECT_LANDSCAPE) -> str: + """Build Omni Ingredients/reference-to-video submit (RPC ``MZZa6b``).""" + refs = [str(mid) for mid in reference_media_ids if str(mid)] + if not refs: + raise ValueError("Omni reference-to-video requires at least one reference image") + if duration_s not in (4, 6, 8, 10): + raise ValueError("Omni duration must be 4, 6, 8 or 10 seconds") + res = str(resolution).strip().lower() + if res not in {"360p", "720p"}: + raise ValueError("Omni resolution must be 360p or 720p") + model = f"abra_r2v_{duration_s}s" + ("_360p" if res == "360p" else "") + request = [ + [None, None, [[[prompt]]]], + [[None, mid] for mid in refs], + model, + resolve_video_aspect(aspect), + None, + [None, None, None, None, _client_uuid(), _client_uuid()], + ] + if res == "360p": + request.extend([None, None, None, None, None, [4]]) + return build_envelope(RPC_GEN_VIDEO_REFERENCES, [ + [request], + _context(project_id), + [_client_uuid(), 2], + ]) + + def text_video_request(prompt: str, project_id: str, aspect: Any = VIDEO_ASPECT_LANDSCAPE, model: str = "abra_t2v_4s") -> str: diff --git a/agent/services/omni_flash.py b/agent/services/omni_flash.py index 8de784447..21895f1dc 100644 --- a/agent/services/omni_flash.py +++ b/agent/services/omni_flash.py @@ -1,13 +1,14 @@ """Gemini Omni Flash video generation through the Google Flow bridge. -What works today: **text-to-video**, on flow.google.com batchexecute. It submits -through ``fb.text_video_request`` like every other migrated call. +Migrated ``flow.google.com`` batch surfaces live-verified on 2026-09-14: -What does not: first-frame, first+last, and reference-to-video. Those three were -only ever implemented against the pre-migration REST endpoints, and that -transport was removed when Flow stopped minting the bearer it needed. Their -migrated payloads have not been captured, so they validate their inputs and then -fail by name — see ``docs/CAPTURE.md`` for how to record one off the new UI. +* text -> video: ``YhhmEf`` + ``abra_t2v_s`` (workflow/media polling) +* first frame -> video: ``eb1hJf`` + ``abra_i2v_s`` +* first + last frame -> video: ``nprQif`` + ``omni_flash_i2v_s_first_last`` +* Ingredients/references -> video: ``MZZa6b`` + ``abra_r2v_s`` + +Reference-conditioned modes return normal batch operation receipts and use the +same operation/media poller as migrated Veo. Duration-specific model keys live in ``agent/models.json`` so a rollout that rotates a wire key does not need a code release. @@ -26,17 +27,6 @@ _MODELS_FILE = Path(__file__).parent.parent / "models.json" -#: Frame and reference generation had only a REST implementation, and that -#: transport is gone. Failing by name here beats dying on a 401 five retries -#: deep, and names the thing a capture would restore. -_UNSUPPORTED_ON_BATCH = ( - "UNSUPPORTED_ON_BATCH_API: Omni Flash frame/reference generation is not yet " - "ported to flow.google.com batchexecute. Omni text-to-video is supported on " - "the batch path; frame-to-video, start+end and reference-to-video still need " - "their migrated payload captures." -) - - OMNI_FLASH_VALID_DURATIONS = (4, 6, 8, 10) OMNI_FLASH_VALID_ASPECTS = { "VIDEO_ASPECT_RATIO_PORTRAIT", @@ -63,6 +53,34 @@ def _validate_aspect(aspect_ratio: str) -> None: ) +def _validate_resolution(resolution: str) -> str: + value = str(resolution or "720p").strip().lower() + if value not in {"360p", "720p"}: + raise ValueError("Omni Flash resolution must be 360p or 720p") + return value + + +def _batch_operation_result(operation, project_id: str, model: str, duration_s: int, resolution: str) -> dict: + pending = { + "operation": {"name": operation.operation_id}, + "status": "MEDIA_GENERATION_STATUS_PENDING", + } + return { + "status": 200, + "data": { + "operations": [pending], + "model": model, + "duration_s": duration_s, + "resolution": resolution, + "flowkitPolling": { + "mode": "batch_operation", + "project_id": project_id, + "operations": [pending], + }, + }, + } + + def _load_model_key(duration_s: int, mode: str = "reference_to_video") -> str: """Resolve a configured Omni Flash model key for ``mode`` + duration.""" _validate_duration(duration_s) @@ -204,19 +222,48 @@ async def _submit_omni_frame_video( project_id: str, scene_id: str = "", duration_s: int = 8, + resolution: str = "720p", aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT", user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, ) -> dict: - """Submit Omni first-frame or First+Last generation.""" - _validate_frame_inputs( - start_image_media_id, - end_image_media_id, - duration_s, - aspect_ratio, - ) - return {"error": _UNSUPPORTED_ON_BATCH} + """Submit Omni first-frame or First+Last generation. + Batch payloads were live-captured from ``flow.google.com`` on 2026-09-14: + first-frame uses ``eb1hJf``; First+Last uses ``nprQif``. Both return normal + batch operations and are polled through ``/api/flow/check-status``. + """ + _validate_frame_inputs(start_image_media_id, end_image_media_id, duration_s, aspect_ratio) + resolution = _validate_resolution(resolution) + mode = "start_end_frame_to_video" if end_image_media_id is not None else "frame_to_video" + client = get_flow_client() + + try: + pid = client._batch_project_id(project_id) + if end_image_media_id is None: + freq = fb.omni_first_frame_request( + prompt, pid, start_image_media_id, duration_s=duration_s, + resolution=resolution, aspect=aspect_ratio, + ) + rpcid = fb.RPC_GEN_VIDEO + batch_model = f"abra_i2v_{duration_s}s" + ("_360p" if resolution == "360p" else "") + else: + freq = fb.omni_first_last_request( + prompt, pid, start_image_media_id, end_image_media_id, + duration_s=duration_s, resolution=resolution, aspect=aspect_ratio, + ) + rpcid = fb.RPC_GEN_VIDEO_FIRST_LAST + batch_model = f"omni_flash_i2v_{duration_s}s_first_last" + ( + "_360p" if resolution == "360p" else "" + ) + payload = await client._batch_payload( + rpcid, freq, fb.CAPTCHA_VIDEO, timeout=120, + ) + operation = fb.read_operation(payload) + client._remember_operation(operation.operation_id, pid) + except Exception as exc: + return {"status": 502, "error": f"{type(exc).__name__}: {exc}"} + return _batch_operation_result(operation, pid, batch_model, duration_s, resolution) async def generate_omni_flash_first_frame_video( start_image_media_id: str, @@ -224,6 +271,7 @@ async def generate_omni_flash_first_frame_video( project_id: str, scene_id: str = "", duration_s: int = 8, + resolution: str = "720p", aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT", user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, @@ -236,6 +284,7 @@ async def generate_omni_flash_first_frame_video( project_id=project_id, scene_id=scene_id, duration_s=duration_s, + resolution=resolution, aspect_ratio=aspect_ratio, user_paygate_tier=user_paygate_tier, seed=seed, @@ -249,6 +298,7 @@ async def generate_omni_flash_first_last_video( project_id: str, scene_id: str = "", duration_s: int = 8, + resolution: str = "720p", aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT", user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, @@ -261,6 +311,7 @@ async def generate_omni_flash_first_last_video( project_id=project_id, scene_id=scene_id, duration_s=duration_s, + resolution=resolution, aspect_ratio=aspect_ratio, user_paygate_tier=user_paygate_tier, seed=seed, @@ -273,19 +324,35 @@ async def generate_omni_flash_video( project_id: str, scene_id: str = "", duration_s: int = 8, + resolution: str = "720p", aspect_ratio: str = "VIDEO_ASPECT_RATIO_PORTRAIT", user_paygate_tier: str = "PAYGATE_TIER_ONE", seed: int | None = None, ) -> dict: - """Submit an Omni Flash reference-to-video generation. + """Submit Omni Flash Ingredients/reference-to-video generation. - Successful responses are annotated with ``data.flowkitPolling`` containing - the workflow names and primary media IDs required by the Omni polling path. - Do not feed Omni operation handles to ``check_video_status``. + The migrated UI uses RPC ``MZZa6b`` with ``abra_r2v_s`` (720p) + or ``abra_r2v_s_360p``. Batch jobs use normal operation polling. """ - _validate_reference_inputs(reference_media_ids, duration_s, aspect_ratio) - return {"error": _UNSUPPORTED_ON_BATCH} + refs = _validate_reference_inputs(reference_media_ids, duration_s, aspect_ratio) + resolution = _validate_resolution(resolution) + client = get_flow_client() + try: + pid = client._batch_project_id(project_id) + freq = fb.omni_reference_video_request( + prompt, pid, refs, duration_s=duration_s, + resolution=resolution, aspect=aspect_ratio, + ) + payload = await client._batch_payload( + fb.RPC_GEN_VIDEO_REFERENCES, freq, fb.CAPTCHA_VIDEO, timeout=120, + ) + operation = fb.read_operation(payload) + client._remember_operation(operation.operation_id, pid) + except Exception as exc: + return {"status": 502, "error": f"{type(exc).__name__}: {exc}"} + batch_model = f"abra_r2v_{duration_s}s" + ("_360p" if resolution == "360p" else "") + return _batch_operation_result(operation, pid, batch_model, duration_s, resolution) async def _check_omni_batch_media( workflows: list[dict], diff --git a/docs/OMNI_FLASH.md b/docs/OMNI_FLASH.md index bf8a2e607..e83bb9805 100644 --- a/docs/OMNI_FLASH.md +++ b/docs/OMNI_FLASH.md @@ -15,33 +15,44 @@ Expected state on the migrated Flow transport: ```json {"status":"ok","extension_connected":true} -{"connected":true,"transport":"batch"} +{"connected":true,"transport":"batch","authenticated":true,"at_token_present":true} ``` Use `http://127.0.0.1:8100` when the caller runs on the FlowKit host. For a remote integration, set `FLOWKIT_BASE_URL` to the protected HTTPS reverse-proxy URL and allow only the required source IPs or private network. Do not expose Chrome, VNC/noVNC, the extension WebSocket, or port 8100 publicly. ## Supported modes -On `flow.google.com`, Omni **text-to-video** is migrated and live-verified. Frame and reference generation had only a REST implementation; that transport has been removed, so they validate their inputs and then fail by name with `UNSUPPORTED_ON_BATCH_API` until their payloads are captured (see `docs/CAPTURE.md`). +On the current `flow.google.com` batch transport, every Omni 1.1 Flash video mode exposed by Flow's Video composer is supported and live-verified: -| Mode | Batch status | Endpoint | Internal model family | +| Mode | Batch status | Endpoint | Current wire | |---|---|---|---| -| Text to video | **supported** | `POST /api/flow/generate-video-omni-text` | `abra_t2v_s` | -| First frame to video | not yet ported | `POST /api/flow/generate-video` | `abra_i2v_s` (legacy only) | -| First + Last frame to video | not yet ported | `POST /api/flow/generate-video` | `abra_i2v_s` (legacy only) | -| References to video | not yet ported | `POST /api/flow/generate-video-omni` | `abra_r2v_s` (legacy only) | +| Text to video | **supported** | `POST /api/flow/generate-video-omni-text` | `YhhmEf` + `abra_t2v_s` | +| First frame to video | **supported** | `POST /api/flow/generate-video` with `model_family=omni_flash` | `eb1hJf` + `abra_i2v_s` | +| First + Last frame to video | **supported** | `POST /api/flow/generate-video` with `model_family=omni_flash` + `end_image_media_id` | `nprQif` + `omni_flash_i2v_s_first_last` | +| Ingredients / references to video | **supported** | `POST /api/flow/generate-video-omni` or `/generate-video-refs` | `MZZa6b` + `abra_r2v_s` | -Text-to-video supports `4`, `6`, `8`, and `10` seconds, with portrait and landscape aspect ratios. The migrated `YhhmEf` wire was live-verified with `abra_t2v_4s`; the downloaded result was exactly 4.000 seconds at 1280×720/24 fps. Completed media resolves through the migrated `as29s` media lookup. +Reference-conditioned modes support durations `4`, `6`, `8`, and `10` seconds, resolutions `360p` and `720p`, and: + +- `VIDEO_ASPECT_RATIO_PORTRAIT` (`9:16`) +- `VIDEO_ASPECT_RATIO_LANDSCAPE` (`16:9`) + +For 360p Flow uses `_360p` model variants. The 360p first-frame and Ingredients payloads also carry the same low-resolution option slots captured from the live UI. First+Last uses its dedicated `nprQif` payload and model family. + +The migrated wires were re-captured from the live Flow UI on 2026-09-14. Live API smoke tests completed successfully for First frame, First+Last, and Ingredients/R2V and resolved signed `flow-content.google` video URLs through the batch operation poller. + +Polling differs only for text-to-video: text-to-video returns `flowkitPolling.mode=batch_media` and uses `/api/flow/check-omni-status`; all image/reference-conditioned modes return `flowkitPolling.mode=batch_operation` and use `/api/flow/check-status` with their `operations` array. ## End-to-end integration flow 1. Check `/health` and `/api/flow/status`. -2. Submit `POST /api/flow/generate-video-omni-text` with prompt, project ID, duration and aspect ratio. -3. Persist the returned `flowkitPolling` object. -4. Poll `/api/flow/check-omni-status` every 10–20 seconds with its `project_id` and `workflows`. -5. On `COMPLETED`, immediately download `workflows[].media.url`; the signed URL is short-lived. +2. Submit the desired Omni mode. +3. Persist the complete `flowkitPolling` object returned by the submit before doing anything else. +4. If `mode=batch_media`, poll `/api/flow/check-omni-status` using `project_id` + `workflows`. +5. If `mode=batch_operation`, poll `/api/flow/check-status` using `project_id` + `operations`. +6. Continue on pending state; stop on failure; on success immediately download the signed video URL. +7. Store the downloaded video in durable storage because Google URLs are signed and short-lived. -Example: +Example text-to-video submit: ```bash curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video-omni-text" \ @@ -54,11 +65,11 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video-omni-text" \ }' ``` -Do not feed Omni workflow names to the legacy Veo operation poller. +Do not convert workflow names into operation handles, and do not convert batch operation handles into workflow names. Persist and replay the polling descriptor exactly as FlowKit returns it. ## Supplying images -This section applies to the legacy frame/reference Omni modes, which are not yet ported to the migrated batch transport. +Frame, First+Last, and Ingredients/R2V all consume Flow media IDs. Upload or reuse the reference images first, then pass their media IDs to the generation endpoint. `POST /api/flow/upload-image` is not a multipart upload endpoint. Its `file_path` is an absolute path on the **FlowKit server**, not on the calling server. @@ -106,12 +117,13 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video" \ "project_id": "FLOW_PROJECT_ID", "scene_id": "JOB_ID", "duration_s": 4, + "resolution": "720p", "aspect_ratio": "VIDEO_ASPECT_RATIO_PORTRAIT", "user_paygate_tier": "PAYGATE_TIER_ONE" }' ``` -This uses `batchAsyncGenerateVideoStartImage`. +This uses batch RPC `eb1hJf` with `abra_i2v_s` (or the `_360p` variant). ## First + Last frame to video @@ -126,12 +138,13 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video" \ "project_id": "FLOW_PROJECT_ID", "scene_id": "JOB_ID", "duration_s": 4, + "resolution": "720p", "aspect_ratio": "VIDEO_ASPECT_RATIO_PORTRAIT", "user_paygate_tier": "PAYGATE_TIER_ONE" }' ``` -This uses `batchAsyncGenerateVideoStartAndEndImage` and sends both `startImage` and `endImage`. +This uses batch RPC `nprQif` with the dedicated `omni_flash_i2v_s_first_last` model family. ## References to video @@ -146,86 +159,71 @@ curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/generate-video-omni" \ "project_id": "FLOW_PROJECT_ID", "scene_id": "JOB_ID", "duration_s": 4, + "resolution": "720p", "aspect_ratio": "VIDEO_ASPECT_RATIO_LANDSCAPE", "user_paygate_tier": "PAYGATE_TIER_ONE" }' ``` -The compatible generic endpoint is `POST /api/flow/generate-video-refs` with the same fields plus `"model_family":"omni_flash"`. +The compatible generic endpoint is `POST /api/flow/generate-video-refs` with the same fields plus `"model_family":"omni_flash"`. Both routes use batch RPC `MZZa6b`. ## Submit response and polling -Persist the entire normalized polling descriptor returned by submit: +Image/reference-conditioned Omni submits return normal batch operation polling: ```json { + "operations": [ + {"operation":{"name":"OPERATION_ID"},"status":"MEDIA_GENERATION_STATUS_PENDING"} + ], + "model": "abra_r2v_4s", + "duration_s": 4, + "resolution": "720p", "flowkitPolling": { - "mode": "project_media", + "mode": "batch_operation", "project_id": "FLOW_PROJECT_ID", - "workflows": [ - { - "name": "WORKFLOW_NAME", - "primary_media_id": "PRIMARY_MEDIA_ID", - "project_id": "FLOW_PROJECT_ID" - } + "operations": [ + {"operation":{"name":"OPERATION_ID"},"status":"MEDIA_GENERATION_STATUS_PENDING"} ] } } ``` -Poll using those values without transforming workflow names into operation handles: +Poll that descriptor without transforming it: ```bash -curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/check-omni-status" \ +curl -fsS -X POST "$FLOWKIT_BASE_URL/api/flow/check-status" \ -H 'Content-Type: application/json' \ -d '{ "project_id": "FLOW_PROJECT_ID", - "workflows": [ - { - "name": "WORKFLOW_NAME", - "primary_media_id": "PRIMARY_MEDIA_ID", - "project_id": "FLOW_PROJECT_ID" - } - ], - "include_encoded_video": false + "operations": [ + {"operation":{"name":"OPERATION_ID"},"status":"MEDIA_GENERATION_STATUS_PENDING"} + ] }' ``` -The generic `POST /api/flow/check-status` endpoint also accepts the same `workflows` and automatically selects Omni project polling. - -Pending response: - -```json -{ - "project_id": "FLOW_PROJECT_ID", - "done": false, - "status": "PENDING", - "workflows": [{"done":false,"status":"PENDING"}] -} -``` - -Successful response: +When complete, the operation contains the generated media ID and signed video URL: ```json { - "project_id": "FLOW_PROJECT_ID", - "done": true, - "status": "COMPLETED", - "workflows": [ + "operations": [ { - "done": true, - "status": "MEDIA_GENERATION_STATUS_SUCCESSFUL", - "media": { - "media_id": "PRIMARY_MEDIA_ID", - "url": "https://flow-content.google/...", - "encoded_video_available": false - } + "operation": { + "name": "OPERATION_ID", + "metadata": { + "video": { + "mediaId": "MEDIA_ID", + "fifeUrl": "https://flow-content.google/video/..." + } + } + }, + "status": "MEDIA_GENERATION_STATUS_SUCCESSFUL" } ] } ``` -Keep `include_encoded_video` set to `false`. FlowKit resolves the signed download URL without buffering the MP4 through Chrome or the extension bridge. +Text-to-video remains the exception: it returns `mode=batch_media` with workflow/media descriptors and is polled through `/api/flow/check-omni-status`. ## Retry and failure policy @@ -267,15 +265,15 @@ Mappings live in `agent/models.json`: } ``` -The mappings can be changed through `PATCH /api/models` if Google rotates internal keys. Treat configured credit-cost estimates as informational only: Google can change pricing, so use `GET /api/flow/credits` and the submit response's `remainingCredits` where available. +The legacy mappings above remain for the pre-migration transport. On the migrated batch transport, current Flow wire names are derived from duration + resolution exactly as live-captured: First frame uses `abra_i2v_*`, First+Last uses `omni_flash_i2v_*_first_last`, and Ingredients uses `abra_r2v_*`, with `_360p` appended for 360p. The mappings can still be changed through `PATCH /api/models` for legacy transport if Google rotates old keys. Treat configured credit-cost estimates as informational only: Google can change pricing, so use `GET /api/flow/credits` and the submit response's `remainingCredits` where available. ## Minimal agent checklist - Use the `/api/flow/...` paths exactly. - Select Omni explicitly with `model_family: "omni_flash"` on shared endpoints. - Upload inputs once and reuse returned Flow media IDs. -- Persist `project_id`, workflow `name`, and `primary_media_id` before polling. -- Poll workflows through project media status, never through legacy Veo operations. +- Persist the entire returned `flowkitPolling` descriptor before polling. +- Use `/check-status` for `batch_operation` and `/check-omni-status` for `batch_media`; never transform one receipt type into the other. - Download signed output URLs immediately into durable project storage. - Do not log Google auth data, extension messages, or complete signed URLs. - Use a stable `scene_id`/job ID for traceability. diff --git a/skills/fk-doctor.md b/skills/fk-doctor.md index a13074b9c..cd505235f 100644 --- a/skills/fk-doctor.md +++ b/skills/fk-doctor.md @@ -138,8 +138,7 @@ Detection lives in `agent/worker/_parsing.py:_is_error`. A result is treated as | `FLOW_TAB_DISCARDED` | Chrome discarded the backgrounded tab and the reload did not revive it | Retried with backoff | Pin the Flow tab, or keep its window visible | | `NO_INJECTION_RESULT` | `chrome.scripting.executeScript` resolved with no frame result (`background.js:540`), so the envelope never ran — the Flow tab went away or was still booting at the moment of the call. Most likely on the first captcha-bearing RPC after an agent restart | Retried with backoff, counts against `MAX_RETRIES` | Usually transient — retry first. Rule out the lookalikes before digging: a failed mint reports `CAPTCHA_FAILED`, an oversized payload is not it (`MAX_RPC_TEXT` is 32 MB), and a structured-clone failure surfaces as 500, not 502. If it repeats, pin the Flow tab and reload the extension | | `NO_FLOW_PROJECT` | No Flow project to scope the RPC to | **Terminal — not retried** | Create a project in the Flow UI, pin its uuid as `FLOW_PROJECT_ID` (or pass `flow_project_id` on `POST /api/projects`) | -| `UNSUPPORTED_ON_BATCH_API` | A capability whose payload was never captured off the new UI: **video upscale**, **r2v**, **start+end-frame chaining** | **Terminal — not retried** | For chaining and r2v, `FLOW_ALLOW_DEGRADED=1` falls back to plain i2v off the start frame. Upscale has no fallback. Real fix: capture the payload — `docs/CAPTURE.md` | -| `UNSUPPORTED_ON_BATCH_API: Omni Flash` | Omni frame/reference had only a REST implementation; no batchexecute payload captured | **Terminal — not retried** | Use `model_family=veo`, or Omni **text-to-video**, which is migrated | +| `UNSUPPORTED_ON_BATCH_API` | A capability whose payload was never captured off the new UI, all on the Veo path: **video upscale**, **Veo r2v**, **Veo start+end-frame chaining**. Every Omni mode is ported, so this never names Omni any more | **Terminal — not retried** | For chaining and r2v, `FLOW_ALLOW_DEGRADED=1` falls back to plain i2v off the start frame. Upscale has no fallback. Real fix: capture the payload — `docs/CAPTURE.md` | | `PUBLIC_ERROR_UNUSUAL_ACTIVITY` | A reCAPTCHA token was replayed — they are single-use | Retried as a captcha error | Usually self-clears; if it persists the extension is reusing a token, reload it | | `no ogiZ0b envelope in response` | The RPC answered but not with the payload we came for — usually a signed-out page returning an HTML redirect | Retried with backoff | Re-sign in on the Flow tab | | `Polling timeout after Ns: Media not found.` | The job never produced media inside the budget | Terminal after `MAX_RETRIES` | The quoted complaint is a **diagnostic, not the cause** — finished jobs report it too. Check the Flow UI: if the clip is there, raise `VIDEO_POLL_TIMEOUT` | diff --git a/tests/unit/test_flow_batch.py b/tests/unit/test_flow_batch.py index db1f3457a..2a0804593 100644 --- a/tests/unit/test_flow_batch.py +++ b/tests/unit/test_flow_batch.py @@ -155,6 +155,42 @@ def test_a_hand_reframed_crop_overrides_the_default(self): crop = [None, 0.1, 1, 0.9] assert inner(fb.video_request("go", self.PID, "mid", crop=crop))[0][0][4][5] == crop + def test_omni_first_frame_360p_matches_live_eb1hjf_shape(self): + payload = inner(fb.omni_first_frame_request( + "move", self.PID, "start-mid", duration_s=4, resolution="360p", + aspect="VIDEO_ASPECT_RATIO_LANDSCAPE", + )) + request = payload[0][0] + assert request[1] == "abra_i2v_4s_360p" + assert request[2] == fb.VIDEO_ASPECT_LANDSCAPE + assert request[4][1] == "start-mid" + assert request[-1] == [4] + assert json.loads(fb.omni_first_frame_request("x", self.PID, "m"))[0][0][0] == fb.RPC_GEN_VIDEO + + def test_omni_first_last_matches_live_nprqif_shape(self): + freq = fb.omni_first_last_request( + "morph", self.PID, "start", "end", duration_s=6, resolution="720p", + aspect="VIDEO_ASPECT_RATIO_PORTRAIT", + ) + assert json.loads(freq)[0][0][0] == fb.RPC_GEN_VIDEO_FIRST_LAST + request = inner(freq)[0][0] + assert request[1] == "omni_flash_i2v_6s_first_last" + assert request[2] == fb.VIDEO_ASPECT_PORTRAIT + assert request[4][1] == "start" + assert request[5][1] == "end" + + def test_omni_reference_matches_live_mzza6b_shape(self): + freq = fb.omni_reference_video_request( + "keep refs", self.PID, ["a", "b"], duration_s=4, resolution="360p", + aspect="VIDEO_ASPECT_RATIO_LANDSCAPE", + ) + assert json.loads(freq)[0][0][0] == fb.RPC_GEN_VIDEO_REFERENCES + request = inner(freq)[0][0] + assert request[1] == [[None, "a"], [None, "b"]] + assert request[2] == "abra_r2v_4s_360p" + assert request[3] == fb.VIDEO_ASPECT_LANDSCAPE + assert request[-1] == [4] + def test_text_video_matches_the_captured_yhhmef_shape(self): payload = inner(fb.text_video_request( "a boat", self.PID, diff --git a/tests/unit/test_omni_flash.py b/tests/unit/test_omni_flash.py index 617054eb1..7bcb137e3 100644 --- a/tests/unit/test_omni_flash.py +++ b/tests/unit/test_omni_flash.py @@ -1,20 +1,9 @@ -"""Unit tests for Gemini Omni Flash submissions and workflow polling. - -Omni has one working surface — text-to-video on batchexecute — and three that -are not ported. So these cover three things: `_load_model_key`, the input -validation that runs before any transport is touched, and that the unported -modes say so instead of reaching for auth that is gone. - -Note on `_load_model_key`: it currently has **no production caller**. Frame and -reference generation used to call it and now return early, and text-to-video -hardcodes `abra_t2v_{n}s` instead of reading models.json. The cases below pin -the mapping for when that is fixed; they are not evidence that model keys are -configurable today. - -The REST and tRPC wire-contract tests that used to live here went with the -transport they asserted. They were pinned to the legacy path by a module-level -autouse fixture, which meant most of this file was testing code that no longer -runs. Git history has them if a capture ever needs a reference. +"""Unit tests for Gemini Omni Flash Flow submissions and workflow polling. + +Every mode here runs on the flow.google.com batch path; the REST implementations +they used to shadow went with the transport. The wire-contract tests lock the +captured envelopes — rpc id, model key and slot order — because a payload that +submits but is shaped wrong buys a render and returns the wrong clip. """ from unittest.mock import AsyncMock, MagicMock, patch @@ -34,11 +23,14 @@ ) -# ─── model key policy (models.json, transport-independent) ────────────────── - @pytest.mark.parametrize( ("duration", "expected"), - [(4, "abra_r2v_4s"), (6, "abra_r2v_6s"), (8, "abra_r2v_8s"), (10, "abra_r2v_10s")], + [ + (4, "abra_r2v_4s"), + (6, "abra_r2v_6s"), + (8, "abra_r2v_8s"), + (10, "abra_r2v_10s"), + ], ) def test_omni_reference_duration_model_keys(duration, expected): assert _load_model_key(duration) == expected @@ -46,7 +38,12 @@ def test_omni_reference_duration_model_keys(duration, expected): @pytest.mark.parametrize( ("duration", "expected"), - [(4, "abra_i2v_4s"), (6, "abra_i2v_6s"), (8, "abra_i2v_8s"), (10, "abra_i2v_10s")], + [ + (4, "abra_i2v_4s"), + (6, "abra_i2v_6s"), + (8, "abra_i2v_8s"), + (10, "abra_i2v_10s"), + ], ) def test_omni_first_frame_model_keys(duration, expected): assert _load_model_key(duration, mode="frame_to_video") == expected @@ -54,11 +51,14 @@ def test_omni_first_frame_model_keys(duration, expected): @pytest.mark.parametrize( ("duration", "expected"), - [(4, "abra_i2v_4s"), (6, "abra_i2v_6s"), (8, "abra_i2v_8s"), (10, "abra_i2v_10s")], + [ + (4, "abra_i2v_4s"), + (6, "abra_i2v_6s"), + (8, "abra_i2v_8s"), + (10, "abra_i2v_10s"), + ], ) def test_omni_first_last_model_keys_are_independently_configured(duration, expected): - """Same family as first-frame today, but its own key so a rollout that - splits them does not need a code release.""" assert _load_model_key(duration, mode="start_end_frame_to_video") == expected @@ -67,20 +67,21 @@ def test_invalid_duration_fails_before_submit(): _load_model_key(5) -# ─── polling descriptors ──────────────────────────────────────────────────── - def test_extract_omni_workflows_uses_primary_media_id(): - """A submit carries operation-looking handles that are NOT pollable as - operations. The workflow's primaryMediaId is the one that is.""" result = { "status": 200, "data": { "operations": [ - {"operation": {"name": "operation-looking-handle"}, - "status": "MEDIA_GENERATION_STATUS_PENDING"} + { + "operation": {"name": "operation-looking-handle"}, + "status": "MEDIA_GENERATION_STATUS_PENDING", + } ], "workflows": [ - {"name": "workflow-1", "metadata": {"primaryMediaId": "media-1"}} + { + "name": "workflow-1", + "metadata": {"primaryMediaId": "media-1"}, + } ], }, } @@ -90,36 +91,21 @@ def test_extract_omni_workflows_uses_primary_media_id(): @pytest.mark.asyncio -async def test_batch_omni_poll_uses_as29s_media(): - client = MagicMock() - client.get_media = AsyncMock(return_value={ - "status": 200, - "data": {"video": {"fifeUrl": "https://flow-content.google/video/media-1?Signature=test"}}, - }) - with patch("agent.services.omni_flash.get_flow_client", return_value=client): - result = await check_omni_flash_status([{ - "name": "workflow-1", "primary_media_id": "media-1", "project_id": "project-1", - }]) - assert result["done"] is True - assert result["workflows"][0]["media"]["resolved_via"] == "as29s" - client.get_media.assert_awaited_once_with("media-1") - - -# ─── text-to-video: the one Omni surface on the batch path ────────────────── - -@pytest.mark.asyncio -async def test_batch_text_video_builds_4s_yhhmef_submit(): - import json - +async def test_batch_text_video_builds_4s_yhhmef_submit(monkeypatch): client = MagicMock() client._batch_project_id.return_value = "11111111-2222-3333-4444-555555555555" client._batch_payload = AsyncMock(return_value=[ - None, 10, [], [[ + None, + 10, + [], + [[ "22222222-3333-4444-5555-666666666666", "11111111-2222-3333-4444-555555555555", - "77777777-8888-9999-aaaa-bbbbbbbbbbbb", "CAE", + "77777777-8888-9999-aaaa-bbbbbbbbbbbb", + "CAE", ]], ]) + with patch("agent.services.omni_flash.get_flow_client", return_value=client): result = await generate_omni_flash_text_video( prompt="A red paper boat drifts across a pond", @@ -127,105 +113,247 @@ async def test_batch_text_video_builds_4s_yhhmef_submit(): duration_s=4, aspect_ratio="VIDEO_ASPECT_RATIO_LANDSCAPE", ) + assert result["status"] == 200 assert result["data"]["model"] == "abra_t2v_4s" assert result["data"]["duration_s"] == 4 assert result["data"]["flowkitPolling"]["mode"] == "batch_media" + assert result["data"]["flowkitPolling"]["workflows"][0]["primary_media_id"] == ( + "22222222-3333-4444-5555-666666666666" + ) rpcid, freq, captcha = client._batch_payload.await_args.args[:3] assert rpcid == omni_flash.fb.RPC_GEN_VIDEO_TEXT assert captcha == omni_flash.fb.CAPTCHA_VIDEO - payload = json.loads(json.loads(freq)[0][0][1]) + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) assert payload[0][0][1] == "abra_t2v_4s" + assert payload[0][0][2] == omni_flash.fb.VIDEO_ASPECT_LANDSCAPE + +@pytest.mark.asyncio +async def test_batch_first_frame_video_uses_eb1hjf_abra_i2v(monkeypatch): + client = MagicMock() + pid = "11111111-2222-3333-4444-555555555555" + client._batch_project_id.return_value = pid + client._batch_payload = AsyncMock(return_value=[ + None, + 50, + [["op-omni-1", pid, "scene-1", None]], + ]) + + with patch("agent.services.omni_flash.get_flow_client", return_value=client): + result = await generate_omni_flash_first_frame_video( + start_image_media_id="media-start", + prompt="Three children clap gently", + project_id=pid, + duration_s=6, + aspect_ratio="VIDEO_ASPECT_RATIO_LANDSCAPE", + ) + + assert result["status"] == 200 + assert result["data"]["model"] == "abra_i2v_6s" + assert result["data"]["duration_s"] == 6 + assert result["data"]["flowkitPolling"]["mode"] == "batch_operation" + assert result["data"]["flowkitPolling"]["project_id"] == pid + assert result["data"]["operations"][0]["operation"]["name"] == "op-omni-1" + client._remember_operation.assert_called_once_with("op-omni-1", pid) + + rpcid, freq, captcha = client._batch_payload.await_args.args[:3] + assert rpcid == omni_flash.fb.RPC_GEN_VIDEO + assert captcha == omni_flash.fb.CAPTCHA_VIDEO + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + request = payload[0][0] + assert request[0][2][0][0][0] == "Three children clap gently" + assert request[1] == "abra_i2v_6s" + assert request[2] == omni_flash.fb.VIDEO_ASPECT_LANDSCAPE + assert request[4][1] == "media-start" -# ─── validation runs before the capability gap is reported ────────────────── -# -# Order matters: a caller with a bad duration AND an unported mode should hear -# about the duration, which it can fix, rather than the gap, which it cannot. @pytest.mark.asyncio async def test_first_frame_rejects_missing_start_before_submit(): with pytest.raises(ValueError, match="requires start_image_media_id"): await generate_omni_flash_first_frame_video( - start_image_media_id="", prompt="test", project_id="p") + start_image_media_id="", + prompt="test", + project_id="p", + ) @pytest.mark.asyncio async def test_first_last_rejects_missing_end_before_submit(): with pytest.raises(ValueError, match="non-empty end_image_media_id"): await generate_omni_flash_first_last_video( - start_image_media_id="start", end_image_media_id="", - prompt="test", project_id="p") + start_image_media_id="start", + end_image_media_id="", + prompt="test", + project_id="p", + ) + + +@pytest.mark.asyncio +async def test_batch_omni_poll_uses_as29s_media(monkeypatch): + client = MagicMock() + client.get_media = AsyncMock(return_value={ + "status": 200, + "data": { + "video": { + "fifeUrl": "https://flow-content.google/video/media-1?Signature=test" + } + }, + }) + + with patch("agent.services.omni_flash.get_flow_client", return_value=client): + result = await check_omni_flash_status([ + { + "name": "workflow-1", + "primary_media_id": "media-1", + "project_id": "project-1", + } + ]) + + assert result["done"] is True + assert result["status"] == "COMPLETED" + assert result["workflows"][0]["media"]["resolved_via"] == "as29s" + client.get_media.assert_awaited_once_with("media-1") + + +@pytest.mark.asyncio +async def test_batch_poll_hands_back_the_signed_url_and_buffers_nothing(): + """The poller must return Flow's signed url, never the bytes: buffering a + finished clip through the agent is what the as29s lookup exists to avoid.""" + client = MagicMock() + client.get_media = AsyncMock(return_value={"status": 200, "data": {"video": { + "fifeUrl": "https://flow-content.google/video/media-1?Signature=test"}}}) + + with patch("agent.services.omni_flash.get_flow_client", return_value=client): + result = await check_omni_flash_status( + [{"name": "workflow-1", "primary_media_id": "media-1"}], + include_encoded_video=True, project_id="project-1") + + media = result["workflows"][0]["media"] + assert result["done"] is True + assert media["url"].startswith("https://flow-content.google/video/") + assert media["encoded_video_available"] is False + assert media["encoded_video"] is None + + +@pytest.mark.asyncio +async def test_batch_poll_treats_a_media_record_without_video_as_pending(): + """A media id exists before its clip does — the record serves the poster + image first. Reading that as done is what saves a still instead of a video.""" + client = MagicMock() + client.get_media = AsyncMock(return_value={"status": 200, "data": {"image": { + "fifeUrl": "https://flow-content.google/image/media-1?Signature=test"}}}) + + with patch("agent.services.omni_flash.get_flow_client", return_value=client): + result = await check_omni_flash_status( + [{"name": "workflow-1", "primary_media_id": "media-1"}], + project_id="project-1") + + assert result["done"] is False + assert result["workflows"][0]["status"] == "PENDING" @pytest.mark.asyncio async def test_submit_rejects_more_than_seven_references(): refs = [f"ref-{i}" for i in range(OMNI_FLASH_MAX_REFERENCE_IMAGES + 1)] + with pytest.raises(ValueError, match="at most 7 reference images"): await generate_omni_flash_video( - reference_media_ids=refs, prompt="test", project_id="p", duration_s=8) + reference_media_ids=refs, + prompt="test", + project_id="p", + duration_s=8, + ) @pytest.mark.asyncio async def test_submit_rejects_empty_reference_set(): with pytest.raises(ValueError, match="requires at least one reference image"): await generate_omni_flash_video( - reference_media_ids=[], prompt="test", project_id="p", duration_s=8) - - -@pytest.mark.asyncio -async def test_seven_references_pass_validation_and_reach_the_capability_gap(): - """Seven is the limit, not one past it — so this must fail on the gap, not - on the count. Guards the boundary against an off-by-one in the validator.""" - refs = [f"ref-{i}" for i in range(OMNI_FLASH_MAX_REFERENCE_IMAGES)] - result = await generate_omni_flash_video( - reference_media_ids=refs, prompt="test", project_id="p", duration_s=4) - assert "UNSUPPORTED_ON_BATCH_API" in result["error"] - + reference_media_ids=[], + prompt="test", + project_id="p", + duration_s=8, + ) -# ─── unported modes ───────────────────────────────────────────────────────── -class TestUnportedOmniModesAreRefusedRatherThanAttempted: - """Frame and reference generation had only a REST implementation, and that - transport is gone. Naming the gap beats a 401 five retries deep — and beats - silently submitting with a Veo key, which is what a careless unification - of these paths would do.""" +class TestMigratedOmniReferenceBatchModes: @pytest.fixture def client(self): with patch("agent.services.omni_flash.get_flow_client") as factory: stub = MagicMock() - stub._send = AsyncMock() - stub.generate_video = AsyncMock() + pid = "11111111-2222-3333-4444-555555555555" + stub._batch_project_id.return_value = pid + stub._batch_payload = AsyncMock(return_value=[ + None, 50, [["op-ref-1", pid, "scene-1", None]], + ]) factory.return_value = stub yield stub - async def test_first_frame_names_the_gap_and_sends_nothing(self, client): - result = await generate_omni_flash_first_frame_video( - start_image_media_id="mid", prompt="go", project_id="pid") - assert "UNSUPPORTED_ON_BATCH_API" in result["error"] - client._send.assert_not_called() - client.generate_video.assert_not_called() - - async def test_first_last_names_the_gap_and_sends_nothing(self, client): + async def test_first_last_uses_nprqif_and_batch_operation_polling(self, client): result = await generate_omni_flash_first_last_video( - start_image_media_id="a", end_image_media_id="b", - prompt="go", project_id="pid") - assert "UNSUPPORTED_ON_BATCH_API" in result["error"] - client._send.assert_not_called() - client.generate_video.assert_not_called() - - async def test_reference_to_video_names_the_gap_and_sends_nothing(self, client): + start_image_media_id="start", end_image_media_id="end", + prompt="morph", project_id="pid", duration_s=4, + resolution="360p", aspect_ratio="VIDEO_ASPECT_RATIO_LANDSCAPE") + assert result["status"] == 200 + assert result["data"]["model"] == "omni_flash_i2v_4s_first_last_360p" + assert result["data"]["resolution"] == "360p" + assert result["data"]["flowkitPolling"]["mode"] == "batch_operation" + rpcid, freq, captcha = client._batch_payload.await_args.args[:3] + assert rpcid == omni_flash.fb.RPC_GEN_VIDEO_FIRST_LAST + assert captcha == omni_flash.fb.CAPTCHA_VIDEO + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + req = payload[0][0] + assert req[1] == "omni_flash_i2v_4s_first_last_360p" + assert req[4][1] == "start" + assert req[5][1] == "end" + client._remember_operation.assert_called_once_with( + "op-ref-1", "11111111-2222-3333-4444-555555555555") + + async def test_seven_references_submit_rather_than_trip_the_validator(self, client): + """Seven is the limit, not one past it. This used to assert the + capability gap; now that r2v submits, it has to reach the wire — + otherwise an off-by-one in the validator looks like a Flow refusal.""" + refs = [f"ref-{i}" for i in range(OMNI_FLASH_MAX_REFERENCE_IMAGES)] result = await generate_omni_flash_video( - reference_media_ids=["a"], prompt="go", project_id="pid") - assert "UNSUPPORTED_ON_BATCH_API" in result["error"] - client._send.assert_not_called() - client.generate_video.assert_not_called() - - async def test_the_message_points_to_supported_text_to_video(self, client): - """The error is the only place a caller learns there IS a working Omni - surface, so it names both the gap and the way round it.""" + reference_media_ids=refs, prompt="seven", project_id="pid", + duration_s=4, resolution="720p") + assert result["status"] == 200 + _rpcid, freq, _captcha = client._batch_payload.await_args.args[:3] + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + assert payload[0][0][1] == [[None, r] for r in refs] + + async def test_reference_to_video_uses_mzza6b_and_all_references(self, client): result = await generate_omni_flash_video( - reference_media_ids=["a"], prompt="go", project_id="pid") - assert "text-to-video is supported" in result["error"] - assert "reference-to-video" in result["error"] + reference_media_ids=["a", "b", "c"], prompt="keep all refs", + project_id="pid", duration_s=6, resolution="720p", + aspect_ratio="VIDEO_ASPECT_RATIO_PORTRAIT") + assert result["status"] == 200 + assert result["data"]["model"] == "abra_r2v_6s" + assert result["data"]["flowkitPolling"]["mode"] == "batch_operation" + rpcid, freq, _captcha = client._batch_payload.await_args.args[:3] + assert rpcid == omni_flash.fb.RPC_GEN_VIDEO_REFERENCES + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + req = payload[0][0] + assert req[1] == [[None, "a"], [None, "b"], [None, "c"]] + assert req[2] == "abra_r2v_6s" + assert req[3] == omni_flash.fb.VIDEO_ASPECT_PORTRAIT + + async def test_reference_360p_uses_live_wire_model_and_quality_slot(self, client): + await generate_omni_flash_video( + reference_media_ids=["a", "b"], prompt="refs", project_id="pid", + duration_s=4, resolution="360p", + aspect_ratio="VIDEO_ASPECT_RATIO_LANDSCAPE") + _rpcid, freq, _captcha = client._batch_payload.await_args.args[:3] + payload = __import__("json").loads(__import__("json").loads(freq)[0][0][1]) + req = payload[0][0] + assert req[2] == "abra_r2v_4s_360p" + assert req[-1] == [4] + + async def test_invalid_resolution_is_rejected_before_submit(self, client): + with pytest.raises(ValueError, match="resolution must be 360p or 720p"): + await generate_omni_flash_first_frame_video( + start_image_media_id="a", prompt="go", project_id="pid", + resolution="1080p") + client._batch_payload.assert_not_called()