Add cam2v app and lingbot integration

apps/cam2v/README.md

@@ -0,0 +1,31 @@
+# FlashDreams Cam2V application
+
+`flashdreams-cam2v` owns the reusable v2 application, session, model-generation
+loop, camera controls, and timing for interactive camera-to-video models.
+Concrete integrations supply an existing runner config plus an input resolver
+that turns their asset format into `Cam2VConditioning`.
+
+The application owns the loaded pipeline. Each session owns its autoregressive
+cache, first frame, keyboard state, camera pose, and SlangPy UI overlay. The
+io-thread runs the UI loop over the current video frame; the
+model-generation-thread runs the model loop and is the only thread that mutates
+rollout state. Model status crosses to the UI loop through `invoke_async`
+messages.
+
+The overlay is enabled by default. Pass `-- --no-ui` after the application
+arguments to use the default model-output blitter for headless or benchmark
+runs.
+
+For UI testing without loading a real model, run the packaged dummy pipeline:
+
+```bash
+uv run flashdreams-run-v2 cam2v-dummy --mode webrtc \
+  --host 0.0.0.0 --port 8089 -- \
+  --step-wait-seconds 0.9 --frames-per-chunk 12
+```
+
+The model-generation-thread waits on a `threading.Event` for each synthetic
+step while the io-thread continues processing and rendering browser input.
+
+See `integrations/lingbot/lingbot/cam2v/app.py` for the minimal specialization
+pattern.

apps/cam2v/__init__.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Reusable interactive camera-to-video application primitives."""
+
+from .application import Cam2VApplication
+from .controls import CameraPoseIntegrator, KeyboardResampler, PoseSegment
+from .defaults import (
+    Cam2VApplicationDefaults,
+    Cam2VConditioning,
+    Cam2VInputResolver,
+)
+from .session import (
+    Cam2VModelLoop,
+    Cam2VModelState,
+    Cam2VSession,
+    Cam2VSessionConfig,
+    CameraControlInput,
+)
+from .ui import Cam2VSlangPyUILoop, Cam2VUIState, Cam2VUIStatus
+
+__all__ = [
+    "Cam2VApplication",
+    "Cam2VApplicationDefaults",
+    "Cam2VConditioning",
+    "Cam2VInputResolver",
+    "Cam2VModelLoop",
+    "Cam2VModelState",
+    "Cam2VSession",
+    "Cam2VSessionConfig",
+    "Cam2VSlangPyUILoop",
+    "Cam2VUIState",
+    "Cam2VUIStatus",
+    "CameraControlInput",
+    "CameraPoseIntegrator",
+    "KeyboardResampler",
+    "PoseSegment",
+]

apps/cam2v/application.py

@@ -0,0 +1,279 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Reusable camera-to-video application on the FlashDreams v2 API."""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+import torch
+
+from flashdreams.api_v2.application import IApplication
+from flashdreams.api_v2.session import ISession
+from flashdreams.infra.config import derive_config
+from flashdreams.runtime_v2.session_desc import SessionDesc
+from flashdreams.runtime_v2.video_tensor import VideoTensorLayout
+
+from .defaults import Cam2VApplicationDefaults, Cam2VConditioning
+from .session import Cam2VSession, Cam2VSessionConfig
+
+
+class Cam2VApplication(IApplication):
+    """Reusable interactive camera-to-video application.
+
+    The shared class owns command-line parsing, pipeline lifetime, session
+    validation, and model-generation-loop construction. A concrete model
+    integration contributes a runner config and an input resolver through
+    :class:`Cam2VApplicationDefaults`.
+    """
+
+    session_type: type[Cam2VSession] = Cam2VSession
+    """Session constructed for each independent camera rollout."""
+
+    def __init__(self, *, defaults: Cam2VApplicationDefaults) -> None:
+        self.defaults = defaults
+        self._pipeline_config = defaults.pipeline_config
+        self._device = defaults.device
+        self._total_blocks = defaults.total_blocks
+        self._log_every_blocks = defaults.log_every_blocks
+        self._warmup_blocks = defaults.warmup_blocks
+        self._use_ui = True
+        self._input_values: dict[str, Any] | None = None
+        self._pipeline: Any | None = None
+
+    @property
+    def pipeline_config(self) -> Any:
+        """Return the model configuration after command-line overrides."""
+        return self._pipeline_config
+
+    def init(self, commandline_args: Sequence[str]) -> None:
+        """Parse shared camera-to-video inputs without loading the model."""
+        parser = argparse.ArgumentParser(
+            prog="flashdreams-run-v2 CAM2V_SLUG --",
+            description="Generate video from a first frame and keyboard camera input.",
+        )
+        input_defaults = self.defaults.input_defaults
+        parser.add_argument("--prompt", default=input_defaults.get("prompt", ""))
+        parser.add_argument(
+            "--prompt-path",
+            type=Path,
+            default=input_defaults.get("prompt_path"),
+        )
+        parser.add_argument(
+            "--image-path",
+            type=Path,
+            default=input_defaults.get("image_path"),
+        )
+        parser.add_argument(
+            "--pose-path",
+            type=Path,
+            default=input_defaults.get("pose_path"),
+        )
+        parser.add_argument(
+            "--intrinsic-path",
+            type=Path,
+            default=input_defaults.get("intrinsic_path"),
+        )
+        parser.add_argument(
+            "--world-scale",
+            type=float,
+            default=input_defaults.get("world_scale"),
+        )
+        parser.add_argument(
+            "--example-data",
+            action=argparse.BooleanOptionalAction,
+            default=bool(input_defaults.get("example_data", False)),
+        )
+        parser.add_argument(
+            "--example-idx",
+            type=int,
+            default=int(input_defaults.get("example_idx", 0)),
+        )
+        parser.add_argument(
+            "--device",
+            default=self.defaults.device,
+            help="Device used by the shared model. Default: %(default)s.",
+        )
+        parser.add_argument(
+            "--total-blocks",
+            type=int,
+            default=self.defaults.total_blocks,
+            help="Autoregressive chunks generated per rollout. Default: %(default)s.",
+        )
+        parser.add_argument(
+            "--log-every-blocks",
+            type=int,
+            default=self.defaults.log_every_blocks,
+            help="Emit live timing every N steady-state chunks.",
+        )
+        parser.add_argument(
+            "--warmup-blocks",
+            type=int,
+            default=self.defaults.warmup_blocks,
+            help="Leading chunks excluded from steady-state FPS.",
+        )
+        parser.add_argument(
+            "--ui",
+            action=argparse.BooleanOptionalAction,
+            default=True,
+            help="Render the shared camera controls and timing overlay.",
+        )
+        parser.add_argument(
+            "--compile",
+            action=argparse.BooleanOptionalAction,
+            default=None,
+        )
+        parser.add_argument("--seed", type=int, default=None)
+        self._configure_argument_parser(parser)
+        args = parser.parse_args(list(commandline_args))
+
+        self._validate_arguments(args)
+        self._apply_parsed_arguments(args)
+        self._pipeline_config = self.defaults.pipeline_config
+        if args.compile is not None:
+            self._pipeline_config = self._apply_compile_override(
+                self._pipeline_config,
+                args.compile,
+            )
+        if args.seed is not None:
+            self._pipeline_config = self._apply_seed_override(
+                self._pipeline_config,
+                args.seed,
+            )
+        self._device = args.device
+        self._total_blocks = args.total_blocks
+        self._log_every_blocks = args.log_every_blocks
+        self._warmup_blocks = args.warmup_blocks
+        self._use_ui = args.ui
+        self._input_values = {
+            "prompt": args.prompt,
+            "prompt_path": args.prompt_path,
+            "image_path": args.image_path,
+            "pose_path": args.pose_path,
+            "intrinsic_path": args.intrinsic_path,
+            "world_scale": args.world_scale,
+            "example_data": args.example_data,
+            "example_idx": args.example_idx,
+            "total_blocks": args.total_blocks,
+        }
+
+    def session_desc(self) -> SessionDesc:
+        """Return the model's default output shape and interactive rates."""
+        return SessionDesc(
+            output_layout=self.defaults.output_layout,
+            backpressure_mode=self.defaults.backpressure_mode,
+            presentation_mode=self.defaults.presentation_mode,
+            frames_per_second_for_ui=self.defaults.ui_fps,
+            frames_per_second_for_step=self.defaults.fps,
+            video_width=self.defaults.pixel_width,
+            video_height=self.defaults.pixel_height,
+        )
+
+    def create_session(self, session_desc: SessionDesc) -> ISession:
+        """Create an isolated rollout after lazily loading the shared pipeline."""
+        input_values = self._input_values
+        if input_values is None:
+            raise RuntimeError(
+                f"{type(self).__name__}.init() must run before create_session()."
+            )
+        self._validate_layout(session_desc)
+        resolved_values = {
+            **input_values,
+            "pixel_height": session_desc.video_height,
+            "pixel_width": session_desc.video_width,
+            "fps": session_desc.frames_per_second_for_step,
+        }
+        conditioning = self.defaults.input_resolver(resolved_values)
+        if not isinstance(conditioning, Cam2VConditioning):
+            raise TypeError(
+                "Cam2VApplicationDefaults.input_resolver must return Cam2VConditioning."
+            )
+
+        pipeline = self._pipeline
+        if pipeline is None:
+            pipeline = self._pipeline_config.setup().to(self._device).eval()
+            self._pipeline = pipeline
+        self._validate_frame_size(session_desc, pipeline)
+        return self.session_type(
+            pipeline=pipeline,
+            session_desc=session_desc,
+            config=Cam2VSessionConfig(
+                conditioning=conditioning,
+                total_blocks=self._total_blocks,
+                device=torch.device(self._device),
+                log_every_blocks=self._log_every_blocks,
+                warmup_blocks=self._warmup_blocks,
+                install_hint=self.defaults.install_hint,
+            ),
+            use_ui=self._use_ui,
+        )
+
+    def close(self) -> None:
+        """Release the application-owned pipeline after all sessions stop."""
+        pipeline = self._pipeline
+        self._pipeline = None
+        self._input_values = None
+        close = getattr(pipeline, "close", None)
+        if callable(close):
+            close()
+
+    def _configure_argument_parser(self, parser: argparse.ArgumentParser) -> None:
+        """Add integration-specific application arguments to ``parser``."""
+
+    def _apply_parsed_arguments(self, args: argparse.Namespace) -> None:
+        """Retain integration-specific arguments after shared validation."""
+
+    def _validate_arguments(self, args: argparse.Namespace) -> None:
+        """Reject invalid rollout and timing settings."""
+        if args.total_blocks <= 0:
+            raise ValueError("--total-blocks must be > 0.")
+        if args.log_every_blocks <= 0:
+            raise ValueError("--log-every-blocks must be > 0.")
+        if args.warmup_blocks < 0:
+            raise ValueError("--warmup-blocks must be >= 0.")
+        if args.world_scale is not None and args.world_scale < 0:
+            raise ValueError("--world-scale must be >= 0 when set.")
+
+    def _apply_compile_override(self, pipeline_config: Any, enabled: bool) -> Any:
+        """Return ``pipeline_config`` with network compilation overridden."""
+        return derive_config(
+            pipeline_config,
+            diffusion_model={"transformer": {"compile_network": enabled}},
+        )
+
+    def _apply_seed_override(self, pipeline_config: Any, seed: int) -> Any:
+        """Return ``pipeline_config`` with diffusion sampling seed overridden."""
+        return derive_config(pipeline_config, diffusion_model={"seed": seed})
+
+    def _validate_layout(self, session_desc: SessionDesc) -> None:
+        """Reject output layouts that differ from the model's declared layout."""
+        if session_desc.output_layout is not self.defaults.output_layout:
+            raise ValueError(
+                "This camera-to-video model only produces "
+                f"{self.defaults.output_layout.value} output, got "
+                f"{session_desc.output_layout.value}."
+            )
+        if self._use_ui and session_desc.output_layout is not VideoTensorLayout.tchw:
+            raise ValueError("The Cam2V SlangPy UI overlay requires tchw output.")
+
+    def _validate_frame_size(self, session_desc: SessionDesc, pipeline: Any) -> None:
+        """Reject frame dimensions that cannot map to integral latents."""
+        decoder = getattr(pipeline, "decoder", None)
+        ratio = getattr(decoder, "spatial_compression_ratio", None)
+        if not isinstance(ratio, int) or ratio <= 0:
+            raise TypeError(
+                "Cam2V requires a decoder with a positive integer "
+                "spatial_compression_ratio."
+            )
+        if session_desc.video_width % ratio or session_desc.video_height % ratio:
+            raise ValueError(
+                f"Frame dimensions must be multiples of {ratio}, got "
+                f"{session_desc.video_width}x{session_desc.video_height}."
+            )
+
+
+__all__ = ["Cam2VApplication"]

apps/cam2v/assets/dummy_frame.ppm

@@ -0,0 +1,5 @@
+P3
+2 2
+255
+34 42 58   52 64 82
+52 64 82   34 42 58