diff --git a/Docs/Code_Documentation/Jobs_Module.md b/Docs/Code_Documentation/Jobs_Module.md index f25ee90766..61e917e43c 100644 --- a/Docs/Code_Documentation/Jobs_Module.md +++ b/Docs/Code_Documentation/Jobs_Module.md @@ -120,10 +120,11 @@ Audio Studio job payloads are sanitized before persistence. Client requests must - `JOBS_INTEGRITY_SWEEP_ENABLED` (default false), `JOBS_INTEGRITY_SWEEP_INTERVAL_SEC` (default 60), `JOBS_INTEGRITY_SWEEP_FIX` (default false). - Periodically flags (and optionally fixes) impossible states like leases on non-processing jobs and expired processing leases. - Postgres RLS (optional): - - `JOBS_PG_RLS_ENABLE` (default false): Enable row-level security policies that scope access to domains in `current_setting('app.domain_allowlist')`. - - To scope a connection/session, set the allowlist before issuing queries/updates (example): - - `SELECT set_config('app.domain_allowlist', 'chatbooks,prompt_studio', true);` - - The policies will then allow access only to rows where `domain` is in that list. + - `JOBS_PG_RLS_ENABLE` (default false): Enable row-level security policies that scope Jobs access by domain/owner and playlist authority tables by owner. + - `JOBS_PG_RLS_ROLE` must be a dedicated `NOLOGIN`, non-superuser group role without `BYPASSRLS` or parent-role memberships. Across every non-system schema, the installer revokes all direct schema, table, and sequence privileges from that role, then grants only current-schema `USAGE` plus the enumerated Jobs/playlist objects. Migration removes any custom direct grants on this dedicated role; place unrelated privileges on another role. Installation fails if `PUBLIC`, inherited, or ownership-derived privileges expose an unrelated object, grant `CREATE` on a non-system schema, or exceed the allowlist ceiling (`SELECT`/`INSERT`/`UPDATE`/`DELETE` on tables and `USAGE`/`SELECT` on sequences); it does not mutate `PUBLIC`. The application login receives membership and `JobManager` assumes the role for managed connections. + - When both RLS settings are configured, managed connections fail closed if `SET ROLE` or any request-scoped RLS setting fails; queries never continue under the application login. + - Use `JobManager.rls_context(...)` around owner-scoped operations. It restores prior context across nesting and exceptions; playlist authority policies fail closed when `app.owner_user_id` is unset or blank. + - RLS is a trusted-application query-bug guardrail. It is not a defense against direct database access, SQL injection, a compromised application login, or database owner/superuser access; enforce network and credential least privilege separately. - Metrics/Tracing buckets: - `JOBS_DURATION_BUCKETS`: CSV of float seconds for `duration_seconds` histogram buckets. - `JOBS_QUEUE_LATENCY_BUCKETS`: CSV of float seconds for `queue_latency_seconds` histogram buckets. @@ -441,6 +442,7 @@ jm.fail_job(job["id"], error="boom", retryable=True, worker_id=worker_id, lease_ - Emits text/event-stream with incremental IDs; clients can resume by passing `after_id`. - Requires `JOBS_EVENTS_OUTBOX=true` to persist events; otherwise events are process-local only. - Admin endpoints set per-request Postgres RLS context automatically when enabled. + - Playlist ingestion store and worker paths set owner-scoped RLS context before opening managed Postgres connections; only genuinely global operations use admin context. - Response: `{ non_processing_with_lease: int, processing_expired: int, fixed: int }` - When `fix=true`, clears stale lease fields on non-processing rows, and re-queues expired processing rows. diff --git a/Docs/Published/Code_Documentation/Jobs_Module.md b/Docs/Published/Code_Documentation/Jobs_Module.md index d488d72c7b..8e91e7711f 100644 --- a/Docs/Published/Code_Documentation/Jobs_Module.md +++ b/Docs/Published/Code_Documentation/Jobs_Module.md @@ -103,10 +103,11 @@ Connector sync jobs are not only deduped at Jobs level. They also reserve a sour - `JOBS_INTEGRITY_SWEEP_ENABLED` (default false), `JOBS_INTEGRITY_SWEEP_INTERVAL_SEC` (default 60), `JOBS_INTEGRITY_SWEEP_FIX` (default false). - Periodically flags (and optionally fixes) impossible states like leases on non-processing jobs and expired processing leases. - Postgres RLS (optional): - - `JOBS_PG_RLS_ENABLE` (default false): Enable row-level security policies that scope access to domains in `current_setting('app.domain_allowlist')`. - - To scope a connection/session, set the allowlist before issuing queries/updates (example): - - `SELECT set_config('app.domain_allowlist', 'chatbooks,prompt_studio', true);` - - The policies will then allow access only to rows where `domain` is in that list. + - `JOBS_PG_RLS_ENABLE` (default false): Enable row-level security policies that scope Jobs access by domain/owner and playlist authority tables by owner. + - `JOBS_PG_RLS_ROLE` must be a dedicated `NOLOGIN`, non-superuser group role without `BYPASSRLS` or parent-role memberships. Across every non-system schema, the installer revokes all direct schema, table, and sequence privileges from that role, then grants only current-schema `USAGE` plus the enumerated Jobs/playlist objects. Migration removes any custom direct grants on this dedicated role; place unrelated privileges on another role. Installation fails if `PUBLIC`, inherited, or ownership-derived privileges expose an unrelated object, grant `CREATE` on a non-system schema, or exceed the allowlist ceiling (`SELECT`/`INSERT`/`UPDATE`/`DELETE` on tables and `USAGE`/`SELECT` on sequences); it does not mutate `PUBLIC`. The application login receives membership and `JobManager` assumes the role for managed connections. + - When both RLS settings are configured, managed connections fail closed if `SET ROLE` or any request-scoped RLS setting fails; queries never continue under the application login. + - Use `JobManager.rls_context(...)` around owner-scoped operations. It restores prior context across nesting and exceptions; playlist authority policies fail closed when `app.owner_user_id` is unset or blank. + - RLS is a trusted-application query-bug guardrail. It is not a defense against direct database access, SQL injection, a compromised application login, or database owner/superuser access; enforce network and credential least privilege separately. - Metrics/Tracing buckets: - `JOBS_DURATION_BUCKETS`: CSV of float seconds for `duration_seconds` histogram buckets. - `JOBS_QUEUE_LATENCY_BUCKETS`: CSV of float seconds for `queue_latency_seconds` histogram buckets. diff --git a/Docs/superpowers/plans/2026-07-12-youtube-playlist-ingest-backend.md b/Docs/superpowers/plans/2026-07-12-youtube-playlist-ingest-backend.md new file mode 100644 index 0000000000..aa87a3d722 --- /dev/null +++ b/Docs/superpowers/plans/2026-07-12-youtube-playlist-ingest-backend.md @@ -0,0 +1,531 @@ +# YouTube Playlist Ingest Backend Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add the owner-scoped backend contracts that inspect a YouTube playlist completely, materialize selected occurrences, and track every selected occurrence through one ingest run without allowing an opaque playlist job. + +**Architecture:** Reuse the existing Jobs SQLite/PostgreSQL database, leases, events, and media-ingest worker. Add focused playlist-ingest tables and a repository beside Jobs, keep yt-dlp in a bounded child process, and let a small service translate run occurrences into existing media jobs or terminal duplicate actions. Keep the synchronous preflight endpoint only as a compatibility surface. + +**Tech Stack:** FastAPI, Pydantic v2, SQLite/PostgreSQL Jobs backends, yt-dlp, existing `JobManager`, Media DB and Collections DB abstractions, pytest, Hypothesis, Bandit. + +**Backlog:** `TASK-12110` + +**Spec:** `Docs/superpowers/specs/2026-07-12-youtube-playlist-per-item-ingest-design.md` + +--- + +## File map + +**Create** + +- `tldw_Server_API/app/api/v1/schemas/media_playlist_ingest.py` — version-2 preflight, materialization, run, item, event, retry, and structured submission models. +- `tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_store.py` — owner-filtered persistence, cursor encoding, expiry cleanup, and atomic state transitions over the Jobs database. +- `tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_preflight_runner.py` — bounded child-process extraction and cancellation/timeout termination. +- `tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_service.py` — preflight scheduling, materialization, run validation, duplicate-action resolution, and job/run reconciliation. +- `tldw_Server_API/app/api/v1/endpoints/media/playlist_ingest.py` — version-2 HTTP and SSE routes. +- `tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py` — SQLite repository and state-machine tests. +- `tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_service.py` — orchestration, duplicate, idempotency, and metadata-patch tests. +- `tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_endpoint.py` — route, auth, cursor, and error-contract tests. +- `tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_store_postgres.py` — repository parity against the existing isolated PostgreSQL fixture. +- `tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_run_workflow.py` — real Jobs DB/worker workflow with media processing faked. +- `tldw_Server_API/tests/MediaIngestion_NEW/property/test_playlist_ingest_properties.py` — pagination, identity, action-resolution, and retry invariants. + +**Modify** + +- `tldw_Server_API/app/core/Jobs/migrations.py` — SQLite tables/indexes for preflights, items, materializations, runs, run items, and run events. +- `tldw_Server_API/app/core/Jobs/pg_migrations.py` — PostgreSQL-equivalent schema and indexes. +- `tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_preflight.py` — configured-limit-plus-one extraction and availability/occurrence normalization. +- `tldw_Server_API/app/api/v1/endpoints/media/__init__.py` — register `playlist_ingest` beside the compatibility preflight router. +- `tldw_Server_API/app/api/v1/endpoints/media/ingest_jobs.py` — reject playlist candidates; accept aligned run/occurrence/attempt bindings; return structured per-occurrence acceptance. +- `tldw_Server_API/app/services/media_ingest_jobs_worker.py` — handle the internal preflight job and require exactly one media result for a concrete occurrence. +- `tldw_Server_API/app/core/DB_Management/media_db/runtime/media_item_update_ops.py` — atomic allowlisted metadata-only patch. +- `tldw_Server_API/app/core/DB_Management/media_db/repositories/media_lookup_repository.py`, `api.py`, and `runtime/query_ops.py` — one owner-database bulk URL lookup shared by preflight enrichment and run validation. +- `tldw_Server_API/app/core/DB_Management/media_db/media_database_impl.py` — bind the metadata-patch method. +- `tldw_Server_API/app/core/DB_Management/Collections_DB.py` — transactional optional collection plus planned-item creation. +- `tldw_Server_API/app/api/v1/endpoints/config_info.py` — advertise contract version/readiness. +- Existing tests under `tldw_Server_API/tests/Jobs/`, `MediaIngestion_NEW/unit/`, `DB_Management/`, and `Media/` — extend migration, worker, collection, compatibility, and capability coverage. + +## Stage 1: Durable owner-scoped storage + +**Goal:** Establish portable persistence and strict state transitions before adding routes. + +**Success Criteria:** Both Jobs backends create the same schema; owner-scoped reads, immutable ordering, expiry, and cursors are deterministic. + +**Tests:** SQLite/PostgreSQL migration tests, repository unit tests, Hypothesis pagination/identity tests. + +**Status:** Complete + +### Task 1: Add contract models and Jobs migrations + +- [x] **Step 1: Write failing migration and schema tests** + +Add table assertions to `tldw_Server_API/tests/Jobs/test_jobs_migrations_sqlite.py` and `test_jobs_migrations_postgres.py`. Add model-validation tests to `test_playlist_ingest_store.py` proving duplicate policies are explicit and `file_reattach_required` is rejected as a server state. + +```python +EXPECTED_PLAYLIST_TABLES = { + "playlist_preflights", + "playlist_preflight_items", + "playlist_materializations", + "playlist_materialization_items", + "media_ingest_runs", + "media_ingest_run_items", + "media_ingest_run_events", +} + +def test_run_state_rejects_client_only_file_reattach_state(): + with pytest.raises(ValidationError): + RunItemSnapshot(occurrence_id="occ-1", ordinal=1, state="file_reattach_required") +``` + +- [x] **Step 2: Run the tests and verify RED** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/Jobs/test_jobs_migrations_sqlite.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py -q` + +Expected: FAIL because the tables and `media_playlist_ingest` schemas do not exist. + +- [x] **Step 3: Add the minimum portable schema** + +Define Pydantic enums/models in `media_playlist_ingest.py`. Add matching `CREATE TABLE IF NOT EXISTS` DDL to both Jobs migration modules. Store bounded display metadata and patches as JSON, but keep owner, status, ordinal, occurrence ID, normalized source ID, job ID, attempt, event ID, and expiry as indexed columns. Add uniqueness constraints for `(preflight_id, ordinal)`, occurrence IDs, `(run_id, occurrence_id)`, and `(run_id, occurrence_id, attempt)` job mappings. + +- [x] **Step 4: Run SQLite and PostgreSQL migration tests** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/Jobs/test_jobs_migrations_sqlite.py tldw_Server_API/tests/Jobs/test_jobs_migrations_postgres.py -q` + +Expected: PASS (PostgreSQL test skips only through its existing fixture when unavailable). + +- [x] **Step 5: Commit** + +```bash +git add tldw_Server_API/app/api/v1/schemas/media_playlist_ingest.py tldw_Server_API/app/core/Jobs/migrations.py tldw_Server_API/app/core/Jobs/pg_migrations.py tldw_Server_API/tests/Jobs tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py +git commit -m "feat: add playlist ingest persistence schema (TASK-12110)" +``` + +### Task 2: Implement the focused repository and cursor contract + +- [x] **Step 1: Write failing repository tests** + +Cover creation, owner isolation, ready-only materialization, selected occurrence copying, immutable ordering, cursor tampering, expiry, event replay, compare-and-set state transitions, and one-query Media DB lookup of normalized URL candidates. + +```python +def test_materialization_copies_identity_but_not_review_policy(store): + ready = store.seed_ready_preflight(owner_id="1", item_count=2) + materialized = store.create_materialization( + owner_id="1", preflight_id=ready.preflight_id, occurrence_ids=["occ-2"] + ) + item = store.list_materialization_items("1", materialized.id)[0] + assert item.occurrence_id == "occ-2" + assert item.source_url.endswith("v=2") + assert "duplicate_policy" not in item.display_metadata + assert "metadata_patch" not in item.display_metadata +``` + +- [x] **Step 2: Run the repository tests and verify RED** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py tldw_Server_API/tests/MediaDB2/test_dedupe_url_normalization.py -q` + +Expected: FAIL because `PlaylistIngestStore` is missing. + +- [x] **Step 3: Implement `PlaylistIngestStore`** + +Accept the existing `JobManager` and use its package-owned `_connect`/`_pg_cursor` helpers so backend/DSN selection is not duplicated. The playlist store itself must not open the Media DB or AuthNZ DB. Every public method requires `owner_user_id`. Use one transaction for snapshot replacement, materialization copying, run creation, event append/version bump, and cleanup. Sign opaque cursors with `AuthNZ.crypto_utils.derive_hmac_key()` and bind them to owner, resource, ordering, and last ordinal. + +In the same red/green cycle, add `get_media_by_urls(urls)` through `media_lookup_repository.py` → `media_db/api.py` → `runtime/query_ops.py` → `MediaDatabase`. Normalize all URL candidates and execute one parameterized query against the already owner-specific Media DB. This helper is shared by preflight duplicate enrichment and Start Processing validation. + +- [x] **Step 4: Add and run property tests** + +For arbitrary unique occurrence lists and page sizes, concatenated pages must equal the source ordering exactly once. Invalid owner/cursor pairs must never disclose whether a resource exists. + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_store_postgres.py tldw_Server_API/tests/MediaDB2/test_dedupe_url_normalization.py tldw_Server_API/tests/MediaIngestion_NEW/property/test_playlist_ingest_properties.py -q` + +Expected: PASS. + +- [x] **Step 5: Commit** + +```bash +git add tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_store.py tldw_Server_API/app/core/DB_Management/media_db/repositories/media_lookup_repository.py tldw_Server_API/app/core/DB_Management/media_db/api.py tldw_Server_API/app/core/DB_Management/media_db/runtime/query_ops.py tldw_Server_API/app/core/DB_Management/media_db/media_database_impl.py tldw_Server_API/tests/MediaDB2/test_dedupe_url_normalization.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_store_postgres.py tldw_Server_API/tests/MediaIngestion_NEW/property/test_playlist_ingest_properties.py +git commit -m "feat: persist playlist ingest resources (TASK-12110)" +``` + +## Stage 2: Asynchronous inspection and materialization + +**Goal:** Replace optional synchronous preview with a bounded, cancellable version-2 resource. + +**Success Criteria:** The API returns 202, worker extraction is terminable, complete snapshots paginate, oversize playlists block, and materialization survives preflight expiry. + +**Tests:** Runner process tests, worker tests, endpoint tests, configured-limit-plus-one tests. + +**Status:** Complete + +### Task 3: Add the bounded preflight child-process runner + +- [x] **Step 1: Write failing runner tests** + +Test successful normalized extraction, child timeout termination, cancellation termination, malformed child payload, `configured_limit + 1` producing `playlist_too_large` without a partial-ready snapshot, owner-library duplicates being marked `duplicate_existing`/counted/deselected before ready, and a failed library lookup producing `unknown` evidence plus a warning rather than falsely reporting `new`. + +- [x] **Step 2: Verify RED** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_preflight.py -k "process or configured_limit" -q` + +Expected: FAIL because the process runner and hard-ceiling behavior are absent. + +- [x] **Step 3: Implement the runner and tighten extraction** + +Use `multiprocessing.get_context("spawn")`, one result pipe, and a small polling loop. On timeout/cancel, call `terminate()`, then `kill()` only if the child remains alive after a bounded join. The child calls the existing yt-dlp normalizer. Request at most `limit + 1`, return unavailable entries visibly, generate opaque occurrence IDs server-side, and never return truncation as success. + +- [x] **Step 4: Extend the media worker** + +Handle `job_type == "playlist_preflight"` before `media_ingest_item`; call the runner, open the owner's Media DB, bulk-resolve extracted URLs with `get_media_by_urls`, and merge `duplicate_existing` evidence with in-snapshot duplicates before atomically storing the complete snapshot. Recompute duplicate/selected counts from the enriched items. A library lookup failure marks otherwise-new evidence `unknown` and adds a typed warning; extraction, capacity, or snapshot-write failure blocks the resource with a safe error code. Keep Jobs leases as the only cross-process extraction claim. + +- [x] **Step 5: Run tests and commit** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_preflight.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_worker.py -q` + +Expected: PASS. + +```bash +git add tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_preflight.py tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_preflight_runner.py tldw_Server_API/app/services/media_ingest_jobs_worker.py tldw_Server_API/tests/MediaIngestion_NEW/unit +git commit -m "feat: run playlist inspection as bounded jobs (TASK-12110)" +``` + +### Task 4: Expose preflight, pages, cancellation, and materialization + +- [x] **Step 1: Write failing route tests** + +Cover POST 202, summary polling, paginated pages, delete/cancel, ready-only materialization, cross-owner 404-equivalent behavior, `preflight_busy`, expiry, and sanitized errors. + +- [x] **Step 2: Verify RED** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_endpoint.py -k preflight -q` + +Expected: FAIL because `/playlist-preflights` routes are missing. + +- [x] **Step 3: Implement the routes and service methods** + +`POST /playlist-preflights` validates the trusted YouTube boundary, reserves capacity transactionally, creates the resource, and enqueues the internal job. Item routes return bounded pages. Materialization accepts only selected occurrence IDs and returns compact identity records; it never accepts policies or patches. `DELETE` requests job cancellation and expires the resource. + +- [x] **Step 4: Register the router and keep compatibility explicit** + +Add `playlist_ingest` to `_MEDIA_ENDPOINT_MODULES`. Leave `/playlists/preflight` working for older clients, but add compatibility tests proving version-2 clients are advertised separately. + +- [x] **Step 5: Run tests and commit** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_endpoint.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_preflight_endpoint.py -q` + +Expected: PASS. + +```bash +git add tldw_Server_API/app/api/v1/endpoints/media/playlist_ingest.py tldw_Server_API/app/api/v1/endpoints/media/__init__.py tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_service.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_endpoint.py +git commit -m "feat: expose playlist preflight resources (TASK-12110)" +``` + +## Stage 3: Run creation and duplicate actions + +**Goal:** Resolve every selected occurrence exactly once before any media job is accepted. + +**Success Criteria:** Mixed materialized URLs/direct URLs/file stubs validate atomically; stale duplicate choices return `review_required`; non-processing policies terminate without jobs. + +**Tests:** Service tests, Media DB patch tests, Collections DB transaction tests, action-resolution properties. + +**Status:** Complete + +### Task 5: Validate and create ingest runs atomically + +- [x] **Step 1: Write failing service tests** + +Test mixed input unions, playlist rejection in `direct_url`, expired materialization, unique occurrence IDs, file `awaiting_upload`, missing/extra review overrides, and a fresh duplicate appearing after Review. + +```python +def test_fresh_duplicate_requires_review_without_side_effects(service, media_db): + materialized = service.seed_materialized_video("youtube:video:abc") + media_db.seed_existing(url="https://www.youtube.com/watch?v=abc") + with pytest.raises(ReviewRequiredError) as exc: + service.create_run(inputs=[materialized], review_overrides={}) + assert exc.value.items[0].occurrence_id == materialized.occurrence_id + assert service.count_runs() == 0 + assert service.count_media_jobs() == 0 +``` + +- [x] **Step 2: Verify RED** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_service.py -k create_run -q` + +Expected: FAIL because run creation is absent. + +- [x] **Step 3: Implement validation and fresh duplicate lookup** + +Resolve materialized identity server-side, canonicalize non-playlist direct URLs, and create file stubs without bytes. Reuse the Stage 1 `get_media_by_urls(urls)` owner-database bulk lookup to refresh library evidence. Resolve in-run repeats by normalized source ID plus occurrence order. Validate Review overrides only after this fresh evidence. Return structured `review_required` before opening the run transaction when choices are stale. + +- [x] **Step 4: Persist the run and initial events in one transaction** + +Store immutable ordinal/identity plus mutable state/outcome/attempt. Initial action is `ingest`, `overwrite`, `skip`, `include_existing`, or `update_metadata_only`; do not create jobs yet. Append one initial event per occurrence and one summary version bump. + +- [x] **Step 5: Run tests and commit** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_service.py tldw_Server_API/tests/MediaIngestion_NEW/property/test_playlist_ingest_properties.py -q` + +Expected: PASS. + +```bash +git add tldw_Server_API/app/api/v1/schemas/media_playlist_ingest.py tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_service.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_service.py tldw_Server_API/tests/MediaIngestion_NEW/property/test_playlist_ingest_properties.py +git commit -m "feat: validate playlist ingest runs (TASK-12112)" +``` + +### Task 6: Execute non-processing duplicate policies and optional collection planning + +- [x] **Step 1: Write failing atomicity tests** + +Add tests proving `skip`, `include_existing`, and `update_metadata_only` create zero media jobs and distinct outcomes. Test allowlisted title/author/keyword union, empty/forbidden patches, optimistic conflict rollback, and collection-plus-items rollback. + +- [x] **Step 2: Verify RED** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/DB_Management/test_media_db_media_item_update_ops.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_service.py -k "metadata or collection or duplicate" -q` + +Expected: FAIL because the atomic patch and action executor are missing. + +- [x] **Step 3: Add the narrow Media DB patch** + +Implement `apply_media_metadata_patch(media_id, title=None, author=None, keywords_add=())` in `media_item_update_ops.py`. In one Media DB transaction, fetch the active row/version and current keywords, validate non-empty values, union keywords case-insensitively, update title/author/version, update FTS when title changes, update keyword links with the same connection, and log one sync event. Do not expose content/type/analysis mutation. + +- [x] **Step 4: Add transactional collection planning and action execution** + +Add one `CollectionsDatabase.create_media_collection_with_items(...)` method using its existing transaction helpers. In the service, execute non-processing actions after the run transaction exists: set terminal outcome/media ID, resolve optional planned items, and append events. On error, use `metadata_update_failed`; never silently fall back to a media job. + +- [x] **Step 5: Run tests and commit** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/DB_Management/test_media_db_media_item_update_ops.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_service.py -q` + +Expected: PASS. + +```bash +git add tldw_Server_API/app/core/DB_Management/media_db/runtime/media_item_update_ops.py tldw_Server_API/app/core/DB_Management/media_db/media_database_impl.py tldw_Server_API/app/core/DB_Management/Collections_DB.py tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_service.py tldw_Server_API/tests +git commit -m "feat: resolve playlist duplicate actions (TASK-12112)" +``` + +## Stage 4: Processing jobs, status, cancellation, and retry + +**Goal:** Bind processing-required occurrences to existing Jobs with durable run reconciliation. + +**Success Criteria:** One accepted job per occurrence attempt, structured partial acceptance, dynamic run events, real cancellation, and reconciled retry. + +**Tests:** Endpoint/worker tests, event replay tests, cancellation races, integration workflow. + +**Status:** Complete + +### Task 7: Tighten media-job submission and worker boundaries + +**Status:** Complete + +**Specification review follow-up:** Complete + +- [x] **Step 1: Write failing endpoint and worker tests** + +Cover aligned URL/file occurrence arrays, length mismatch, owner/run/state validation, derived idempotency, repeated ambiguous submit, opaque playlist 422, client URL mismatch, canonical server-authoritative payload, and worker rejection when processing returns zero or multiple items. + +- [x] **Step 2: Verify RED** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_worker.py -q` + +Expected: FAIL on the new occurrence contract. + +- [x] **Step 3: Extend `/media/ingest/jobs` minimally** + +Accept `run_id`, aligned `occurrence_ids`, `attempts`, and optional planned IDs for URLs; use equivalent `file_*` arrays for uploads. Derive the Jobs idempotency key as an HMAC/hash over authenticated owner, run, occurrence, and attempt. Validate run membership/state before staging files. Resolve the authoritative concrete URL from the run item; reject any non-matching client URL with `occurrence_source_mismatch`, then write only the stored URL into the job payload. Return one structured accepted/rejected record per occurrence while retaining legacy response fields during deprecation. + +- [x] **Step 4: Enforce one concrete worker result** + +Reject `classify_playlist_url(source).is_playlist` before job creation and again defensively in the worker. Replace `results[0]` projection with a length check: exactly one dict is required for a media occurrence. Include `run_id`, `occurrence_id`, and attempt in the payload/result for reconciliation. + +- [x] **Step 5: Run tests and commit** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_worker.py tldw_Server_API/tests/Media/test_media_ingest_jobs_endpoint_sanitization.py -q` + +Expected: PASS. + +```bash +git add tldw_Server_API/app/api/v1/endpoints/media/ingest_jobs.py tldw_Server_API/app/services/media_ingest_jobs_worker.py tldw_Server_API/tests/MediaIngestion_NEW/unit tldw_Server_API/tests/Media/test_media_ingest_jobs_endpoint_sanitization.py +git commit -m "feat: bind media jobs to ingest occurrences (TASK-12110)" +``` + +- [x] Strictly parse run-bound identity/integer arrays without coercion. +- [x] Reconcile ambiguous job creation before exact-reservation reset. +- [x] Stop chunks on typed global quota/rate failures and preserve retry metadata. +- [x] Re-run Task 7 and prior-stage verification, then commit the follow-up. + +**Final recovery follow-up:** Complete + +- [x] Recover an exact preserved reservation by idempotently creating with its stored batch and identity. +- [x] Cover two-request URL/file recovery, repeated ambiguity, concurrency ownership, and typed quota reset. +- [x] Prove file recovery leaves only the authoritative committed staging path. +- [x] Re-run focused, PostgreSQL, property, static, and Bandit verification, then commit. + +**Shared-reservation race follow-up:** Complete + +- [x] Add deterministic RED coverage for recovery after upload staging failure, typed quota rejection, and confirmed no-job creation failure. +- [x] Reproduce a cooperative retry creating the exact job before the original submitter reaches its former reset-before-bind path. +- [x] Preserve run-bound submit-pending reservations on every retryable submission failure while leaving legacy non-run behavior unchanged. +- [x] Re-run focused, PostgreSQL, property, static, and Bandit verification, update TASK-12112, and commit. + +**Cooperative-retry status propagation follow-up:** Complete + +- [x] Add RED coverage proving generic HTTP 429/503 remains global after a preserved-reservation retry confirms no exact job. +- [x] Preserve Retry-After, stop later entries, and leave the exact reservation submit-pending. +- [x] Keep exact committed-job reconciliation authoritative over the prior create exception. +- [x] Re-run focused, PostgreSQL, property, static, and Bandit verification, update TASK-12112, and commit. + +**Quality-review safety follow-up:** Complete + +- [x] **Group A — Bound decoding and legacy-worker compatibility** + + In `test_media_ingest_jobs_endpoint.py` and `test_media_ingest_jobs_worker.py`, add behavior tests for malformed, non-list, over-500, and encoded-over-256-KiB arrays, plus `JSONDecodeError`, `RecursionError`, and `MemoryError` sanitization before run lookup or upload staging. Add legacy worker cases proving zero results retain the old empty-dict projection, multiple results use the first item, and a non-dict first result returns the old safe error, while occurrence-bound jobs remain exactly-one-dict strict. + + RED: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py -k "run_bound and (encoded or malformed or recursion or memory or more_than_500)" tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_worker.py -k "legacy_media_worker or bound_media_worker_requires" -q` — expected failures are unbounded/unsanitized decoding and changed legacy projection. + + Implement the 256-KiB pre-decode cap, immediate top-level-list/500-element checks, exact exception handling, and conditional worker projection in `ingest_jobs.py` and `media_ingest_jobs_worker.py`. GREEN: rerun the RED command, then both complete test files. + +- [x] **Group B — Durable reservation authority, encrypted binding views, and migration parity** + + In the SQLite/PostgreSQL migration tests, playlist store tests, endpoint tests, and PostgreSQL store parity tests, add upgrade coverage for nullable `submission_queue` and `staging_temp_dir`, dataclass/row-converter parity, old rows with `submission_queue IS NULL`, secret rotation, heavy/default queue config drift, mixed URL/file ingest+overwrite actions, malicious opposite client options, and encrypted URL/file create/retry/bind cleanup. New reservations atomically persist the current HMAC identity and queue selected by `_resolve_media_ingest_queue`; pending/queued retries return the stored identity without recomputing it and always use the stored queue. For an upgraded NULL queue reservation, adopt the queue of one exact existing job when present; otherwise initialize the current resolver choice once with a CAS, after which configuration is never consulted for that reservation. + + Add a public owner-authorized, job-scoped `JobManager` binding-view normalizer that returns only immutable job metadata plus a safely decrypted dict payload. It must fail closed on owner/payload mismatch, never log decrypted content, and never serialize the view into responses. New create results, idempotent-create results, exact lookups, endpoint file cleanup, and store binding all use this normalized view. Per-entry copied options force `overwrite_existing = (reserved.action == "overwrite")` for URL and file submissions; store binding defensively verifies the decrypted option against the reserved action. + + RED: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/Jobs/test_jobs_migrations_sqlite.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py -k "submission_queue or staging_temp_dir or rotation or queue_drift or overwrite or encrypted" -q` — expected failures are absent columns/record fields, recomputed authority, shared options, and encrypted-envelope rejection. + + Modify `migrations.py`, `pg_migrations.py`, `manager.py`, `playlist_ingest_store.py`, and `ingest_jobs.py`. GREEN: rerun the RED command, then the complete SQLite migration/store/endpoint files and `RUN_JOBS=1 python -m pytest tldw_Server_API/tests/Jobs/test_jobs_migrations_postgres.py tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_store_postgres.py -k "submission_queue or encrypted or binding" -q`. + +- [x] **Group C — Non-acquirable creation, transactional bind/publish, and release repair** + + Following the existing preflight sentinel, create occurrence jobs with far-future `available_at`. In the store binding transaction, recheck the unexpired accepting run, exact attempt/reservation/queue/action, normalized binding view, and held Jobs row; atomically bind the run item and publish via database-clock `available_at`. Add an idempotent repair branch for a deliberately seeded `queued` run item whose exact job remains held: retry revalidates owner/run/occurrence/attempt/queue/payload and performs release-only CAS. Already-published exact jobs are a no-op; expired/nonaccepting runs and ambiguous rows remain held. + + RED: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py -k "held or acquisition_before_bind or bind_publish or release_repair or run_expired_during_bind" -q` — expected failures show a pre-bind acquisition and absent repair. Modify only the existing create helper/available-at field and store bind transaction; add no new Jobs state/framework. GREEN: rerun RED plus complete store/endpoint/worker files, then `RUN_JOBS=1 python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_store_postgres.py -k "bind_publish or held or release" -q`. + +- [x] **Group D — PostgreSQL quota serialization and structured quota codes** + + In `test_jobs_quotas_postgres.py`, add barrier-synchronized real-PostgreSQL tests proving concurrent max-queued and submissions-per-minute requests admit no more than the configured limit. In `JobManager.create_job`, within the existing transaction and before both counts and insert, take one stable `pg_advisory_xact_lock` keyed by the exact owner/domain quota scope; all submissions use that fixed single-lock order, and SQLite is unchanged. Endpoint tests assert a typed quota failure uses `detail.code` and `detail.message` in the global 429 payload and the same code in a structured rejected submission if that response path is used; no raw-message matching. + + RED: `source .venv/bin/activate && RUN_JOBS=1 python -m pytest tldw_Server_API/tests/Jobs/test_jobs_quotas_postgres.py -k "concurrent and (max_queued or submits_per_minute)" -q` and `python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py -k "quota_code" -q` — expected failures are quota oversubscription and absent `detail.code`. GREEN: rerun both, the complete SQLite/PostgreSQL quota files, and endpoint tests. + +- [x] **Group E — Persisted, bounded abandoned-staging cleanup and complete verification** + + Persist each reservation's exact `staging_temp_dir` immediately after its task-specific directory is created. A bounded store query supplies at most the configured deletion cap of owner-scoped `submit_pending` candidates older than the retention cutoff; cleanup never discovers candidates by walking the temp root. For each candidate, require the resolved path's parent to equal `tempfile.gettempdir()`, its basename to match the reservation-derived prefix, and no live run-item or Jobs binding view to reference it. Delete only that exact directory, then CAS-clear the exact stored path. Live reservations/jobs and bound authoritative paths survive; malformed/cross-owner/out-of-root paths fail closed. + + RED: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py -k "abandoned_staging or staging_cleanup" -q` — expected failures are no persisted candidates or retention cleanup. Modify the store and existing run-bound endpoint mutation seam only. GREEN: rerun RED and complete endpoint/store files. + + Final gate: run endpoint/store/worker/manager/migration/encryption/cleanup suites, the complete Task 1–7 and property matrix, `/process-videos` legacy coverage, explicitly enabled real PostgreSQL concurrency/encryption, Black, Ruff, compileall, `git diff --check`, and Bandit on every touched production path. Record exact RED/GREEN evidence and any pre-existing baseline in TASK-12112, then commit coherent follow-up(s) with `TASK-12112` in each message. + + Verification evidence: endpoint `110 passed`; store `122 passed`; worker plus SQLite migrations `70 passed`; PostgreSQL migrations, quota concurrency, and playlist-store integration `21 passed`. Ruff, Black, compileall, and `git diff --check` exited zero. Bandit scanned all six touched production paths and reported zero findings. Focused RED/GREEN evidence is recorded in TASK-12112. + + Independent-review remediation: lifecycle-safe already-bound retries; fail-closed encrypted binding views; retain staging metadata after deletion failure; reuse completed staging across cooperative retries; reject shared-path deletion and case-insensitive manifest-name collisions; keep pre-record retries pending; release only exact owner reservations after confirmed pre-manifest cleanup with a NULL-staging CAS; sanitize JSON decoder integer-limit `ValueError` before lookup/staging; strengthen overwrite, malformed-JSON, and file-race coverage. Status: Complete. Independent re-review found no remaining Critical or Important issues. + +**Final submission-lifecycle safety follow-up:** Complete + +- [x] **Group F — Durable submission lease, heartbeat, and generation CAS** + + Add additive SQLite/PostgreSQL migration coverage and record/store tests for an opaque lease token, bounded expiry, and monotonic generation on every acquired or taken-over `submit_pending` reservation. Use the injected Jobs clock and short configured test leases without sleeps. An active retry with a valid complete manifest may reuse it through an explicit generation CAS; a retry seeing a NULL or incomplete staging pointer waits only while the current lease is valid, then atomically takes over after expiry with a caller-proposed new token. Require the current token for pointer/manifest recording and held-job creation/binding, not only for renewal around those steps. Prove pre-record and pre-manifest crash recovery, one-winner expiry takeover, lease renewal during upload chunks and create/bind critical sections, and generation-owned staging paths. A stale owner must fail record/clear/reset/bind CAS, may delete only its own abandoned uncommitted directory, and must never delete the new owner's directory. Preserve the original batch, idempotency identity, and queue across takeover. + + RED: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/Jobs/test_jobs_migrations_sqlite.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py -k "submission_lease or lease_takeover or upload_heartbeat or stale_owner" -q`; expected failures are absent lease columns/record fields and mutation CAS authority. GREEN: rerun that command, then the complete migration/store/endpoint suites and `RUN_JOBS=1 python -m pytest tldw_Server_API/tests/Jobs/test_jobs_migrations_postgres.py tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_store_postgres.py -k "submission_lease or lease_takeover" -q`. + +- [x] **Group G — Expired-run held-job cancellation and quota release** + + Add SQLite and real-PostgreSQL tests proving the `cleanup_expired_resources`/`cleanup_expired` path locks expired run/item/job authority, cancels only an exact owner/run/occurrence/attempt/job/batch/idempotency/queue match that is still queued at the held sentinel, releases queued quota and decrements scheduled counters, and only then deletes run authority in the same Jobs transaction. Assert a fresh quota-limited submit is admitted after cleanup. Published, processing, and terminal jobs must survive. Inject a failure after cancellation and prove the entire transaction rolls back before any authority is deleted. Only after the transaction commits may staging cleanup retire an unreferenced directory; a published job's staging reference stays protected after run authority disappears. + + RED: `source .venv/bin/activate && JOBS_COUNTERS_ENABLED=true python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py -k "cleanup_expired and (held_job or published_job or rollback or quota)" -q`; expected failure is the current child-first deletion without held-job reconciliation. GREEN: implement the transaction contract while reusing existing cancellation/counter semantics where practical, rerun that command and the complete store suite, then `RUN_JOBS=1 JOBS_COUNTERS_ENABLED=true python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_store_postgres.py -k "cleanup_expired and (held_job or published_job or rollback or quota)" -q`. + +- [x] **Group H — Canonical staging paths and exact candidate reference lookup** + + Add real filesystem tests for dot/dot-dot aliases, symlinked directories and components, root escapes, casefold-based manifest filename collisions with deterministic behavior across case-sensitive/case-insensitive platforms, and platform separator edge cases. Validate lexical canonical form before resolution or deletion, reject every symlink component, and canonicalize candidate and live-reference paths through one fail-closed helper. Replace the broad owner `list_jobs(limit=101)` scan with bounded exact candidate lookups so more than 100 unrelated active jobs do not starve cleanup and an encrypted exact reference still preserves its candidate without broad unbounded decryption. + + RED: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py -k "staging_path_alias or symlink or root_escape or manifest_casefold or more_than_100_jobs or encrypted_exact_reference" -q`; expected failures are alias acceptance and over-100 cleanup starvation. GREEN: implement the shared validator and bounded exact candidate binding lookup, then rerun the RED selector and all staging/cleanup/encryption endpoint tests. + +- [x] **Group I — Already-bound future schedule compatibility** + + Add a store test showing an exact already-bound queued job with any non-sentinel `available_at`, including a future retry/backoff timestamp, is a valid idempotent binding. Keep the sentinel as the only held-repair state and continue rejecting non-queued unbound jobs. + + RED: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py -k "already_bound_future_available_at" -q`; expected failure is the current wall-clock `is_published` check. GREEN: simplify the already-bound lifecycle predicate and rerun complete store tests plus PostgreSQL binding parity. + +- [x] **Group J — Final verification, independent review, and commit** + + Run endpoint/store/worker/migrations/manager/cleanup, Tasks 1–7/property coverage, real PostgreSQL lease+cleanup parity, encrypted binding coverage, Black, Ruff, compileall, `git diff --check`, and Bandit over every touched production path. Request an independent correctness/security re-review, remediate any Critical or Important findings with another RED/GREEN cycle, record exact evidence in TASK-12112, and commit as `fix: finalize playlist submission lifecycle safety (TASK-12112)`. + + Final evidence: endpoint/store `264 passed`; SQLite JobManager/idempotency/admin-counter/fault `31 passed, 2 skipped`; specification focus `27 passed`; Ruff and Black clean on all 16 touched Python files; compileall clean on seven production modules; `git diff --check` clean; Bandit zero findings across 15,690 LOC. Real PostgreSQL migrations/integration passed `18 passed` before the final SQLite-only counter-attribution correction. The post-correction PostgreSQL reprovisioning retry reached the three-attempt infrastructure cap (two unreachable skips and one aborted provisioning run), so it is recorded as not freshly rerun. Independent reviews returned `✅ Spec compliant` and `Ready to proceed? Yes`. + +### Task 8: Add run routes, reconciliation, event replay, cancellation, and retry + +**Status:** Complete + +- [x] **Step 1: Write failing run-route tests** + +Cover run POST, summary, paginated items, SSE initial snapshot/replay/resync, a stream-only client observing job progress without polling, later chunk jobs appearing in the same stream, occurrence-scoped cancellation of unsent/accepted work, whole-run cancellation, `status_unavailable`, and retry after media reconciliation. + +- [x] **Step 2: Verify RED** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_endpoint.py -k run -q` + +Expected: FAIL because run routes are missing. + +- [x] **Step 3: Implement run routes and reconciliation** + +Run POST calls the service, executes terminal duplicate actions, and returns processing occurrences ready for bounded client chunks. Summary/items reconcile Jobs by stored mappings and append occurrence events only on actual changes. On every SSE cycle, call `reconcile_run_jobs(owner_id, run_id)` before reading new run events; it must query the current run/job mappings dynamically, compare state/progress/result, and transactionally append occurrence events/version changes. SSE then reads events by monotonically increasing ID; expired replay emits `resync_required`. + +Define `POST /ingest/runs/{run_id}/cancel` with optional body `{ occurrence_ids?: string[], reason?: string }`. Before a run exists, cancellation is client-local. Once a run exists, supplied occurrence IDs terminalize unsent items and cancel their accepted jobs; an omitted list cancels the whole run. Repeated requests are idempotent and completion may win the race. + +- [x] **Step 4: Implement retry with media-first reconciliation** + +Before incrementing attempt, query the current user's Media DB by normalized URL and planned item. If media exists, resolve terminally without a new job. Otherwise increment once with compare-and-set, clear the prior job mapping, and return the occurrence for resubmission. + +- [x] **Step 5: Run tests and commit** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_endpoint.py tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_run_workflow.py -q` + +Expected: PASS. + +```bash +git add tldw_Server_API/app/api/v1/endpoints/media/playlist_ingest.py tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_service.py tldw_Server_API/tests/MediaIngestion_NEW +git commit -m "feat: track playlist ingest runs (TASK-12112)" +``` + +Final evidence: Task 8 route/workflow `64 passed`; complete Task 1–8 affected matrix `534 passed, 18 skipped`; Black and Ruff clean on eight touched Python files; four production modules compiled; `git diff --check` clean; Bandit zero findings across 7,267 LOC. The skips are fixture-declared PostgreSQL migration/store tests; the parity assertions were collected but not executed because the local fixture was unavailable. Independent reviews returned `✅ Spec compliant` and `Ready to proceed? Yes`. + +## Stage 5: Capability rollout and release gates + +**Goal:** Make version negotiation, cleanup, compatibility, security, and operational limits explicit. + +**Success Criteria:** Version-2 capability is truthful, expired resources clean up, compatibility remains covered, all focused gates and Bandit pass. + +**Tests:** Config capability tests, cleanup tests, owner isolation, `/process-videos` compatibility, full focused suite. + +**Status:** Complete + +### Task 9: Finish rollout, cleanup, and verification + +- [x] **Step 1: Add failing capability and cleanup tests** + +Assert `mediaPlaylistIngestContractVersion == 2` only when preflight/run routes and worker readiness are enabled. Test bounded cleanup of expired preflights/materializations/runs/events and lease release after worker crash. Keep `/process-videos` multi-result playlist behavior covered. + +- [x] **Step 2: Implement capability and cleanup wiring** + +Update `config_info.py` with granular flags. Invoke bounded cleanup from preflight/run mutations and worker startup; do not add a new scheduler. Emit only counts/error codes in logs and metrics, never full playlist URLs. + +- [x] **Step 3: Run the complete focused backend gate** + +Run: `source .venv/bin/activate && python -m pytest tldw_Server_API/tests/Jobs/test_jobs_migrations_sqlite.py tldw_Server_API/tests/Jobs/test_jobs_migrations_postgres.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_preflight.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_store.py tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_store_postgres.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_service.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_playlist_ingest_endpoint.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_endpoint.py tldw_Server_API/tests/MediaIngestion_NEW/unit/test_media_ingest_jobs_worker.py tldw_Server_API/tests/MediaIngestion_NEW/integration/test_playlist_ingest_run_workflow.py tldw_Server_API/tests/MediaIngestion_NEW/property/test_playlist_ingest_properties.py -q` + +Expected: PASS with only fixture-declared PostgreSQL skips. + +- [x] **Step 4: Run security and diff gates** + +Run: `source .venv/bin/activate && python -m bandit -r tldw_Server_API/app/api/v1/endpoints/media/playlist_ingest.py tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_store.py tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_preflight_runner.py tldw_Server_API/app/core/Ingestion_Media_Processing/Video/playlist_ingest_service.py tldw_Server_API/app/api/v1/endpoints/media/ingest_jobs.py tldw_Server_API/app/services/media_ingest_jobs_worker.py -f json -o /tmp/bandit_task_12110.json` + +Run: `git diff --check` + +Expected: Bandit exits 0 with no new findings; diff check exits 0. + +- [x] **Step 5: Update docs/task and commit** + +Record exact test counts, PostgreSQL skips, Bandit result, touched files, and compatibility result in `TASK-12112`. + +```bash +git add tldw_Server_API Docs backlog/tasks +git commit -m "test: verify playlist ingest backend contract (TASK-12112)" +``` + +Final evidence: the complete focused backend gate passed `570 passed, 18 skipped` with exit status 0; the exact Task 9 changed-module matrix passed `319 passed, 15 skipped` with exit status 0; and the complete store file passed `172 passed`. All skips were fixture-declared PostgreSQL tests because a live local PostgreSQL fixture was unavailable. The missing pytest footer was traced to the command wrapper discarding nested child-process metadata and obscuring captured output; redirected logs plus explicit child status files proved normal pytest completion, so no application-code change was required. Black and py_compile/import passed on the nine Task 9 Python files; `git diff --check` passed; Ruff introduced no findings beyond the five unchanged `config_info.py` baseline findings; Bandit reported zero findings on the four touched production files. Legacy `/process-videos` playlist behavior passed its six focused controls. Independent final reviews returned `✅ Spec compliant` and `✅ Quality approved` with no actionable findings. diff --git a/Docs/superpowers/plans/2026-07-12-youtube-playlist-ingest-shared-frontend.md b/Docs/superpowers/plans/2026-07-12-youtube-playlist-ingest-shared-frontend.md new file mode 100644 index 0000000000..2bffb978c0 --- /dev/null +++ b/Docs/superpowers/plans/2026-07-12-youtube-playlist-ingest-shared-frontend.md @@ -0,0 +1,495 @@ +# YouTube Playlist Ingest Shared Frontend Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the shared WebUI/browser-extension Quick Ingest flow inspect every YouTube playlist before queueing and show every selected video through review, processing, recovery, and results. + +**Architecture:** Keep one controller, contract client, queue model, and run-status adapter in `@tldw/ui`; WebUI and extension differ only in transport preference and runtime lifetime. Reuse the backend's version-2 preflight/run resources, TanStack Virtual, Dexie, Zustand, and the current Quick Ingest wizard. Delete the direct playlist-to-queue bypass instead of adding guards to each caller. + +**Tech Stack:** React 18, TypeScript, Ant Design/design-system primitives, Zustand, Dexie, `@tanstack/react-virtual`, Vitest/Testing Library, Playwright, WXT extension runtime. + +**Backlog:** `TASK-12113` (depends on backend contract task `TASK-12112`) + +**Spec:** `Docs/superpowers/specs/2026-07-12-youtube-playlist-per-item-ingest-design.md` + +--- + +## File map + +**Create** + +- `apps/packages/ui/src/services/tldw/playlist-ingest.ts` — normalized version-2 HTTP models, cursor paging, run transport, and typed public errors. +- `apps/packages/ui/src/components/Common/QuickIngest/usePlaylistInspection.ts` — the single mandatory Add/Enter/extension-seed inspection controller. +- `apps/packages/ui/src/db/dexie/quick-ingest.ts` — compact session persistence, migration, retention, quota reporting, and single-writer lease helpers. +- `apps/packages/ui/src/services/tldw/__tests__/playlist-ingest.test.ts` — contract normalization, paging, run, SSE/poll, and error tests. +- `apps/packages/ui/src/components/Common/QuickIngest/__tests__/AddContentStep.playlist-ingest.test.tsx` — fail-closed shared entry tests. +- `apps/packages/ui/src/components/Common/QuickIngest/__tests__/PlaylistPreflightPanel.virtualization.test.tsx` — scale, selection, focus, and ARIA tests. +- `apps/packages/ui/src/store/__tests__/quick-ingest-indexeddb.test.ts` — migration, quota, cleanup, and multi-tab tests. +- `apps/tldw-frontend/e2e/workflows/quick-ingest-playlist.spec.ts` — WebUI journey against deterministic routes/fixtures. + +**Modify** + +- `apps/packages/ui/src/services/tldw/domains/media.ts` — expose version-2 playlist/run methods through the existing media-domain mixin on `TldwApiClient`. +- `apps/packages/ui/src/services/tldw/openapi-guard.ts`, `server-capabilities.ts`, and their tests — allow and require the version-2 route/version signal. +- `apps/packages/ui/src/services/tldw/playlist-preflight.ts` — compatibility exports only; occurrence IDs come from the server. +- `apps/packages/ui/src/components/Common/QuickIngest/types.ts` — materialization, occurrence, lifecycle/outcome, run, and client-derived file reattachment types. +- `apps/packages/ui/src/components/Common/QuickIngest/AddContentStep.tsx` — remove the playlist bypass; delegate all candidate handling to the controller. +- `apps/packages/ui/src/components/Common/QuickIngest/PlaylistPreflightPanel.tsx` — asynchronous summary, complete pagination, virtual rows, selection tools, and typed recovery. +- `apps/packages/ui/src/components/Common/QuickIngest/IngestWizardContext.tsx`, `ReviewStep.tsx`, and `ItemMetadataTable.tsx` — preserve identities, virtualize/filter Review, and submit Review overrides. +- `apps/packages/ui/src/components/Common/QuickIngestWizardModal.tsx` — create/reconcile runs and bridge wizard state to shared services. +- `apps/packages/ui/src/components/Common/QuickIngest/ProcessingStep.tsx`, `WizardResultsStep.tsx`, `FloatingProgressWidget.tsx`, and `QueuedFileRow.tsx` — truthful lifecycle/outcome UI, run cancellation, retry, and client-derived reattachment. +- `apps/packages/ui/src/services/tldw/quick-ingest-batch.ts` and `quick-ingest-session-reattach.ts` — delegate version-2 runs to the shared run client; keep legacy handling only for non-playlist/old-server compatibility. +- `apps/packages/ui/src/store/quick-ingest-session.ts` — async IndexedDB-backed persistence with visible failures. +- `apps/packages/ui/src/db/dexie/schema.ts` and `types.ts` — version-15 `quickIngestSessions` table. +- `apps/packages/ui/src/entries/background.ts`, `entries/shared/quick-ingest-session-runtime.ts`, and `entries/shared/background-init.ts` — extension polling/reattachment over the same run contract. +- `apps/packages/ui/src/components/Sidepanel/Chat/ControlRow.tsx` and tests — active-tab handoff only; no separate ingest logic. +- Existing Quick Ingest service/component/store/integration tests and English locale strings. + +## Stage 1: Contract client and mandatory inspection + +**Goal:** Make playlist classification and version-2 inspection one shared, fail-closed path. + +**Success Criteria:** Add, Enter, WebUI paste, and extension seed cannot create an opaque playlist queue row; typed inspection state survives component rerenders. + +**Tests:** Media-domain tests, capability tests, controller tests, AddContent integration tests. + +**Status:** Complete + +### Task 1: Add version-2 client models and capability gating + +- [x] **Step 1: Write failing client/capability tests** + +Extend `tldw-api-client.media-ingest.test.ts` and `server-capabilities.test.ts` to cover create/get/page/materialize/run/items/cancel/retry routes and `mediaPlaylistIngestContractVersion >= 2`. + +```ts +it("requires the complete playlist ingest contract for version 2", async () => { + mockDocsInfo({ capabilities: { mediaPlaylistIngestContractVersion: 2 } }) + mockOpenApiPaths([ + "/api/v1/media/playlist-preflights", + "/api/v1/media/playlist-preflights/{preflight_id}", + "/api/v1/media/playlist-preflights/{preflight_id}/items", + "/api/v1/media/playlist-preflights/{preflight_id}/materializations", + "/api/v1/media/ingest/runs", + "/api/v1/media/ingest/runs/{run_id}", + "/api/v1/media/ingest/runs/{run_id}/items", + "/api/v1/media/ingest/runs/{run_id}/events/stream", + "/api/v1/media/ingest/runs/{run_id}/cancel", + "/api/v1/media/ingest/runs/{run_id}/retry", + ]) + expect((await getServerCapabilities()).hasMediaPlaylistIngestV2).toBe(true) +}) +``` + +- [x] **Step 2: Run and verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/services/__tests__/tldw-api-client.media-ingest.test.ts ../packages/ui/src/services/__tests__/server-capabilities.test.ts --maxWorkers=1 --no-file-parallelism` + +Expected: FAIL because the version-2 client and capability are missing. + +- [x] **Step 3: Implement the contract client** + +Add exact server wire types and normalized camelCase types in `playlist-ingest.ts`. Use `bgRequest`/`bgUpload` through `domains/media.ts`; extend `openapi-guard.ts` for only the required paths. Keep cursor strings opaque. Map typed server codes to stable UI errors without exposing raw extractor output. + +- [x] **Step 4: Run tests and commit** + +Run the command from Step 2. + +Expected: PASS. + +```bash +git add apps/packages/ui/src/services/tldw apps/packages/ui/src/services/__tests__ +git commit -m "feat: add playlist ingest v2 client (TASK-12113)" +``` + +Verification before commit: initial RED was 11 failed / 49 passed; review-remediation RED was 8 failed / 68 passed. Final focused Vitest passed 76/76, ESLint exited 0 with no errors, Prettier and `git diff --check` passed. Full TypeScript checking remains blocked by the unrelated repository baseline after the required three-attempt audit. Specification and code-quality re-reviews both approved the final Task 1 diff. + +### Task 2: Route every playlist candidate through one inspection controller + +- [x] **Step 1: Write failing behavior tests** + +Test mixed ordinary/playlist URLs, Add click, Enter, unavailable capability, inspection failure, multiple candidates with bounded concurrency, extension `playlist_preflight` seed, and a session duplicate index spanning queued direct URLs plus multiple playlists. Assert queue mutation never receives a candidate URL directly and Configure/Quick Process stays disabled while any candidate is unresolved. + +```tsx +it("blocks Add until every playlist candidate is materialized", async () => { + renderWizard({ urlInput: `${ordinaryUrl}\n${playlistUrl}` }) + await user.click(screen.getByRole("button", { name: /add urls/i })) + expect(screen.getByText(/inspection required/i)).toBeInTheDocument() + expect(queueItems()).toEqual([expect.objectContaining({ url: ordinaryUrl })]) + expect(queueItems()).not.toEqual(expect.arrayContaining([expect.objectContaining({ url: playlistUrl })])) + expect(screen.getByRole("button", { name: /configure/i })).toBeDisabled() +}) +``` + +- [x] **Step 2: Run and verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/components/Common/QuickIngest/__tests__/AddContentStep.playlist-ingest.test.tsx ../packages/ui/src/components/Common/QuickIngest/__tests__/AddContentStep.url-detection.test.ts --maxWorkers=1 --no-file-parallelism` + +Expected: FAIL because `handleAddUrls` still queues the playlist directly. + +- [x] **Step 3: Implement `usePlaylistInspection` and delete the bypass** + +The hook owns candidate records keyed by the original line, a maximum concurrent inspection count, resource polling, cancel/retry, seed handling, and a session duplicate index built from existing queue rows plus all loaded candidates. `handleAddUrls` delegates parsed lines once: ordinary lines become staged rows; playlist lines become inspection records. Enter calls the same handler. Candidate detection remains the small trusted-host helper already tested in `AddContentStep.tsx`. + +- [x] **Step 4: Run tests and commit** + +Run the command from Step 2. + +Expected: PASS. + +```bash +git add apps/packages/ui/src/components/Common/QuickIngest/AddContentStep.tsx apps/packages/ui/src/components/Common/QuickIngest/usePlaylistInspection.ts apps/packages/ui/src/components/Common/QuickIngest/__tests__ +git commit -m "fix: require playlist inspection before queueing (TASK-12113)" +``` + +Verification before commit: behavior RED was 13 failed / 7 passed; Strict Mode seed review RED was 1 failed / 25 passed; quality-remediation RED was 7 failed / 24 passed. Final focused Vitest passed 31/31. Targeted ESLint exited 0 with no errors, the two new files passed the frontend Prettier configuration, and `git diff --check` passed. Full TypeScript was not rerun because the repository-baseline audit reached its three-attempt cap in Task 1. Specification and code-quality re-reviews both approved the final Task 2 diff. Bandit is not applicable to this TypeScript-only task. + +## Stage 2: Complete virtual preview and stable queue identity + +**Goal:** Show the complete ordered snapshot without unbounded DOM/network work, then queue server-authoritative occurrences. + +**Success Criteria:** Every page loads exactly once, large lists stay virtualized/accessibly navigable, and Add materializes selected occurrence IDs before queue mutation. + +**Tests:** Paging normalizer tests, virtual list tests, selection reconciliation, queue serialization tests. + +**Status:** Complete + +### Task 3: Paginate and virtualize the preflight panel + +- [x] **Step 1: Write failing paging and virtualization tests** + +Use a 500-item fixture. Assert cursors are followed until null, item order/occurrence IDs are stable, unavailable rows remain visible/disabled, Select all/none/new works, mounted rows remain bounded, keyboard focus survives scroll, and `aria-setsize`/`aria-posinset` are correct. With two playlists and a queued direct URL containing repeated videos, assert the first occurrence remains selected, later occurrences use session-level duplicate evidence, and explicit refresh reconciles selection by normalized source ID plus occurrence index among repeats. + +- [x] **Step 2: Run and verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/services/tldw/__tests__/playlist-ingest.test.ts ../packages/ui/src/components/Common/QuickIngest/__tests__/PlaylistPreflightPanel.virtualization.test.tsx --maxWorkers=1 --no-file-parallelism` + +Expected: FAIL because the panel eagerly renders the compatibility response. + +- [x] **Step 3: Implement bounded page loading and TanStack Virtual rows** + +Poll summary until `ready`/blocked terminal state, then request pages sequentially by opaque cursor with an `AbortController`. Merge only by server `occurrenceId`; reject duplicates or count mismatch as `preflight_incomplete`. Reconcile the complete snapshot against the session duplicate index using normalized source ID plus repeat index, retaining unambiguous selections and surfacing reorder/add/remove ambiguity after explicit refresh. Use `useVirtualizer` with stable occurrence keys and overscan. Render ordinal/title first and channel/duration/availability/duplicate second; keep URL in details and do not load thumbnails by default. + +- [x] **Step 4: Run tests and commit** + +Run the command from Step 2. + +Expected: PASS with a bounded mounted-row assertion. + +```bash +git add apps/packages/ui/src/services/tldw/playlist-ingest.ts apps/packages/ui/src/components/Common/QuickIngest/PlaylistPreflightPanel.tsx apps/packages/ui/src/components/Common/QuickIngest/__tests__ apps/packages/ui/src/services/tldw/__tests__ +git commit -m "feat: show complete virtualized playlist previews (TASK-12113)" +``` + +Verification before commit: the initial Task 3 paging/panel RED was 13/13 failures, hardening RED was 10 failed / 13 passed, specification-remediation RED was 3 failed / 30 passed plus 1 failed / 9 passed, and quality-remediation RED was 6 failed / 45 passed. Final focused Vitest passed 66/66 across the paging service, controller integration, virtualized panel, and legacy panel suites. Repository-pinned ESLint exited 0 (apart from the existing Next.js pages-directory informational message), scoped Prettier and `git diff --check` passed, and the final specification and code-quality re-reviews approved the diff. Full TypeScript was not rerun after the Task 1 three-attempt repository-baseline cap. Bandit is not applicable to this TypeScript-only task. Task 4 remains intentionally unimplemented: playlist rows are not materialized or added to the queue yet. + +### Task 4: Materialize occurrences and carry Review-time overrides + +- [x] **Step 1: Write failing queue/review tests** + +Assert Add sends selected occurrence IDs, creates no rows on materialization failure, stores materialization ID/token plus occurrence ID, renders title/ordinal as primary text, and lets Review edit per-occurrence duplicate policy and only title/author/keywords-add metadata. Add 500-item tests proving both the queue and Review render bounded row counts with filters, and an expired-materialization test proving Start Processing requires reinspection rather than cached URL fallback. + +- [x] **Step 2: Run and verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/components/Common/QuickIngest/__tests__/IngestWizardContext.test.tsx ../packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.integration.test.tsx --maxWorkers=1 --no-file-parallelism --testNamePattern="playlist|materialization|review"` + +Expected: FAIL because queue rows contain client-generated IDs and preflight URLs. + +- [x] **Step 3: Extend queue types and materialization handling** + +Add a discriminated source reference to `WizardQueueItem`: + +```ts +type WizardSourceRef = + | { kind: "materialized_playlist_item"; materializationId: string; token: string; occurrenceId: string } + | { kind: "direct_url"; occurrenceId: string; url: string } + | { kind: "file_stub"; occurrenceId: string } +``` + +Playlist row `id` equals the server occurrence ID. Preserve compact title/playlist/ordinal/channel/duration display data, but never treat cached playlist URLs as authoritative after materialization expiry. Keep duplicate policy/metadata patch in Review state, not materialization state. Virtualize the Add-step queue and `ItemMetadataTable` with stable occurrence keys. Add queue filters for playlist/type/duplicate state and Review filters for selected/duplicates/policy; filters change visibility only, never selection. + +- [x] **Step 4: Build the exact Start Processing payload** + +Serialize selected input records plus `review_overrides[occurrenceId]`. Include an explicit duplicate policy for every current duplicate and include a patch only for explicitly edited allowlisted fields. On backend `review_required`, merge refreshed duplicate evidence and return to Review without marking rows as submitted. + +- [x] **Step 5: Run tests and commit** + +Run the command from Step 2 without the name filter. + +Expected: PASS. + +```bash +git add apps/packages/ui/src/components/Common/QuickIngest/types.ts apps/packages/ui/src/components/Common/QuickIngest/IngestWizardContext.tsx apps/packages/ui/src/components/Common/QuickIngest/ReviewStep.tsx apps/packages/ui/src/components/Common/QuickIngest/ItemMetadataTable.tsx apps/packages/ui/src/components/Common/QuickIngest/AddContentStep.tsx apps/packages/ui/src/components/Common/QuickIngest/__tests__ +git commit -m "feat: preserve playlist occurrence identity in review (TASK-12113)" +``` + +Verification before commit: Task 4 was implemented through focused RED/GREEN cycles covering authoritative occurrence materialization, atomic queue mutation, virtualized queue/Review navigation, allowlisted per-occurrence overrides, exact run-request serialization, fail-closed persistence recovery, the 500-input limit, stale duplicate evidence, cached-authority rejection, and evidence-none Review recovery. Final frontend Vitest passed 200/200 across the five Task 4 suites. Backend playlist service and endpoint tests passed 149/149. Repository-pinned production ESLint exited 0 with zero rule findings (apart from the existing Next.js pages-directory informational message), Bandit reported zero findings across the touched 1,334-line backend service, and `git diff --check` passed. Final specification and code-quality re-reviews approved the diff. Full TypeScript was not rerun after the Task 1 three-attempt repository-baseline cap. A blanket Prettier write was not used because the shared package has conflicting frontend/extension configurations; changed hunks were reviewed directly and the whitespace gate passed. + +## Stage 3: Shared run submission and status transport + +**Goal:** Replace WebUI/extension submission differences with one occurrence-aware run client. + +**Success Criteria:** Both clients create the same run, submit bounded chunks, merge status by occurrence, and reattach without per-item polling fan-out. + +**Tests:** Run client tests, ambiguous retry, dynamic events, extension runtime tests. + +**Status:** Complete + +### Task 5: Implement the shared run client and bounded submission + +- [x] **Step 1: Write failing run-client tests** + +Test run creation, Review-required response, processing-only chunk selection, structured partial acceptance, global stop with `Retry-After`, URL/file aligned arrays, server-returned authoritative URLs overriding cached queue display URLs, same-attempt ambiguous retry, and polling/SSE snapshots merged by occurrence ID. + +```ts +expect(submitCalls[0].fields).toMatchObject({ + run_id: "run-1", + occurrence_ids: ["occ-1", "occ-2"], + attempts: [1, 1], + urls: [video1, video2], +}) +``` + +- [x] **Step 2: Run and verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/services/tldw/__tests__/playlist-ingest.test.ts ../packages/ui/src/services/__tests__/quick-ingest-batch.test.ts --maxWorkers=1 --no-file-parallelism --testNamePattern="run|occurrence|chunk|ambiguous"` + +Expected: FAIL because Quick Ingest submits/waits through legacy item logic. + +- [x] **Step 3: Implement one run client** + +In `playlist-ingest.ts`, add `createRun`, `submitPendingChunks`, `getRun`, `listRunItems`, `streamRunEvents`, `cancelRun`, and `retryRunItems`. Use a conservative exported chunk-size constant. Build URL submission fields only from the authoritative processing occurrences returned by run creation/items, never from cached queue URLs. A run item is merged only by `occurrenceId`; state and terminal outcome are separate. Treat `resync_required` as a full summary/items reload. + +- [x] **Step 4: Delegate legacy services without duplicating logic** + +In `quick-ingest-batch.ts`, detect version-2 run payloads and call the shared run client; retain the current legacy branch for non-playlist old-server sessions only. In `quick-ingest-session-reattach.ts`, prefer `runId` snapshots over job-ID fan-out. WebUI prefers SSE with polling fallback; extension supplies a polling preference. + +- [x] **Step 5: Run tests and commit** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/services/tldw/__tests__/playlist-ingest.test.ts ../packages/ui/src/services/__tests__/quick-ingest-batch.test.ts ../packages/ui/src/services/__tests__/quick-ingest-session-reattach.test.ts --maxWorkers=1 --no-file-parallelism` + +Expected: PASS. + +```bash +git add apps/packages/ui/src/services/tldw/playlist-ingest.ts apps/packages/ui/src/services/tldw/quick-ingest-batch.ts apps/packages/ui/src/services/tldw/quick-ingest-session-reattach.ts apps/packages/ui/src/services/__tests__ apps/packages/ui/src/services/tldw/__tests__ +git commit -m "feat: submit quick ingest through shared runs (TASK-12113)" +``` + +Task 5 verification before review/commit: the new delegate RED timed out in legacy per-job polling, while run-ID reattachment returned `interrupted`; the shared-client cases initially lacked the high-level run operations. Review remediation moved normalized processing options, conference collection creation, and per-occurrence conference metadata into the authoritative run-create body without cached materialized URLs; preserved monitoring after a later chunk is rate-limited while cancelling only unsent occurrences; made run cancellation fail explicitly except for genuine old-server fallback; and made SSE reload version-advanced items and ignore its unchanged initial snapshot. A final cleanup-failure RED proved that an occurrence-cancel failure could otherwise reattach forever; the service now returns an explicit cleanup-failed state and the Modal preserves tracking/accepted work in an interrupted recovery state without starting that loop. The Modal gates run-ID reattachment on explicit submit acknowledgement rather than timing, and the regression keeps submission pending after publishing tracking to prove the race is closed; the ordinary accepted-run control proves acknowledgement enables reattachment. The final expanded Vitest gate passed 224/224 across nine suites, including the three required run/delegate suites, both WizardModal suites, bounded persisted run identity/runtime, background upload Retry-After propagation, terminal no-job outcomes, partial chunk acceptance, cleanup failure, and refreshed run cancellation. Repository-pinned focused ESLint exited 0 with zero errors, and `git diff --check` passed. Final code re-review approved the remediated diff. Full TypeScript was not rerun after the Task 1 three-attempt repository-baseline cap. Bandit is not applicable to this TypeScript-only task. A blanket Prettier write was not used because the shared package still reports conflicting/baseline formatting across every touched file. Step 5 remains open only for the root-agent commit. + +Task 5 formal blocker remediation completed test-first, with Step 5 and final approval still open pending both requested re-reviews. The durable tracking contract now persists `creating_run` before create, publishes `run_created` before upload, and merges accepted batch/job mappings after each chunk; reload reconstructs materialized authority and reattaches post-create/partial runs without cached-URL fallback. Missing file/unusable processing occurrences stop submission and are cancelled while accepted work remains attached. Run paging fails closed above 500 occurrences or a 4096-character cursor, retained SSE events cannot regress authoritative lifecycle/progress, and reattachment falls back to legacy jobs only for explicit 404/405/501 compatibility while 429/503 remain retryable and 401/403 surface authorization recovery. Cleanup failure interrupts even when the first chunk accepted nothing. Direct cancellation is observed while create is pending and between chunks, preventing later uploads and cancelling the server run. Formal RED gate: 17 failed / 118 passed across five files. GREEN gate: 135/135. Expanded Task 5 gate: 241/241 across nine files. Repository-pinned scoped ESLint `--quiet` and `git diff --check` exit 0 (only the existing Next pages-directory informational output). Full TypeScript remains skipped under the Task 1 three-attempt repository-baseline cap; Bandit is not applicable to TypeScript-only changes. No files were staged or committed. + +Task 5 second formal-review remediation completed test-first, with Step 5 and final approval still open pending both final re-reviews. A restored `creating_run` marker now fails closed as interrupted and cannot reconstruct or restart a request, including after materialization expiry. A dedicated, sanitized `submissionOccurrenceIds` field preserves at most 500 occurrence identities without overloading collection planning metadata. Reload of `run_created` or `submitting` tracking cancels only server-authoritative unsent states (`staged`, `awaiting_upload`, `submit_pending`) and repolls; accepted/running work is preserved, and cleanup 503 remains retryable without false terminalization. SSE streams without a trustworthy event boundary reload authoritative summary/items for occurrence events, preventing same- or higher-state retained replay from regressing metadata. Second formal RED gate: 8 failed / 132 passed across five files; the additional persisted-bound control brought the final focused gate to 141/141. Expanded Task 5 gate: 247/247 across nine files. Repository-pinned scoped ESLint `--quiet` and `git diff --check` exit 0 (only the existing Next pages-directory informational output). Full TypeScript remains skipped under the Task 1 three-attempt repository-baseline cap; Bandit is not applicable to TypeScript-only changes. No files were staged or committed. + +Task 5 bounded-transport remediation completed test-first, with both final re-reviews still pending. The 500-item unknown-cursor RED proved that replaying retained occurrence events caused 1002 authoritative REST requests instead of the expected two. Reattachment now returns the complete authoritative poll snapshot whenever no trustworthy event high-water mark exists and does not open SSE in that state; the existing stale-terminal control confirms retained events cannot weaken authoritative state. The focused five-file gate passes 142/142 and the expanded nine-file gate passes 248/248. Repository-pinned scoped ESLint `--quiet` exits 0 with only the existing Next pages-directory informational output. Full TypeScript remains skipped under the Task 1 three-attempt baseline cap; Bandit is not applicable to TypeScript-only changes. No files were staged or committed. + +Task 5 final formal specification/code-quality re-reviews: approved with no actionable findings. The reviewers independently confirmed fail-closed restored-create recovery, bounded dedicated submission occurrence tracking, authoritative unsent-only cleanup with retryable cancellation failure, bounded unknown-cursor polling, and cursor-backed SSE correctness. One reviewer reran the requested focused and expanded gates at 142/142 and 248/248; the other independently expanded the nine-file verification to 254/254. Both reported scoped ESLint `--quiet` and `git diff --check` clean. The root agent independently reran the expanded nine-file gate at 248/248, linted every changed TypeScript file with the repository-pinned ESLint configuration, and confirmed `git diff --check` clean before commit. + +### Task 6: Make the extension runtime a thin transport adapter + +- [x] **Step 1: Write failing runtime parity tests** + +Assert the background runtime stores only `runId`/compact mappings, polls the run endpoint after worker recreation, emits occurrence-aware events, delegates cancellation to run cancel, and never independently expands/classifies a playlist. + +- [x] **Step 2: Run and verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/entries/shared/__tests__/quick-ingest-session-runtime.test.ts ../packages/ui/src/entries/__tests__/background.web-clipper.test.ts ../packages/ui/src/components/Sidepanel/Chat/__tests__/ControlRow.chat-handoff.test.tsx --maxWorkers=1 --no-file-parallelism` + +Expected: FAIL on run reattachment/parity assertions. + +- [x] **Step 3: Update the runtime and active-tab handoff** + +`ControlRow` continues to pass only typed open detail. The wizard controller owns inspection. `background.ts` calls the same run client with `transportPreference: "poll"`; remove its duplicate submit/poll loop for version-2 runs. Runtime context tracks `runId` and cancellation, not an in-memory job list as the recovery source of truth. + +- [x] **Step 4: Run extension tests and commit** + +Run the command from Step 2, then run: `cd apps/tldw-frontend && bun run test:extension -- --run ../packages/ui/src/entries/shared/__tests__/quick-ingest-session-runtime.test.ts` + +Expected: PASS. + +```bash +git add apps/packages/ui/src/entries apps/packages/ui/src/components/Sidepanel/Chat/ControlRow.tsx apps/packages/ui/src/components/Sidepanel/Chat/__tests__ +git commit -m "feat: align extension playlist run transport (TASK-12113)" +``` + +Task 6 implementation and verification completed test-first; Step 4 remains open only for the root-owned review/commit. The initial exact RED gate collected 23 tests with 4 intended runtime parity failures. Follow-up RED cycles covered versioned and bounded compact persistence, stale cleanup, the v2 background delegate, continued terminal polling, immediate cancellation visibility, direct background poll/cancel transport, and serialized storage cleanup. The final exact three-file gate passes 32/32, the extension-specific runtime gate passes 12/12, and the three shared run-service regression suites pass 90/90. The extension runtime persists only versioned `sessionId`/`runId`/occurrence mappings, rejects malformed or oversized records (255-character identities, 500 occurrences, 500 job mappings), surfaces storage access/write failures, serializes storage mutations, and uses compare-by-run cleanup so stale terminal work cannot delete or resurrect a replacement run. Version-2 background work returns before the legacy classifier and delegates create/submit, direct run polling, and run cancellation to the shared clients; legacy payload behavior is unchanged. `ControlRow` remains a typed active-tab handoff only. Repository-pinned scoped ESLint `--quiet` and `git diff --check` both exit 0. Full TypeScript remains skipped under the Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only task. No files were staged or committed. + +Task 6 formal-review remediation completed test-first for all nine requested blockers; Step 4 and formal re-approval remain open. The combined RED gate was 12 failed / 119 passed across the run client, extension runtime, MV3 transport, and Modal session suites. Direct transport now survives summary/paged-item resync; compact records preserve bounded submission state; interrupted runs remain recoverable; cancel/create and restore/cancel races are coordinated; MV3 pending v2 work stays in the durable worker across recreation; persisted UI sessions query worker replay instead of restarting; runtime progress and partial terminal failures carry occurrence-aware results; and accepted cancellation retains/polls the authoritative run until terminal. Self-review also corrected cancelled-item normalization when the item lacked its own error string. Final remediation gate: 133/133. Fresh exact Task 6 gate: 38/38; extension runtime: 18/18; shared Task 5 service regressions: 92/92. Repository-pinned scoped ESLint `--quiet` and `git diff --check` pass. Full TypeScript remains skipped under the Task 1 three-attempt cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 6 third formal-review remediation completed test-first for all six blocker groups; Step 4 and formal re-approval remain open. The combined RED gate was 20 failed / 118 passed. The runtime now treats `cleanup_required` as recoverable, retains and reconciles a known post-create run after storage failure, retries failed reconciliation polling, and surfaces/retries startup storage reads. Caller-generated extension session IDs plus a compact pre-create fingerprint marker make repeat delivery idempotent, reject conflicting reuse, and prevent direct submission after ambiguous delivery. Bounded 24-hour terminal tombstones survive worker recreation and require an explicit session/run/generation replay acknowledgement before compare-safe deletion. The Modal keeps authoritative cancellation nonterminal until the runtime reports completed, failed, or cancelled, preserves local no-run cancellation, and queries interrupted extension sessions on reopen. The final five-file remediation gate passes 139/139; the fresh exact Task 6 gate passes 47/47; the standalone extension runtime passes 26/26; and shared Task 5 service regressions pass 95/95. Final self-review confirmed that persisted fingerprints contain only a bounded hash and occurrence identities, terminal events whitelist/bound result fields and enforce TTL/ack identity, storage failures retain cleanup authority, and legacy direct-session paths remain isolated. Repository-pinned scoped ESLint `--quiet` and `git diff --check` pass. Full TypeScript remains skipped under the Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 6 fourth formal-review remediation completed test-first for all eleven requested blocker groups; Step 4 and formal re-approval remain open. The combined RED gate was 24 failed / 118 passed across runtime, background, transport, and Modal suites. Automatic Modal replay acknowledgement was removed so terminal tombstones remain until explicit acknowledgement, TTL expiry, or deterministic capacity eviction. Runtime start acknowledgement now waits for the durable pre-create marker; caller-owned opaque attempt tokens replace payload fingerprints; generation-aware CAS protects marker-to-run/terminal transitions, polling, and explicit replay acknowledgement; failed active-record persistence retries during later reconciliation. Structured Review-required recovery now crosses the background/runtime/Modal boundary. Storage is bounded to 64 unique sessions, 512 KiB per complete terminal record, and 2 MiB aggregate terminal data while never evicting active cleanup authority. Extension cancellation fails closed when the runtime is unavailable or times out. Ambiguous start plus replay timeout retains one stable interrupted extension identity, and Modal replay is open-gated, bounded to three attempts, actionable on exhaustion, repeatable on each reopen/recovery request, and cannot restart a tracked extension run. Self-review added two further RED/GREEN controls: durable replay-ack CAS rejection retains the in-memory tombstone, and the 512 KiB limit measures the whole persisted record rather than only its event. Final combined gate: 144/144; exact Task 6 gate: 60/60; standalone extension runtime: 33/33; shared Task 5 service regressions: 97/97; context regression control: 54/54. Repository-pinned scoped ESLint `--quiet` and `git diff --check` pass. Full TypeScript remains skipped under the Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 6 fifth formal-review remediation completed test-first for all five requested blocker groups; Step 4 and formal re-approval remain open. The combined behavioral RED gate was 8 failed / 95 passed across the runtime, background adapter, and Modal session suites. The real background adapter now preserves structured `reviewRequired` results. Review recovery is a bounded 24-hour, generation-CAS replay tombstone that survives lost responses and worker recreation, participates in the 64-session/2 MiB deterministic replay cap, and cannot evict active recovery authority. The Modal clears its started flag, active runtime identity, in-memory persisted tracking, and durable extension tracking before applying corrected Review state, so a manual corrected submission starts exactly once. Valid 500-item terminal snapshots that exceed the rich 512 KiB representation now deterministically fall back to an essential form retaining every occurrence ID, terminal status, and outcome instead of entering an interrupt/poll loop. Restore isolates a failed run poll, schedules that session's retry, and continues restoring later records. The first production run passed 102/103; the remaining failure was a test-only Review mock that lacked the real manual Start action, and correcting the harness produced the final 103/103 combined gate without another production change. Fresh verification passed the exact Task 6 gate 67/67, standalone runtime 37/37, shared Task 5 services 97/97, Modal 47/47, context 54/54, and expanded combined gate 153/153. Repository-pinned full touched-scope ESLint `--quiet` and `git diff --check` pass. Self-review confirmed Review TTL/whole-record bounds/generation CAS, deterministic replay eviction before active authority, no replay acknowledgement for Review tombstones, complete essential terminal identity/status/outcome retention, per-session restore retry isolation, exactly-once corrected restart, and unchanged legacy delegation behavior. Full TypeScript remains skipped under the Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 6 sixth formal-review remediation completed test-first for the four requested blocker groups; Step 4 and formal re-approval remain open. The accepted combined RED gate was 6 failed / 103 passed across runtime, background, and Modal session behavior, followed by focused atomic-handoff and cleanup-timer RED controls. Restore now isolates every record, including expired terminal/Review cleanup, and retains only the failed record for a bounded retry that is cancelled when a later restore succeeds. Pending polls re-check generation/run authority, terminal CAS is limited to active records, rejected terminal writers stop without emission or rescheduling, and terminal tombstones cannot replace one another. Oversized essential terminal outcomes normalize through the same Modal consumer as rich results. Review handoff now computes one pure Review snapshot, synchronously writes and exact-reads a Zustand-compatible persistence envelope before mutating store/reducer/ref state, guards stale Provider writes by snapshot revision, and leaves extension replay authority untouched when persistence fails or throws. Final self-review confirmed thrown handoff guards are released, manual persistence rehydrates through the normal store, stale cleanup timers are cleared, and delayed/rejected terminal writers cannot duplicate events. Fresh verification passes exact Task 6 72/72, standalone runtime 41/41, shared Task 5 services 97/97, Modal/context/store 123/123, and the expanded five-file gate 180/180. Repository-pinned full touched-scope ESLint `--quiet` and `git diff --check` pass. Full TypeScript remains skipped under the Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 6 final formal specification and code-quality re-reviews approved the sixth-pass diff with no actionable findings. The reviewers independently confirmed per-record restore isolation, canonical compact outcomes, exactly-once terminal authority, active-only CAS, atomic Review persistence and guard cleanup, MV3 recovery, bounded storage, privacy, cancellation, replay, and legacy-path invariants. The root agent independently reran the exact Task 6 gate at 72/72, shared Task 5 service regressions at 97/97, Modal/context/store controls at 123/123, repository-pinned ESLint across every changed TypeScript file, and `git diff --check`; all passed before commit. + +## Stage 4: Truthful lifecycle UI and durable local recovery + +**Goal:** Render server evidence directly and survive reload/runtime loss without silently losing a large session. + +**Success Criteria:** Queue/progress/results use occurrence identity and state/outcome axes; IndexedDB migration and failures are visible; file reattachment remains client-derived. + +**Tests:** Component lifecycle tests, Dexie migration/quota/cleanup tests, multi-tab tests. + +**Status:** Complete + +### Task 7: Update processing, cancellation, retry, and result groups + +- [x] **Step 1: Write failing lifecycle UI tests** + +Cover `awaiting_upload`, client-derived file reattach, submit pending/queued/running, cancellation requested, status unavailable, all eight terminal result groups, real row/run cancellation, and retry reconciliation. Assert no fabricated analyzing/storing stage when the backend supplies only generic progress. At 500 items, assert bounded mounted rows and useful Active/Needs attention/Terminal plus outcome filters. + +- [x] **Step 2: Run and verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.integration.test.tsx ../packages/ui/src/components/Common/QuickIngest/__tests__/WizardResultsStep.navigation.test.tsx ../packages/ui/src/components/Common/QuickIngest/__tests__/FloatingProgressWidget.test.tsx --maxWorkers=1 --no-file-parallelism` + +Expected: FAIL because existing progress uses fabricated stage tokens and fewer outcomes. + +- [x] **Step 3: Replace UI status inference with run snapshots** + +Extend `ItemProgress`/`WizardResultItem` with server lifecycle and terminal outcome. Derive `file_reattach_required` only when server state is `awaiting_upload` and no local `File` exists. Keep `status_unavailable` recoverable with Check again/Reconnect. Virtualize processing/results above the same scale threshold used by preview and preserve title/ordinal primary labels. + +- [x] **Step 4: Wire real cancellation and retry** + +Before run creation, cancellation is local removal. After run creation, row cancellation always calls `POST /runs/{runId}/cancel` with that occurrence ID: the server terminalizes unsent items or cancels accepted jobs. Retain `cancellation_requested` until terminal. Whole-run cancellation calls the same route without occurrence IDs. Retry sends eligible occurrence IDs and waits for the server's reconciled attempt/action response. + +- [x] **Step 5: Run tests and commit** + +Run the command from Step 2. + +Expected: PASS. + +```bash +git add apps/packages/ui/src/components/Common/QuickIngest apps/packages/ui/src/components/Common/QuickIngestWizardModal.tsx +git commit -m "feat: show per-occurrence ingest lifecycle (TASK-12113)" +``` + +Task 7 implementation and verification completed test-first; Step 5 remains open for the root-owned formal review and commit. The accepted exact RED gate was 9 failed / 73 passed, with two additional service REDs proving occurrence-scoped cancellation and canonical reattach metadata. Processing now renders authoritative lifecycle/message/percentage evidence without the former simulated analyzing/storing timer; file reattachment is derived only from `awaiting_upload` plus a missing local `File`; status-unavailable rows expose Check again/Reconnect; and Processing/Floating Progress share Active/Needs attention/Terminal semantics. Results preserve all eight canonical terminal outcomes, canonical retry eligibility, and bounded outcome filtering. Row/run cancellation use the shared run-cancel client, retry uses the shared run-retry client followed by authoritative reconciliation, and an exact-signature guard prevents duplicate immediate retry reattachment while allowing changed tracking signatures to poll. Small lists render normally and lists at the existing 100-row threshold use stable-key TanStack virtualization with list position semantics; 500-row gates remain bounded. Final exact Task 7 gate: 83/83; service/playlist gates: 98/98; Modal/context/store controls: 123/123; extension runtime/background controls: 61/61. Repository-pinned touched-scope ESLint `--quiet` and `git diff --check` pass. Full TypeScript remains skipped under the Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only task. No files were staged or committed. + +Task 7 formal-review remediation completed test-first for all eleven requested finding groups; Step 5 and formal specification/code-quality approval remain open. The accepted combined RED was 21 failed / 294 passed across eight files. Canonical lifecycle/progress/retry/run identity now crosses the runtime boundary; occurrence cancel/retry use the shared authority path; unchanged retry snapshots continue polling; reattach transport failures remain recoverable; pre-run, cancellation, results, file reselection, local removal, late-callback, virtual keyboard/focus, and translated floating-summary behavior are all covered. First combined GREEN attempt: 311/315; final combined GREEN: 315/315. Fresh final gates: exact Task 7 89/89; playlist/batch/reattach services 100/100; Modal/context/store 126/126; extension runtime/background/handoff 76/76. Repository-pinned touched-scope ESLint `--quiet` and `git diff --check` pass. Full TypeScript was not rerun under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 7 second formal-review remediation completed test-first for all eleven follow-up finding groups; Step 5 and root-owned review/commit remain open. The accepted combined RED was 22 failed / 275 passed across eight files. The first GREEN attempt was 286/297; after correcting the four expectation/fixture mismatches, the second attempt was 297/297. A final retry-generation CAS regression brought the fresh combined gate to 298/298 with zero skips. Fresh supporting gates are exact Task 7 95/95, playlist/batch/reattach services 105/105, Modal/context/store 197/197, and extension runtime/background/handoff 127/127, all with zero skips. Canonical run IDs and retryability now survive runtime replay, retry accepted by the backend re-arms the same retained session/run through a generation CAS, and persistence/CAS failure retains the terminal tombstone. Retryable run-status transport failures remain recoverable; pre-run occurrence cancellation is live-filtered before creation; same-run file reselection submits exactly one occurrence and reconciles authoritatively; durable Retry all excludes canonical non-retryable failures; late whole-run cancellation callbacks preserve terminal authority; direct and extension results retain ordinal/title identity; ETA copy uses translation keys; occurrence-scoped cancellation cannot fall back to whole-batch cancellation; and Check again reconciles both direct and extension sessions. Repository-pinned touched-scope ESLint `--quiet` and `git diff --check` pass. Full TypeScript was not rerun under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 7 third formal-review remediation completed test-first for all eight requested blocker groups; Step 5 and root-owned formal approval/commit remain open. The accepted four-file RED gate was 15 failed / 225 passed with zero skips. Direct-session cancellation is now reserved before backend create, distinguishes whole-stop from occurrence cancellation, cancels newly created occurrence rows before upload, rechecks row cancellation before every chunk, and always releases the in-memory registry. Extension retry uses a bounded persisted `retrying` generation reservation before backend mutation, permits one concurrent winner, rolls rejection back through generation CAS, reconciles CAS loss, and keeps accepted work polling if active persistence throws. File reads fall back when `File.arrayBuffer` is unavailable and retained bytes can be retried explicitly. The Modal records row/run cancellation before authority exists, applies retry generation before reconciliation, ignores stale whole-run cancellation settlement from an older generation, and routes direct Check again through one refresh nonce. The first combined GREEN exposed one whole-stop unsent-reporting compatibility regression; after separating whole-stop from row cancellation, the fresh combined gate passed 240/240. Fresh supporting gates pass exact Task 7 96/96, playlist/batch/reattach services 107/107, Modal/context/store 203/203, and extension runtime/background/handoff 84/84, all with zero skips. Repository-pinned full dirty-TypeScript ESLint `--quiet` and `git diff --check` pass. Full TypeScript remains skipped under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 7 fourth formal-review remediation completed test-first for all nine requested authority and recovery groups; Step 5 and root-owned formal approval/commit remain open. The accepted six-file RED gate was 14 failed / 278 passed with zero skips and no unhandled errors. Pending row cancellation is forwarded only after acknowledged or indeterminate extension identity; direct submission responses are guarded by session/run/generation or pre-authority cancellation keys; row and whole cancellation settlements are generation-scoped; accepted degraded retries return and adopt their reserved generation while live monitoring continues with a visible warning. Existing-run file reselection preserves the authoritative occurrence attempt. Retry reservations receive a fresh bounded recovery TTL, count as non-evictable records in the 2 MiB recovery budget, and fail closed at capacity. Timeout/lost-response retries remain indeterminate under the reserved generation and reconcile without a second backend mutation, while explicit HTTP rejection alone rolls back. Accepted direct retries return a fresh opaque generation so later cancellation keys cannot match old callbacks. The final combined gate passes 292/292; fresh supporting gates pass exact Task 7 96/96, playlist/batch/reattach services 108/108, Modal/context/store 211/211, extension runtime/background/handoff 88/88, and standalone runtime 51/51, all zero-skip with no unhandled errors. Repository-pinned full dirty-TypeScript ESLint `--quiet` and `git diff --check` pass. Full TypeScript remains skipped under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to this TypeScript-only remediation. No files were staged or committed. + +Task 7 fifth formal-review remediation completed test-first for the final four authority findings; Step 5 and root-owned formal approval/commit remain open. The accepted four-file RED gate was 11 failed / 200 passed with no unhandled errors. Authoritative upload attempts now flow from extension polling through the runtime event and Modal store and survive validation, file selection, upload failure, explicit replacement retry, and queued success; both replacement submissions retain attempt 4. Extension cancellation carries an optional expected generation, and runtime/background cancellation rejects stale generation before abort or cancellation mutation; direct retry authority is also generation-fenced. HTTP 500/502/503/504 and status-less timeout retry responses remain indeterminate and retain/reconcile the reserved generation, while explicit 409 rejection remains determinate. A live retry owner prevents a staggered second caller from invoking generic restore, promoting its retrying reservation, polling early, or mutating the backend twice. Final combined gate: 211/211. Supporting gates: exact Task 7 96/96, playlist/batch/reattach services 109/109, Modal/context/store 211/211, extension runtime/background/handoff 95/95, and standalone runtime 53/53. Repository-pinned full dirty-TypeScript ESLint `--quiet` and `git diff --check` pass. Full TypeScript remains skipped under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to TypeScript-only work. No files were staged or committed. + +Task 7 sixth formal-review remediation completed test-first for authoritative retry resubmission and reservation recovery; Step 5 and both root-owned formal approvals/commit remain open. The initial focused RED gate was 14 failed / 144 passed across the batch service, extension background, and extension runtime; source self-review then added three more failing controls to a 163-test gate. Retry success now consumes only the backend's authoritative staged non-file occurrences and attempts, creates jobs, and monitors them while file stubs remain awaiting reselection; cached/display URLs cannot enter resubmission. Status-less and HTTP 500/502/503/504 failures retain the reserved generation, deterministic rejection alone rolls back, replacement upload cannot clear retry authority, and stale-generation cancellation is rejected. Direct reservations are bounded to 64 live non-evictable entries with 24-hour expiry and fail closed before backend mutation. Direct and durable extension reservations reconcile a fresh manifest before any second retry, retain authority through unavailable or partially advanced mixed manifests, submit newly staged occurrences idempotently, and release only after every selected non-file occurrence is active or resolved. Final focused retry gate: 163/163. Supporting gates: exact Task 7 96/96; playlist/batch/reattach services 123/123; Modal/context/store 211/211; extension runtime/background/handoff 99/99; standalone runtime 55/55. Repository-pinned full dirty-TypeScript ESLint `--quiet` across 23 files and `git diff --check` pass. Expected test-harness output was limited to Node localStorage warnings, the deliberate outside-provider exception, mocked unconfigured-server reconciliation, and simulated extension listener delivery retries. Full TypeScript remains skipped under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to TypeScript-only work. The formatter check was not rerun because its earlier `bunx prettier --check` invocation was blocked before analysis by sandbox tempdir access; no formatter mutation occurred. No files were staged or committed. + +Task 7 seventh formal-review remediation completed test-first for retry-owner, mixed-manifest, retry-cadence, lost-delivery, pending-cancellation, and generation-lifecycle follow-ups; Step 5 and root-owned approval/commit remain open. The accepted four-file RED was 9 failed / 237 passed, and source review added focused RED controls for direct empty-response classification (1 failed / 81 skipped), extension empty-response classification (1 failed / 34 skipped), and Modal unmount cleanup (1 failed / 73 skipped). One shared authoritative manifest classifier now gives URL status-unavailable evidence precedence over file awaiting-upload, distinguishes not-advanced terminal retryable URLs from active/resolved rows, submits only fresh staged non-files, and handles successful empty retry responses consistently in direct and extension paths. Direct g2 owners coalesce concurrent calls, reservations survive partial advancement and explicit retry, expiry pruning runs before generation fencing, and active/terminal/cancelled authority retires without weakening stale-generation rejection. Extension lost-before-delivery preserves `notAdvanced`, re-owners exactly one later POST, and pending g2 scoped cancellation reaches the authoritative run while stale g1 is rejected and the durable reservation remains fenced. Modal recovery retries the reservation on its existing cadence with one in-flight owner, a three-POST bound, actionable terminal fallback, and unmount cleanup that prevents late retries; ordinary reattach remains the rendering authority. Final focused gate: 248/248. Supporting gates: exact Task 7 96/96; playlist/batch/reattach services 130/130; Modal/context/store 212/212; extension runtime/background/handoff 103/103; standalone runtime 57/57; focused retry 174/174. Repository-pinned frontend ESLint `--quiet` across all 23 dirty TypeScript files and `git diff --check` pass. Expected output was limited to Node localStorage, deliberate provider-boundary, mocked unconfigured-server, and simulated extension-delivery warnings. Full TypeScript remains skipped under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to TypeScript-only work. No formatter, staging, or commit action was taken. + +Task 7 eighth formal-review remediation completed test-first for ambiguous file-only retries, mixed-manifest advancement, whole-run pending-generation cancellation, Modal reattach ownership, and direct-generation retirement; Step 5 and root-owned approval/commit remain open. The accepted four-file RED gate was 6 failed / 248 passed. The first GREEN attempt was 3 failed / 251 passed; after making the manual retry path the atomic single reattach owner, the focused gate passed 254/254. Source self-review added an interrupted-status non-retirement RED (1 failed / 78 skipped), after which the Modal gate passed 79/79 and the final focused four-file gate passed 257/257. File-only and partially advanced mixed retries now remain explicitly not advanced until file reselection submits exactly one later retry request without creating jobs. Pending whole-run cancellation omits occurrence IDs, preserves the current-generation reservation until genuine terminal reconciliation, and rejects stale-generation cancellation. Modal retry publication and reattachment have one owner even under StrictMode, ignore delayed work after unmount, and retire direct generation authority only through a CAS-safe helper after genuine completed, cancelled, or partial-failure reattachment—not after interrupted or unavailable status. Fresh broad gates pass exact Task 7 96/96; playlist/batch/reattach services 133/133; Modal/context/store 217/217; extension runtime/background/handoff 104/104; standalone runtime 58/58; focused retry batch/background/runtime 178/178; and the expanded batch/runtime/background/Modal gate 257/257. A preliminary repository-pinned ESLint invocation exited 2 before code analysis because the repo-root call omitted an explicit config; the corrected repository-pinned invocation with `apps/tldw-frontend/eslint.config.mjs` passed `--quiet` across all 23 dirty TypeScript files, with only the existing Next pages-directory informational output. `git diff --check` passes. Expected test-harness output was limited to Node localStorage, deliberate or mocked unconfigured-server reconciliation, and simulated extension-delivery warnings. Full TypeScript remains skipped under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to TypeScript-only work. No formatter, staging, or commit action was taken. + +Task 7 ninth formal-review remediation completed test-first for selected-occurrence resolution while an unselected sibling remains active; Step 5 and root-owned approval/commit remain open. The accepted focused RED was 1 failed / 3 passed / 84 skipped: an empty idempotent retry returned a fresh g2 after the selected occurrence reached terminal completed attempt 2, but selected-only reconciliation retired whole-run authority despite the authoritative summary and sibling both remaining running, so stale g1 cancellation reached the backend. The minimal fix separates retry-reservation release from whole-session generation retirement: selected resolution always releases its reservation, while g2 retires only when the complete authoritative run summary is `completed`, `cancelled`, or `partial_failure` and the generation CAS still matches. Running sibling authority therefore keeps stale g1 fenced with no cancellation request while current g2 cancellation succeeds; interrupted/unavailable reconciliation and Modal terminal CAS retirement remain unchanged. Focused lifecycle controls pass 4/4, the complete batch service passes 88/88, and the focused batch/runtime/background/Modal gate passes 260/260. Fresh broad gates pass exact Task 7 96/96; playlist/batch/reattach services 136/136; Modal/context/store 217/217; extension runtime/background/handoff 104/104; standalone runtime 58/58; focused retry batch/background/runtime 181/181; and the expanded four-file gate 260/260. Repository-pinned frontend ESLint with explicit `apps/tldw-frontend/eslint.config.mjs` passes `--quiet` across all 23 dirty TypeScript files with only the existing Next pages-directory informational output; `git diff --check` passes. Expected test-harness output remains limited to Node localStorage, the deliberate provider-context exception, mocked unconfigured-server reconciliation, and simulated extension-delivery warnings. Full TypeScript remains skipped under the documented Task 1 three-attempt repository-baseline cap; Bandit is not applicable to TypeScript-only work. No Prettier, unpinned ESLint, staging, or commit action was taken. + +Task 7 final closeout approved. Both formal reviewers approved the exact ninth-pass diff with no actionable findings. Root verification is green: exact UI 96/96, playlist/batch/reattach services 136/136, Modal/context/store 217/217, and extension runtime/background/handoff 104/104. Repository-pinned ESLint with the explicit frontend config passed all 23 dirty TypeScript files, including the untracked `apps/packages/ui/src/components/Common/QuickIngest/file-bytes.ts`, and `git diff --check` passed. Task 7 Step 5 is complete; implementation commit: `cabc72f550`. TASK-12113 remains In Progress for Tasks 8 and 9. + +### Task 8: Move compact sessions to IndexedDB with visible recovery failures + +- [x] **Step 1: Write failing persistence tests** + +Test Dexie v15 migration, one-time sessionStorage import, interruption idempotency, compact 500-item storage, exclusion of `File`/thumbnail bytes, quota/write failure surfaced in store state, active-run retention, terminal cleanup, and two-tab single-writer lease behavior. + +- [x] **Step 2: Run and verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/store/__tests__/quick-ingest-session.test.ts ../packages/ui/src/store/__tests__/quick-ingest-indexeddb.test.ts --maxWorkers=1 --no-file-parallelism` + +Expected: FAIL because persistence is synchronous sessionStorage and ignores writes. + +- [x] **Step 3: Add the Dexie record and adapter** + +Add `QuickIngestSessionDbRecord` and `quickIngestSessions: "id, lifecycle, updatedAt, expiresAt"` to schema version 15. `db/dexie/quick-ingest.ts` implements async Zustand `StateStorage`, migration marker, retention cleanup, and an atomic lease field. Persist compact display/mapping data only. Keep origins isolated naturally; do not attempt WebUI-extension IndexedDB sharing. + +- [x] **Step 4: Surface persistence and coordination state** + +Add `persistenceStatus: "ready" | "migrating" | "unavailable" | "quota_error"` and `isSubmissionOwner` to the session store. Block Start Processing in non-owner tabs, allow takeover after lease expiry, and show a recovery warning when persistence cannot guarantee resume. + +Task 8 implementation and implementation-agent verification completed test-first; Step 5 remains open for root-owned review and commit. The accepted initial persistence RED was 17 failed / 18 passed, with focused Review recovery RED at 4 failed / 66 skipped and awaited Modal handoff RED at 2 failed / 79 skipped. Self-review added three narrow RED/GREEN cycles: identical failed background-write retry (1 failed / 1 passed, then 2/2), lease preservation across ordinary session writes (1 failed, then 1/1), and post-initialization read/delete failure visibility (2 failed, then 2/2). The Dexie v15 adapter now performs durable-before-remove legacy migration, bounded lifecycle retention, one quota cleanup retry without evicting processing authority, visible read/write/delete failures, compact sanitized Zustand envelopes, and transaction-backed per-tab leases whose columns survive session updates. Review blocks on migration/storage/ownership, automatically acquires on entry, revalidates immediately before Start, supports expired takeover through Check again, and awaits durable Review handoff before changing replay identity. Final gates: exact store/IndexedDB 40/40; expanded Quick Ingest stores 45/45; full Modal integration/session 151/151; existing Dexie helpers/migrations 10/10. Repository-pinned ESLint with the explicit frontend config passed all 10 changed TypeScript files with only the known Next pages-directory informational output. Full TypeScript remains skipped under the documented Task 1 three-attempt baseline cap; Bandit is not applicable to TypeScript-only work. The temporary `antd` link was restored, no dependency or lockfile changed, and no files were staged or committed. + +Task 8 formal-review remediation completed test-first for the three requested persistence and authority hardening groups; Step 5 remains open for root-owned approval and commit. Accepted RED was 12 failed / 45 passed in the 57-test store/IndexedDB gate and 1 failed / 152 passed in the 153-test Modal gate, with no unhandled errors. Malformed legacy and normal writes now fail closed through one injected production normalizer, preserve the legacy source/current durable authority, and publish unavailable status. The whitelist sanitizer strips transient payloads across every persisted envelope surface, retains the compact recovery fields consumed by the UI, and caps tracking collections/maps at 500 entries with 255-character identities. Lease acquisition is a transactional draft-only lifecycle CAS; rejected acquisition reloads durable non-draft authority; and the live Modal guard blocks only stale Review step-3 persistence against tracked non-draft step-4/5 authority, while preserving legitimate Start-over and retry/cancel backsteps. The first full UI GREEN exposed an overbroad guard at 4 failed / 149 passed; the focused four regressions then passed and final full UI was 153/153. Fresh final gates: exact store/IndexedDB 57/57, expanded Quick Ingest stores 62/62, existing Dexie helpers/migrations 10/10, and full Modal integration/session 153/153, all with zero unhandled errors. Repository-pinned explicit-config ESLint passed all 10 dirty TypeScript/TSX files with only the known Next pages-directory informational output; `git diff --check` passed. Full TypeScript remains skipped under the Task 1 three-attempt baseline cap; Bandit is not applicable to this TypeScript-only remediation. The temporary `antd` link was restored exactly; no dependency/lockfile change, staging, or commit was performed. + +Task 8 second formal-review remediation completed test-first for the default-storage failure boundary, failed-lease ordering, and deterministic schema-v15 harness; Step 5 remains open pending external reviews and root-owned approval/commit. The frontend-config schema reproduction failed 1/57 because its setup mock replaced the real Dexie constructor. After the test-only harness imported the actual schema, the two behavioral controls failed as intended (2 failed / 57 skipped), while a combined focused gate showed the real schema control passing and both behaviors still RED. Default legacy storage is now resolved inside initialization so a throwing `sessionStorage` getter rejects initialization, publishes unavailable status, and leaves durable authority untouched. A rejected lease reads and merges durable non-draft authority before any owner-status write, preventing stale hydrated drafts from being queued or exposed. The schema test now deterministically imports the actual constructor under the frontend setup. Focused GREEN passed 3/3; the exact frontend-config store/IndexedDB gate passed 59/59; full Modal integration/session passed 153/153; expanded Quick Ingest stores passed 64/64; and existing Dexie helpers/migrations passed 10/10, all with zero unhandled errors. Repository-pinned explicit-config ESLint passed all 10 Task 8 TypeScript/TSX files with only the known Next pages-directory informational output, and `git diff --check` passed before tracking updates. Full TypeScript was not run per the Task 1 three-attempt baseline cap; Bandit is not applicable to this TypeScript-only remediation. The temporary `antd` link was restored to its exact original target; no dependency/lockfile change, staging, or commit was performed. + +Task 8 quality review found four remaining authority issues: `tracking.jobIds` was capped before deduplication; ordinary adapter writes and clears could weaken non-draft authority instead of requiring an explicit Review transition and compare-and-swap clear; mounted acquisition did not fully replace stale wizard state from durable authority; and processing shortcuts could cross the Start boundary before lease renewal and a durable `creating_run` handoff. The first three fixes are green: job-ID sanitization passed 4/4 focused controls (25 skipped); adapter authority/CAS and store coverage passed 65/65; full external-state acquisition reconciliation passed 5/5 focused controls (116 skipped) and the Context/session-store gate passed 84/84. The focused authority-only Modal gate also passed 14/14 controls (148 skipped) with zero unhandled errors. The remaining render-loop remediation is not green. Root tracing identified queued stale effect-driven `startRun` executions crossing the synchronous terminal/abort dispatch-and-store window. Three bounded attempts—effect-boundary plus abort handling, latest-callback/live-step guards, and a render-confirmed latch—each finished the same focused gate at 3 failed / 14 passed / 147 skipped with 3 unhandled maximum-update-depth errors. The ordinary start case and both pre-submit abort cases remain failing, so the full Modal suite is not green and was not rerun. Step 5 remains open, TASK-12113 remains In Progress, and the failed-defense production/test changes are retained uncommitted for a root architectural decision. + +Task 8 explicit-command architecture was approved after the three-attempt stop and a read-only architecture review. The implementation follows `Docs/superpowers/plans/2026-07-14-quick-ingest-explicit-submission-command.md`: Step 4 remains display state only; Review, quick, and auto entry share one explicit command; the command resolves durable authority before creating an immutable snapshot, persists `creating_run` before backend mutation, fences stale continuations with a transient token, and routes pre-submit failures to Review. Restored processing remains reattach-only. No IndexedDB schema, dependency, or extension-runtime change is in scope. + +Task 8 explicit-command implementation stopped at its own three-attempt limit before Task 2 could be marked green. The pure transition task passed 61/61 plus specification and quality review. The command-boundary RED was 22 failed / 81 passed with 16 unhandled depth errors; after replacing the Step-4 starter, command-order and restored-state assertions passed, but a focused durable-handoff control still exited with an unhandled maximum-depth error. Instrumentation proved one queued `UPDATE_ITEM_PROGRESS` reducer action was reprocessed 52 times whenever the Provider's passive `onStateChange` synchronously updated the parent session store. Provider key/mount, callback identity, and external authority revision stayed stable, and both faithful and deferred handoffs reproduced it. Render-time session-ref synchronization and a complete persisted-projection equality guard failed and were reverted. The dedicated plan records exact attempts. Task 8 remains In Progress and requires an approved architecture change that decouples ordinary wizard persistence from the component owning the active reducer before implementation resumes. + +Task 8 resumed after user approval of that architecture revision. The refined boundary keeps the explicit command, exact durable-handoff guard, and token-fenced runner unchanged. `QuickIngestWizardModal` becomes a session-ID shell; the component owning `IngestWizardProvider` subscribes only to the primitive external-authority revision; and ordinary session, persistence-status, ownership, and tracking subscriptions move into a descendant bridge below the Provider. This directly removes the proven synchronous parent-rerender edge without adding reducer-wide equality guards, a persistence queue, schema fields, dependencies, or extension contract changes. The focused failing durable-handoff control remains the required RED before implementation. + +Task 8 explicit-command Tasks 1-3 are now complete after the approved boundary revision. The architectural control passed 1/1 without an unhandled render loop; Task 2 authority controls passed 5/5; and the final Review/Quick/Auto plus cancellation gate passed 180/180 (integration 74/74, session 106/106) with zero unhandled errors. The command claims its attempt before the first await, resolves current durable authority, renews ownership, commits the exact `creating_run` snapshot before backend mutation, and transfers that immutable snapshot to a token-fenced runner. Preparation cancellation closes before renewal and one frozen occurrence set filters the payload, tracking, durable handoff, applied UI, and runner inputs. Restored processing remains reattach/display-only. The focused final re-review approved the implementation with no findings. Step 5 remains open for the full Task 8 persistence/context/Dexie/lint gates and whole-diff review. + +The first Task 8 whole-diff specification and quality reviews remain blocking despite the fresh green gates. Both found that the generic authoritative IndexedDB write is not an atomic lease/revision CAS, so a suspended owner can resume after lease takeover and still start duplicate backend work; Review handoff has the same stale-write risk. They also require runner checks against session plus accepted external revision, synchronous exposure of replacement Provider state, hydration-safe draft creation/merge, and a fresh durable session ID for Results Start over. One unused context authority escape hatch is minor cleanup. These findings are now explicit test-first remediation steps in the dedicated command plan; Step 5 remains open and no commit is permitted until both reviewers approve the corrected diff. + +Task 8 final remediation and whole-diff review are complete at implementation commit `e97cc15b8b`. Transactional exact-envelope Review/processing CAS, draft-only lease renewal, token/session/revision runner fences, synchronous external Provider snapshots, hydration-safe merge/fallback creation, and durable fresh-ID Start over are covered by focused and full regressions. The final side-panel fix defers playlist opens until persistence hydration, preserves the exact sender detail/options through the existing singleton, retains the durable draft queue/preset/Review state, and removes duplicate callbacks. The same final specification and quality reviewers both APPROVED with no findings. Fresh root gates pass persistence/context `135/135`, Modal `184/184`, Results/pending-open `26/26`, side-panel/pending-open `22/22`, and existing Dexie helpers `10/10`; repository-pinned ESLint passes all 22 dirty TypeScript/TSX files; whitespace checks are clean; and the test symlink is restored. Full TypeScript remains skipped under the established baseline cap; Bandit is not applicable. The internal-QA debug side-panel dual-host behavior is recorded as a non-blocking follow-up and does not affect production WebUI or extension host ownership. + +- [x] **Step 5: Run tests and commit** + +Run the command from Step 2. + +Expected: PASS. + +```bash +git add apps/packages/ui/src/db/dexie apps/packages/ui/src/store/quick-ingest-session.ts apps/packages/ui/src/store/__tests__ +git commit -m "feat: persist quick ingest runs in indexeddb (TASK-12113)" +``` + +## Stage 5: Cross-client, accessibility, and release gates + +**Goal:** Prove one behavior across WebUI and extension at realistic playlist scale. + +**Success Criteria:** Focused suites, type/lint checks, deterministic WebUI browser journey, extension parity tests, and accessibility checks pass. + +**Tests:** Vitest focused gate, Playwright journey, axe/keyboard/virtual-list tests, no fan-out structural assertions. + +**Status:** Complete + +### Task 9: Add final browser journeys and verification + +- [x] **Step 1: Add deterministic browser fixtures and failing E2E** + +Mock the version-2 API with the existing 34-item conference fixture. Test paste → inspect → complete preview → select → materialize → Review → run → progress → reload/reattach → results. Add an extension-level integration test for active-tab seed and background recreation. Do not hit live YouTube in required CI. + +- [x] **Step 2: Run the browser test and verify RED** + +Run: `cd apps/tldw-frontend && bunx playwright test e2e/workflows/quick-ingest-playlist.spec.ts --project=chromium --reporter=line` + +Expected: FAIL until route mocks and final UI wiring are complete. + +- [x] **Step 3: Complete route mocks, copy, and accessibility details** + +Add English Quick Ingest strings for typed errors/actions. Verify live-region summary deduplication, virtual row position metadata, keyboard selection/focus recovery, no-referrer thumbnail opt-in, bounded mounted rows for 500 items, bounded chunk requests, and zero per-item status polling. + +- [x] **Step 4: Run the focused frontend gate** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/services/tldw/__tests__/playlist-ingest.test.ts ../packages/ui/src/services/__tests__/quick-ingest-batch.test.ts ../packages/ui/src/services/__tests__/quick-ingest-session-reattach.test.ts ../packages/ui/src/store/__tests__/quick-ingest-session.test.ts ../packages/ui/src/store/__tests__/quick-ingest-indexeddb.test.ts ../packages/ui/src/components/Common/QuickIngest/__tests__ ../packages/ui/src/entries/shared/__tests__/quick-ingest-session-runtime.test.ts --maxWorkers=1 --no-file-parallelism` + +Run: `cd apps/tldw-frontend && bunx tsc --noEmit` + +Run: `cd apps/tldw-frontend && bunx eslint ../packages/ui/src/services/tldw/playlist-ingest.ts ../packages/ui/src/components/Common/QuickIngest ../packages/ui/src/store/quick-ingest-session.ts ../packages/ui/src/db/dexie/quick-ingest.ts` + +Run: `cd apps/tldw-frontend && bunx playwright test e2e/workflows/quick-ingest-playlist.spec.ts --project=chromium --reporter=line` + +Expected: all commands exit 0. + +- [x] **Step 5: Verify diff, update task, and commit** + +Run: `git diff --check` + +Record test counts, browser result, accessibility assertions, and any explicit skips in `TASK-12113`. Bandit is not applicable to this TypeScript-only plan; record the skip. + +```bash +git add apps/packages/ui apps/tldw-frontend/e2e backlog/tasks +git commit -m "test: verify shared playlist ingest experience (TASK-12113)" +``` + +Task 9 completed with deterministic cross-client coverage and root-owned review hardening. The Chromium journey uses a dedicated 34-item version-2 fixture and verifies two-page inspection, virtual-list first/last position evidence, selection/materialization, one bounded 31-occurrence submission, authoritative Running/45% progress, durable reload reattachment without resubmission, 30 completed plus one failed terminal outcome, zero per-item polls, zero eager thumbnail requests, and zero serious/critical axe findings. The extension integration control preserves the typed active-tab playlist seed and reattaches all 34 occurrence identities after worker recreation. Typed English playlist copy uses native ICU variables; thumbnails are explicit opt-in, lazy, no-referrer, and cosmetic on failure; the drop zone has one interactive browse control; and primary Add styling applies only while enabled. + +Final review added two test-first reliability fixes beyond the journey itself: persisted run-marker publication now waits for its IndexedDB write and propagates durability failures, and React Strict Mode cleanup releases only its own reattachment signature so the replacement effect can resume a persisted run. The shared v15 Dexie test double now includes `quickIngestSessions`, and the virtual-list browser assertion scrolls and checks visibility atomically. Fresh release gates pass 23 files / 695 Vitest tests with zero unhandled failures and Chromium Playwright 1/1. Repository-pinned ESLint passes the full touched TypeScript/TSX scope with no errors (known Next pages-directory informational output only); configured Prettier checks pass for the new/touched configured files; tracked and new-file whitespace checks are clean; the temporary `antd` symlink is restored exactly. Full frontend TypeScript remains skipped under the documented Task 1 three-attempt repository-baseline cap. Bandit is not applicable to this TypeScript-only task. + +The final pre-commit audit found and fixed one additional concurrency defect in the durability barrier: two callers waiting on the same failed queued IndexedDB write could receive different results because the first waiter consumed a shared error slot. The accepted RED was 1 failed / 40 passed; the minimal fix makes every waiter share the latest operation promise while retaining failure-tolerant queue sequencing. Focused GREEN passed 41/41, and the adjacent IndexedDB/session gate passed 77/77. Repository-pinned ESLint and whitespace checks pass for the two remediation files; the temporary `antd` symlink is restored exactly. diff --git a/Docs/superpowers/plans/2026-07-14-quick-ingest-explicit-submission-command.md b/Docs/superpowers/plans/2026-07-14-quick-ingest-explicit-submission-command.md new file mode 100644 index 0000000000..52d05a62c9 --- /dev/null +++ b/Docs/superpowers/plans/2026-07-14-quick-ingest-explicit-submission-command.md @@ -0,0 +1,734 @@ +# Quick Ingest Explicit Submission Command Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace effect-driven Quick Ingest submission with one explicit, authority-checked command that starts each WebUI or extension run at most once. + +**Architecture:** A pure transition builder produces an immutable processing snapshot without side effects. Review, quick-process, and `autoProcessQueued` pass that snapshot through one `beginProcessing` command, which validates, acquires and reconciles the lease, builds the payload, renews ownership, durably commits `creating_run`, applies the UI state once, and invokes a token-fenced runner. The component that owns `IngestWizardProvider` subscribes only to the primitive external-authority revision; ordinary session, persistence-status, ownership, and tracking subscriptions live in a descendant bridge below the Provider so a persistence write cannot synchronously rerender the active reducer owner. Processing-state effects remain reattach/status consumers only and never create backend work. + +**Tech Stack:** React 18, TypeScript, Zustand, Dexie, Vitest, Testing Library. + +## Global Constraints + +- Do not add dependencies, change the Dexie v15 schema, or change extension-runtime message contracts. +- Restored `processing` or `creating_run` state must never create a new backend session. +- Pre-submit storage, ownership, payload, or durable-handoff failure must not call `startQuickIngestSession` or `submitQuickIngestBatch`. +- Quick/auto pre-submit failure must land on Review so existing persistence and ownership alerts are visible. +- Preserve the existing `START_PROCESSING` and `SKIP_TO_PROCESSING` context APIs for compatibility during this task; production entry points stop using them directly. +- An ordinary Quick Ingest session-store write must not rerender the component that owns `IngestWizardProvider`; only a changed external-authority revision may replace Provider state. +- Full TypeScript remains skipped under the recorded Task 1 three-attempt repository-baseline cap. +- The tracked `antd` symlink must be restored to `../../../node_modules/.bun/antd@6.2.1+6dbf9a050bc9aadb/node_modules/antd` after every UI test command. +- Never read, edit, stage, or delete the two protected untracked watchlist templates. + +--- + +## File map + +- `apps/packages/ui/src/components/Common/QuickIngest/IngestWizardContext.tsx`: pure processing transition and exact transition application. +- `apps/packages/ui/src/components/Common/QuickIngestWizardModal.tsx`: non-subscribing Provider owner, descendant session-store bridge, explicit command, durable handoff guard, immutable runner, and removal of the Step-4 submission effect. +- `apps/packages/ui/src/components/Common/QuickIngest/ReviewStep.tsx`: async explicit Start callback and loading state. +- `apps/packages/ui/src/components/Common/QuickIngest/__tests__/IngestWizardContext.test.tsx`: transition-unit coverage. +- `apps/packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.session.test.tsx`: command ordering, exactly-once, restore, and stale-continuation coverage. +- `apps/packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.integration.test.tsx`: Review/quick/auto UX and visible failure coverage. +- `apps/packages/ui/src/store/quick-ingest-session.ts`: reuse the existing acquisition reconciliation and awaited `commitProcessingHandoff`; no new persisted phase. + +### Task 1: Pure processing transition + +**Files:** +- Modify: `apps/packages/ui/src/components/Common/QuickIngest/IngestWizardContext.tsx` +- Test: `apps/packages/ui/src/components/Common/QuickIngest/__tests__/IngestWizardContext.test.tsx` + +**Interfaces:** +- Produces: `buildProcessingTransition(state: IngestWizardState): { ok: true; nextState: IngestWizardState } | { ok: false; nextState: IngestWizardState }`. +- Produces: `applyProcessingTransition(nextState: IngestWizardState): void` on `IngestWizardContextValue`. +- Preserves: `startProcessing()` and `skipToProcessing()` as compatibility wrappers around the same builder. + +- [x] **Step 1: Write failing transition tests** + +```tsx +const ready = buildProcessingTransition(seedWithValidOccurrences) +expect(ready).toMatchObject({ + ok: true, + nextState: { + currentStep: 4, + processingState: { status: "running" }, + }, +}) + +const blocked = buildProcessingTransition(seedWithExpiredMaterialization) +expect(blocked).toMatchObject({ + ok: false, + nextState: { + currentStep: 3, + processingBlock: { code: "materialization_expired" }, + }, +}) +``` + +- [x] **Step 2: Verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/components/Common/QuickIngest/__tests__/IngestWizardContext.test.tsx --maxWorkers=1 --no-file-parallelism` + +Expected: FAIL because `buildProcessingTransition` and `applyProcessingTransition` do not exist. + +- [x] **Step 3: Extract the shared pure builder** + +```tsx +export const buildProcessingTransition = ( + state: IngestWizardState +): ProcessingTransition => { + const { request, block } = buildPlaylistIngestRunRequest(state.queueItems) + if (!request) { + return { + ok: false, + nextState: { + ...state, + currentStep: 3, + highestStep: Math.max(state.highestStep, 3) as WizardStep, + pendingRunRequest: null, + processingBlock: block, + }, + } + } + const perItemProgress = buildInitialProgress(state.queueItems) + if (perItemProgress.length === 0) return { ok: false, nextState: state } + return { + ok: true, + nextState: { + ...state, + pendingRunRequest: request, + processingBlock: null, + currentStep: 4, + highestStep: Math.max(state.highestStep, 4) as WizardStep, + processingState: { + status: "running", + perItemProgress, + elapsed: 0, + estimatedRemaining: 0, + }, + results: [], + }, + } +} +``` + +- [x] **Step 4: Verify GREEN** + +Run the command from Step 2. + +Expected: all context tests pass with zero unhandled errors. + +### Task 2: Explicit authority command and immutable runner + +**Files:** +- Modify: `apps/packages/ui/src/components/Common/QuickIngestWizardModal.tsx` +- Test: `apps/packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.session.test.tsx` + +**Interfaces:** +- Consumes: `buildProcessingTransition` and `applyProcessingTransition` from Task 1. +- Consumes: existing `acquireSubmissionLease(): Promise`, `renewSubmissionLease(): Promise`, and `commitProcessingHandoff(state, tracking): Promise`. +- Produces: a session-id shell whose Provider owner subscribes only to `externalAuthorityRevision`, plus a descendant store bridge for ordinary session/display updates. +- Produces: `beginProcessing(candidateState: IngestWizardState): Promise`. +- Produces: an internal `runSubmission(snapshot, payload, attemptToken): Promise` that never infers work from rendered Step 4 state. + +- [x] **Step 1: Write failing command-boundary tests** + +```tsx +await clickQuickProcess() +expect(commitProcessingHandoff.mock.invocationCallOrder[0]).toBeLessThan( + startQuickIngestSession.mock.invocationCallOrder[0] +) +expect(startQuickIngestSession).toHaveBeenCalledTimes(1) + +rerenderRestoredCreatingRun() +expect(startQuickIngestSession).not.toHaveBeenCalled() + +resolveOlderAttemptAfterReplacement() +expect(screen.getByTestId("wizard-results")).toHaveTextContent("new-attempt") +expect(screen.getByTestId("wizard-results")).not.toHaveTextContent("old-attempt") +``` + +Add controls for unavailable, quota, non-owner, lease loss after payload construction, durable-handoff failure, two simultaneous commands, terminal completion, cancellation, and unmount while start acknowledgement is pending. Include row and whole-run cancellation while file bytes are still being prepared. The row case omits only the cancelled occurrence from the eventual payload; the whole-run case performs no backend start or submit mutation. + +- [x] **Step 2: Verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.session.test.tsx --maxWorkers=1 --no-file-parallelism` + +Expected: the existing maximum-update-depth controls fail and Step-4 state can still trigger submission. + +- [x] **Step 3: Implement one explicit command** + +```tsx +const activeAttemptRef = useRef(null) + +const showReviewRecovery = useCallback( + (sourceState: IngestWizardState) => + applyProcessingTransition({ + ...sourceState, + currentStep: 3, + highestStep: Math.max(sourceState.highestStep, 3) as WizardStep, + }), + [applyProcessingTransition] +) + +const beginProcessing = useCallback(async (candidateState: IngestWizardState) => { + if (activeAttemptRef.current) return false + if (persistenceStatus !== "ready") { + showReviewRecovery(candidateState) + return false + } + const transitionBeforeLease = buildProcessingTransition(candidateState) + if (!transitionBeforeLease.ok) { + applyProcessingTransition(transitionBeforeLease.nextState) + return false + } + const attemptToken = Symbol("quick-ingest-submission") + activeAttemptRef.current = attemptToken + let runnerStarted = false + let recoveryState = candidateState + const attemptIsCurrent = () => + activeAttemptRef.current === attemptToken && isMountedRef.current + try { + const revisionBeforeAcquire = externalAuthorityRevisionRef.current + const acquired = await acquireSubmissionLease() + if (!attemptIsCurrent()) return false + const live = useQuickIngestSessionStore.getState() + const durableState = + live.session?.id === session.id + ? buildInitialWizardState(live.session) + : null + if (!acquired) { + showReviewRecovery(durableState ?? candidateState) + return false + } + const authoritativeState = + live.externalAuthorityRevision > revisionBeforeAcquire && durableState + ? durableState + : candidateState + recoveryState = authoritativeState + const transition = buildProcessingTransition(authoritativeState) + if (!transition.ok) { + applyProcessingTransition(transition.nextState) + return false + } + const selectedItems = transition.nextState.queueItems.filter( + (item) => + item.validation.valid && + item.conferenceOverride?.selected !== false && + item.playlistReview?.selected !== false + ) + const config = transition.nextState.presetConfig + const payload = await buildQuickIngestPayload( + selectedItems, + transition.nextState.conferenceBatchMetadata, + transition.nextState.pendingRunRequest, + { + ...config.common, + storeRemote: config.storeRemote, + reviewBeforeStorage: config.reviewBeforeStorage, + advancedValues: config.advancedValues, + typeDefaults: config.typeDefaults, + }, + { + isCancelled: () => activeAttemptRef.current !== attemptToken, + isOccurrenceCancelled: (occurrenceId) => + preAuthorityCancelledOccurrenceIdsRef.current.has(occurrenceId), + } + ) + if (!attemptIsCurrent()) return false + const renewed = await renewSubmissionLease() + if (!attemptIsCurrent()) return false + if (!renewed) { + showReviewRecovery(recoveryState) + return false + } + const tracking = buildCreatingRunTracking(payload, selectedItems) + const committed = await commitProcessingHandoff( + transition.nextState, + tracking + ) + if (!attemptIsCurrent()) return false + if (!committed) { + showReviewRecovery(recoveryState) + return false + } + applyProcessingTransition(transition.nextState) + runnerStarted = true + void runSubmission(transition.nextState, payload, attemptToken) + return true + } catch (error) { + if (!attemptIsCurrent()) return false + setSubmissionStartError(safeSubmissionStartError(error)) + showReviewRecovery(recoveryState) + return false + } finally { + if (!runnerStarted && activeAttemptRef.current === attemptToken) { + activeAttemptRef.current = null + } + } +}, [ + acquireSubmissionLease, + applyProcessingTransition, + commitProcessingHandoff, + persistenceStatus, + renewSubmissionLease, + session.id, + showReviewRecovery, +]) +``` + +Claim `activeAttemptRef` before the first await so simultaneous commands cannot both cross lease acquisition. A duplicate command returns without changing visible state; unavailable persistence still routes to Review. `showReviewRecovery` applies an exact local Step-3 snapshot instead of `goToStep(3)`, whose backward-only guard cannot move a quick intent forward from Step 1. When lease acquisition exposes a same-session durable record, build the recovery snapshot from that authority; the outer persistence guard prevents this local Review display from weakening a non-draft durable run. Before the claim, validation is pure and cannot mutate backend or durable state. `runnerStarted` transfers token ownership only after the durable handoff and exact UI application; every earlier exit clears only its own matching token. + +`buildCreatingRunTracking` is a local helper that returns the existing bounded `PersistedQuickIngestTracking` fields: `mode: "unknown"`, `submissionState: "creating_run"`, selected occurrence IDs, and `startedAt`. `safeSubmissionStartError` returns one fixed safe message for non-`Error` values and the existing sanitized `Error.message` path otherwise. The unmount cleanup invalidates `activeAttemptRef` before any pending continuation settles. + +The outer persistence adapter owns `processingHandoffGuardRef`, because `persistWizardState` and `commitSessionProcessingHandoff` live outside `WizardModalContent`. Before its awaited durable commit, `commitSessionProcessingHandoff` sets the guard to the exact `transition.nextState` object. In `persistWizardState`, ignore every state while the guard is set; when `state === processingHandoffGuardRef.current`, clear the guard and return without another write because that exact snapshot is already durable. The adapter clears the guard on throw or false commit; a successful commit leaves it until that exact UI snapshot is observed. This closes the store/UI transition window without a revision serializer. Remove `hasStartedRunRef`, `startRunRef`, the render-confirmed latch, `abortPreSubmit`, and the effect whose Step-4 predicate calls `startRun`. + +Check the attempt token after every await and before each result, tracking, warning, Review-return, interruption, or terminal mutation: + +```tsx +if (activeAttemptRef.current !== attemptToken || !isMountedRef.current) return +``` + +`runSubmission` owns the token after the durable handoff. Wrap the runner body in +`try/finally` and clear `activeAttemptRef.current` only when it still equals that +runner's token. Cancellation and every terminal path invalidate the matching token; +unmount invalidates it before any pending continuation can mutate UI or storage. + +- [x] **Step 4: Historical full-suite gate (superseded by the approved resume amendment)** + +Run the command from Step 2. + +The original gate mixed Task 2 command controls with Task 3 entry-path wiring. +The resume amendment's Steps 7-8 replace it: Task 2 must make its focused +command/authority controls green and leave only the enumerated Task 3 entry +controls red. The full session suite becomes green in Task 3. + +**STOP RECORD (2026-07-14):** Task 2 is blocked at the repository's +three-attempt limit. The initial RED was 22 failed / 81 passed with 16 +unhandled maximum-update-depth errors. The explicit runner replacement made +the command-boundary assertions pass but its first full attempt ended at 32 +failed / 71 passed with 29 unhandled depth errors. A render-time session-ref +sync and then a complete persisted-wizard projection equality guard each +failed the isolated durable-handoff control with the same unhandled error and +were reverted. Diagnostic instrumentation showed one +`UPDATE_ITEM_PROGRESS` action being reprocessed 52 times: the Provider mounted +once, `onStateChange` stayed referentially stable, external authority revision +stayed unchanged, and each synchronous session-store write re-entered the +Provider before React finished settling the queued reducer action. Both a +deferred test double and the faithful store-backed handoff reproduced it. +Task 2 production/tests remain uncommitted and not green; the full suite was +not rerun after the focused third-attempt failure. Further work requires an +approved architecture change that prevents ordinary wizard persistence from +synchronously rerendering the component that owns the active reducer. + +**APPROVED RESUME AMENDMENT (2026-07-14):** The user approved the architecture +revision after the stop. This amendment overrides conflicting Task 2 example +signatures above; the retained implementation's actual +`buildCreatingRunTracking(payload, selectedItems, startedAt)` and +`runSubmission(snapshot, payload, selectedItems, attemptToken, authority)` +signatures remain authoritative. Do not rewrite the command or add reducer-wide +idempotence. The next change is limited to moving ordinary store subscriptions +below the Provider boundary. + +- [x] **Step 5: Preserve the focused architectural RED** + +Use the existing focused control `awaits durable creating-run authority before +the backend start mutation` as the regression test. Its command-order assertions +already pass, but Vitest must remain RED before the composition change because +the persisted `UPDATE_ITEM_PROGRESS` dispatch produces an unhandled +maximum-update-depth error. + +Run: +`cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.session.test.tsx --maxWorkers=1 --no-file-parallelism -t "awaits durable creating-run authority before the backend start mutation"` + +Expected: the assertion passes but the process exits nonzero with exactly the +known unhandled reducer/store feedback-loop error. Restore the tracked `antd` +symlink immediately after the command. + +- [x] **Step 6: Split ordinary store subscriptions below the Provider** + +Keep the public component as a session-id shell. It subscribes only to the +primitive session ID so ordinary updates to the same record do not rerender it: + +```tsx +export const QuickIngestWizardModal: React.FC = + (props) => { + const sessionId = useQuickIngestSessionStore( + (store) => store.session?.id ?? null + ) + + useEffect(() => { + if (!props.open || sessionId) return + useQuickIngestSessionStore.getState().createDraftSession() + }, [props.open, sessionId]) + + const initialSession = useQuickIngestSessionStore.getState().session + if (!sessionId || initialSession?.id !== sessionId) return null + return ( + + ) + } +``` + +`QuickIngestWizardSession` owns the persistence refs, handoff guards, local +request nonces, and `IngestWizardProvider`. It subscribes only to the primitive +`externalAuthorityRevision`. Capture the same-session initial record once, and +derive a new external snapshot only when that revision changes: + +```tsx +const externalAuthorityRevision = useQuickIngestSessionStore( + (store) => store.externalAuthorityRevision +) +const initialSessionRef = useRef(initialSession) +const initialStateRef = useRef( + buildInitialWizardState(initialSessionRef.current) +) +const authoritativeSession = useMemo(() => { + const current = useQuickIngestSessionStore.getState().session + return current?.id === sessionId ? current : initialSessionRef.current +}, [externalAuthorityRevision, sessionId]) +const externalState = useMemo( + () => buildInitialWizardState(authoritativeSession), + [authoritativeSession] +) +sessionRef.current = authoritativeSession +``` + +The shell validates the captured record's ID before mounting the session +component; a changed/cleared ID unmounts it through the primitive selector. +Persistence and handoff callbacks +must invoke the latest actions through +`useQuickIngestSessionStore.getState()` and refresh `sessionRef.current` from the +store after each synchronous mutation; they must not subscribe the Provider +owner to `session`, `persistenceStatus`, `isSubmissionOwner`, tracking, or store +action objects. + +Render a `QuickIngestSessionStoreBridge` as a child of +`IngestWizardProvider`. The bridge subscribes to the same-session record, +`persistenceStatus`, and `isSubmissionOwner`, derives +`shouldAttemptPersistedReattach`, and passes them to `WizardModalContent`. +Those descendant rerenders are allowed because they cannot re-enter the +component that owns the Provider reducer: + +```tsx + + + +``` + +Do not add a queue, serializer, reducer-wide equality guards, schema fields, +dependencies, or exported APIs. Preserve the exact processing-handoff object +guard and every current command token check. + +- [x] **Step 7: Verify the focused GREEN** + +Run the exact focused command from Step 5. + +Expected: 1/1 passes, `startQuickIngestSession` is called once after the durable +handoff, and Vitest reports zero unhandled errors. Restore the tracked `antd` +symlink immediately after the command. + +- [x] **Step 8: Verify scoped Task 2 GREEN and preserve Task 3 RED** + +Run the five Task 2 command/authority controls with this exact name filter: + +```text +claims one attempt before awaiting acquisition when two commands race|fences stale tracking, warning, result, and token cleanup after cancellation replacement|replaces the mounted wizard with a newer durable draft after acquisition reconciliation|rechecks ownership through the real Modal callback and does not duplicate an authoritative run|forwards a pending 'whole' cancellation once an 'acknowledged' extension start reveals authority +``` + +Then run the complete session suite from Step 2 once as a sequencing control. + +Expected focused result: 5/5 pass with zero unhandled errors. Expected complete +suite at this boundary: 97/103 pass with zero unhandled errors; the only six +failures are `keeps blocked auto-process retry available until Review resolves +the block`, `clears extension review authority and starts exactly one corrected +retry`, `omits a row cancelled while its direct payload bytes are still being +prepared`, `does not start a direct run cancelled while payload bytes are still +being prepared`, `does not pre-seed direct tracking item identities before +backend submissions are acknowledged`, and `restarts direct processing after +refresh when tracking exists without persisted job ids`. These are Task 3's +accepted RED. Restore the tracked `antd` symlink immediately after each command. + +### Task 3: Route every product entry through the command + +**Files:** +- Modify: `apps/packages/ui/src/components/Common/QuickIngestWizardModal.tsx` +- Modify: `apps/packages/ui/src/components/Common/QuickIngest/ReviewStep.tsx` +- Modify: `apps/packages/ui/src/components/Common/QuickIngest/ProcessingStep.tsx` +- Test: `apps/packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.integration.test.tsx` +- Test: `apps/packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.session.test.tsx` + +**Interfaces:** +- Consumes: `beginProcessing(candidateState): Promise` from Task 2. +- Produces: `ReviewStep.onBeginProcessing(state): Promise`. + +- [x] **Step 1: Write failing entry and UX tests** + +```tsx +await startFromReview() +await startFromQuickProcess() +rerenderWithAutoProcessQueued() +expect(startQuickIngestSession).toHaveBeenCalledTimes(3) + +setPersistenceStatus("unavailable") +await startFromQuickProcess() +expect(screen.getByTestId("wizard-review")).toBeVisible() +expect(screen.getByText(/local recovery is unavailable/i)).toBeVisible() +expect(startQuickIngestSession).not.toHaveBeenCalled() +``` + +Cover quota, other-tab ownership, payload-build failure, automatic retry only after readiness changes, and Review loading state. Replace legacy restored-Step-4 expectations that call `startQuickIngestSession` or `submitQuickIngestBatch`: restored `processing` and `creating_run` records are display/reattach-only and issue zero new backend start/submit mutations, whether tracking is absent or contains a direct session without persisted job IDs. + +Use the real `ProcessingStep` in a focused preparation control: before durable +handoff, it must list the selected queue rows and expose per-row and whole-run +cancellation. After a row is cancelled, assert that the outbound payload, the +durable `pendingRunRequest`, the persisted queue/progress snapshot, and the +applied UI snapshot all omit that occurrence. Strengthen the payload-failure +control to require a visible generic Review alert and assert that the raw thrown +value is absent from both UI and persistence. Add an `autoProcessQueued` restored +Step-4/non-draft control that performs zero lease/start/submit calls and remains +display-only. + +- [x] **Step 2: Verify RED** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.integration.test.tsx ../packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.session.test.tsx --maxWorkers=1 --no-file-parallelism` + +Expected: quick and auto paths bypass the new command or fail without visible Review recovery. + +- [x] **Step 3: Wire the three entry paths** + +```tsx +const stateRef = useRef(state) +stateRef.current = state + +const handleQuickProcess = useCallback(() => { + void beginProcessing(stateRef.current) +}, [beginProcessing]) + +const lastAutoAttemptKeyRef = useRef("") + +const buildAutoProcessIntentKey = ( + candidateState: IngestWizardState +): string => + JSON.stringify({ + sessionId: session.id, + persistenceStatus, + isSubmissionOwner, + externalAuthorityRevision, + queueItems: buildPersistedQueueItems(candidateState.queueItems), + selectedPreset: candidateState.selectedPreset, + customBasePreset: candidateState.customBasePreset, + presetConfig: candidateState.presetConfig, + customOptions: candidateState.customOptions, + conferenceBatchMetadata: candidateState.conferenceBatchMetadata, + }) + +useEffect(() => { + if (!autoProcessQueued || !isOnlineForIngest) return + const attemptKey = buildAutoProcessIntentKey(state) + if (lastAutoAttemptKeyRef.current === attemptKey) return + lastAutoAttemptKeyRef.current = attemptKey + void beginProcessing(state) +}, [ + autoProcessQueued, + beginProcessing, + externalAuthorityRevision, + isOnlineForIngest, + isSubmissionOwner, + persistenceStatus, + session.id, + state, +]) +``` + +The key is written before the async command starts, so rerenders cannot consume the same automatic intent twice. A readiness, ownership, durable-authority, or bounded submission-input change produces a new key and permits one retry. It intentionally excludes display-only step, progress, and result state so a running render cannot manufacture a fresh intent. Review calls the same command and keeps its button loading until the pre-submit command settles. Quick/auto storage or ownership failure uses `showReviewRecovery` so the existing Review alert is visible even when the intent started before Step 3. A local safe error alert covers payload-build failure without exposing raw transport output. + +Pass Review the same command: + +```tsx + +``` + +`ReviewStep` awaits `onBeginProcessing`, keeps Start loading only while the +command settles, and does not call `startProcessing()` when the callback is +present; it passes its current context state to the callback. Preserve the existing compatibility path for other `ReviewStep` +consumers. Auto-process must call `beginProcessing(state)` directly and must +not call `skipToProcessing()`. + +During quick/auto payload preparation, render +``. `ProcessingStep` derives transient +per-item progress with the existing `buildProcessingTransition(state)` builder; +it must not dispatch or persist Step 4 before `commitProcessingHandoff` succeeds. +After payload construction, filter the immutable transition snapshot by the +recorded pre-authority cancelled occurrence IDs and rebuild the processing +transition. Use that rebuilt snapshot consistently for tracking, the durable +handoff, UI application, and `runSubmission`; never combine a filtered payload +with the stale pre-cancel snapshot. + +Preparation cancellation handlers are owned by `WizardModalContent`, not the +ordinary context cancellation callbacks. They synchronously record accepted row +or whole-run cancellation while an `acceptingPreparationCancellationRef` is +true. Immediately after payload construction, close that gate and hide the +preparation UI before lease renewal. Build one frozen remaining-ID set, then +re-filter `entries`, `files`, `pendingRunRequest.inputs`, and per-occurrence +conference metadata from that set before tracking or handoff. A cancellation +attempt after the gate closes is ignored and cannot mutate the live queue; +cancellation accepted before the gate closes is absent everywhere. Add a +two-file regression where file A finishes reading, file B remains blocked, and +file A is then cancelled; A must be absent from the final payload and every +authoritative snapshot. Add a renewal-pending control proving preparation +cancellation controls are gone and a captured stale callback cannot change the +frozen set. + +Pass the persisted safe pre-submit warning to Review as a dedicated error prop +and render it in an error alert. `safeSubmissionStartError` must not include a +raw thrown value. Clear/replace the warning only through the existing session +adapter. Auto eligibility requires a draft session with a current wizard step +below Step 4; restored, processing, interrupted, terminal, and result states do +not invoke `beginProcessing`, lease acquisition, or backend mutation merely +because `autoProcessQueued` is true. + +- [x] **Step 4: Verify GREEN** + +Run the command from Step 2. + +Expected: both Modal suites pass with zero unhandled errors and every blocked entry shows a visible Review explanation. + +Task 3 completed test-first after the approved Provider/store-boundary revision. Review, Quick, and Auto now share the explicit authority command; restored non-draft Step 4 remains display-only; preparation is a transient projection; and pre-authority cancellation freezes one remaining occurrence set before lease renewal. That set filters payload entries/files, pending inputs, conference metadata, the durable snapshot, tracking, applied UI, and runner inputs. Focused race controls passed 3/3, including completed-file cancellation and a stale callback during renewal. The final combined Modal gate passed 180/180 (integration 74/74, session 106/106) with zero unhandled errors. The final specification/quality re-review approved the occurrence-freeze boundary with no critical, important, or minor findings. No schema, dependency, serializer, backend, or extension-runtime contract changed. + +### Task 4: Full Task 8 verification and review + +**Files:** +- Modify: `Docs/superpowers/plans/2026-07-12-youtube-playlist-ingest-shared-frontend.md` +- Modify through Backlog MCP: `TASK-12113` + +- [x] **Step 1: Run focused persistence and context gates** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/store/__tests__/quick-ingest-session.test.ts ../packages/ui/src/store/__tests__/quick-ingest-indexeddb.test.ts ../packages/ui/src/components/Common/QuickIngest/__tests__/IngestWizardContext.test.tsx --maxWorkers=1 --no-file-parallelism` + +Expected: all tests pass with zero unhandled errors. + +- [x] **Step 2: Run full Modal gates** + +Run: `cd apps/tldw-frontend && bunx vitest run ../packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.integration.test.tsx ../packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.session.test.tsx --maxWorkers=1 --no-file-parallelism` + +Expected: all tests pass with zero maximum-update-depth or unhandled errors. + +- [x] **Step 3: Run existing Dexie, scoped lint, and whitespace gates** + +Run the existing Dexie suite, repository-pinned ESLint with `apps/tldw-frontend/eslint.config.mjs` across every Task 8 TypeScript/TSX file, and `git diff --check`. + +Expected: all commands exit 0. Restore the tracked `antd` symlink immediately after Vitest. + +Fresh root verification passed the focused persistence/context gate 129/129 (Context 61, IndexedDB 36, session store 32), the full Modal gate 180/180 (integration 74, session 106), and the unchanged Dexie helper/migration gate 10/10, all with zero unhandled errors and exit 0. Repository-pinned ESLint with the explicit frontend config passed all 13 Task 8 TypeScript/TSX files with zero findings. `git diff --check` passed for tracked changes; the two new TypeScript files emitted no whitespace diagnostics under `git diff --no-index --check` and will receive the authoritative cached-diff check after staging. The tracked `antd` symlink was restored and verified after every Vitest command. Full frontend TypeScript was not rerun under the documented Task 1 three-attempt repository-baseline cap. Bandit is not applicable to this TypeScript-only scope. + +#### Final whole-diff review remediation + +The first whole-diff specification and quality reviews both returned NEEDS FIXES. The following gates are required before Task 4 Step 4 can be checked: + +- [x] **Remediation Step 1: Add focused RED controls** + + Cover an expired renewed lease that another tab takes before the durable processing handoff; stale Review handoff against newer durable authority; same-session external-authority replacement while the runner awaits; an external replacement render with stale reducer queue/new tracking; pre-hydration draft mutation against durable non-draft authority; and Start over through the real IndexedDB adapter. + +- [x] **Remediation Step 2: Make durable handoffs transactional CAS operations** + + Replace the generic authoritative write with separate adapter operations. Processing handoff must atomically require the exact expected durable envelope, draft lifecycle, this adapter owner's unexpired lease, and a processing target before writing `creating_run`. Renewal must require draft lifecycle. Review handoff must atomically require the exact expected durable envelope before replacing it with the explicit draft Review target. CAS loss returns `false`, publishes no captured session, and never permits backend mutation. + +- [x] **Remediation Step 3: Fence runner and external replacement state** + + Capture accepted post-acquisition authority, then require the attempt token, wizard session ID, and live external-authority revision after every runner await and in tracking callbacks. Invalidate the active attempt when accepted authority is replaced. While a Provider external replacement is pending, synchronously expose the external snapshot to descendants so no render combines an old reducer queue with new session tracking. Remove the unused `replaceExternalState` context escape hatch. + +- [x] **Remediation Step 4: Preserve hydration and Start-over authority** + + Do not create a Modal-owned draft before Zustand persistence hydration completes, retain durable non-draft authority during hydration merge, and queue the early open until hydration settles. Route Results Start over/Ingest More through `replaceWithNewDraft` so the prior terminal/processing row is compare-and-set cleared and the new draft has a fresh session ID before the Provider restarts. + +- [x] **Remediation Step 4a: Preserve persisted drafts from side-panel opens** + + Gate the side-panel `handleOpenQuickIngest` entry point on the session store's persistence hydration. Before hydration, retain the exact open detail through the existing pending-open request mechanism and replay it only after hydration, so a newly created in-memory draft cannot overwrite a durable draft's queue, preset, or Review state. Add a delayed-hydration regression with a persisted draft and remove the two unused duplicate side-panel callbacks. + + Completed test-first on 2026-07-14. RED failed in two focused files because the side-panel hydration hook and identity-preserving singleton helper did not exist (`1 failed | 7 passed`, zero unhandled errors). GREEN passed the side-panel delayed-hydration behavior, exact detail/options replay, competing-host singleton claim, unmount/rejection lifecycle, form contract, QuickIngestButton resume, and utility suites (`4 files | 22 tests`, zero unhandled errors). The persisted draft retained its queue, deep preset, and Review step; no draft was created before hydration; replay occurred once; and another mounted host can win the singleton consume race without the side panel recreating it. Relevant store/Modal hydration controls also passed (`3 files | 3 passed | 215 skipped`). Repository-pinned explicit-config ESLint passed the seven touched TypeScript/TSX files, scoped whitespace checks passed, and the Ant Design symlink was restored exactly. No schema, dependency, backend, extension-runtime, staging, commit, or full-TypeScript change was made. + +- [x] **Remediation Step 5: Re-run focused and full gates, then re-review** + + Run the new focused controls, the 129-test persistence/context gate, both full Modal suites, existing Dexie tests, scoped ESLint, symlink restoration, and whitespace checks. Re-dispatch the same two final reviewers against the remediation diff and leave Step 4 open until both approve. + +- [x] **Step 4: Request specification and code-quality review** + +The specification reviewer must confirm all Task 8 requirements. A fresh quality reviewer must inspect command ordering, stale continuation fencing, two-tab behavior, restored-session behavior, failure UX, and unnecessary abstractions. + +Final closeout review is approved. After Step 4a, the same specification and code-quality reviewers both returned APPROVE with no critical, important, or minor findings. Fresh root verification passed persistence/context `135/135`, full Modal `184/184`, Results/pending-open `26/26`, side-panel/pending-open `22/22`, and existing Dexie migration helpers `10/10`, all with exit 0 and no unhandled test failures. Repository-pinned explicit-config ESLint passed all 22 dirty TypeScript/TSX files; tracked and untracked whitespace checks were clean; and the Ant Design symlink was restored exactly. Full TypeScript remains skipped under the documented Task 1 three-attempt baseline cap, and Bandit is not applicable to this TypeScript-only scope. The internal-QA debug side-panel can co-mount two already-hydrated modal hosts, but both reviewers confirmed this pre-existing debug-only ownership issue is a non-blocking follow-up: production WebUI uses the global host and the production extension side panel uses the local host. + +- [x] **Step 5: Update tracking and commit** + +Use Backlog MCP to record exact verification counts, review results, and touched files. Check Task 8 Step 5 only after both reviews approve. + +Task 8 implementation committed as `e97cc15b8b` (`feat: persist quick ingest runs in indexeddb (TASK-12113)`). The verified staged set contained exactly the 26 Task 8 implementation, test, design, plan, and Backlog files; cached whitespace was clean and both protected watchlist templates remained untracked and excluded. + +```bash +git add Docs/superpowers/specs/2026-07-12-youtube-playlist-per-item-ingest-design.md \ + Docs/superpowers/plans/2026-07-12-youtube-playlist-ingest-shared-frontend.md \ + Docs/superpowers/plans/2026-07-14-quick-ingest-explicit-submission-command.md \ + apps/packages/ui/src/components/Common/QuickIngest/IngestWizardContext.tsx \ + apps/packages/ui/src/components/Common/QuickIngest/ProcessingStep.tsx \ + apps/packages/ui/src/components/Common/QuickIngest/ReviewStep.tsx \ + apps/packages/ui/src/components/Common/QuickIngest/WizardResultsStep.tsx \ + apps/packages/ui/src/components/Common/QuickIngestWizardModal.tsx \ + apps/packages/ui/src/components/Common/QuickIngest/__tests__/IngestWizardContext.test.tsx \ + apps/packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.integration.test.tsx \ + apps/packages/ui/src/components/Common/QuickIngest/__tests__/QuickIngestWizardModal.session.test.tsx \ + apps/packages/ui/src/components/Common/QuickIngest/__tests__/WizardResultsStep.navigation.test.tsx \ + apps/packages/ui/src/components/Layouts/QuickIngestButton.tsx \ + apps/packages/ui/src/components/Sidepanel/Chat/form.tsx \ + apps/packages/ui/src/components/Sidepanel/Chat/useSidepanelQuickIngestOpen.ts \ + apps/packages/ui/src/components/Sidepanel/Chat/__tests__/form.queue.contract.test.tsx \ + apps/packages/ui/src/components/Sidepanel/Chat/__tests__/useSidepanelQuickIngestOpen.test.tsx \ + apps/packages/ui/src/db/dexie/schema.ts \ + apps/packages/ui/src/db/dexie/types.ts \ + apps/packages/ui/src/db/dexie/quick-ingest.ts \ + apps/packages/ui/src/store/quick-ingest-session.ts \ + apps/packages/ui/src/store/__tests__/quick-ingest-session.test.ts \ + apps/packages/ui/src/store/__tests__/quick-ingest-indexeddb.test.ts \ + apps/packages/ui/src/utils/quick-ingest-open.ts \ + apps/packages/ui/src/utils/__tests__/quick-ingest-open.test.ts \ + "backlog/tasks/task-12113 - Implement-shared-WebUI-and-extension-per-video-playlist-ingestion.md" +git commit -m "feat: persist quick ingest runs in indexeddb (TASK-12113)" +``` diff --git a/Docs/superpowers/specs/2026-07-12-youtube-playlist-per-item-ingest-design.md b/Docs/superpowers/specs/2026-07-12-youtube-playlist-per-item-ingest-design.md new file mode 100644 index 0000000000..f1ba968ca2 --- /dev/null +++ b/Docs/superpowers/specs/2026-07-12-youtube-playlist-per-item-ingest-design.md @@ -0,0 +1,543 @@ +# YouTube Playlist Per-Item Ingest Design + +**Date:** 2026-07-12 + +**Status:** Approved by requester after three-iteration written-spec review + +**Backlog:** TASK-12109 + +**Scope:** Shared WebUI/browser-extension Quick Ingest plus the backend contracts required to expose every playlist item throughout ingestion + +## Summary + +When a user adds a YouTube playlist URL, the WebUI can currently represent it as one queue row and one ingest job even though video processing later expands the URL into many videos. The frontend therefore reports the lifecycle of the original playlist URL rather than the videos actually being processed. + +The target behavior is fail-closed and per-item: + +1. Every playlist-shaped URL is inspected by the server before it can enter the queue. +2. The complete, ordered playlist is shown to the user. +3. Every selected video becomes one concrete queue occurrence. Each occurrence resolves exactly once, and only an action that requires media processing creates one media ingest job per attempt. +4. WebUI and extension show the same per-video queue, progress, cancellation, recovery, and result states. +5. Large playlists remain bounded, virtualized, resumable, and honest about server limits. + +The frontend must not parse YouTube playlists. The server owns URL classification, metadata extraction, duplicate lookup, snapshot consistency, and job execution. + +## Current-State Evidence + +The repository already contains most of the product skeleton, but the ordinary URL-add path can bypass it: + +- `apps/packages/ui/src/components/Common/QuickIngest/AddContentStep.tsx` detects playlist candidates and offers `PlaylistPreflightPanel`, but `handleAddUrls` still converts every input line directly into one `WizardQueueItem`. +- The extension active-tab action in `apps/packages/ui/src/components/Sidepanel/Chat/ControlRow.tsx` seeds the shared playlist preflight flow, so extension capture is safer than ordinary WebUI paste. +- `apps/packages/ui/src/components/Common/QuickIngest/PlaylistPreflightPanel.tsx` already renders metadata returned by the server, but preview is optional and the panel eagerly renders a short list. +- `tldw_Server_API/app/api/v1/endpoints/media/ingest_jobs.py` creates one job per submitted URL. A playlist URL is therefore initially treated as one item. +- `tldw_Server_API/app/core/Ingestion_Media_Processing/Video/Video_DL_Ingestion_Lib.py` expands YouTube playlists inside video processing. +- `tldw_Server_API/app/services/media_ingest_jobs_worker.py` then projects only `results[0]` into the job result, losing the remaining child-result identities at the Jobs boundary. +- The WebUI direct path in `apps/packages/ui/src/services/tldw/quick-ingest-batch.ts` submits and waits on URL entries sequentially. The extension background path submits entries before polling, so the two clients also differ in scale behavior. +- `apps/packages/ui/src/store/quick-ingest-session.ts` persists Quick Ingest state in `sessionStorage`, whose quota is unsuitable for large playlists and whose failures are currently silent. + +The earlier bulk conference workflow established a useful metadata-only preflight and durable collection model. This design tightens the active playlist path and addresses the remaining identity, scale, and recovery gaps without creating a separate playlist product. + +## Product Decisions + +| Decision | Required behavior | +|---|---| +| Expansion timing | Expand before confirmation; never after the UI has committed to one playlist row. | +| Failure policy | Block playlist ingestion when inspection is unavailable or incomplete. Do not offer an opaque fallback. | +| Large playlists | Load the complete playlist through a bounded server snapshot and paginated client reads. Never silently truncate. | +| Row presentation | Show playlist position and video title first; show channel, duration, availability, duplicate state, and progress second. Keep the concrete URL in row details. | +| Client parity | WebUI paste, extension active-tab import, Add, and Enter use the same shared controller and state model. | +| Execution identity | Every selected occurrence resolves exactly once. A processing-required action maps to one media ingest job per attempt; duplicate reuse, skip, and metadata-only actions terminate without a processing job. | +| Collections | A media collection is optional. Run tracking cannot depend on collection creation. | + +## Goals + +- Make every selected video visible before submission and throughout its ingest lifecycle. +- Preserve stable mapping among preflight occurrence, queue row, planned collection item when present, job, media result, and retry attempt. +- Support multiple playlists and ordinary URLs in the same staged Quick Ingest session. +- Keep large playlist rendering and network behavior bounded. +- Recover accepted work after WebUI reload or extension service-worker restart. +- Give users truthful statuses for inspection, submission, processing, cancellation, and recovery. +- Preserve the documented multi-result playlist behavior of the no-database `/process-videos` endpoint. + +## Non-Goals + +- Parsing YouTube pages or playlist APIs in the frontend. +- Creating a general workflow engine or replacing the existing media Jobs system. +- Guaranteeing access to private, deleted, region-blocked, or authentication-gated videos. +- Claiming that all playlist sizes are safe. The server retains an administrator-configurable hard ceiling. +- Automatically loading third-party thumbnails for every row. +- Requiring a media collection for ordinary playlist ingestion. + +## Target Architecture + +```mermaid +flowchart LR + A["WebUI paste or extension active tab"] --> B["Shared playlist detector"] + B -->|"ordinary URL"| C["Staged ordinary queue row"] + B -->|"playlist context"| D["Owner-scoped preflight resource"] + D --> E["Immutable metadata snapshot"] + E --> F["Paginated virtual preview"] + F --> G["Confirmed concrete occurrences"] + G --> H["Owner-scoped ingest run"] + H --> I["Bounded job-submission chunks"] + I --> J["One media job per processing occurrence"] + J --> K["SSE or run-level polling"] + K --> L["Per-video progress and results"] + H -. "optional" .-> M["Media collection and planned items"] +``` + +The new contracts are deliberately narrow: + +- an asynchronous, temporary playlist-preflight resource; +- a small owner-scoped playlist ingest run manifest; +- structured occurrence-aware submission fields and results on the existing media Jobs endpoint; +- one shared client controller and normalized run-status transport. + +The existing Jobs worker remains responsible for processing each concrete video URL. + +## Playlist Detection and Mandatory Inspection + +The shared client classifies every submitted line before mutating the queue. + +A URL is a playlist candidate when it is a trusted YouTube/youtu.be host and contains a non-empty `list` parameter. This includes `/playlist`, `/watch?...&list=...`, and youtu.be URLs with playlist context. Lookalike and suffix-appended hosts are ordinary web URLs. + +Rules: + +- Add and Enter invoke playlist inspection instead of `handleAddUrls` for candidates. +- The extension active-tab action invokes exactly the same shared controller. +- The original input remains visible until inspection succeeds or the user removes it. +- Ordinary URLs may appear as staged rows while playlist inspection runs, but Configure, Quick Process, and Start Processing remain disabled while any candidate is unresolved. +- Multiple playlist candidates may inspect concurrently only up to a small client-side bound; the backend applies its own global/per-owner capacity limits. +- No client path may convert a candidate directly into an opaque `WizardQueueItem`. + +## Preflight Resource Contract + +The current synchronous endpoint remains available as a compatibility surface during rollout. The version-2 client uses an asynchronous resource contract advertised by `mediaPlaylistIngestContractVersion >= 2`. + +Required version-2 routes: + +- `POST /api/v1/media/playlist-preflights` +- `GET /api/v1/media/playlist-preflights/{preflight_id}` +- `GET /api/v1/media/playlist-preflights/{preflight_id}/items?cursor=...&limit=...` +- `POST /api/v1/media/playlist-preflights/{preflight_id}/materializations` +- `DELETE /api/v1/media/playlist-preflights/{preflight_id}` + +`POST` validates the playlist URL, creates an owner-scoped resource, starts bounded extraction, and returns promptly with HTTP 202. It does not create media, collections, runs, or media ingest jobs. The implementation may use an internal `playlist_preflight` Jobs-domain task for multi-worker-safe leasing and execution; that task is not a media ingest job and has no library side effects. + +The resource state is one of: + +- `pending` +- `ready` +- `blocked` +- `cancelled` +- `expired` + +The summary exposes: + +- `preflight_id` +- `status` +- `source_url` +- `source_kind` +- `playlist_id` +- `playlist_title` +- `total_count`, when trustworthy +- `loaded_count` +- `ingestible_count` +- `unavailable_count` +- `duplicate_count` +- `expires_at` +- typed warnings or error code + +The item endpoint returns an immutable ordered page plus an opaque `next_cursor`. A cursor is bound to owner, preflight ID, and snapshot version. Tampered, cross-owner, expired, or mismatched cursors fail safely. + +### Extraction and capacity + +- Extraction runs outside the API event loop in a bounded, terminable process so timeout or cancellation can reclaim capacity. +- The extractor requests at most `configured_limit + 1` entries. Seeing the extra entry produces `playlist_too_large`; it must not first materialize an unbounded playlist. +- If YouTube provides a trustworthy total, the UI may show it. Otherwise the oversized error says the playlist contains more than the configured limit rather than inventing an exact count. +- Preflight resource metadata and normalized snapshot items are stored in a shared owner-scoped temporary repository backed by the deployment's configured SQLite/PostgreSQL data layer. A process-local TTL/LRU may be only a read-through optimization, never the source of truth. +- Extraction claims use database-backed leases and enforce global and per-owner capacity across API/worker processes. A local child process performs the blocking yt-dlp call only after a worker owns the lease. +- Successful snapshots are immutable and TTL-bound. Cleanup releases leases and removes expired snapshot rows or temporary artifacts. +- Server restart may expire temporary preflights. That is a recoverable state, not data loss, because no ingest has begun. +- Deleting a preflight terminates or marks extraction cancelled, releases temporary artifacts, and makes future item reads unavailable. + +### Preflight item fields + +Each item includes: + +- `occurrence_id` +- `ordinal` +- `occurrence_index_for_source` +- `source_url`, when ingestible +- `normalized_source_id` +- `source_kind` +- `title` +- `channel_or_uploader` +- `duration_seconds` +- `published_at` +- `thumbnail_url` +- `availability` +- `duplicate_status` +- `duplicate_of_occurrence_id`, when applicable +- `selected_by_default` + +Unavailable entries remain visible but cannot be selected. A response is `ready` only when the snapshot is complete. An interrupted extraction with unknown missing entries is `blocked`, not partial-ready. + +### Queue materialization + +`Add N videos` calls the materialization route with selected occurrence IDs. The server re-reads the completed owner-scoped snapshot and copies only the selected source identity: concrete URLs, occurrence IDs, normalized source IDs, and compact display metadata. It must not freeze duplicate evidence, duplicate-policy choices, or metadata patches because library state and user choices can change through Review. The route returns `materialization_id`, an owner-bound opaque materialization token, compact occurrence records, and `expires_at`. + +The client creates queue rows only after this request succeeds and persists the materialization reference with them. A preflight snapshot may then expire without invalidating those rows because Start Processing resolves playlist inputs from the materialization record, not from the deleted preflight snapshot and not from client-supplied URLs. + +Queue materialization retention is at least as long as the supported Quick Ingest draft-retention window and may be renewed while the draft remains active, subject to an administrator-configured maximum. If the materialization itself expires before Start Processing, playlist rows become blocked and require reinspection; the client must not submit its cached URL copies as authoritative replacements. + +## Identity and Duplicate Semantics + +Identity must distinguish an occurrence from its underlying video: + +- **Occurrence ID:** an opaque server-generated identifier returned on every preflight item. It is unique within the authenticated owner's preflight namespace, stable for the lifetime of that immutable snapshot, and copied unchanged into the queue and run. Clients must not derive it. It is used for queue, job, progress, cancellation, and result mapping. +- **Normalized source ID:** canonical video identity, such as a YouTube video ID, used for duplicate detection. + +The same video may appear more than once in one playlist or across several staged playlists. Those occurrences remain independently visible while sharing a dedupe identity. + +Duplicate evaluation occurs across: + +1. repeated items in one snapshot; +2. all playlists and ordinary concrete URLs staged in the current Quick Ingest session; +3. the authenticated user's existing Media DB records through an owner-scoped bulk lookup. + +If the library lookup cannot establish existing state, the status is `unknown`, not `new`. + +Default selection keeps the first new occurrence and deselects later duplicates. Existing duplicate policies remain explicit: skip, include existing, update metadata only, or overwrite. Overwrite is never selected implicitly. + +Refresh reconciliation uses normalized source ID plus occurrence index among repeats. Unambiguous selection is retained. Ambiguous additions, removals, or reordering are disclosed before confirmation. + +## Ingest Run Contract + +Confirmation creates a lightweight owner-scoped run before any job submission. The run is not a replacement for Jobs; it is a manifest that preserves the relationship between selected occurrences and the job batches that execute them. + +Required routes: + +- `POST /api/v1/media/ingest/runs` +- `GET /api/v1/media/ingest/runs/{run_id}` +- `GET /api/v1/media/ingest/runs/{run_id}/items?cursor=...&limit=...` +- `GET /api/v1/media/ingest/runs/{run_id}/events/stream?after_id=...` +- `POST /api/v1/media/ingest/runs/{run_id}/cancel` +- `POST /api/v1/media/ingest/runs/{run_id}/retry` + +`Add N videos` materializes selected preflight items into the owner-scoped queue materialization described above and then creates concrete client queue records plus compact IndexedDB persistence. It does not create the server run. The server run is created only when the user chooses Start Processing, after all playlists, ordinary URLs, files, configuration, and review choices are final. Preflight expiry after queue materialization therefore does not invalidate the queued playlist rows. + +Run creation receives normalized processing options, optional collection metadata, a list of selected input records, and `review_overrides` keyed by occurrence ID. An override contains the final duplicate policy when duplicate evidence requires a choice and an optional metadata patch conforming to the allowlist below. At Start Processing, the server refreshes owner-scoped duplicate evidence, then validates that every override belongs to an input occurrence, that every selected duplicate has an explicit valid policy, and that metadata patches are allowed for that policy. Unknown, missing, conflicting, or extra overrides fail run creation before any collection, metadata mutation, or job submission. If refreshed evidence changes a required choice, the API returns structured `review_required` details and the client returns to Review without side effects. + +Input records are a discriminated union: + +- `materialized_playlist_item`: owner-bound materialization ID/token plus the server-issued occurrence ID; +- `direct_url`: a client-generated opaque occurrence ID, concrete non-playlist URL, normalized type, and compact display metadata; +- `file_stub`: a client-generated opaque occurrence ID plus file name/type/size metadata, later bound to the staged upload job without storing file bytes in the run manifest. + +For `materialized_playlist_item`, the server resolves the occurrence from the owner-scoped unexpired queue materialization and treats its stored source identity as authoritative, refreshes duplicate evidence, and applies only the validated Review-time override supplied at Start Processing. For `direct_url`, the server validates and canonicalizes the URL, rejects any playlist candidate, requires occurrence IDs to be unique within the run, and validates its Review-time override against the same fresh duplicate lookup. For `file_stub`, the server records metadata and an `awaiting_upload` lifecycle state but no bytes. This gives mixed sessions one identity contract while keeping local file bytes in the existing upload path. + +The run stores: + +- owner and `run_id` +- playlist summaries +- occurrence IDs, normalized source IDs, concrete URLs, compact display metadata, and selection +- optional collection and planned-item IDs +- submission chunk states and batch IDs +- job-to-occurrence and job-to-planned-item mappings +- attempt number and endpoint-derived idempotency identity +- per-occurrence and aggregate states +- retention timestamps + +A collection is optional. When requested, collection plus planned-item creation is one transactional bulk operation. Failure leaves the run unsubmitted and does not create a partial collection. + +Run retention outlives active jobs and remains long enough for reload, extension restart, retry, and result inspection. Cleanup removes only run metadata; it never deletes referenced media or collections. + +### Run status snapshot and events + +`GET /ingest/runs/{run_id}` returns the owner-scoped run summary, aggregate counts, current version, update timestamp, optional collection ID, and all known batch IDs. It does not embed an unbounded item list. + +`GET /ingest/runs/{run_id}/items` returns an ordered occurrence page with lifecycle `state`, terminal `outcome` when present, progress percentage/message when known, job and media IDs when available, attempt, retryability, and an opaque `next_cursor`. Occurrence ordering is immutable after run creation, so the cursor is bound to owner, run ID, sort order, and the last stable occurrence position rather than the frequently changing status version. Each page reports the current run version; clients merge by occurrence ID and refresh active rows if versions changed during a multi-page read. + +The run event stream emits an initial summary snapshot followed by occurrence-aware events with: + +- monotonically increasing `event_id` +- `run_id` +- `occurrence_id` +- optional `job_id` and `batch_id` +- event type, lifecycle state, and terminal outcome when present +- progress percentage/message when known +- `occurred_at` + +Clients resume with `Last-Event-ID` or `after_id`. The producer queries run/job events by owner and `run_id` on every cycle; it must not freeze the tracked job-ID set at connection time. Jobs accepted by later submission chunks therefore appear in the same stream. If replay history has expired, the server emits `resync_required` and the client reloads the summary and paginated item snapshot before continuing. + +### Lifecycle state and terminal outcome + +Run items expose two separate axes. `state` describes current lifecycle position: + +- `staged` +- `preparing` +- `awaiting_upload` +- `submit_pending` +- `queued` +- `running` +- `cancellation_requested` +- `status_unavailable` +- `terminal` + +`outcome` is null until `state=terminal` and is then exactly one of: + +- `completed` +- `included_existing` +- `metadata_updated` +- `skipped_existing` +- `submit_failed` +- `processing_failed` +- `metadata_update_failed` +- `cancelled` + +Progress phase/message is optional evidence from the worker, not a third lifecycle state. This separation is used consistently by the run snapshot, events, IndexedDB record, filters, counts, and result groups. + +`file_reattach_required` is a client presentation state, not a backend lifecycle value. The client derives it only when the server reports `awaiting_upload` and the current browser runtime no longer has the corresponding local file bytes. + +## Job Submission and Idempotency + +Selected concrete URLs are submitted to the existing `/api/v1/media/ingest/jobs` endpoint in bounded chunks. The initial implementation should use a configurable chunk size with a conservative default rather than one request per video or one unbounded request. + +Each submitted occurrence carries: + +- `run_id` +- `occurrence_id` +- optional planned collection item ID +- attempt number +- a client attempt token used only as input to endpoint-derived idempotency + +The endpoint derives the actual Jobs idempotency key from authenticated owner, run, occurrence, and attempt. Raw client keys are never trusted as globally scoped Jobs keys. + +The submit response returns a structured record for every occurrence: + +- `occurrence_id` +- `accepted` or `rejected` +- `job_id`, when accepted +- `batch_id` +- safe error code and message, when rejected +- `retryable` +- attempt reference + +String-only error lists are insufficient because they cannot safely map failures back to repeated URLs. + +URL chunks use aligned occurrence/attempt/planned-item arrays. File chunks use multipart `files` plus aligned `file_occurrence_ids`, `file_attempts`, and optional `file_planned_item_ids`; array lengths must exactly match the uploaded file count. The server validates that each file occurrence belongs to the owner/run and is currently `awaiting_upload`, stages the upload, writes `run_id`, `occurrence_id`, and attempt into the job payload, and returns the same structured per-occurrence acceptance record used for URLs. + +If the UI/runtime restarts before a file job is accepted, browser file bytes are not assumed recoverable. The server item remains `awaiting_upload`; the client presents `file_reattach_required` while the bytes are absent. The user may reselect the file, preserving occurrence identity, or cancel it. Once the job is accepted, normal run/job reattachment applies. + +Global failures stop later chunks: authentication/authorization, quota, worker unavailability, shutdown/draining, invalid run ownership, and rate limiting. Rate-limited clients honor `Retry-After`. Isolated invalid, unavailable, or duplicate entries do not stop unrelated occurrences. + +Ambiguous network failure retries the same occurrence attempt. Jobs idempotency returns the original job rather than creating another. A deliberate processing retry uses a new attempt number after reconciling whether the prior attempt already created media. + +### Duplicate policy actions + +Duplicate policies have distinct server actions whether or not a collection exists: + +| Policy | Job action | Terminal outcome | Optional collection behavior | +|---|---|---|---| +| `skip` | Do not submit a job or mutate media metadata. | `skipped_existing` with the resolved existing media ID when available. | Do not add membership. | +| `include_existing` | Do not submit a job. Resolve and reuse the existing media item. | `included_existing`. | Resolve the planned item/membership to the existing media ID. | +| `update_metadata_only` | Do not submit a media-processing job. Apply only the reviewed metadata patch contract through the Media DB abstraction. | `metadata_updated` or `metadata_update_failed`. | Resolve membership to the existing media ID when the update succeeds. | +| `overwrite` | Submit one concrete job with overwrite enabled. | Normal completed/processing-failed outcome. | Resolve the planned item to the resulting media ID. | + +Without a collection, `include_existing` still creates a run result linking the existing media item, while `update_metadata_only` still performs and reports the metadata operation. These policies must not collapse into the same `skipped_existing` state. + +The metadata patch is built at Review time only from fields the user explicitly edited or explicitly applied as shared tags. Extracted playlist metadata that the user did not edit is not permission to overwrite an existing record. + +The allowed patch schema is: + +- `title`: non-empty validated string; replace the existing title; +- `author`: validated string sourced from an explicitly edited speaker/author field; replace the existing author; +- `keywords_add`: validated keyword list sourced from explicitly applied shared/item tags; case-insensitive union with existing keywords. + +Content, media type, analysis, prompt, deletion state, and keyword set/remove operations are forbidden. An empty patch makes `update_metadata_only` unavailable in the UI and invalid at the API. The backend applies title/author plus keyword union through one Media DB abstraction transaction with normal version/conflict handling; a conflict produces `metadata_update_failed` and does not silently partially apply the patch. + +## Worker Boundary + +Every selected run occurrence resolves to exactly one action and terminal outcome. The media Jobs endpoint enforces one job per attempt only for occurrences whose resolved action requires media processing. `skip`, `include_existing`, and `update_metadata_only` resolve through the run without creating a media-processing job. An opaque playlist candidate submitted directly to the Jobs endpoint fails with HTTP 422 and `playlist_preflight_required`. + +The worker receives one concrete video URL and returns one media result. It must not expand a playlist within a media ingest job. + +The no-database `/process-videos` endpoint may continue accepting a playlist and returning multiple results because its response already represents a processing batch and does not require one job or one planned-item identity. + +## Shared Frontend UX + +### Inspection + +Pasting a playlist changes the primary action to **Inspect playlist**. Extension active-tab import begins the same inspection automatically because the extension action itself is explicit. + +The preflight card shows: + +- inspecting progress, such as `Loaded 100 of 742`; +- playlist title and availability summary; +- a virtualized ordered item list; +- title and playlist position as primary text; +- channel, duration, availability, and duplicate state as secondary text; +- Select all, Select none, Select new, and per-row controls; +- typed blocking guidance, retry, cancel, and refresh. + +`Add N videos` remains disabled until the snapshot is complete and every selected occurrence has a concrete URL. + +An expired snapshot can be restarted. Selection reconciliation is shown only after explicit refresh or expiry recovery; immutable snapshots do not continuously claim the source changed. + +### Queue + +Confirmation converts selected occurrences into ordinary `WizardQueueItem` records. The queue uses flat per-video rows with a lightweight playlist heading rather than a collapsible parent that hides active work. + +Each row preserves: + +- occurrence ID and normalized source ID +- title +- playlist ID/title and ordinal +- channel/uploader and duration +- concrete URL +- duplicate state and chosen policy +- optional planned collection item + +The title/ordinal is primary. The URL and optional thumbnail appear only in details. Thumbnails are not eagerly loaded; explicit loading uses no-referrer behavior and failure is cosmetic. + +### Processing + +The first state is **Preparing N videos** while run/collection records and jobs are created. Rows do not claim to be processing before job acceptance. + +The processing UI uses the lifecycle `state` and terminal `outcome` axes defined by the run contract. In particular, included-existing and metadata-only results are terminal outcomes and `status_unavailable` is a recoverable server lifecycle state. `file_reattach_required` is a recoverable client presentation of server `awaiting_upload` when local bytes are missing. + +Known backend progress and messages are displayed. The UI must not fabricate precise Analyze or Store stages when the server supplied only generic processing evidence. + +Preflight, queue, processing, and result lists are virtualized and offer useful filters for large runs. Virtual rows preserve keyboard navigation, stable focus, `aria-setsize`, `aria-posinset`, and live summary announcements. + +### Cancellation + +- Before submission, cancelling a row removes it from unsent chunks. +- After job acceptance, cancelling a row invokes the actual job-cancel endpoint. +- The row remains `cancellation_requested` until the server reports a terminal state. +- If completion wins the race, the final state is completed. +- Cancelling the whole run stops unsent chunks and requests cancellation for all accepted jobs. + +### Results and retry + +Terminal result groups are Completed, Included existing, Metadata updated, Skipped existing, Not submitted, Failed during processing, Metadata update failed, and Cancelled. + +`status_unavailable` is not terminal. It retains Check again and Reconnect actions and becomes interrupted only when the user abandons recovery or the server proves the job record is no longer recoverable. + +Retry first reconciles by normalized source/media identity and optional planned item because a failed job may have created media before failing later. Only eligible occurrences receive a new attempt. + +## Persistence and Runtime Recovery + +Large Quick Ingest sessions move from silent `sessionStorage` persistence to compact IndexedDB records shared by the common UI package. The WebUI and extension keep separate origin-local databases; the design does not assume cross-origin storage. + +Persisted state includes compact display metadata, occurrence/run/job mappings, selections, and terminal summaries. It excludes thumbnail bytes and other bulky artifacts. + +Requirements: + +- migrate the current session-storage record safely; +- make migration interruption recoverable; +- surface quota or write failure instead of silently losing resume guarantees; +- retain active/interrupted runs and recent terminal results for a bounded period; +- clean expired preflight state and stale terminal runs; +- coordinate multiple tabs so one run is not submitted twice. + +### Explicit submission command boundary + +`currentStep === 4` and `processingState.status === "running"` describe the UI; they are not a command to create backend work. Review Start, quick-process, and `autoProcessQueued` must call one explicit `beginProcessing(candidateState)` command. No effect may infer a new submission from restored or rendered processing state. + +The command validates a pure processing transition before taking a lease. After transactional lease acquisition it chooses the caller snapshot unless acquisition advanced `externalAuthorityRevision`, in which case it rebuilds from the reconciled durable session. It then builds the immutable payload, renews the lease, durably writes the exact `processing` / `creating_run` handoff, applies the same processing snapshot to the Provider once, and invokes the backend runner. A handoff guard prevents the Provider's previous state from being persisted across the durable-commit/UI-transition window. + +The runner receives only the immutable processing snapshot, payload, session authority, and a transient attempt token. It checks that token and the existing session/revision authority after every await before publishing state. Cancellation, unmount, or a replacement attempt invalidates the token. The existing extension runtime attempt token remains authoritative for extension delivery; the UI token only fences stale component continuations. + +Pre-submit storage, ownership, payload-build, or handoff failure performs no backend mutation and does not enter Processing. Quick and automatic entry move to Review so the existing persistence/ownership alerts are visible; a safe local message covers payload-build failure. Restored `processing` and `creating_run` records are reattach/interruption inputs only and can never create a fresh submission. The sole remaining automatic effect consumes `autoProcessQueued` once; it never observes Step 4 as a trigger. + +Status transport is normalized but platform-aware: + +- WebUI prefers run-scoped SSE and falls back to paginated run polling. +- Extension treats polling/reattachment as first-class because a service worker may be suspended; SSE is opportunistic. +- Reopening either UI rehydrates the local run record, asks the server for the owner-scoped run/job snapshot, and reconciles by occurrence ID. + +Temporary status-fetch failure produces `status_unavailable`, not failure. + +## Error Model + +Stable public preflight errors include: + +- `invalid_playlist_url` +- `playlist_not_found` +- `playlist_private_or_auth_required` +- `playlist_metadata_unavailable` +- `playlist_too_large` +- `preflight_busy` +- `preflight_timeout` +- `preflight_expired` +- `preflight_cancelled` +- `preflight_incomplete` +- `materialization_expired` +- `server_unreachable` + +Submission/run errors distinguish authentication, authorization, quota, rate limit, worker unavailable, server draining, invalid ownership, structured per-occurrence rejection, and terminal processing failure. + +The UI preserves the source input and gives typed retry/troubleshooting guidance. It never exposes raw yt-dlp output. Unknown provider failures use a generic safe public message with sanitized operator diagnostics. + +## Security and Privacy + +- Validate trusted YouTube host boundaries; reject lookalike domains. +- Bind preflights, queue materializations, cursors, runs, and jobs to the authenticated owner. +- Derive Jobs idempotency keys with owner scope. +- Redact complete playlist URLs and query secrets in logs and metrics. +- Never persist browser cookies or credentials in preflight/run records. +- Never return another owner's existence through cursor, run, batch, or idempotency behavior. +- Apply existing rate, billing, storage, and worker-readiness controls before accepting work. +- Record counts and typed outcome categories in metrics, not complete source URLs. + +## Compatibility and Capability Signaling + +The server advertises `mediaPlaylistIngestContractVersion` and granular readiness for asynchronous preflight resources, run tracking, media Jobs, worker availability, and event streaming. + +Version-2 clients require the new preflight/run contract for playlist candidates. Older clients that submit a playlist directly to Jobs receive a structured 422 with actionable preflight guidance. The existing synchronous preflight may remain during a deprecation window, but it is not the version-2 path. + +The `/process-videos` playlist contract remains separate and covered by compatibility tests. + +## Verification Strategy + +### Fast PR gate + +- Backend unit tests for classification, snapshot state, queue materialization/expiry, capacity, pagination, duplicate lookup and Review-time refresh, metadata-patch validation, error sanitization, Jobs rejection, run transitions, file binding, state/outcome separation, and owner-derived idempotency. +- Hypothesis property/state-machine tests for pagination completeness, occurrence uniqueness, chunk coverage, exactly-once action resolution, zero media jobs for non-processing duplicate actions, at-most-one accepted job per processing attempt, idempotent ambiguous retry, cancellation/completion races, and terminal-count invariants. +- Shared Vitest tests for mandatory inspection, mixed input blocking, metadata persistence, duplicate selection, filters, virtualization, cancellation dispatch, and transport normalization. +- TypeScript checks and Bandit on touched backend scope. + +### Integration gate + +- Temporary real Jobs database and worker with only expensive media processing replaced by a deterministic fake. +- Preflight process timeout, cancellation, crash, shutdown, capacity release, and orphan cleanup. +- Owner isolation across preflight, queue materializations, cursors, runs, jobs, optional collections, and idempotency. +- Structured partial acceptance, global chunk-stop behavior, retry reconciliation, run-level list/cancel, and optional transactional collection planning. +- IndexedDB migration, interrupted migration, quota failure, cleanup, retention, multi-tab coordination, and runtime recreation. + +### Browser gate + +- One complete WebUI journey using the existing 34-item conference fixture: paste, inspect, select, queue, submit, progress, reload/reattach, and results. +- One extension journey: active-tab handoff into the same inspection state, background runtime recreation, polling reattachment, and completion. + +Large 500-item behavior is verified below full E2E through bounded mounted-row counts, bounded chunk requests, run-level status retrieval, and the absence of per-item polling fan-out. The configured-limit-plus-one case is a backend contract test. + +### External and benchmark checks + +Required CI uses deterministic sanitized yt-dlp-shaped fixtures. An optional/nightly external test may verify current YouTube compatibility. Non-blocking benchmarks may track extraction and rendering timings, but normal PR gates assert structural bounds rather than flaky wall-clock thresholds. + +Accessibility checks combine axe with explicit tests for virtual-list position metadata, keyboard selection, focus recovery, filter announcements, and live-region deduplication. + +## Acceptance Criteria + +- No WebUI or extension entry path can queue a YouTube playlist as one opaque item. +- A complete ordered preview appears before confirmation, or ingestion remains blocked with typed guidance. +- Every selected occurrence becomes one concrete queue row and resolves exactly once; only processing-required actions create at most one accepted job per attempt. +- Queue, processing, cancellation, recovery, and results preserve occurrence identity and title-first presentation. +- WebUI and extension expose the same normalized states and outcomes. +- Large playlists use bounded extraction, virtualized rendering, chunked submission, and run-level status retrieval. +- Reload, extension runtime restart, partial submission, ambiguous retry, and cancellation do not lose or duplicate accepted work. +- Owner isolation, safe idempotency, redacted diagnostics, and configurable limits are verified. +- `/process-videos` retains its documented multi-result playlist behavior while Jobs ingestion fails clearly when preflight is bypassed. diff --git a/apps/packages/ui/src/assets/locale/en/option.json b/apps/packages/ui/src/assets/locale/en/option.json index c04e18746e..ce9cf70f57 100644 --- a/apps/packages/ui/src/assets/locale/en/option.json +++ b/apps/packages/ui/src/assets/locale/en/option.json @@ -970,6 +970,70 @@ }, "quickIngest": { "title": "Quick Ingest", + "playlistInspection": { + "title": "PLAYLIST INSPECTION", + "regionAria": "Playlist inspections", + "readyLabel": "Inspection ready", + "readyMessage": "{count, plural, one {# playlist item loaded for review.} other {# playlist items loaded for review.}}", + "unavailableLabel": "Inspection unavailable", + "unavailableMessage": "Playlist inspection is unavailable on this server. Update the server or remove this playlist.", + "failedLabel": "Inspection failed", + "failedMessage": "Playlist inspection failed. Try again.", + "blockedLabel": "Inspection blocked", + "blockedMessage": "Playlist inspection was blocked. Remove it or try again.", + "expiredLabel": "Inspection expired", + "expiredMessage": "Playlist inspection expired. Try again.", + "cancelledLabel": "Inspection cancelled", + "cancelledMessage": "Playlist inspection was cancelled.", + "inspectingLabel": "Inspecting playlist", + "inspectingMessage": "Loading playlist details from the server.", + "cancel": "Cancel", + "cancelAria": "Cancel playlist inspection for {url}", + "retry": "Retry", + "retryAria": "Retry playlist inspection for {url}", + "remove": "Remove", + "removeAria": "Remove playlist inspection for {url}", + "moreNotLoaded": "More playlist items are not loaded yet.", + "sessionDuplicates": "{count, plural, one {# item also appears elsewhere in this queue.} other {# items also appear elsewhere in this queue.}}" + }, + "playlistPreflight": { + "detected": "Playlist detected", + "details": "Playlist details", + "refresh": "Refresh", + "refreshAria": "Refresh playlist inspection", + "itemCount": "{count, plural, one {# item} other {# items}}", + "selectedCount": "{count} selected", + "duplicateCount": "{count, plural, one {# duplicate} other {# duplicates}}", + "selectionWarning": "Playlist items were added, removed, reordered, or could not be matched. Review your selection before continuing.", + "selectAll": "Select all", + "selectNone": "Select none", + "selectNew": "Select new", + "filterAria": "Filter playlist items", + "filterAll": "All items", + "filterNew": "New", + "filterDuplicates": "Duplicates", + "filterUnavailable": "Unavailable", + "listAria": "Playlist videos", + "untitled": "Untitled video", + "duplicate": "duplicate", + "duplicateUnknown": "duplicate status unknown", + "availabilityAvailable": "Available", + "availabilityDeleted": "Deleted", + "availabilityNeedsAuth": "Authentication required", + "availabilityPremiumOnly": "Premium only", + "availabilityPrivate": "Private", + "availabilitySubscriberOnly": "Subscribers only", + "availabilityUnavailable": "Unavailable", + "availabilityUnknown": "Availability unknown", + "itemSelectionAria": "Select playlist item {ordinal}: {title}", + "itemDetails": "Details", + "selectionStatus": "{count} selected", + "addVideos": "{count, plural, one {Add # video} other {Add # videos}}", + "loadThumbnailAria": "Load thumbnail for {title}", + "loadThumbnail": "Load thumbnail", + "thumbnailAlt": "Thumbnail for {title}", + "thumbnailUnavailable": "Thumbnail unavailable" + }, "tabs": { "queue": "Queue", "options": "Options", @@ -1146,7 +1210,7 @@ "dropzoneRelease": "Release to add files", "dropzoneReleaseSingle": "Release to add 1 file", "dropzoneReleaseMultiple": "Release to add {{count}} files", - "dropzoneAriaLabel": "File upload zone. Drag and drop files or press Enter to browse.", + "dropzoneAriaLabel": "File upload zone. Drag and drop files or use Browse files.", "addFiles": "Browse files", "filesRejected": "{{count}} files rejected", "pasteFromClipboard": "Paste URLs from clipboard", diff --git a/apps/packages/ui/src/components/Common/QuickIngest/AddContentStep.tsx b/apps/packages/ui/src/components/Common/QuickIngest/AddContentStep.tsx index e93f78e5e6..19d1992a5f 100644 --- a/apps/packages/ui/src/components/Common/QuickIngest/AddContentStep.tsx +++ b/apps/packages/ui/src/components/Common/QuickIngest/AddContentStep.tsx @@ -1,5 +1,6 @@ -import React, { useCallback, useMemo, useState } from "react" +import React, { useCallback, useEffect, useMemo, useRef, useState } from "react" import { Button, Input, Tooltip, Typography } from "antd" +import { useVirtualizer } from "@tanstack/react-virtual" import { useTranslation } from "react-i18next" import { AlertTriangle, @@ -14,25 +15,28 @@ import { Plus, } from "lucide-react" import type { - ConferenceDuplicatePolicy, DetectedMediaType, - WizardQueueItem, QueueItemValidation, + WizardQueueItem, } from "./types" import { useIngestWizard } from "./IngestWizardContext" import { useServerCapabilities } from "@/hooks/useServerCapabilities" import { Alert as DesignSystemAlert, Badge } from "@/components/ui/primitives" -import { tldwClient } from "@/services/tldw/TldwApiClient" -import type { PlaylistPreflightResult } from "@/services/tldw/playlist-preflight" import { FileDropZone } from "./QueueTab/FileDropZone" -import { PlaylistPreflightPanel } from "./PlaylistPreflightPanel" import { BatchMetadataPanel } from "./BatchMetadataPanel" +import { usePlaylistInspection } from "./usePlaylistInspection" +import { PlaylistPreflightPanel } from "./PlaylistPreflightPanel" import { QUICK_INGEST_MAX_FILE_SIZE_LABEL, QUICK_INGEST_MAX_FILE_SIZE, } from "./constants" import { normalizeUrlForDedupe } from "@/entries/shared/ingest-payloads" -import { isQuickIngestPlaylistPreflightDetail } from "@/utils/quick-ingest-open" +import { tldwClient } from "@/services/tldw/TldwApiClient" +import { + PlaylistIngestPublicError, + toPlaylistIngestPublicError, +} from "@/services/tldw/playlist-ingest" +import type { PlaylistInspectionCandidate } from "./usePlaylistInspection" // --------------------------------------------------------------------------- // Helpers @@ -74,7 +78,9 @@ const detectTypeFromExtension = (name: string): DetectedMediaType => { } const detectTypeFromMime = (mimeType: string | undefined): DetectedMediaType => { - const normalized = String(mimeType || "").trim().toLowerCase() + const normalized = String(mimeType || "") + .trim() + .toLowerCase() if (!normalized) return "unknown" if (normalized.startsWith("audio/")) return "audio" if (normalized.startsWith("video/")) return "video" @@ -139,9 +145,6 @@ export const detectPlaylistPreflightCandidate = (url: string): boolean => { } } -const isDuplicatePreflightStatus = (status: string | undefined): boolean => - status === "duplicate_existing" || status === "duplicate_in_batch" - const isValidUrl = (raw: string): boolean => { const trimmed = raw.trim() if (!trimmed) return false @@ -153,6 +156,24 @@ const isValidUrl = (raw: string): boolean => { } } +const playlistSourceAliases = ( + normalizedSourceId?: string | null, + sourceUrl?: string | null +): string[] => { + const aliases = [ + normalizedSourceId?.trim() || "", + sourceUrl ? normalizeUrlForDedupe(sourceUrl) : "", + ] + return [...new Set(aliases.filter(Boolean))] +} + +const queueSourceAliases = (item: WizardQueueItem): string[] => [ + ...playlistSourceAliases(item.playlist?.normalizedSourceId, item.url), + ...(item.sourceRef?.kind === "direct_url" + ? playlistSourceAliases(null, item.sourceRef.url) + : []), +] + const formatFileSize = (bytes: number): string => { if (bytes === 0) return "0 B" if (bytes < 1024) return `${bytes} B` @@ -243,29 +264,69 @@ export const AddContentStep: React.FC = ({ quickProcessWarning = null, }) => { const { t } = useTranslation(["option"]) - const { - state, - setQueueItems, - setPlaylistPreflightSeed, - setConferenceBatchMetadata, - goNext, - } = useIngestWizard() - const { - queueItems, - conferenceBatchMetadata, - playlistPreflightSeed, - firstSourceAddMode, - } = state + const { state, updateQueueItems, setPlaylistPreflightSeed, goNext } = useIngestWizard() + const { queueItems, playlistPreflightSeed, firstSourceAddMode } = state const [urlInput, setUrlInput] = useState("") const [pastedTextInput, setPastedTextInput] = useState("") - const [playlistPreflightUrl, setPlaylistPreflightUrl] = useState("") - const [playlistPreflight, setPlaylistPreflight] = useState(null) - const [duplicatePolicy, setDuplicatePolicy] = useState("skip") - const [playlistPreflightLoading, setPlaylistPreflightLoading] = useState(false) - const [playlistPreflightError, setPlaylistPreflightError] = useState(null) - const { capabilities } = useServerCapabilities() - const seededPlaylistUrlRef = React.useRef(null) + const [materializingKeys, setMaterializingKeys] = useState>(() => new Set()) + const [materializationErrors, setMaterializationErrors] = useState>({}) + const [pendingMaterializationCommits, setPendingMaterializationCommits] = useState< + Array<{ candidateKey: string; materializationId: string; occurrenceIds: string[] }> + >([]) + const [queuePlaylistFilter, setQueuePlaylistFilter] = useState("all") + const [queueTypeFilter, setQueueTypeFilter] = useState("all") + const [queueDuplicateFilter, setQueueDuplicateFilter] = useState("all") + const materializingKeysRef = useRef(new Set()) + const queueListRef = useRef(null) + const queueRowRefs = useRef(new Map()) + const queueListOwnsFocusRef = useRef(false) + const activeQueueRowRef = useRef<{ id: string; index: number } | null>(null) + const [activeQueueId, setActiveQueueId] = useState(null) + const unlockMaterialization = useCallback((candidateKey: string) => { + materializingKeysRef.current.delete(candidateKey) + setMaterializingKeys(new Set(materializingKeysRef.current)) + }, []) + const { capabilities, loading: capabilitiesLoading } = useServerCapabilities() + const clearPlaylistPreflightSeed = useCallback( + () => setPlaylistPreflightSeed(null), + [setPlaylistPreflightSeed] + ) + const playlistInspection = usePlaylistInspection({ + enabled: + capabilities === null && capabilitiesLoading + ? null + : capabilities?.hasMediaPlaylistIngestV2 === true, + queueItems, + seed: playlistPreflightSeed, + clearSeed: clearPlaylistPreflightSeed, + }) + const addPlaylistCandidates = playlistInspection.addCandidates + useEffect(() => { + if (pendingMaterializationCommits.length === 0) return + for (const commit of pendingMaterializationCommits) { + const committed = commit.occurrenceIds.every((occurrenceId) => + queueItems.some( + (item) => + item.id === occurrenceId && + item.sourceRef?.kind === "materialized_playlist_item" && + item.sourceRef.materializationId === commit.materializationId && + item.sourceRef.occurrenceId === occurrenceId + ) + ) + if (committed) { + playlistInspection.removeCandidate(commit.candidateKey) + } else { + const message = new PlaylistIngestPublicError("invalid_occurrence_selection").message + setMaterializationErrors((current) => ({ + ...current, + [commit.candidateKey]: message, + })) + } + unlockMaterialization(commit.candidateKey) + } + setPendingMaterializationCommits([]) + }, [pendingMaterializationCommits, playlistInspection, queueItems, unlockMaterialization]) const shouldShowPastedTextInput = firstSourceAddMode === "paste_text" const shouldFocusUrlInput = firstSourceAddMode === "web_url" @@ -280,11 +341,12 @@ export const AddContentStep: React.FC = ({ // Add files from the drop zone const handleFilesAdded = useCallback( (files: File[]) => { - const newItems: WizardQueueItem[] = [] - for (const file of files) { + const newItems = files.map((file): WizardQueueItem => { const detectedType = detectTypeFromFile(file) - const item: WizardQueueItem = { - id: crypto.randomUUID(), + const id = crypto.randomUUID() + return { + id, + sourceRef: { kind: "file_stub", occurrenceId: id }, fileName: file.name, file, detectedType, @@ -293,12 +355,19 @@ export const AddContentStep: React.FC = ({ mimeType: file.type || undefined, validation: { valid: true }, } - item.validation = validateQueueItem(item, [...queueItems, ...newItems]) - newItems.push(item) - } - setQueueItems([...queueItems, ...newItems]) + }) + updateQueueItems((current) => { + const validated: WizardQueueItem[] = [] + for (const item of newItems) { + validated.push({ + ...item, + validation: validateQueueItem(item, [...current, ...validated]), + }) + } + return [...current, ...validated] + }) }, - [queueItems, setQueueItems] + [updateQueueItems] ) // Add URLs from the multi-line input @@ -310,38 +379,58 @@ export const AddContentStep: React.FC = ({ if (lines.length === 0) return - const newItems: WizardQueueItem[] = [] + const playlistCandidates: string[] = [] + const ordinaryLines: string[] = [] for (const url of lines) { + if (detectPlaylistPreflightCandidate(url)) { + playlistCandidates.push(url) + } else { + ordinaryLines.push(url) + } + } + const newItems: WizardQueueItem[] = [] + for (const url of ordinaryLines) { const detectedType = detectTypeFromUrl(url) + const id = crypto.randomUUID() const item: WizardQueueItem = { - id: crypto.randomUUID(), + id, + sourceRef: { kind: "direct_url", occurrenceId: id, url }, url, detectedType, icon: ICON_NAME_MAP[detectedType], fileSize: 0, validation: { valid: true }, } - item.validation = validateQueueItem(item, [...queueItems, ...newItems]) newItems.push(item) } - setQueueItems([...queueItems, ...newItems]) + if (newItems.length > 0) { + updateQueueItems((current) => { + const validated: WizardQueueItem[] = [] + for (const item of newItems) { + validated.push({ + ...item, + validation: validateQueueItem(item, [...current, ...validated]), + }) + } + return [...current, ...validated] + }) + } + addPlaylistCandidates(playlistCandidates) setUrlInput("") - setPlaylistPreflight(null) - setPlaylistPreflightUrl("") - setDuplicatePolicy("skip") - setPlaylistPreflightError(null) - }, [urlInput, queueItems, setQueueItems]) + }, [addPlaylistCandidates, updateQueueItems, urlInput]) const handleAddPastedText = useCallback(() => { if (!pastedTextInput.trim()) return const file = new File([pastedTextInput], "pasted-text.txt", { - type: "text/plain" + type: "text/plain", }) const detectedType = detectTypeFromFile(file) + const id = crypto.randomUUID() const item: WizardQueueItem = { - id: crypto.randomUUID(), + id, + sourceRef: { kind: "file_stub", occurrenceId: id }, fileName: file.name, file, detectedType, @@ -350,178 +439,157 @@ export const AddContentStep: React.FC = ({ mimeType: file.type || undefined, validation: { valid: true }, } - item.validation = validateQueueItem(item, queueItems) - setQueueItems([...queueItems, item]) + updateQueueItems((current) => [ + ...current, + { ...item, validation: validateQueueItem(item, current) }, + ]) setPastedTextInput("") - }, [pastedTextInput, queueItems, setQueueItems]) - - const playlistCandidateUrls = useMemo( - () => - urlInput - .split("\n") - .map((line) => line.trim()) - .filter(Boolean) - .filter(detectPlaylistPreflightCandidate), - [urlInput] - ) - const primaryPlaylistCandidateUrl = playlistCandidateUrls[0] || "" - const shouldOfferPlaylistPreflight = - Boolean(capabilities?.hasMediaPlaylistPreflight) && Boolean(primaryPlaylistCandidateUrl) - - const handlePreviewPlaylist = useCallback(async () => { - if (!primaryPlaylistCandidateUrl) return - setPlaylistPreflightLoading(true) - setPlaylistPreflightError(null) - try { - const result = await tldwClient.preflightPlaylist({ - url: primaryPlaylistCandidateUrl, - max_items: 100, - timeoutMs: 60_000 - }) - setPlaylistPreflight(result) - setPlaylistPreflightUrl(primaryPlaylistCandidateUrl) - setDuplicatePolicy("skip") - } catch (error) { - setPlaylistPreflight(null) - setPlaylistPreflightUrl(primaryPlaylistCandidateUrl) - setPlaylistPreflightError( - error instanceof Error && error.message - ? error.message - : "Playlist preview failed." - ) - } finally { - setPlaylistPreflightLoading(false) - } - }, [primaryPlaylistCandidateUrl]) + }, [pastedTextInput, updateQueueItems]) - React.useEffect(() => { - if (!isQuickIngestPlaylistPreflightDetail(playlistPreflightSeed)) { - return - } - - const seededUrl = playlistPreflightSeed.url.trim() - if (!seededUrl) return - - seededPlaylistUrlRef.current = seededUrl - setUrlInput(seededUrl) - setPlaylistPreflight(null) - setPlaylistPreflightUrl("") - setDuplicatePolicy("skip") - setPlaylistPreflightError(null) - setPlaylistPreflightSeed(null) - }, [playlistPreflightSeed, setPlaylistPreflightSeed]) - - React.useEffect(() => { - if (!seededPlaylistUrlRef.current) return - if (!shouldOfferPlaylistPreflight) return - if (primaryPlaylistCandidateUrl !== seededPlaylistUrlRef.current) return - - seededPlaylistUrlRef.current = null - void handlePreviewPlaylist() - }, [handlePreviewPlaylist, primaryPlaylistCandidateUrl, shouldOfferPlaylistPreflight]) - - const handleAddPreflightItems = useCallback(() => { - if (!playlistPreflight) return - const newItems: WizardQueueItem[] = [] - const selectedItems = playlistPreflight.items.filter( - (item) => item.selected && item.sourceUrl - ) - for (const preflightItem of selectedItems) { - const detectedType = detectTypeFromUrl(preflightItem.sourceUrl) - const item: WizardQueueItem = { - id: crypto.randomUUID(), - url: preflightItem.sourceUrl, - detectedType, - icon: ICON_NAME_MAP[detectedType], - fileSize: 0, - validation: { valid: true }, - playlist: { - playlistId: playlistPreflight.playlistId, - playlistTitle: playlistPreflight.playlistTitle, - ordinal: preflightItem.ordinal, - normalizedSourceId: preflightItem.normalizedSourceId, - duplicateStatus: preflightItem.duplicateStatus - }, - conferenceOverride: { - selected: true, - ...(isDuplicatePreflightStatus(preflightItem.duplicateStatus) - ? { duplicatePolicy } - : {}) - } + const handleMaterializeCandidate = useCallback( + async (candidate: PlaylistInspectionCandidate) => { + if ( + candidate.status !== "ready" || + !candidate.preflightId || + materializingKeysRef.current.has(candidate.key) + ) { + return } - item.validation = validateQueueItem(item, [...queueItems, ...newItems]) - newItems.push(item) - } - if (newItems.length === 0) return - setQueueItems([...queueItems, ...newItems]) - setConferenceBatchMetadata({ - collectionName: - conferenceBatchMetadata?.collectionName || - playlistPreflight.playlistTitle || - "", - conferenceName: conferenceBatchMetadata?.conferenceName, - eventDate: conferenceBatchMetadata?.eventDate, - eventYear: conferenceBatchMetadata?.eventYear, - sharedTags: conferenceBatchMetadata?.sharedTags ?? [], - sourcePlaylistUrl: - conferenceBatchMetadata?.sourcePlaylistUrl || playlistPreflightUrl, - }) - setUrlInput((current) => - current - .split("\n") - .map((line) => line.trim()) - .filter((line) => line && line !== playlistPreflightUrl) - .join("\n") - ) - setPlaylistPreflight(null) - setPlaylistPreflightUrl("") - setDuplicatePolicy("skip") - setPlaylistPreflightError(null) - }, [ - conferenceBatchMetadata, - duplicatePolicy, - playlistPreflight, - playlistPreflightUrl, - queueItems, - setConferenceBatchMetadata, - setQueueItems, - ]) - - const handlePreflightItemSelectionChange = useCallback( - (ordinal: number, selected: boolean) => { - setPlaylistPreflight((current) => { - if (!current) return current - const items = current.items.map((item) => - item.ordinal === ordinal ? { ...item, selected } : item + const selectedOccurrenceIds = candidate.items + .filter( + (item) => + candidate.selectedOccurrenceIds.has(item.occurrenceId) && + Boolean(item.sourceUrl) && + (item.availability === null || item.availability === "available") ) - return { - ...current, - selectedCount: items.filter((item) => item.selected && item.sourceUrl).length, - items - } + .sort((left, right) => left.ordinal - right.ordinal) + .map((item) => item.occurrenceId) + if (selectedOccurrenceIds.length === 0) return + + materializingKeysRef.current.add(candidate.key) + setMaterializingKeys(new Set(materializingKeysRef.current)) + setMaterializationErrors((current) => { + if (!(candidate.key in current)) return current + const next = { ...current } + delete next[candidate.key] + return next }) - }, - [] - ) + try { + const materialization = await tldwClient.materializePlaylistPreflight( + candidate.preflightId, + selectedOccurrenceIds + ) + const expected = new Set(selectedOccurrenceIds) + const returned = new Set(materialization.items.map((item) => item.occurrenceId)) + if ( + materialization.preflightId !== candidate.preflightId || + !materialization.materializationId || + materialization.items.length !== selectedOccurrenceIds.length || + returned.size !== expected.size || + [...expected].some((occurrenceId) => !returned.has(occurrenceId)) || + materialization.items.some((item) => !item.sourceUrl) + ) { + throw new PlaylistIngestPublicError("invalid_occurrence_selection") + } + const existingQueueIds = new Set(queueItems.map((item) => item.id)) + if ([...returned].some((occurrenceId) => existingQueueIds.has(occurrenceId))) { + throw new PlaylistIngestPublicError("invalid_occurrence_selection") + } - const handleDuplicatePolicyChange = useCallback( - (policy: ConferenceDuplicatePolicy) => { - setDuplicatePolicy(policy) - setPlaylistPreflight((current) => { - if (!current) return current - const items = current.items.map((item) => { - if (!item.sourceUrl) return { ...item, selected: false } - if (!isDuplicatePreflightStatus(item.duplicateStatus)) return item - return { ...item, selected: policy !== "skip" } + const inspectedByOccurrence = new Map( + candidate.items.map((item) => [item.occurrenceId, item] as const) + ) + const sortedMaterializedItems = [...materialization.items].sort( + (left, right) => left.ordinal - right.ordinal + ) + updateQueueItems((current) => { + if ( + sortedMaterializedItems.some((item) => + current.some((row) => row.id === item.occurrenceId) + ) + ) { + return current + } + const authoritativeAliases = new Set(current.flatMap(queueSourceAliases)) + const newItems: WizardQueueItem[] = sortedMaterializedItems.map((item) => { + const inspected = inspectedByOccurrence.get(item.occurrenceId) + const normalizedSourceId = item.normalizedSourceId?.trim() || null + const aliases = new Set([ + ...playlistSourceAliases(normalizedSourceId, item.sourceUrl), + ...playlistSourceAliases(inspected?.normalizedSourceId, inspected?.sourceUrl), + ]) + const overlapsSelectedOrQueued = [...aliases].some((alias) => + authoritativeAliases.has(alias) + ) + const duplicateTargetWasExcluded = Boolean( + inspected?.duplicateStatus === "duplicate_in_batch" && + inspected.duplicateOfOccurrenceId && + !expected.has(inspected.duplicateOfOccurrenceId) + ) + const duplicateStatus = + inspected?.duplicateStatus === "duplicate_existing" + ? "duplicate_existing" + : overlapsSelectedOrQueued + ? "duplicate_in_batch" + : inspected?.duplicateStatus === "unknown" + ? "unknown" + : inspected?.duplicateStatus === "duplicate_in_batch" && + !duplicateTargetWasExcluded + ? "duplicate_in_batch" + : aliases.size > 0 + ? "new" + : "unknown" + for (const alias of aliases) authoritativeAliases.add(alias) + return { + id: item.occurrenceId, + kind: "url", + sourceRef: { + kind: "materialized_playlist_item", + materializationId: materialization.materializationId, + occurrenceId: item.occurrenceId, + }, + // Display-only cache. Run serialization uses sourceRef exclusively. + url: item.sourceUrl, + detectedType: "video", + icon: ICON_NAME_MAP.video, + fileSize: 0, + validation: { valid: true }, + playlist: { + playlistId: item.displayMetadata.playlistId ?? candidate.summary?.playlistId, + playlistTitle: + item.displayMetadata.playlistTitle ?? candidate.summary?.summary?.playlistTitle, + ordinal: item.ordinal, + title: item.displayMetadata.title, + channelOrUploader: item.displayMetadata.channelOrUploader, + durationSeconds: item.displayMetadata.durationSeconds, + normalizedSourceId, + duplicateStatus, + sourceUrl: item.sourceUrl, + materializationExpiresAt: materialization.expiresAt, + }, + playlistReview: { selected: true }, + } + }) + return [...current, ...newItems] }) - return { + setPendingMaterializationCommits((current) => [ + ...current.filter((commit) => commit.candidateKey !== candidate.key), + { + candidateKey: candidate.key, + materializationId: materialization.materializationId, + occurrenceIds: sortedMaterializedItems.map((item) => item.occurrenceId), + }, + ]) + } catch (error) { + const publicError = toPlaylistIngestPublicError(error) + setMaterializationErrors((current) => ({ ...current, - selectedCount: items.filter((item) => item.selected && item.sourceUrl).length, - items - } - }) + [candidate.key]: publicError.message, + })) + unlockMaterialization(candidate.key) + } }, - [] + [queueItems, unlockMaterialization, updateQueueItems] ) // Handle Enter key in URL input @@ -538,15 +606,15 @@ export const AddContentStep: React.FC = ({ // Remove an item from the queue const handleRemoveItem = useCallback( (id: string) => { - setQueueItems(queueItems.filter((item) => item.id !== id)) + updateQueueItems((current) => current.filter((item) => item.id !== id)) }, - [queueItems, setQueueItems] + [updateQueueItems] ) // Clear all items const handleClearAll = useCallback(() => { - setQueueItems([]) - }, [setQueueItems]) + updateQueueItems(() => []) + }, [updateQueueItems]) const hasItems = queueItems.length > 0 const selectedItems = useMemo( @@ -558,7 +626,7 @@ export const AddContentStep: React.FC = ({ [selectedItems] ) const invalidItemCount = selectedItems.length - validItemCount - const canProceed = validItemCount > 0 + const canProceed = validItemCount > 0 && !playlistInspection.hasUnresolvedCandidates const canStartProcessing = canProceed && isOnlineForIngest && !isCheckingConnection const hasLargeFiles = useMemo( @@ -568,12 +636,158 @@ export const AddContentStep: React.FC = ({ const ffmpegMissing = capabilities?.ffmpegAvailable === false const hasAvMediaItems = useMemo( + () => queueItems.some((item) => item.detectedType === "audio" || item.detectedType === "video"), + [queueItems] + ) + const hasPlaylistItems = useMemo( + () => queueItems.some((item) => Boolean(item.playlist)), + [queueItems] + ) + useEffect(() => { + if (hasPlaylistItems) return + setQueuePlaylistFilter("all") + setQueueTypeFilter("all") + setQueueDuplicateFilter("all") + }, [hasPlaylistItems]) + const queuePlaylistOptions = useMemo( () => - queueItems.some( - (item) => item.detectedType === "audio" || item.detectedType === "video" + Array.from( + new Set( + queueItems + .map((item) => item.playlist?.playlistTitle?.trim()) + .filter((title): title is string => Boolean(title)) + ) + ).sort((left, right) => left.localeCompare(right)), + [queueItems] + ) + const queueTypeOptions = useMemo( + () => + Array.from(new Set(queueItems.map((item) => item.detectedType))).sort((left, right) => + left.localeCompare(right) ), [queueItems] ) + const filteredQueueItems = useMemo( + () => + queueItems.filter((item) => { + if (queuePlaylistFilter !== "all" && item.playlist?.playlistTitle !== queuePlaylistFilter) { + return false + } + if (queueTypeFilter !== "all" && item.detectedType !== queueTypeFilter) { + return false + } + const duplicateStatus = item.playlist?.duplicateStatus ?? "unknown" + if (queueDuplicateFilter === "duplicates") { + return ( + duplicateStatus === "duplicate_existing" || duplicateStatus === "duplicate_in_batch" + ) + } + if (queueDuplicateFilter !== "all" && duplicateStatus !== queueDuplicateFilter) { + return false + } + return true + }), + [queueDuplicateFilter, queueItems, queuePlaylistFilter, queueTypeFilter] + ) + const queueVirtualizer = useVirtualizer({ + count: filteredQueueItems.length, + getScrollElement: () => queueListRef.current, + estimateSize: () => 76, + overscan: 6, + getItemKey: (index) => filteredQueueItems[index]?.id ?? index, + measureElement: (element) => element?.getBoundingClientRect().height || 76, + }) + const queueVirtualItems = queueVirtualizer.getVirtualItems() + const restoreQueueRowFocus = useCallback((id: string) => { + const attempt = (remaining: number) => { + if (!queueListOwnsFocusRef.current) return + const row = queueRowRefs.current.get(id) + if (row) { + row.focus() + return + } + if (remaining > 0) window.requestAnimationFrame(() => attempt(remaining - 1)) + } + window.requestAnimationFrame(() => attempt(2)) + }, []) + + useEffect(() => { + const handleFocusIn = (event: FocusEvent) => { + const target = event.target + if (target instanceof Node && queueListRef.current?.contains(target)) return + queueListOwnsFocusRef.current = false + } + document.addEventListener("focusin", handleFocusIn) + return () => document.removeEventListener("focusin", handleFocusIn) + }, []) + + useEffect(() => { + if (filteredQueueItems.length === 0) { + activeQueueRowRef.current = null + setActiveQueueId(null) + return + } + const active = activeQueueRowRef.current + if (!active) { + const id = filteredQueueItems[0].id + activeQueueRowRef.current = { id, index: 0 } + setActiveQueueId(id) + return + } + const currentIndex = filteredQueueItems.findIndex((item) => item.id === active.id) + if (currentIndex >= 0) { + active.index = currentIndex + if ( + queueListOwnsFocusRef.current && + !queueRowRefs.current.has(active.id) && + queueVirtualItems.length > 0 + ) { + const nearest = queueVirtualItems.reduce((best, row) => + Math.abs(row.index - currentIndex) < Math.abs(best.index - currentIndex) ? row : best + ) + const nearestItem = filteredQueueItems[nearest.index] + if (nearestItem) { + activeQueueRowRef.current = { id: nearestItem.id, index: nearest.index } + setActiveQueueId(nearestItem.id) + restoreQueueRowFocus(nearestItem.id) + } + } + return + } + const index = Math.min(active.index, filteredQueueItems.length - 1) + const id = filteredQueueItems[index].id + activeQueueRowRef.current = { id, index } + setActiveQueueId(id) + if (queueListOwnsFocusRef.current) { + queueVirtualizer.scrollToIndex(index, { align: "auto" }) + restoreQueueRowFocus(id) + } + }, [filteredQueueItems, queueVirtualItems, queueVirtualizer, restoreQueueRowFocus]) + + const handleQueueRowKeyDown = useCallback( + (event: React.KeyboardEvent, index: number) => { + if (event.target !== event.currentTarget) { + if (event.key === "Escape") { + event.preventDefault() + event.currentTarget.focus() + } + return + } + if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return + event.preventDefault() + const nextIndex = Math.max( + 0, + Math.min(filteredQueueItems.length - 1, index + (event.key === "ArrowDown" ? 1 : -1)) + ) + const nextItem = filteredQueueItems[nextIndex] + if (!nextItem) return + activeQueueRowRef.current = { id: nextItem.id, index: nextIndex } + setActiveQueueId(nextItem.id) + queueVirtualizer.scrollToIndex(nextIndex, { align: "auto" }) + restoreQueueRowFocus(nextItem.id) + }, + [filteredQueueItems, queueVirtualizer, restoreQueueRowFocus] + ) return (
@@ -675,22 +889,59 @@ export const AddContentStep: React.FC = ({
- {shouldOfferPlaylistPreflight && ( - + {playlistInspection.candidates.length > 0 && ( +
+ + {qi("playlistInspection.title", "PLAYLIST INSPECTION")} + + {playlistInspection.candidates.map((candidate) => ( + playlistInspection.cancelCandidate(candidate.key)} + onRetry={() => playlistInspection.retryCandidate(candidate.key)} + onRemove={() => { + if (materializingKeysRef.current.has(candidate.key)) return + playlistInspection.removeCandidate(candidate.key) + }} + onRefresh={() => { + if (materializingKeysRef.current.has(candidate.key)) return + playlistInspection.refreshCandidate(candidate.key) + }} + onAdd={() => void handleMaterializeCandidate(candidate)} + isAdding={materializingKeys.has(candidate.key)} + addError={materializationErrors[candidate.key] ?? null} + onSelectionChange={(occurrenceId, selected) => { + if (materializingKeysRef.current.has(candidate.key)) return + playlistInspection.setCandidateSelection(candidate.key, occurrenceId, selected) + }} + onSelectionBatchChange={(updates) => { + if (materializingKeysRef.current.has(candidate.key)) return + playlistInspection.setCandidateSelections(candidate.key, updates) + }} + /> + ))} + {playlistInspection.hasTruncatedCandidates && ( + + {qi("playlistInspection.moreNotLoaded", "More playlist items are not loaded yet.")} + + )} + {playlistInspection.sessionDuplicateCount > 0 && ( + + )} +
)} @@ -752,14 +1003,10 @@ export const AddContentStep: React.FC = ({ {invalidItemCount > 0 && ( - {qi( - "queueValiditySummary", - "{{valid}} valid / {{invalid}} invalid", - { - valid: validItemCount, - invalid: invalidItemCount, - } - )} + {qi("queueValiditySummary", "{{valid}} valid / {{invalid}} invalid", { + valid: validItemCount, + invalid: invalidItemCount, + })} )} @@ -774,94 +1021,165 @@ export const AddContentStep: React.FC = ({ -
- {queueItems.map((item) => ( -
+ + + + + Showing {filteredQueueItems.length} of {queueItems.length} queued items + +
+ )} - {/* Name/URL and metadata */} -
-
- {item.fileName || item.url || qi("untitledItem", "Untitled")} -
-
- {item.fileSize > 0 && ( - {formatFileSize(item.fileSize)} - )} - {ffmpegMissing && - (item.detectedType === "audio" || - item.detectedType === "video") ? ( - +
+ {queueVirtualItems.map((virtualRow) => { + const item = filteredQueueItems[virtualRow.index] + if (!item) return null + return ( +
{ + queueVirtualizer.measureElement(element) + if (element) queueRowRefs.current.set(item.id, element) + else queueRowRefs.current.delete(item.id) + }} + role="listitem" + tabIndex={activeQueueId === item.id ? 0 : -1} + aria-setsize={filteredQueueItems.length} + aria-posinset={virtualRow.index + 1} + data-occurrence-id={item.sourceRef?.occurrenceId || item.id} + data-index={virtualRow.index} + onFocusCapture={() => { + queueListOwnsFocusRef.current = true + activeQueueRowRef.current = { id: item.id, index: virtualRow.index } + setActiveQueueId(item.id) + }} + onKeyDown={(event) => handleQueueRowKeyDown(event, virtualRow.index)} + className={`absolute left-0 top-0 flex w-full items-center gap-3 rounded-md border px-3 py-2 ${ + !item.validation.valid + ? "border-danger/30 bg-danger/5" + : item.validation.warnings?.length + ? "border-warn/30 bg-warn/5" + : "border-border" + }`} + style={{ transform: `translateY(${virtualRow.start}px)` }} + > + {/* Type icon */} + + {MEDIA_TYPE_ICONS[item.detectedType]} + + + {/* Name/URL and metadata */} +
+
+ {item.playlist?.ordinal && item.playlist?.title + ? `${item.playlist.ordinal}. ${item.playlist.title}` + : item.fileName || item.url || qi("untitledItem", "Untitled")} +
+
+ {item.playlist?.playlistTitle && ( + {item.playlist.playlistTitle} )} - > - - - - ) : ( - - {item.detectedType === "web" - ? "Web page" - : item.detectedType.charAt(0).toUpperCase() + - item.detectedType.slice(1)} - - )} - {item.detectedType !== "unknown" && ( - (auto) - )} -
- {/* Validation errors/warnings */} - {item.validation.errors?.map((err, i) => ( -
- {err} -
- ))} - {item.validation.warnings?.map((warn, i) => ( -
- {warn} + {item.fileSize > 0 && {formatFileSize(item.fileSize)}} + {ffmpegMissing && + (item.detectedType === "audio" || item.detectedType === "video") ? ( + + + + + ) : ( + + {item.detectedType === "web" + ? "Web page" + : item.detectedType.charAt(0).toUpperCase() + + item.detectedType.slice(1)} + + )} + {item.detectedType !== "unknown" && ( + (auto) + )} +
+ {/* Validation errors/warnings */} + {item.validation.errors?.map((err, i) => ( +
+ {err} +
+ ))} + {item.validation.warnings?.map((warn, i) => ( +
+ {warn} +
+ ))} + {item.sourceRef?.kind === "materialized_playlist_item" && item.url && ( +
+ Source details + {item.url} +
+ )}
- ))} -
- - {/* Remove button */} - -
- ))} + + {/* Remove button */} + +
+ ) + })} +
)} @@ -880,7 +1198,7 @@ export const AddContentStep: React.FC = ({ {/* Action buttons */}
- {hasItems && onQuickProcess && ( + {(hasItems || playlistInspection.hasUnresolvedCandidates) && onQuickProcess && (
) diff --git a/apps/packages/ui/src/components/Common/QuickIngest/PlaylistPreflightPanel.tsx b/apps/packages/ui/src/components/Common/QuickIngest/PlaylistPreflightPanel.tsx index 7de9e5d2a3..9e1afe7512 100644 --- a/apps/packages/ui/src/components/Common/QuickIngest/PlaylistPreflightPanel.tsx +++ b/apps/packages/ui/src/components/Common/QuickIngest/PlaylistPreflightPanel.tsx @@ -1,149 +1,708 @@ -import React from "react" -import { Button, Checkbox, List, Radio, Typography } from "antd" -import { ListVideo, Plus, RefreshCw } from "lucide-react" -import { Alert, Badge } from "@/components/ui/primitives" -import type { PlaylistPreflightResult } from "@/services/tldw/playlist-preflight" -import type { ConferenceDuplicatePolicy } from "./types" +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import { Button, Typography } from "antd"; +import { useVirtualizer } from "@tanstack/react-virtual"; +import { ListVideo, RefreshCw } from "lucide-react"; -const isDuplicatePreflightStatus = (status: string | undefined): boolean => - status === "duplicate_existing" || status === "duplicate_in_batch" +import type { + PlaylistInspectionCandidate, + PlaylistSelectionUpdate, +} from "./usePlaylistInspection"; +import type { PlaylistPreflightItem } from "@/services/tldw/playlist-ingest"; +import { Alert, Badge } from "@/components/ui/primitives"; + +export type PlaylistPreflightText = ( + key: string, + fallback: string, + options?: Record, +) => string; type PlaylistPreflightPanelProps = { - candidateUrl: string - loading?: boolean - error?: string | null - result?: PlaylistPreflightResult | null - duplicatePolicy?: ConferenceDuplicatePolicy - onPreview: () => void - onAddItems: () => void - onItemSelectionChange?: (ordinal: number, selected: boolean) => void - onDuplicatePolicyChange?: (policy: ConferenceDuplicatePolicy) => void -} + candidate: PlaylistInspectionCandidate; + qi?: PlaylistPreflightText; + onCancel: () => void; + onRetry: () => void; + onRemove: () => void; + onRefresh: () => void; + onAdd?: () => void; + isAdding?: boolean; + addError?: string | null; + onSelectionChange: (occurrenceId: string, selected: boolean) => void; + onSelectionBatchChange: (updates: readonly PlaylistSelectionUpdate[]) => void; +}; + +type PlaylistFilter = "all" | "new" | "duplicates" | "unavailable"; + +const fallbackText: PlaylistPreflightText = (_key, fallback, options) => + fallback.replace(/\{\{(\w+)\}\}/g, (_match, token: string) => + String(options?.[token] ?? ""), + ); + +const isEligible = (item: PlaylistPreflightItem): boolean => + Boolean(item.sourceUrl) && + (item.availability === null || item.availability === "available"); + +const isDuplicate = ( + item: PlaylistPreflightItem, + sessionDuplicates: ReadonlySet, +): boolean => + item.duplicateStatus === "duplicate_existing" || + item.duplicateStatus === "duplicate_in_batch" || + sessionDuplicates.has(item.occurrenceId); + +const isNew = ( + item: PlaylistPreflightItem, + sessionDuplicates: ReadonlySet, +): boolean => + item.duplicateStatus === "new" && !sessionDuplicates.has(item.occurrenceId); + +const statusBadge = ( + status: PlaylistInspectionCandidate["status"], +): "info" | "success" | "warning" | "danger" => { + if (status === "ready") return "success"; + if (status === "queued" || status === "inspecting") return "info"; + if (status === "failed" || status === "blocked") return "danger"; + return "warning"; +}; + +const statusCopy = ( + candidate: PlaylistInspectionCandidate, + qi: PlaylistPreflightText, +): { label: string; message: string } => { + switch (candidate.status) { + case "ready": + return { + label: qi("playlistInspection.readyLabel", "Inspection ready"), + message: qi( + "playlistInspection.readyMessage", + "{{count}} playlist items loaded for review.", + { count: candidate.items.length }, + ), + }; + case "unavailable": + return { + label: qi( + "playlistInspection.unavailableLabel", + "Inspection unavailable", + ), + message: qi( + "playlistInspection.unavailableMessage", + "Playlist inspection is unavailable on this server. Update the server or remove this playlist.", + ), + }; + case "failed": + return { + label: qi("playlistInspection.failedLabel", "Inspection failed"), + message: + candidate.error?.message ?? + qi( + "playlistInspection.failedMessage", + "Playlist inspection failed. Try again.", + ), + }; + case "blocked": + return { + label: qi("playlistInspection.blockedLabel", "Inspection blocked"), + message: + candidate.error?.message ?? + qi( + "playlistInspection.blockedMessage", + "Playlist inspection was blocked. Remove it or try again.", + ), + }; + case "expired": + return { + label: qi("playlistInspection.expiredLabel", "Inspection expired"), + message: + candidate.error?.message ?? + qi( + "playlistInspection.expiredMessage", + "Playlist inspection expired. Try again.", + ), + }; + case "cancelled": + return { + label: qi("playlistInspection.cancelledLabel", "Inspection cancelled"), + message: + candidate.error?.message ?? + qi( + "playlistInspection.cancelledMessage", + "Playlist inspection was cancelled.", + ), + }; + default: + return { + label: qi("playlistInspection.inspectingLabel", "Inspecting playlist"), + message: qi( + "playlistInspection.inspectingMessage", + "Loading playlist details from the server.", + ), + }; + } +}; + +const formatDuration = (seconds: number | null | undefined): string | null => { + if (seconds === null || seconds === undefined || seconds < 0) return null; + const minutes = Math.floor(seconds / 60); + return `${minutes}:${String(Math.floor(seconds % 60)).padStart(2, "0")}`; +}; + +const availabilityCopy = ( + availability: PlaylistPreflightItem["availability"], + qi: PlaylistPreflightText, +): string => { + switch (availability) { + case "available": + return qi("playlistPreflight.availabilityAvailable", "Available"); + case "deleted": + return qi("playlistPreflight.availabilityDeleted", "Deleted"); + case "needs_auth": + return qi( + "playlistPreflight.availabilityNeedsAuth", + "Authentication required", + ); + case "premium_only": + return qi("playlistPreflight.availabilityPremiumOnly", "Premium only"); + case "private": + return qi("playlistPreflight.availabilityPrivate", "Private"); + case "subscriber_only": + return qi( + "playlistPreflight.availabilitySubscriberOnly", + "Subscribers only", + ); + case "unavailable": + return qi("playlistPreflight.availabilityUnavailable", "Unavailable"); + default: + return qi( + "playlistPreflight.availabilityUnknown", + "Availability unknown", + ); + } +}; export const PlaylistPreflightPanel: React.FC = ({ - candidateUrl, - loading = false, - error = null, - result = null, - duplicatePolicy = "skip", - onPreview, - onAddItems, - onItemSelectionChange, - onDuplicatePolicyChange + candidate, + qi = fallbackText, + onCancel, + onRetry, + onRemove, + onRefresh, + onAdd, + isAdding = false, + addError = null, + onSelectionChange, + onSelectionBatchChange, }) => { - const selectedCount = - result?.items.filter((item) => item.selected && item.sourceUrl).length ?? 0 - const duplicateCount = - result?.duplicateCount ?? - result?.items.filter((item) => isDuplicatePreflightStatus(item.duplicateStatus)).length ?? - 0 + const [filter, setFilter] = useState("all"); + const [requestedThumbnails, setRequestedThumbnails] = useState>( + () => new Set(), + ); + const [failedThumbnails, setFailedThumbnails] = useState>( + () => new Set(), + ); + const scrollParentRef = useRef(null); + const rowRefs = useRef(new Map()); + const listOwnsFocusRef = useRef(false); + const activeListRowRef = useRef<{ + occurrenceId: string; + index: number; + } | null>(null); + const copy = statusCopy(candidate, qi); + const isActive = + candidate.status === "queued" || candidate.status === "inspecting"; + const canRetry = + ((candidate.status === "failed" || candidate.status === "blocked") && + candidate.error?.retryable === true) || + candidate.status === "expired" || + candidate.status === "cancelled"; + const selectedCount = candidate.selectedOccurrenceIds.size; + const sessionDuplicates = candidate.sessionDuplicateOccurrenceIds; + const duplicateCount = candidate.items.filter((item) => + isDuplicate(item, sessionDuplicates), + ).length; + const filteredItems = useMemo( + () => + candidate.items.filter((item) => { + if (filter === "new") + return isNew(item, sessionDuplicates) && isEligible(item); + if (filter === "duplicates") + return isDuplicate(item, sessionDuplicates); + if (filter === "unavailable") return !isEligible(item); + return true; + }), + [candidate.items, filter, sessionDuplicates], + ); + // TanStack Virtual exposes an imperative object that React Compiler intentionally skips. + // eslint-disable-next-line react-hooks/incompatible-library + const virtualizer = useVirtualizer({ + count: filteredItems.length, + getScrollElement: () => scrollParentRef.current, + estimateSize: () => 76, + overscan: 5, + getItemKey: (index) => filteredItems[index]?.occurrenceId ?? index, + measureElement: (element) => element?.getBoundingClientRect().height || 76, + }); + const virtualItems = virtualizer.getVirtualItems(); + + const applyVisibleSelection = useCallback( + (mode: "all" | "none" | "new") => { + const updates = filteredItems.filter(isEligible).map((item) => ({ + occurrenceId: item.occurrenceId, + selected: + mode === "all" || (mode === "new" && isNew(item, sessionDuplicates)), + })); + if (updates.length > 0) onSelectionBatchChange(updates); + }, + [filteredItems, onSelectionBatchChange, sessionDuplicates], + ); + + const restoreRowFocus = useCallback((occurrenceId: string) => { + const attempt = (remaining: number) => { + if (!listOwnsFocusRef.current) return; + const row = rowRefs.current.get(occurrenceId); + if (row) { + row.focus(); + return; + } + if (remaining > 0) window.setTimeout(() => attempt(remaining - 1), 0); + }; + window.setTimeout(() => attempt(2), 0); + }, []); + + useEffect(() => { + const handleFocusIn = (event: FocusEvent) => { + const target = event.target; + if (target instanceof Node && scrollParentRef.current?.contains(target)) { + return; + } + listOwnsFocusRef.current = false; + }; + document.addEventListener("focusin", handleFocusIn); + return () => document.removeEventListener("focusin", handleFocusIn); + }, []); + + useEffect(() => { + const activeRow = activeListRowRef.current; + if ( + !listOwnsFocusRef.current || + !activeRow || + rowRefs.current.has(activeRow.occurrenceId) || + virtualItems.length === 0 + ) { + return; + } + const nearestVirtualRow = virtualItems.reduce((nearest, current) => + Math.abs(current.index - activeRow.index) < + Math.abs(nearest.index - activeRow.index) + ? current + : nearest, + ); + const nearestItem = filteredItems[nearestVirtualRow.index]; + if (nearestItem) restoreRowFocus(nearestItem.occurrenceId); + }, [filteredItems, restoreRowFocus, virtualItems]); + + const handleRowKeyDown = useCallback( + (event: React.KeyboardEvent, index: number) => { + if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return; + event.preventDefault(); + const targetIndex = Math.max( + 0, + Math.min( + filteredItems.length - 1, + index + (event.key === "ArrowDown" ? 1 : -1), + ), + ); + const target = filteredItems[targetIndex]; + if (!target) return; + virtualizer.scrollToIndex(targetIndex, { align: "auto" }); + restoreRowFocus(target.occurrenceId); + }, + [filteredItems, restoreRowFocus, virtualizer], + ); return (
-
- {error && ( - - )} +
+
+ + {copy.label} + + + {copy.message} + +
+
+ {isActive && ( + + )} + {canRetry && ( + + )} + {!isActive && ( + + )} +
+
- {result && ( -
+ {candidate.status === "ready" && ( +
- {result.itemCount} items - {selectedCount} selected + + {qi("playlistPreflight.itemCount", "{{count}} items", { + count: candidate.items.length, + })} + + + {qi("playlistPreflight.selectedCount", "{{count}} selected", { + count: selectedCount, + })} + {duplicateCount > 0 && ( - {duplicateCount} duplicates + + {qi( + "playlistPreflight.duplicateCount", + "{{count}} duplicates", + { count: duplicateCount }, + )} + )}
- {duplicateCount > 0 && ( -
- - Duplicate policy - - + )} + + {addError && } + +
+ + + + +
- ( - - - onItemSelectionChange?.(item.ordinal, event.target.checked) - } - /> -
- - {item.ordinal}. {item.title} - - - {item.sourceUrl} - -
- {isDuplicatePreflightStatus(item.duplicateStatus) && ( - - duplicate - - )} - - )} - /> +
+
+ {virtualItems.map((virtualRow) => { + const item = filteredItems[virtualRow.index]; + if (!item) return null; + const eligible = isEligible(item); + const title = + item.displayMetadata.title || + qi("playlistPreflight.untitled", "Untitled video"); + const duration = formatDuration( + item.displayMetadata.durationSeconds, + ); + const thumbnailUrl = item.displayMetadata.thumbnailUrl; + const thumbnailKey = thumbnailUrl + ? `${item.occurrenceId}\u0000${thumbnailUrl}` + : null; + const secondary = [ + item.displayMetadata.playlistTitle, + item.displayMetadata.channelOrUploader, + duration, + availabilityCopy(item.availability, qi), + isDuplicate(item, sessionDuplicates) + ? qi("playlistPreflight.duplicate", "duplicate") + : item.duplicateStatus === "unknown" + ? qi( + "playlistPreflight.duplicateUnknown", + "duplicate status unknown", + ) + : null, + ].filter(Boolean); + return ( +
{ + if (element) { + rowRefs.current.set(item.occurrenceId, element); + virtualizer.measureElement(element); + } else { + rowRefs.current.delete(item.occurrenceId); + } + }} + role="listitem" + tabIndex={0} + aria-setsize={filteredItems.length} + aria-posinset={virtualRow.index + 1} + data-occurrence-id={item.occurrenceId} + data-index={virtualRow.index} + onFocusCapture={() => { + listOwnsFocusRef.current = true; + activeListRowRef.current = { + occurrenceId: item.occurrenceId, + index: virtualRow.index, + }; + }} + onKeyDown={(event) => + handleRowKeyDown(event, virtualRow.index) + } + className="absolute left-0 top-0 flex w-full items-start gap-2 border-b border-border px-2 py-2" + style={{ transform: `translateY(${virtualRow.start}px)` }} + > + + onSelectionChange( + item.occurrenceId, + event.target.checked, + ) + } + /> +
+ + {item.ordinal}. {title} + + + {secondary.join(" • ")} + + {(item.sourceUrl || thumbnailUrl) && ( +
+ + {qi("playlistPreflight.itemDetails", "Details")} + + {item.sourceUrl && {item.sourceUrl}} + {thumbnailUrl && thumbnailKey && ( +
+ {!requestedThumbnails.has(thumbnailKey) ? ( + + ) : failedThumbnails.has(thumbnailKey) ? ( + + {qi( + "playlistPreflight.thumbnailUnavailable", + "Thumbnail unavailable", + )} + + ) : ( + // This component also runs in the browser extension, + // where Next.js Image is unavailable. + // eslint-disable-next-line @next/next/no-img-element + {qi( { + setFailedThumbnails((current) => + new Set(current).add(thumbnailKey), + ); + }} + /> + )} +
+ )} +
+ )} +
+
+ ); + })} +
+
-
+
+ + {qi("playlistPreflight.selectionStatus", "{{count}} selected", { + count: selectedCount, + })} +
)}
- ) -} + ); +}; -export default PlaylistPreflightPanel +export default PlaylistPreflightPanel; diff --git a/apps/packages/ui/src/components/Common/QuickIngest/ProcessingStep.tsx b/apps/packages/ui/src/components/Common/QuickIngest/ProcessingStep.tsx index 710b34f3dc..3622b8aa1c 100644 --- a/apps/packages/ui/src/components/Common/QuickIngest/ProcessingStep.tsx +++ b/apps/packages/ui/src/components/Common/QuickIngest/ProcessingStep.tsx @@ -1,8 +1,8 @@ -import React, { useCallback, useMemo, useState } from "react" +import React, { useCallback, useEffect, useMemo, useRef, useState } from "react" import { useTranslation } from "react-i18next" +import { useVirtualizer } from "@tanstack/react-virtual" import { Check, - Circle, Loader2, X, Minimize2, @@ -17,19 +17,20 @@ import { AlertTriangle, } from "lucide-react" import type { ItemProgress, ItemProgressStatus, WizardQueueItem } from "./types" -import { useIngestWizard } from "./IngestWizardContext" +import { + buildProcessingTransition, + useIngestWizard, +} from "./IngestWizardContext" import { useQuickIngestSessionStore } from "@/store/quick-ingest-session" +import { QUICK_INGEST_ACCEPT_STRING } from "./constants" +import { validateQuickIngestFile } from "./QueueTab/FileDropZone" +import { submitQuickIngestBatch } from "@/services/tldw/quick-ingest-batch" +import { readQuickIngestFileBytes } from "./file-bytes" // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- -/** - * Ordered stages for per-item multi-stage progress indicator. - */ -const PROCESSING_STAGES = ["uploading", "processing", "analyzing", "storing"] as const -type ProcessingStage = (typeof PROCESSING_STAGES)[number] - /** * Map of detected media types to lucide icon components. */ @@ -53,6 +54,8 @@ const TERMINAL_STATUSES = new Set([ "cancelled", ]) +const LIST_VIRTUALIZATION_THRESHOLD = 100 + type FailedProcessingItem = { id: string label: string @@ -61,6 +64,9 @@ type FailedProcessingItem = { error?: string } +type LifecycleGroup = "active" | "attention" | "terminal" +type ProcessingFilter = "all" | LifecycleGroup + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -85,79 +91,31 @@ const formatEstimated = (seconds: number): string => { return `~${m} min remaining` } -/** - * Determine the visual state of each stage dot relative to the item's current status. - */ -const getStageState = ( - stage: ProcessingStage, - itemStatus: ItemProgressStatus -): "done" | "active" | "pending" | "failed" => { - const stageIndex = PROCESSING_STAGES.indexOf(stage) - const activeIndex = PROCESSING_STAGES.indexOf(itemStatus as ProcessingStage) - - if (itemStatus === "complete") return "done" - if (itemStatus === "failed") { - // Stages before the failure point are done; the failure point is failed; rest pending - if (activeIndex >= 0) { - if (stageIndex < activeIndex) return "done" - if (stageIndex === activeIndex) return "failed" - return "pending" - } - // If status doesn't map to a stage (e.g. failed during queued), mark first as failed - return stageIndex === 0 ? "failed" : "pending" +const lifecycleGroup = ( + progress: ItemProgress, + item?: WizardQueueItem +): LifecycleGroup => { + if ( + progress.lifecycleState === "terminal" || + (!progress.lifecycleState && TERMINAL_STATUSES.has(progress.status)) + ) { + return "terminal" } - if (itemStatus === "cancelled" || itemStatus === "queued") return "pending" - - // Active processing: check relative position - if (activeIndex < 0) return "pending" - if (stageIndex < activeIndex) return "done" - if (stageIndex === activeIndex) return "active" - return "pending" -} - -// --------------------------------------------------------------------------- -// Sub-components -// --------------------------------------------------------------------------- - -type StageIndicatorProps = { - stage: ProcessingStage - visualState: "done" | "active" | "pending" | "failed" - label: string + if ( + progress.lifecycleState === "status_unavailable" || + progress.lifecycleState === "cancellation_requested" || + (progress.lifecycleState === "awaiting_upload" && !item?.file) + ) { + return "attention" + } + return "active" } -const StageIndicator: React.FC = ({ - stage: _stage, - visualState, - label, -}) => { - return ( -
- - {visualState === "done" ? ( - - - {label} - -
- ) +const itemDisplayName = (item: WizardQueueItem): string => { + if (item.playlist?.ordinal && item.playlist.title) { + return `${item.playlist.ordinal}. ${item.playlist.title}` + } + return item.playlist?.title || item.fileName || item.url || item.id } // --------------------------------------------------------------------------- @@ -169,29 +127,68 @@ type ItemRowProps = { progress: ItemProgress qi: (key: string, defaultValue: string, options?: Record) => string onCancel: (id: string) => void + onCheckStatus: (id: string) => void + onReconnect: () => void + onReselectFile: (id: string, file: File) => void + onRetryUpload: (id: string) => void } -const ItemRow: React.FC = ({ item, progress, qi, onCancel }) => { +const ItemRow: React.FC = ({ + item, + progress, + qi, + onCancel, + onCheckStatus, + onReconnect, + onReselectFile, + onRetryUpload, +}) => { const IconComponent = TYPE_ICON_MAP[item.detectedType] || File - const displayName = item.fileName || item.url || item.id - const isTerminal = TERMINAL_STATUSES.has(progress.status) - const isActive = - progress.status !== "queued" && - progress.status !== "complete" && - progress.status !== "failed" && - progress.status !== "cancelled" - - const stageLabels: Record = useMemo( - () => ({ - uploading: qi("processing.stage.upload", "Upload"), - processing: qi("processing.stage.process", "Process"), - analyzing: qi("processing.stage.analyze", "Analyze"), - storing: qi("processing.stage.store", "Store"), - }), - [qi] - ) + const displayName = itemDisplayName(item) + const group = lifecycleGroup(progress, item) + const isTerminal = group === "terminal" + const isActive = group === "active" + const replacementInputRef = useRef(null) const statusLabel = useMemo(() => { + switch (progress.lifecycleState) { + case "staged": + case "preparing": + return qi("processing.status.preparing", "Preparing") + case "awaiting_upload": + return item.file + ? qi("processing.status.awaitingUpload", "Awaiting upload") + : qi("processing.status.fileReattachRequired", "File reattach required") + case "submit_pending": + return qi("processing.status.submitPending", "Submit pending") + case "queued": + return qi("processing.status.queued", "Queued") + case "running": + return qi("processing.status.running", "Running") + case "cancellation_requested": + return qi("processing.status.cancellationRequested", "Cancellation requested") + case "status_unavailable": + return qi("processing.status.statusUnavailable", "Status unavailable") + case "terminal": + switch (progress.terminalOutcome) { + case "included_existing": + return qi("processing.outcome.includedExisting", "Included existing") + case "metadata_updated": + return qi("processing.outcome.metadataUpdated", "Metadata updated") + case "skipped_existing": + return qi("processing.outcome.skippedExisting", "Skipped existing") + case "submit_failed": + return qi("processing.outcome.submitFailed", "Submit failed") + case "processing_failed": + return qi("processing.outcome.processingFailed", "Processing failed") + case "metadata_update_failed": + return qi("processing.outcome.metadataUpdateFailed", "Metadata update failed") + case "cancelled": + return qi("processing.status.cancelled", "Cancelled") + default: + return qi("processing.outcome.completed", "Completed") + } + } switch (progress.status) { case "queued": return qi("processing.status.queued", "Queued") @@ -212,7 +209,7 @@ const ItemRow: React.FC = ({ item, progress, qi, onCancel }) => { default: return "" } - }, [progress.status, qi]) + }, [item.file, progress.lifecycleState, progress.status, progress.terminalOutcome, qi]) const handleCancel = useCallback( (e: React.MouseEvent) => { @@ -279,46 +276,16 @@ const ItemRow: React.FC = ({ item, progress, qi, onCancel }) => {
- {/* Multi-stage progress indicator */} - {progress.status !== "queued" && ( -
- {/* Stage dots with connectors */} -
- {PROCESSING_STAGES.map((stage, idx) => { - const state = getStageState(stage, progress.status) - return ( - - {idx > 0 && ( - + {progress.currentStage && progress.currentStage !== statusLabel && ( +

{progress.currentStage}

+ )} - {/* Progress bar for active items */} - {isActive && ( -
-
-
- )} + {isActive && progress.progressPercent > 0 && ( +
+
)} @@ -328,8 +295,75 @@ const ItemRow: React.FC = ({ item, progress, qi, onCancel }) => { )}
+ {progress.lifecycleState === "status_unavailable" && ( +
+ + +
+ )} + + {progress.lifecycleState === "awaiting_upload" && !item.file && ( +
+ { + const file = event.currentTarget.files?.[0] + if (file) onReselectFile(item.id, file) + event.currentTarget.value = "" + }} + /> + +
+ )} + + {progress.lifecycleState === "awaiting_upload" && item.file && ( + + )} + {/* Cancel button */} - {!isTerminal && ( + {!isTerminal && progress.lifecycleState !== "cancellation_requested" && (
)} @@ -661,23 +1012,113 @@ export const ProcessingStep: React.FC = ({ onCancelAll }) =
)} +
+ + +
+ {/* Item list */} -
- {processingState.perItemProgress.map((progress) => { - const queueItem = queueItemMap.get(progress.id) - if (!queueItem) return null - - return ( -
- -
- ) - })} +
+ {!usesVirtualProcessing && ( +
+ {filteredProgress.map((progress, index) => { + const queueItem = queueItemMap.get(progress.id) + if (!queueItem) return null + const group = lifecycleGroup(progress, queueItem) + return ( +
+ +
+ ) + })} +
+ )} + {usesVirtualProcessing &&
+ {processingVirtualItems.map((virtualRow) => { + const progress = filteredProgress[virtualRow.index] + const queueItem = progress ? queueItemMap.get(progress.id) : undefined + if (!progress || !queueItem) return null + const group = lifecycleGroup(progress, queueItem) + + return ( +
setFocusedProcessingIndex(virtualRow.index)} + onKeyDown={(event) => + handleProcessingRowKeyDown(event, virtualRow.index) + } + className="absolute left-0 top-0 w-full pb-2" + style={{ transform: `translateY(${virtualRow.start}px)` }} + > + +
+ ) + })} +
}
{/* Summary bar */} diff --git a/apps/packages/ui/src/components/Common/QuickIngest/QueueTab/FileDropZone.tsx b/apps/packages/ui/src/components/Common/QuickIngest/QueueTab/FileDropZone.tsx index b1493b772d..4ff1bf8ad1 100644 --- a/apps/packages/ui/src/components/Common/QuickIngest/QueueTab/FileDropZone.tsx +++ b/apps/packages/ui/src/components/Common/QuickIngest/QueueTab/FileDropZone.tsx @@ -23,6 +23,36 @@ interface FileDropZoneProps { className?: string } +const QUICK_INGEST_ACCEPT_ENTRIES = QUICK_INGEST_ACCEPT_STRING.split(",") + .map((entry) => entry.trim().toLowerCase()) + .filter(Boolean) +const QUICK_INGEST_VALID_EXTENSIONS = QUICK_INGEST_ACCEPT_ENTRIES.filter( + (entry) => entry.startsWith(".") +) +const QUICK_INGEST_VALID_MIME_TYPES = QUICK_INGEST_ACCEPT_ENTRIES.filter( + (entry) => entry.includes("/") +) + +const quickIngestMimeTypeIsAllowed = (mimeType: string): boolean => { + const normalized = mimeType.toLowerCase() + if (!normalized) return false + if (QUICK_INGEST_VALID_MIME_TYPES.includes(normalized)) return true + const category = normalized.split("/")[0] + return QUICK_INGEST_VALID_MIME_TYPES.includes(`${category}/*`) +} + +export const validateQuickIngestFile = (file: File): string | null => { + if (file.size > QUICK_INGEST_MAX_FILE_SIZE) { + return `${file.name} exceeds ${QUICK_INGEST_MAX_FILE_SIZE_LABEL} quick-ingest limit` + } + const fileExtension = `.${file.name.split(".").pop()?.toLowerCase() || ""}` + const validExtension = QUICK_INGEST_VALID_EXTENSIONS.includes(fileExtension) + const validMime = file.type ? quickIngestMimeTypeIsAllowed(file.type) : false + return validExtension || validMime + ? null + : `${file.name} is not a supported file type` +} + /** * Drag-and-drop file zone for Quick Ingest. * Based on pattern from Flashcards/FileDropZone.tsx with: @@ -57,7 +87,7 @@ export const FileDropZone: React.FC = ({ return } if (disabled || hasAutoFocusedRef.current) return - dropZoneRef.current?.focus() + dropZoneRef.current?.querySelector("button")?.focus() hasAutoFocusedRef.current = true }, [autoFocus, disabled]) @@ -67,51 +97,14 @@ export const FileDropZone: React.FC = ({ [t] ) - const acceptEntries = React.useMemo(() => { - return QUICK_INGEST_ACCEPT_STRING.split(",") - .map((entry) => entry.trim().toLowerCase()) - .filter(Boolean) - }, []) - - // Validate extensions from accept string - const validExtensions = React.useMemo(() => { - return acceptEntries.filter((entry) => entry.startsWith(".")) - }, [acceptEntries]) - - const validMimeTypes = React.useMemo(() => { - return acceptEntries.filter((entry) => entry.includes("/")) - }, [acceptEntries]) - const isMimeTypeAllowed = useCallback( - (mimeType: string) => { - const normalized = mimeType.toLowerCase() - if (!normalized) return false - if (validMimeTypes.includes(normalized)) return true - const category = normalized.split("/")[0] - return validMimeTypes.includes(`${category}/*`) - }, - [validMimeTypes] + (mimeType: string) => quickIngestMimeTypeIsAllowed(mimeType), + [] ) const validateFile = useCallback( - (file: File): string | null => { - // Size check - if (file.size > QUICK_INGEST_MAX_FILE_SIZE) { - return `${file.name} exceeds ${QUICK_INGEST_MAX_FILE_SIZE_LABEL} quick-ingest limit` - } - - // Type check - allow if extension matches OR MIME type matches - const fileExt = "." + (file.name.split(".").pop()?.toLowerCase() || "") - const isValidExtension = validExtensions.includes(fileExt) - const isValidMime = file.type ? isMimeTypeAllowed(file.type) : false - - if (!isValidExtension && !isValidMime) { - return `${file.name} is not a supported file type` - } - - return null - }, - [isMimeTypeAllowed, validExtensions] + (file: File): string | null => validateQuickIngestFile(file), + [] ) const resetDragState = useCallback(() => { @@ -241,25 +234,18 @@ export const FileDropZone: React.FC = ({ } }, [disabled]) - const handleKeyDown = (event: React.KeyboardEvent) => { - if (event.currentTarget !== event.target) return - if (event.key !== "Enter" && event.key !== " ") return - event.preventDefault() - handleBrowseClick(event) - } - // Determine visual state const getStateStyles = () => { if (disabled) { return "border-border bg-surface2 opacity-50 cursor-not-allowed" } if (isDragging && isDragReject) { - return "border-danger/50 bg-danger/10 cursor-pointer" + return "border-danger/50 bg-danger/10" } if (isDragging) { - return "border-primary bg-primary/5 cursor-pointer" + return "border-primary bg-primary/5" } - return "border-border bg-surface2 hover:border-primary/50 cursor-pointer" + return "border-border bg-surface2 hover:border-primary/50" } const releaseLabel = @@ -288,17 +274,14 @@ export const FileDropZone: React.FC = ({ onDragOver={handleDragOver} onDragLeave={handleDragLeave} onDrop={handleDrop} - onClick={handleBrowseClick} - onKeyDown={handleKeyDown} className={`mt-3 w-full rounded-md border border-dashed px-4 py-4 text-center transition-colors ${getStateStyles()} ${className || ""}`} data-testid="qi-file-dropzone" - role="button" + role="group" aria-label={qi( "dropzoneAriaLabel", - "File upload zone. Drag and drop files or press Enter to browse." + "File upload zone. Drag and drop files or use Browse files." )} aria-disabled={disabled} - tabIndex={disabled ? -1 : 0} >
{/* Icon based on state */} diff --git a/apps/packages/ui/src/components/Common/QuickIngest/ReviewStep.tsx b/apps/packages/ui/src/components/Common/QuickIngest/ReviewStep.tsx index 4d7c98c26e..0750c490bc 100644 --- a/apps/packages/ui/src/components/Common/QuickIngest/ReviewStep.tsx +++ b/apps/packages/ui/src/components/Common/QuickIngest/ReviewStep.tsx @@ -1,5 +1,6 @@ -import React, { useCallback, useMemo } from "react" +import React, { useCallback, useEffect, useMemo, useRef, useState } from "react" import { useTranslation } from "react-i18next" +import { useVirtualizer } from "@tanstack/react-virtual" import { Alert as DesignSystemAlert } from "@/components/ui/primitives" import { AlertTriangle, @@ -14,8 +15,15 @@ import { File, } from "lucide-react" import type { DetectedMediaType, IngestPreset, PresetConfig, WizardQueueItem } from "./types" -import { useIngestWizard } from "./IngestWizardContext" +import { + buildPlaylistIngestRunRequest, + MAX_PLAYLIST_RUN_INPUTS, + useIngestWizard, + type IngestWizardState, +} from "./IngestWizardContext" import { estimateTotalSeconds, formatEstimate } from "./timeEstimation" +import { ItemMetadataTable } from "./ItemMetadataTable" +import type { QuickIngestPersistenceStatus } from "@/db/dexie/quick-ingest" // --------------------------------------------------------------------------- // Helpers @@ -25,6 +33,8 @@ const LARGE_FILE_THRESHOLD = 50 * 1024 * 1024 // 50 MB const LONG_TIME_THRESHOLD = 15 * 60 // 15 minutes in seconds const LARGE_BATCH_THRESHOLD = 5 +type ReviewFilter = "selected" | "duplicates" | "policy" + /** * Return a human-readable file size string (e.g., "42 MB", "1.2 GB"). */ @@ -79,6 +89,9 @@ const TYPE_ICONS: Record = { unknown: File, } +const getQueueItemOccurrenceId = (item: WizardQueueItem): string => + item.sourceRef?.occurrenceId || item.id + // --------------------------------------------------------------------------- // Component // --------------------------------------------------------------------------- @@ -88,6 +101,11 @@ type ReviewStepProps = { isCheckingConnection?: boolean connectionRecoveryMessage?: string onRetryConnection?: () => void + persistenceStatus?: QuickIngestPersistenceStatus + isSubmissionOwner?: boolean + onCheckSubmissionOwnership?: () => Promise + onBeginProcessing?: (state: IngestWizardState) => Promise + submissionStartError?: string | null } export const ReviewStep: React.FC = ({ @@ -95,9 +113,24 @@ export const ReviewStep: React.FC = ({ isCheckingConnection = false, connectionRecoveryMessage, onRetryConnection, + persistenceStatus = "ready", + isSubmissionOwner = true, + onCheckSubmissionOwnership, + onBeginProcessing, + submissionStartError, }) => { const { t } = useTranslation(["option"]) - const { state, goBack, goNext, startProcessing } = useIngestWizard() + const { state, goBack, startProcessing } = useIngestWizard() + + const [filter, setFilter] = useState("selected") + const reviewListRef = useRef(null) + const reviewRowRefs = useRef(new Map()) + const reviewListOwnsFocusRef = useRef(false) + const activeReviewRowRef = useRef<{ id: string; index: number } | null>(null) + const [activeReviewId, setActiveReviewId] = useState(null) + const [isCheckingSubmissionOwnership, setIsCheckingSubmissionOwnership] = + useState(false) + const [isStartingProcessing, setIsStartingProcessing] = useState(false) const qi = useCallback( (key: string, defaultValue: string, options?: Record) => @@ -109,9 +142,134 @@ export const ReviewStep: React.FC = ({ const { queueItems, selectedPreset, presetConfig, conferenceBatchMetadata } = state const selectedQueueItems = useMemo( - () => queueItems.filter((item) => item.conferenceOverride?.selected !== false), + () => + queueItems.filter( + (item) => item.conferenceOverride?.selected !== false && item.playlistReview?.selected !== false + ), [queueItems] ) + const filteredReviewItems = useMemo( + () => + queueItems.filter((item) => { + const selected = + item.conferenceOverride?.selected !== false && item.playlistReview?.selected !== false + const duplicate = + item.playlist?.duplicateStatus === "duplicate_existing" || + item.playlist?.duplicateStatus === "duplicate_in_batch" || + item.playlistReview?.duplicateEvidence?.kind === "library" || + item.playlistReview?.duplicateEvidence?.kind === "in_run" + if (filter === "duplicates") return duplicate + if (filter === "policy") return Boolean(item.playlistReview?.duplicatePolicy) + return selected + }), + [filter, queueItems] + ) + const visibleReviewItemIds = useMemo( + () => new Set(filteredReviewItems.map((item) => item.id)), + [filteredReviewItems] + ) + // TanStack Virtual exposes an imperative object that React Compiler skips. + // eslint-disable-next-line react-hooks/incompatible-library + const reviewVirtualizer = useVirtualizer({ + count: filteredReviewItems.length, + getScrollElement: () => reviewListRef.current, + estimateSize: () => 62, + overscan: 6, + getItemKey: (index) => filteredReviewItems[index]?.id ?? index, + measureElement: (element) => element?.getBoundingClientRect().height || 62, + }) + const reviewVirtualItems = reviewVirtualizer.getVirtualItems() + const restoreReviewRowFocus = useCallback((id: string) => { + const attempt = (remaining: number) => { + if (!reviewListOwnsFocusRef.current) return + const row = reviewRowRefs.current.get(id) + if (row) { + row.focus() + return + } + if (remaining > 0) window.requestAnimationFrame(() => attempt(remaining - 1)) + } + window.requestAnimationFrame(() => attempt(2)) + }, []) + + useEffect(() => { + const handleFocusIn = (event: FocusEvent) => { + const target = event.target + if (target instanceof Node && reviewListRef.current?.contains(target)) return + reviewListOwnsFocusRef.current = false + } + document.addEventListener("focusin", handleFocusIn) + return () => document.removeEventListener("focusin", handleFocusIn) + }, []) + + useEffect(() => { + if (filteredReviewItems.length === 0) { + activeReviewRowRef.current = null + setActiveReviewId(null) + return + } + const active = activeReviewRowRef.current + if (!active) { + const id = getQueueItemOccurrenceId(filteredReviewItems[0]) + activeReviewRowRef.current = { id, index: 0 } + setActiveReviewId(id) + return + } + const currentIndex = filteredReviewItems.findIndex( + (item) => getQueueItemOccurrenceId(item) === active.id + ) + if (currentIndex >= 0) { + active.index = currentIndex + if ( + reviewListOwnsFocusRef.current && + !reviewRowRefs.current.has(active.id) && + reviewVirtualItems.length > 0 + ) { + const nearest = reviewVirtualItems.reduce((best, row) => + Math.abs(row.index - currentIndex) < Math.abs(best.index - currentIndex) ? row : best + ) + const nearestItem = filteredReviewItems[nearest.index] + if (nearestItem) { + const id = getQueueItemOccurrenceId(nearestItem) + activeReviewRowRef.current = { id, index: nearest.index } + setActiveReviewId(id) + restoreReviewRowFocus(id) + } + } + return + } + const index = Math.min(active.index, filteredReviewItems.length - 1) + const id = getQueueItemOccurrenceId(filteredReviewItems[index]) + activeReviewRowRef.current = { id, index } + setActiveReviewId(id) + if (reviewListOwnsFocusRef.current) { + reviewVirtualizer.scrollToIndex(index, { align: "auto" }) + restoreReviewRowFocus(id) + } + }, [filteredReviewItems, restoreReviewRowFocus, reviewVirtualItems, reviewVirtualizer]) + + const handleReviewRowKeyDown = useCallback( + (event: React.KeyboardEvent, index: number) => { + if (event.target !== event.currentTarget) return + if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return + event.preventDefault() + const targetIndex = Math.max( + 0, + Math.min( + filteredReviewItems.length - 1, + index + (event.key === "ArrowDown" ? 1 : -1) + ) + ) + const target = filteredReviewItems[targetIndex] + if (!target) return + const id = getQueueItemOccurrenceId(target) + activeReviewRowRef.current = { id, index: targetIndex } + setActiveReviewId(id) + reviewVirtualizer.scrollToIndex(targetIndex, { align: "auto" }) + restoreReviewRowFocus(id) + }, + [filteredReviewItems, restoreReviewRowFocus, reviewVirtualizer] + ) // Compute total estimated time const totalEstimatedSeconds = useMemo( @@ -136,14 +294,63 @@ export const ReviewStep: React.FC = ({ () => selectedQueueItems.filter((item) => item.validation.valid).length, [selectedQueueItems] ) + const runRequestBuild = useMemo(() => buildPlaylistIngestRunRequest(queueItems), [queueItems]) + const currentProcessingBlock = state.processingBlock ?? runRequestBuild.block + const materializationExpired = currentProcessingBlock?.code === "materialization_expired" + const reviewRequired = currentProcessingBlock?.code === "review_required" + const exceedsRunInputLimit = + currentProcessingBlock?.code === "invalid_run_request" && + validItemCount > MAX_PLAYLIST_RUN_INPUTS + const canStartWithoutPersistence = + validItemCount > 0 && + runRequestBuild.request !== null && + state.processingBlock === null && + isOnlineForIngest && !isCheckingConnection const canStartProcessing = - validItemCount > 0 && isOnlineForIngest && !isCheckingConnection - - const handleStartProcessing = useCallback(() => { - if (!canStartProcessing) return + canStartWithoutPersistence && + persistenceStatus === "ready" && + isSubmissionOwner && + !isCheckingSubmissionOwnership && + !isStartingProcessing + + const handleStartProcessing = useCallback(async () => { + if (!canStartProcessing || persistenceStatus !== "ready") return + if (onBeginProcessing) { + setIsStartingProcessing(true) + try { + await onBeginProcessing(state) + } finally { + setIsStartingProcessing(false) + } + return + } + if (onCheckSubmissionOwnership) { + setIsCheckingSubmissionOwnership(true) + try { + if (!(await onCheckSubmissionOwnership())) return + } finally { + setIsCheckingSubmissionOwnership(false) + } + } startProcessing() - goNext() - }, [canStartProcessing, goNext, startProcessing]) + }, [ + canStartProcessing, + onBeginProcessing, + onCheckSubmissionOwnership, + persistenceStatus, + state, + startProcessing, + ]) + + const handleCheckSubmissionOwnership = useCallback(async () => { + if (!onCheckSubmissionOwnership || persistenceStatus !== "ready") return + setIsCheckingSubmissionOwnership(true) + try { + await onCheckSubmissionOwnership() + } finally { + setIsCheckingSubmissionOwnership(false) + } + }, [onCheckSubmissionOwnership, persistenceStatus]) // Contextual warnings const warnings = useMemo(() => { @@ -188,6 +395,11 @@ export const ReviewStep: React.FC = ({ // Item display name const getItemLabel = useCallback((item: WizardQueueItem): string => { + if (item.playlist?.title) { + return item.playlist.ordinal + ? `${item.playlist.ordinal}. ${item.playlist.title}` + : item.playlist.title + } if (item.conferenceOverride?.title) return item.conferenceOverride.title if (item.fileName) return item.fileName if (item.url) { @@ -225,9 +437,7 @@ export const ReviewStep: React.FC = ({ {conferenceBatchMetadata.collectionName || "Conference batch"} - - {selectedQueueItems.length} selected - + {selectedQueueItems.length} selected
{conferenceBatchMetadata.conferenceName && ( @@ -245,40 +455,105 @@ export const ReviewStep: React.FC = ({
)} -
    item.playlist) && ( +
    + + + Showing {filteredReviewItems.length} of {queueItems.length} review items + +
    + )} +
    - {selectedQueueItems.map((item) => { - const IconComponent = TYPE_ICONS[item.detectedType] ?? File - const ops = getOperationDescription(item.detectedType, selectedPreset, presetConfig) - const label = getItemLabel(item) - - return ( -
  • -
    • - ) - })} -
+
+ {reviewVirtualItems.map((virtualRow) => { + const item = filteredReviewItems[virtualRow.index] + if (!item) return null + const IconComponent = TYPE_ICONS[item.detectedType] ?? File + const ops = getOperationDescription(item.detectedType, selectedPreset, presetConfig) + const label = getItemLabel(item) + + return ( +
{ + const id = getQueueItemOccurrenceId(item) + if (element) { + reviewRowRefs.current.set(id, element) + reviewVirtualizer.measureElement(element) + } else { + reviewRowRefs.current.delete(id) + } + }} + role="listitem" + tabIndex={activeReviewId === getQueueItemOccurrenceId(item) ? 0 : -1} + aria-setsize={filteredReviewItems.length} + aria-posinset={virtualRow.index + 1} + data-occurrence-id={getQueueItemOccurrenceId(item)} + data-index={virtualRow.index} + onFocusCapture={() => { + const id = getQueueItemOccurrenceId(item) + reviewListOwnsFocusRef.current = true + activeReviewRowRef.current = { id, index: virtualRow.index } + setActiveReviewId(id) + }} + onKeyDown={(event) => handleReviewRowKeyDown(event, virtualRow.index)} + className="absolute left-0 top-0 flex w-full items-center gap-3 border-b border-border px-3 py-2.5 text-sm" + style={{ transform: `translateY(${virtualRow.start}px)` }} + > +
+ ) + })} +
+
+ + {queueItems.some((item) => item.sourceRef?.kind === "materialized_playlist_item") && ( + + )} {/* Storage mode */}

@@ -304,16 +579,102 @@ export const ReviewStep: React.FC = ({ : undefined } > - { - connectionRecoveryMessage || + {connectionRecoveryMessage || qi( "wizard.offline.description", "Reconnect to your tldw server before processing. You can go back and keep editing the queue." - ) - } + )} )} + {persistenceStatus === "migrating" && ( + + )} + + {persistenceStatus === "unavailable" && ( + + )} + + {persistenceStatus === "quota_error" && ( + + )} + + {persistenceStatus === "ready" && !isSubmissionOwner && ( + void handleCheckSubmissionOwnership(), + loading: isCheckingSubmissionOwnership, + disabled: isCheckingSubmissionOwnership, + } + : undefined + } + /> + )} + + {submissionStartError && ( + + )} + + {materializationExpired && ( + + )} + + {reviewRequired && ( + + )} + + {exceedsRunInputLimit && ( + + )} + {/* Contextual warnings */} {warnings.length > 0 && (

@@ -347,8 +708,9 @@ export const ReviewStep: React.FC = ({ + ), + Input: { + TextArea: ({ + autoSize: _autoSize, + ...props + }: React.TextareaHTMLAttributes & { + autoSize?: unknown; + }) =>