diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 49829717..64906f63 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -21,6 +21,7 @@ FreeVideo code uses [Apache-2.0](LICENSE). Its dependencies and models retain th | [Eigen](https://gitlab.com/libeigen/eigen) 5.0.1, compiled into sherpa-onnx and ONNX Runtime; source: https://gitlab.com/libeigen/eigen/-/tree/5.0.1 | MPL-2.0 | | [Silero VAD](https://github.com/snakers4/silero-vad) v5.0 voice-activity model, © 2020-present Silero Team (optional download with subtitles, license file downloaded with the model; the onnx-community export is downloaded with the music model) | [MIT](freevideo_engine/licenses/Silero-VAD-MIT.txt) | | [ACE-Step 1.5](https://github.com/ace-step/ACE-Step-1.5) XL turbo music model, © 2026 ACEStep (optional download for music; inference subset adapted in `freevideo_engine/music/acestep_model.py`) | [MIT](freevideo_engine/licenses/ACE-Step-MIT.txt) | +| [Bandit Plus](https://zenodo.org/records/10160698) cocktail-party separation model, weights by Watcharasupat et al. (optional download for sound separation, attribution required, non-commercial); code from [Bandit](https://github.com/kwatcharasupat/source-separation), © the Bandit authors, reduced to an inference subset in `freevideo_engine/stems/bandit/` (modified) | [CC BY-NC 4.0](freevideo_engine/licenses/Bandit-CC-BY-NC-4.0.txt) (weights), [Apache-2.0](freevideo_engine/licenses/Bandit-Apache-2.0.txt) (code) | | [vector-quantize-pytorch](https://github.com/lucidrains/vector-quantize-pytorch), © 2020 Phil Wang (residual FSQ decoding adapted in `freevideo_engine/music/acestep_model.py`) | [MIT](freevideo_engine/licenses/Vector-Quantize-PyTorch-MIT.txt) | | [Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B), © Alibaba Cloud (text encoder of the music model, downloaded with it) | [Apache-2.0](freevideo_engine/licenses/Qwen3-Embedding-Apache-2.0.txt) | | [NVIDIA NeMo stt_en_conformer_ctc_small](https://huggingface.co/nvidia/stt_en_conformer_ctc_small), © NVIDIA, exported to ONNX and quantized to int8 by the sherpa-onnx project, changed from the original; not endorsed by NVIDIA (optional download for English subtitles; [notice](freevideo_engine/licenses/NeMo-stt_en_conformer_ctc_small-NOTICE.txt)) | [CC-BY-4.0](https://creativecommons.org/licenses/by/4.0/) | diff --git a/freevideo_engine/adaptive.py b/freevideo_engine/adaptive.py index 9963b8ef..beb0fadd 100644 --- a/freevideo_engine/adaptive.py +++ b/freevideo_engine/adaptive.py @@ -785,6 +785,11 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry): output_frames = (count * canvas['fps'] + rate - 1) // rate if hidden or not 0 < output_frames <= expected: raise ValueError('Original audio exceeds the generated canvas') + if isinstance(original_audio.get('segment'), dict): + # A chain part keeps its whole picture grid; the join trims the tail. + output_frames = expected + if hidden or not 0 < output_frames <= expected: + raise ValueError('Original audio exceeds the generated canvas') if continuation is not None: from .continuation import OVERLAP_FRAMES if (not isinstance(continuation, dict) or continuation.get('overlap_frames') != OVERLAP_FRAMES diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index 6c03fe6b..8f11a97f 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -121,6 +121,10 @@ 'submit the music job again with a new ref; if it fails again, write the score yourself.'), 'music_failed': (409, True, 'Composing the music did not finish.', 'Read the project, then submit the music job again with a new ref; if it fails again, write the score yourself.'), + 'stems_download_failed': (409, True, 'The sound separation model could not be downloaded.', + 'Tell the user the model needs a network connection, then submit the job again with a new ref.'), + 'stems_failed': (409, True, 'Splitting the sound did not finish.', + 'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'), 'clip_source_unusable': (409, False, 'A clip\'s source is missing, or cannot be read as far as the clip uses it.', 'Read the project and check each clip\'s source: it must exist and play, and in + frames must stay ' 'within its real length. Correct the project, then export the new revision.'), @@ -235,6 +239,8 @@ def recovery(code, *, kind=None, stage='request', resumable=True): return 'Tell the user to free disk space, then submit the picture again with a new ref.' if kind == 'music' and code == 'out_of_memory': return 'Close other GPU applications, then submit the music job again with a new ref.' + if kind == 'stems' and code == 'out_of_memory': + return 'Close other GPU applications, then submit the job again with a new ref.' if kind == 'image' and code == 'no_memory': return 'Lower the picture dimensions or use fewer reference pictures, then submit with a new ref.' if kind in RECOVERY and stage == 'terminal': diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index f0095b93..81726d2d 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -144,6 +144,7 @@ 'On 409 project_changed, recompute edits from error.project and its current_revision, then send a new request. On clip_busy wait for generation to finish.', 'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.', 'Music: POST /projects/{id}/music with the revision you read and a caption; wait for the job, then read the project: result.clip_ids are the new clips on a2. Run it after the cut is joined; a second run needs the earlier score removed from a2.', + 'Split sound: POST /projects/{id}/stems with the revision you read and the clip_id of a sound clip; wait for the job. The clip\'s sound is split into dialogue, music and ambience on tracks a4, a5 and a6 over the clip\'s range, and the clip itself goes silent. result.parts lists each track with its name, clip_id, start, end and frames. Offered only where features lists projects.stems; a clip is split once, and deleting the three parts gives its sound back. To save the three tracks and the original mix as four wav files, POST /projects/{id}/stems/export with the revision and the split clip\'s clip_id; the answer lists folder and files at once.', 'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.', 'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.', 'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.', @@ -377,6 +378,9 @@ '没有安装语音模型时返回 no_speech_model，改为自己写字幕行', ('POST', '/projects/{id}/music'): '为剪辑时间线作一段与其等长的配乐，放在第二条声音轨道的开头，音量低于视频声音并带淡入淡出。' '剪辑拼好后先读取 revision，并删去该轨道上原有的配乐；首次使用会下载音乐模型（约 20 GB）', + ('POST', '/projects/{id}/stems'): '把一个声音片段的声音分成人声、音乐和环境声，放在其下方的 a4、a5、a6 三条轨道上，原片段静音。' + '先读取 revision 与片段 ID，等待返回的任务完成，结果 parts 给出各轨道的名称、起止帧；仅在 features 含 projects.stems 时可用', + ('POST', '/projects/{id}/stems/export'): '把已拆开片段的人声、音乐、环境声和原混音保存为四个 wav 文件；先读取 revision 与被拆片段的 ID，返回立即给出文件夹和文件名', ('POST', '/images'): '像图片编辑页面一样打开图片；提交 source，会话会保留源图片副本和图层文档', ('GET', '/images/{id}'): '读取图片编辑会话的图层文档、选区和 revision；编辑前先读取 revision 和图层标识', ('PUT', '/images/{id}'): '替换整个图片文档；先读取会话并提交其 revision。' diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 9cd7488e..53d5b346 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -32,7 +32,7 @@ ACTIVE = ('waiting', 'queued', 'running', 'cancelling') CAST_AGAIN = 'POST /freevideo/agent/v1/cast/%s/generate to make the reference pictures again.' DONE = ('succeeded', 'failed', 'cancelled', 'interrupted') -GPU_KINDS = ('vlm', 'enhance', 'image', 'music') +GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems') IMAGE_ERRORS = errors.IMAGE_ERRORS IMAGE_WAIT_ERRORS = ('busy', 'busy_video') IMAGE_WAIT_NOTICES = { @@ -1485,6 +1485,9 @@ def run(): elif row['kind'] == 'music': from .music import run_music result = run_music(self, row) + elif row['kind'] == 'stems': + from .stems import run_stems + result = run_stems(self, row) else: result = self.run_assemble(row) return result diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py index dd7e2f2f..619935cb 100644 --- a/freevideo_engine/agent/projects.py +++ b/freevideo_engine/agent/projects.py @@ -11,6 +11,7 @@ import hashlib import json import logging +from pathlib import Path import re import secrets import threading @@ -413,6 +414,70 @@ def music(self, identity, value, *, jobs, ref=None, client=None): self._remember(ref, digest, result, client) return result + def stems(self, identity, value, *, jobs, ref=None, client=None): + """Queue a sound-splitting job for one sound clip of the revision the caller read.""" + schema.validate(schema.project_schema('stems'), value) + with self.listen_lock: + digest, replay = self._receipt('projects:stems', dict(id=identity, body=value), ref, client) + if replay is not None: + return replay + stored = self._load(identity)['project'] + if stored['revision'] != value['revision']: + raise ProjectError('project_changed', project=stored) + clip = _clips(stored).get(value['clip_id']) + if clip is None or not clip.get('source') or clip.get('stem_of'): + raise _invalid('/clip_id', 'a sound clip with media that is not itself a separated part', value['clip_id']) + job = jobs.submit(dict(kind='stems', editor_project=identity, + **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0] + result = dict(job=jobs.view(job), jobs=[jobs.view(job)]) + self._remember(ref, digest, result, client) + return result + + def stems_export(self, identity, value, *, ref=None, client=None): + """Copy the four wav files of a split clip into the saved sounds; the clip must be one that was split.""" + from ..stems.separate import SeparationError, export_tracks + schema.validate(schema.project_schema('stems_export'), value) + with self.listen_lock: + digest, replay = self._receipt('projects:stems_export', dict(id=identity, body=value), ref, client) + if replay is not None: + return replay + stored = self._load(identity)['project'] + if stored['revision'] != value['revision']: + raise ProjectError('project_changed', project=stored) + parts = [clip for clip in _clips(stored).values() if clip.get('stem_of') == value['clip_id']] + if not parts: + raise _invalid('/clip_id', 'a clip that was split into dialogue, music and ambience', value['clip_id']) + folder = parts[0]['source']['id'].rsplit('/', 1)[0] + if not folder.startswith('freevideo-stems/') or folder.count('/') != 1: + raise _invalid('/clip_id', 'a clip split by this version', value['clip_id']) + name = folder.split('/', 1)[1] + try: + files = export_tracks(Path(self.root) / folder, Path(self.root) / 'freevideo-stems-saved' / name) + except SeparationError as error: + raise JobError('stems_failed', 'The separated sound cannot be saved (%s)' % error) from error + result = dict(editor_project=identity, revision=stored['revision'], clip_id=value['clip_id'], + folder='freevideo-stems-saved/' + name, files=files) + self._remember(ref, digest, result, client) + return result + + def lay_stems(self, identity, clip_id, sources): + """Put the three parts on sub-tracks under the clip and silence the clip; returns the save and the new clip IDs.""" + before, refused = {}, {} + def change(project): + before['ids'] = set(_clips(project)) + reply = editor_runtime.runtime().call('placeStems', project, [clip_id, { + name: dict(kind='generated', id=source.split(':', 1)[1]) for name, source in sources.items()}], + module='model', serialize=True)[0] + if not reply['ok']: + refused.update(reply['error']) + return dict(ok=True, project=project) + return dict(ok=True, project=reply['value']) + saved = self._update(identity, change) + if refused: + code = STATE_CODES.get(refused.get('code'), 'stems_failed') + raise JobError(code, 'The separated sound cannot be laid under the clip (%s)' % refused.get('code')) + return saved, sorted(set(_clips(saved)) - before['ids']) + def lay_music(self, identity, source): """Put a finished score on a2 from frame 0 of the project as it is now; returns the save and the new clip IDs.""" found = comfy_editor.sources(self.root, [source])[source] diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index c95fcdd1..79c82b5c 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -172,7 +172,13 @@ def submit_batch(jobs, body, client): return jobs.submit_request(body, client=client)[0] -def features(continuation, vlm): +def stems_supported(machine): + """Sound separation is offered on NVIDIA machines with enough video memory, never on a Mac.""" + from ..stems.service import supported + return supported(machine.get('vram_gib'), uuid=machine.get('gpu_uuid')) and machine.get('device_backend', 'cuda') == 'cuda' + + +def features(continuation, vlm, stems=False): """What an agent can use here (ready says whether the engine can run now). A missing name is not offered.""" from .editor_runtime import available rules = available() @@ -183,7 +189,8 @@ def features(continuation, vlm): 'queue.shared': True, 'queue.busy': True, 'chain.auto': bool(continuation), 'enhance': bool(vlm.get('ready')), 'drama.film': True, 'images.edit': rules, 'images.layers': rules, 'images.heal': rules, 'images.render': rules, 'projects.read': True, - **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen', 'music')}} + **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen', 'music')}, + 'projects.stems': bool(rules and stems)} def whitebox_support(jobs): @@ -221,7 +228,7 @@ def capabilities(port, jobs): return dict( api_version=API_VERSION, freevideo_version=release or __version__, build=__version__, ready=ready, problem=problem, continuation=continuation, - features=features(continuation, vlm), + features=features(continuation, vlm, stems=stems_supported(machine)), whitebox=whitebox_support(jobs), device=dict(backend=machine.get('device_backend', 'cuda') if machine else None, vram_limit_gib=machine.get('vram_gib'), ram_limit_gib=machine.get('ram_gib')), @@ -233,6 +240,7 @@ def capabilities(port, jobs): 'export': 'Render a saved revision of an editing project to a video', 'listen': 'Listen to the sound of an editing project for subtitle lines and times', 'music': 'Compose music for an editing project and lay it on its second sound track', + 'stems': 'Split the sound of a clip into dialogue, music and ambience on sub-tracks', 'image_edit': 'Render a saved revision of an image session to a picture'}, limits=dict(width=[256, 4096, 32], height=[256, 4096, 32], seconds=[1.625, 60 if continuation else BLOCK_SECONDS], seconds_tested=[4, 15], @@ -507,7 +515,9 @@ async def respond(request): raise JobError('page_session_required', 'Page session required') header = request.headers.get('Authorization', '') given = header[7:].strip() if header[:7].lower() == 'bearer ' else request.headers.get('X-FreeVideo-Agent', '') - if definition.get('auth', True) or given: + # The editor page may drive a few routes with its page session instead of the token. + by_page = bool(definition.get('page_ok')) and not given and valid_page_session(request) + if (definition.get('auth', True) and not by_page) or given: try: token = expected() except (OSError, ValueError): @@ -2215,7 +2225,7 @@ async def handler(request): raise JobError('idempotency_conflict', 'Conflicting keys') args = [] if operation == 'create' else [identity] extra = dict(client=request.headers.get('X-FreeVideo-Client', '')[:80] or None) - if operation in ('generate', 'export', 'listen', 'music'): + if operation in ('generate', 'export', 'listen', 'music', 'stems'): extra['jobs'] = jobs if operation == 'generate': def capability(size): @@ -2225,9 +2235,9 @@ def capability(size): return None extra['capability'] = capability result = await asyncio.to_thread(getattr(projects, operation), *args, value, ref=key or ref, **extra) - if operation in ('generate', 'export', 'listen', 'music'): + if operation in ('generate', 'export', 'listen', 'music', 'stems'): await jobs.tick() - return reply(result, 201 if operation in ('create', 'generate', 'export', 'listen', 'music') else 200) + return reply(result, 201 if operation in ('create', 'generate', 'export', 'listen', 'music', 'stems') else 200) return guarded(handler) for operation, (method, suffix, _) in contract.PROJECT_OPERATIONS.items(): diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 905a4d0e..ecbef87a 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -120,12 +120,12 @@ def edit_recipe(): 'then': {'required': ['from', 'to']}, 'else': {'required': ['width', 'height']}}) operations.append(obj(props, ['op'] + required.get(name, []), **({'allOf': rules} if rules else {}))) return obj({'operations': array({'oneOf': operations}, 1, MAX_OPERATIONS)}, ('operations',), - description='x-editor-recipe uses the shared CSS colour/ratio parsers. Source geometry and layer references also require editor preflight.', + description='Colours and ratios use CSS syntax; geometry and layer links are checked on submit.', **{'x-editor-recipe': True}) def edit_document(): - return {'type': 'object', 'required': ['version', 'layers'], 'properties': {'version': {'const': 2}}, 'description': 'Editor document version 2; x-editor-document uses the renderer pure document validator (geometry and layer relationships).', + return {'type': 'object', 'required': ['version', 'layers'], 'properties': {'version': {'const': 2}}, 'description': 'Editor document version 2.', 'x-editor-document': True} @@ -155,11 +155,11 @@ def media(image=False, *, local=False, clip=False, video=False, early=False): rules.append({'if': {'anyOf': [{'required': [k]} for k in ('from_job', 'library')]}, 'then': {'required': ['frame']}}) return {'oneOf': [FILE, obj(properties, allOf=rules)], 'x-media': dict(image=image, clip=clip, video=video), - 'description': 'Exactly one source. x-media additionally checks clip end > start, a comparison JSON Schema cannot express.'} + 'description': 'Exactly one source; end after start.' if clip else 'Exactly one source.'} def common(): - return dict(label=string(120), project=SLUG, metadata={'x-json-bytes': 16 * 1024, 'description': 'x-json-bytes: serialized finite JSON of at most 16384 bytes.'}, + return dict(label=string(120), project=SLUG, metadata={'x-json-bytes': 16 * 1024, 'description': 'Finite JSON, at most 16384 bytes.'}, after=array(JOB), ref=REF) @@ -337,6 +337,20 @@ def music(*, local=False): **{'x-job-kind': 'music'}) +STEMS_FIELDS = dict( + clip_id=string(64)) +# False leaves the three parts unplaced: the editor page lays them itself so its own saves never clash with the job's. +STEMS_LAY = dict(lay={'type': 'boolean', 'default': True}) + + +def stems(*, local=False): + return obj(dict(kind={'const': 'stems', 'type': 'string'}, + editor_project=string(34, pattern=r'^e-[0-9a-f]{32}$'), + revision=number(1, 2**53 - 1, integer=True), + **STEMS_FIELDS, **STEMS_LAY, **common()), ('kind', 'editor_project', 'revision', 'clip_id'), + **{'x-job-kind': 'stems'}) + + def image_edit(*, local=False): return obj(dict(kind={'const': 'image_edit', 'type': 'string'}, image=dict(IMAGE, description='The image session to render (POST /images).'), @@ -347,12 +361,12 @@ def image_edit(*, local=False): KINDS = dict(generate=generate, assemble=assemble, vlm=vlm, enhance=enhance, image=image, - export=export, listen=listen, music=music, image_edit=image_edit) + export=export, listen=listen, music=music, stems=stems, image_edit=image_edit) def job_schema(*, local=False, refs=False, batch=False): # Music scores a joined cut, so it is not a storyboard row. - value = {'oneOf': [f(local=local) for kind, f in KINDS.items() if not (batch and kind == 'music')], 'x-discriminator': 'kind', + value = {'oneOf': [f(local=local) for kind, f in KINDS.items() if not (batch and kind in ('music', 'stems'))], 'x-discriminator': 'kind', 'description': 'x-discriminator selects field diagnostics by kind; all accepted branches use standard oneOf.'} return batch_references(value) if refs else value @@ -469,7 +483,7 @@ def route(summary, body=None, query=None, **kw): if method == 'GET': extra['query'] = shape else: - extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen', 'music') else {})) + extra.update(body=shape, idempotent=True, **({'status': 201} if name in ('create', 'generate', 'export', 'listen', 'music', 'stems') else {})) routes[method, suffix] = route(summary, **extra) for name, (method, suffix, summary) in WHITEBOX_OPERATIONS.items(): extra = dict(path=obj({'id': WHITEBOX}, ('id',))) if '{id}' in suffix else {} @@ -546,6 +560,10 @@ def route(summary, body=None, query=None, **kw): auth=False, page_or_token=True) routes['PUT', '/confirmations'] = route('Only the page user may change which operations need approval.', body=confirm_settings(), auth=False, page_or_token=True) + # The editor page separates sound itself: these four also accept its page session, which never holds the token. + for key in (('POST', '/projects/{id}/stems'), ('POST', '/projects/{id}/stems/export'), + ('GET', '/jobs/{id}'), ('POST', '/jobs/{id}/cancel')): + routes[key]['page_ok'] = True return routes @@ -854,12 +872,15 @@ def image_schema(name): 'generate': ('POST', '/projects/{id}/generate', 'Make new picture and sound for selected clips or a frame range, as the page Generate does. First get revision and clip IDs; the parts are reserved at once and fill in as the returned job finishes.'), 'export': ('POST', '/projects/{id}/export', 'Render one saved revision to an MP4 in the saved videos. First get the revision; wait for the returned job, then read its result video.'), 'listen': ('POST', '/projects/{id}/listen', 'Listen to the timeline sound for subtitles: mode align times lines you give, transcribe writes down what is said. First get the revision; wait for the returned job, then add the lines with the subtitles edit. Nothing is saved; with no speech model for the language the call fails with no_speech_model, so write the lines by hand.'), + 'stems': ('POST', '/projects/{id}/stems', 'Split one clip\'s sound into dialogue, music and ambience on tracks a4-a6. First get the revision and clip ID; wait for the job, then read result.parts.'), + 'stems_export': ('POST', '/projects/{id}/stems/export', 'Save the separated dialogue, music and ambience of a split clip, with its original mix, as four wav files in the saved sounds. First get the revision and the split clip\'s ID; the answer lists the files at once.'), 'music': ('POST', '/projects/{id}/music', 'Compose music as long as the joined cut and lay it on sound track a2 at frame 0, under the video sound. First get the revision; a2 must be empty. Wait for the returned job; the first use downloads the model (20 GB).'), } PROJECT_TOOLS = dict(list='freevideo_projects', get='freevideo_project_read', create='freevideo_project_create', write='freevideo_project_write', operations='freevideo_editor_operations', generate='freevideo_project_generate', export='freevideo_export', - listen='freevideo_project_listen', music='freevideo_project_music') + listen='freevideo_project_listen', music='freevideo_project_music', + stems='freevideo_project_stems', stems_export='freevideo_project_stems_export') SESSION_OPERATIONS = { 'create': ('POST', '/sessions', 'Create a work conversation for messages and plans shared with the user.'), @@ -980,6 +1001,8 @@ def project_schema(name): return obj(dict(revision=revision, **LISTEN_FIELDS, ref=REF), ('revision', 'mode')) if name == 'music': return obj(dict(revision=revision, **MUSIC_FIELDS, ref=REF), ('revision', 'caption')) + if name in ('stems', 'stems_export'): + return obj(dict(revision=revision, **STEMS_FIELDS, **(STEMS_LAY if name == 'stems' else {}), ref=REF), ('revision', 'clip_id')) if name != 'operations': return EMPTY fields = { @@ -1433,6 +1456,8 @@ def openapi(): if contract.get('raw_body'): operation['requestBody'] = dict(required=True, content={'application/octet-stream': {'schema': { 'type': 'string', 'format': 'binary', 'maxLength': MAX_CHAT_ATTACHMENT_BYTES}}}) + if contract.get('page_ok'): + operation['security'] = [{'bearer': []}, {'page': []}] if contract.get('page_or_token'): operation['security'] = [{'page': []}] if suffix.endswith(('/stop', '/resume')) else [{'bearer': []}, {'page': []}] if contract['body'] is not None: @@ -1452,6 +1477,9 @@ def openapi(): if suffix == '/freevideo/agent/chat/attach': operation['parameters'].append(dict(name='X-FreeVideo-CSRF', **{'in': 'header'}, required=True, schema=string(128), description='The page session CSRF token.')) + if contract.get('page_ok'): + operation['parameters'].append(dict(name='X-FreeVideo-CSRF', **{'in': 'header'}, required=False, + schema=string(128), description='With the freevideo_agent_page cookie, in place of the token.')) if contract.get('page_or_token'): operation['parameters'].append(dict(name='X-FreeVideo-CSRF', **{'in': 'header'}, required=False, schema=string(128), description='Required with the freevideo_agent_page cookie, including reads.')) @@ -1576,7 +1604,7 @@ def openapi(): description='Work session; omit for latest.') -SHAPE_MIN_BYTES = 200 +SHAPE_MIN_BYTES = 60 RECIPE_TOOL = 'freevideo_edit_image' @@ -1635,9 +1663,10 @@ def rebuild(node, path): node = dict(node) for keyword, index, child in list(schema_children(node)): here, text = path + [(keyword, index)], key(child) - if counts.get(text, 0) > 1 and text in first: - child = dict({'type': child['type']} if 'type' in child else {}, - description='Same shape as %s.' % first[text]) + named = text in first and dict({'type': child['type']} if 'type' in child else {}, + description='Same as %s.' % first[text]) + if named and len(json.dumps(named, ensure_ascii=False)) < len(json.dumps(child, ensure_ascii=False)): + child = named else: if counts.get(text, 0) > 1: first[text] = shape_label(here) diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index 46846f8d..f19581f8 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -297,6 +297,9 @@ def normalize(value, *, port=None): elif kind == 'music': result.update(editor_project=value['editor_project'], revision=value['revision'], caption=value['caption'], **{k: value[k] for k in ('seed', 'quiet') if k in value}) + elif kind == 'stems': + result.update(editor_project=value['editor_project'], revision=value['revision'], clip_id=value['clip_id'], + lay=value.get('lay', True)) elif kind == 'image_edit': result.update(image=value['image'], revision=value['revision']) else: diff --git a/freevideo_engine/agent/stems.py b/freevideo_engine/agent/stems.py new file mode 100644 index 00000000..cecaa304 --- /dev/null +++ b/freevideo_engine/agent/stems.py @@ -0,0 +1,85 @@ +"""Stem jobs: the sound of one clip split into dialogue, music and ambience, laid on three sub-tracks.""" +import json +import secrets +import time + +from .jobs import JobError + +FOLDER = 'freevideo-stems' +# A video that starts while the sound is split stops it; the job waits for the video and splits again. +PREEMPTIONS = 8 +NAMES = ('dialogue', 'music', 'ambience') + + +def _busy(jobs): + port = jobs.port + return jobs.runner_busy() or port.queue_busy() or port.image_busy() or port.vlm_busy() + + +def run_stems(jobs, row, *, poll=.5): + from ..editor_export import resolve_source + from ..stems import service + spec, port = row['spec'], jobs.port + try: + project = jobs.projects.get(spec['editor_project'])['project'] + except FileNotFoundError as error: + raise JobError('not_found', 'The project does not exist') from error + clip = next((clip for track in project['tracks'] if track['kind'] == 'audio' for clip in track['clips'] + if clip['id'] == spec['clip_id']), None) + if clip is None or not clip.get('source'): + raise JobError('invalid_request', 'The clip is not a sound clip with media') + try: + found = resolve_source('%s:%s' % (clip['source']['kind'], clip['source']['id'])) + except (OSError, ValueError) as error: + raise JobError('stems_failed', 'The clip media is not readable', kind='stems') from error + source = found['audio'] + if source is None: + raise JobError('stems_failed', 'The clip has no sound', kind='stems') + folder = '%s/%s-%s' % (FOLDER, row['id'], secrets.token_hex(4)) + out = port.output_path(folder + '/dialogue.mp4', create=True).parent + + def phase(name, fraction=None): + row['_phase'] = name + row['_detail'] = json.dumps(dict(fraction=round(fraction, 3))) if fraction is not None else None + + try: + numbers = _split(jobs, row, service, source, out, phase, poll) + sources = {name: 'generated:%s/%s.mp4' % (folder, name) for name in NAMES} + if not spec.get('lay', True): + return dict(editor_project=spec['editor_project'], clip_id=spec['clip_id'], sources=sources, parts=[], clip_ids=[], + seconds=numbers.get('seconds'), separate_seconds=numbers.get('elapsed')) + phase('placing') + saved, clip_ids = jobs.projects.lay_stems(spec['editor_project'], spec['clip_id'], sources) + except BaseException: + for path in out.glob('*'): + path.unlink(missing_ok=True) + out.rmdir() + raise + fresh = set(clip_ids) + parts = [dict(track=track['id'], name=track['stem'], clip_id=clip['id'], start=clip['start'], frames=clip['frames'], + end=clip['start'] + clip['frames']) + for track in saved['tracks'] if track.get('stem') for clip in track['clips'] if clip['id'] in fresh] + return dict(editor_project=spec['editor_project'], revision=saved['revision'], clip_ids=clip_ids, sources=sources, + parts=parts, seconds=numbers.get('seconds'), + separate_seconds=numbers.get('elapsed')) + + +def _split(jobs, row, service, source, out, phase, poll): + for attempt in range(PREEMPTIONS + 1): + try: + return service.separate(source, out, server=getattr(jobs.port, 'server', None), + cancelled=lambda: bool(row.get('_cancel')), phase=phase) + except service.Preempted: + if attempt == PREEMPTIONS: + raise JobError('stems_failed', 'Videos kept starting while the sound was being split', kind='stems') from None + phase('waiting') + while True: + if row.get('_cancel'): + raise JobError('cancelled', 'Cancelled') from None + time.sleep(poll) + if not _busy(jobs): + break + except InterruptedError: + raise JobError('cancelled', 'Cancelled') from None + except service.StemsError as error: + raise JobError(error.code, 'The sound was not split (%s)' % (error.reason or error.code), kind='stems') from None diff --git a/freevideo_engine/backends/mps_vae_encode.py b/freevideo_engine/backends/mps_vae_encode.py index fc747ce1..57431b06 100644 --- a/freevideo_engine/backends/mps_vae_encode.py +++ b/freevideo_engine/backends/mps_vae_encode.py @@ -1,4 +1,6 @@ """Bound the pending Metal work and the live tensors of the original FP32 input-video encoder.""" +import json +import time from contextlib import contextmanager @@ -175,12 +177,13 @@ def forward(hidden_states): return forward -def _residency(model, device, torch): +def _residency(model, target, torch): """Hooks that hold each encoder stage's weights on `device` only while it runs. The weights are the same FP32 values read from the checkpoint; only the stage being computed is resident on the device, so the large late-stage weights are not alive while the early stages hold the largest activations. + `target` holds the device the stages upload to and whether a CPU rescue runs. Returns (hooks, host copies still on the device, uploaded byte counter). """ encoder = model.encoder @@ -194,8 +197,9 @@ def upload(unit, inputs): # A failed run leaves its stage here with the device copy in place; # the host copy kept from the first upload stays the source. host.setdefault(parameter, parameter.data) - parameter.data = host[parameter].to(device) - uploaded[0] += parameter.numel() * parameter.element_size() + parameter.data = host[parameter].to(target['device']) + if not target['rescue']: + uploaded[0] += parameter.numel() * parameter.element_size() def release(unit, inputs, output): for parameter in unit.parameters(): parameter.data = host.pop(parameter) @@ -205,6 +209,92 @@ def release(unit, inputs, output): return handles, host, uploaded +CPU_TILE_LIMIT = 6 + + +def _finite(tensor): + return bool(tensor.isfinite().all()) + + +def _guarded_tiles(encoder, patched, target, device, staged, stats, torch): + """Check every tile the encoder returns and recompute a non-finite one. + + A reference clip takes many minutes to encode and the pinned H3 pipeline + only notices a non-finite latent after the last tile. Each tile output is + checked here instead. A tile with non-finite values is recomputed from the + same input: once more on the device (a transient fault), then with the + original operators and the same weights, then in FP32 on the CPU for a + bounded number of tiles. The first level that succeeds stays in use for the + following tiles, except the CPU, which is only ever a per-tile rescue. A + tile that no level makes finite stops the encode at once with a clear error. + Tile numbers and the CPU limit count over every reference encoded in the + same `bounded_encoder` context. Only the encoder's own output is checked; + the layers after it (quant_conv, sampling, stitching) are not. + """ + inner = encoder.forward + levels = ['device'] + (['plain'] if stats.get('lean_resnet_blocks') else []) + (['cpu'] if staged else []) + stats.update(nonfinite_tiles=0, nonfinite_recoveries={}) + state = {'start': 0, 'tile': 0} + def sweep(): + torch.mps.synchronize() + torch.mps.empty_cache() + def run(level, hidden_states): + if level == 'device': + return inner(hidden_states) + saved = {m: m.__dict__.pop('forward') for m in patched if 'forward' in m.__dict__} + try: + if level == 'plain': + return encoder(hidden_states) + target.update(device='cpu', rescue=True) + try: + return encoder(hidden_states.to('cpu')).to(device) + finally: + target.update(device=device, rescue=False) + finally: + encoder.__dict__.pop('forward', None) + for module, forward in saved.items(): + module.__dict__['forward'] = forward + def forward(hidden_states): + state['tile'] += 1 + output = inner(hidden_states) if state['start'] == 0 else run(levels[state['start']], hidden_states) + if _finite(output): + return output + del output + stats['nonfinite_tiles'] += 1 + first = stats.setdefault('nonfinite_first_tile', dict(index=state['tile'], shape=list(hidden_states.shape), + level=levels[state['start']])) + if first['index'] == state['tile']: + print(json.dumps(dict(event='encoder_nonfinite_tile', **first)), flush=True) + attempts = [(0, 'device-retry')] if state['start'] == 0 else [] + attempts += [(index, level) for index, level in enumerate(levels) if index > state['start']] + tried = [] + for index, level in attempts: + if level == 'cpu' and stats.get('cpu_tiles', 0) >= CPU_TILE_LIMIT: + break + tried.append(level) + sweep() + started = time.monotonic() + if level == 'cpu': + stats['cpu_tiles'] = stats.get('cpu_tiles', 0) + 1 + print(json.dumps(dict(event='encoder_cpu_rescue', phase='start', tile=state['tile'], + rescued=stats['cpu_tiles'])), flush=True) + output = run(levels[index], hidden_states) + if level == 'cpu': + print(json.dumps(dict(event='encoder_cpu_rescue', phase='end', tile=state['tile'], + seconds=round(time.monotonic() - started, 1))), flush=True) + if _finite(output): + stats['nonfinite_recoveries'][level] = stats['nonfinite_recoveries'].get(level, 0) + 1 + if level == 'plain': + state['start'] = 1 + return output + del output + raise RuntimeError('The reference video encoder returned non-finite values for tile %d since the reference ' + 'encoder was prepared (%s) at every level tried (%s); the encode was stopped instead of finishing ' + 'the remaining tiles.' + % (state['tile'], 'x'.join(str(n) for n in hidden_states.shape), ', '.join(tried) or 'none')) + return forward + + @contextmanager def bounded_encoder(model, device=None): """Finish each convolution before submitting the next encoder operation. @@ -227,6 +317,7 @@ def bounded_encoder(model, device=None): if not modules: raise ValueError('Expected the H3 input encoder convolution modules') handles, patched, host, staged, uploaded = [], [], {}, None, [0] + target = dict(device=device, rescue=False) stats = dict(policy='Complete original FP32 convolutions and release unused MPS allocations', convolution_completions=0, convolution_modules=len(modules), lean_resnet_blocks=0, staged_weight_bytes=0) @@ -253,13 +344,18 @@ def complete(module, inputs, output): patched.append(model.encoder) stats['lean_resnet_blocks'] = sum(len(b.resnets) for b in model.encoder.down_blocks) if device is not None: - staged = _residency(model, device, torch) + staged = _residency(model, target, torch) if staged is not None: hooks, host, uploaded = staged handles.extend(hooks) if device is not None and staged is None: # Not the pinned structure: whole model on the device, fences only. model.to(device) + encoder = getattr(model, 'encoder', None) + if device is not None and isinstance(encoder, torch.nn.Module): + if encoder not in patched: + patched.append(encoder) + encoder.forward = _guarded_tiles(encoder, patched, target, device, staged is not None, stats, torch) yield stats finally: if staged is not None: diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index a42794dd..e9f68f0e 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -415,6 +415,12 @@ def host_compiler_choice(below, path=None, env=None, find=shutil.which, major=gc return None +def pinned_compiler_identity(spec=None): + """How FreeVideo's own GCC is named in the wheel cache key and in a build record.""" + spec = spec or json.loads((PACKAGE / 'gcc_linux.json').read_text(encoding='utf-8')) + return 'conda-forge gcc ' + spec['version'] + ' static-libstdc++' + + def existing_parent(path): while not path.exists(): path = path.parent @@ -1907,6 +1913,9 @@ def install_sage(self, uv, python): build_env = dict(self.env, CUDA_HOME=str(toolkit), PATH=str(toolkit / 'bin') + os.pathsep + self.env['PATH'], CPATH=headers, TORCH_CUDA_ARCH_LIST=identity['arch'], MAX_JOBS=str(build['jobs']), EXT_PARALLEL='1', NVCC_APPEND_FLAGS='--threads=' + str(build['nvcc_threads'])) + headers_conflict = compiler['env'] is not None and self.host_headers_conflict(toolkit, dict(build_env, **compiler['env'])) + if headers_conflict: + compiler = dict(compiler, env=None, built_with=pinned_compiler_identity()) build_env.update(compiler['env'] if compiler['env'] is not None else self.install_compiler_linux()) if compiler['env'] is None: # The private compiler is newer than an older system's libstdc++, which the Python process loads first. @@ -1922,6 +1931,8 @@ def install_sage(self, uv, python): wheel = wheels[0] record = {'identity': identity, 'filename': wheel.name, 'sha256': digest(wheel), 'bytes': wheel.stat().st_size, 'build_run': str(self.run_dir), 'kernel_validation': 'pending'} + if headers_conflict: + record.update(headers='private sysroot', built_with=compiler['built_with']) save(manifest, record) # Install from the verified wheel path, avoiding architecture-blind source caches. self.command('sage2-wheel-install', [uv, 'pip', 'install', '--python', python, '--no-deps', @@ -1929,6 +1940,30 @@ def install_sage(self, uv, python): save(self.run_dir / 'sage-wheel.json', dict(record, reused=cached, cache=str(wheel_dir))) return manifest + def host_headers_conflict(self, toolkit, env): + """Whether the system's C library headers clash with the toolkit's, so only a private sysroot can build Sage2. + + A new glibc (rsqrt from 2.42, sinpi and cospi from 2.41) declares math functions noexcept, which + the toolkit declares without. Every compiler that reads the system headers fails on it, whatever its + version, so a small CUDA file that includes is compiled with the chosen compiler first. Any + other outcome, including a probe that cannot run, leaves the build as it was. + """ + probe = self.run_dir / 'host-headers-probe.cu' + probe.write_text('#include \n__global__ void probe(double* p) { p[0] = rsqrt(p[0]); }\n', encoding='utf-8') + command = [str(toolkit / 'bin' / 'nvcc'), '-std=c++17', '-cuda', str(probe), '-o', str(probe.with_suffix('.cpp'))] + if env.get('CC'): + command[1:1] = ['-ccbin', env['CC']] + try: + output, conflict = subprocess_output.output_external(command, env=env, stderr=subprocess.STDOUT, timeout=300), False + except subprocess.CalledProcessError as error: + output = error.output if isinstance(error.output, str) else str(error.output or '') + conflict = 'exception specification is incompatible' in output + except (OSError, subprocess.SubprocessError) as error: + output, conflict = str(error), False + save(self.run_dir / 'sage-host-headers.json', {'conflict': conflict, 'compiler': env.get('CC', 'g++'), 'glibc': list(platform.libc_ver()), + 'output': output[-4000:]}) + return conflict + def host_compiler(self, cuda): """The compiler pair Sage is built with: the system's, a versioned one beside it, or FreeVideo's own. @@ -1943,7 +1978,7 @@ def host_compiler(self, cuda): else: choice = host_compiler_choice(below, self.env['PATH'], self.env) if below else {} if choice is None: - return {'env': None, 'identity': 'conda-forge gcc ' + spec['version'] + ' static-libstdc++'} + return {'env': None, 'identity': pinned_compiler_identity(spec)} cxx = choice.get('CXX', 'g++') return {'env': choice, 'identity': subprocess_output.output_external([cxx, '--version']).splitlines()[0]} diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index 3c6b1995..7db24c3d 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -30,7 +30,7 @@ from .editor_limits import PROJECT_BYTES from .editor_speed import validate_clip, register_reverse from .editor_transport import marker_fields -from .editor_audio import clip_fields, track_fields +from .editor_audio import STEM_TRACKS, clip_fields, track_fields FPS = 24 MAX_IMPORT_BYTES = 8 * 1024**3 @@ -187,14 +187,20 @@ def bounded(value, depth=0): if not _text(json.dumps(settings, ensure_ascii=False, allow_nan=False, separators=(',', ':')), 1024 * 1024): raise ValueError('Settings are too large') raw_tracks = record.get('tracks') - expected = [('v1', 'video'), ('a1', 'audio'), ('a2', 'audio')] - if not isinstance(raw_tracks, list) or len(raw_tracks) not in (2, 3): + # v1 and a1, then an optional second sound track (a2), the recording sub-track under a1 (a3) and, only together with both, the three separated-sound tracks (a4-a6), in that order. + if not isinstance(raw_tracks, list) or len(raw_tracks) not in (2, 3, 4, 7): raise ValueError('Invalid tracks') + later = [raw.get('id') if isinstance(raw, dict) else None for raw in raw_tracks[2:]] + if later not in ([], ['a2'], ['a3'], ['a2', 'a3'], ['a2', 'a3', 'a4', 'a5', 'a6']): + raise ValueError('Invalid tracks') + expected = [('v1', 'video'), ('a1', 'audio')] + [(track_id, 'audio') for track_id in later] tracks, seen = [], {} for raw_track, (track_id, kind) in zip(raw_tracks, expected): if (not isinstance(raw_track, dict) or raw_track.get('id') != track_id or raw_track.get('kind') != kind or not isinstance(raw_track.get('clips'), list) or len(raw_track['clips']) > 500): raise ValueError('Invalid track') + if (raw_track.get('stem') != (STEM_TRACKS[len(tracks) - 4] if len(tracks) >= 4 else None)): + raise ValueError('Invalid track') clips, end = [], 0 for raw in raw_track['clips']: if not isinstance(raw, dict): @@ -740,6 +746,8 @@ def sources(root, keys): text = _work_text(root, relative) if text: media['text'] = text + if source['kind'] in ('work', 'generated') and _is_preview(root, relative): + media['preview'] = True if source['kind'] == 'import': metadata = _read_import(root, source['id']) media.update(preview_width=media['width'], preview_height=media['height']) @@ -769,6 +777,16 @@ def sources(root, keys): return result +def _is_preview(root, relative): + """A finished first-pass preview whose second pass can be run from its saved request.""" + try: + from .comfy_bridge import preview_request + preview_request(root, relative) + return True + except (ImportError, OSError, ValueError, TypeError): + return False + + def _work_text(root, relative): """The text a saved video was generated from; a block made from it starts with this description.""" try: diff --git a/freevideo_engine/comfy_launcher_runtime.py b/freevideo_engine/comfy_launcher_runtime.py index 367c3120..dd444694 100644 --- a/freevideo_engine/comfy_launcher_runtime.py +++ b/freevideo_engine/comfy_launcher_runtime.py @@ -1202,6 +1202,7 @@ def _install(self, accepted): if queue_busy(loopback_url(self.server_address or self.selection['url'])): raise RuntimeError(ENGINE_UPDATE_WAITS + '. Try again after it finishes; the job is left running.') self._stop_server_for_setup(selected['url']) + self.stage('nodes', label='Connecting workflow nodes') engine_binding.rebind(selected['engine'], machine, selected['source']) self.stage('nodes', label='Install FreeVideo workflow') deployed = deploy(selected['root'], selected['source'], selected['engine']) diff --git a/freevideo_engine/comfy_library.py b/freevideo_engine/comfy_library.py index 590ca999..c3eef4fa 100644 --- a/freevideo_engine/comfy_library.py +++ b/freevideo_engine/comfy_library.py @@ -1,5 +1,6 @@ """Browse completed FreeVideo outputs across ComfyUI/browser restarts.""" import asyncio +import hashlib from contextlib import nullcontext import io import json @@ -66,6 +67,77 @@ def _agent(folder): return {key: str(value[key])[:120] for key in ('job', 'label', 'project', 'client') if value.get(key)} +def _identity_of(root, value): + """The library identity of a video path named by a run, or None for a file elsewhere.""" + try: + relative = Path(value).resolve().relative_to(Path(root).resolve() / 'FreeVideo') + except (OSError, ValueError, TypeError): + return None + identity = '/'.join(relative.parts[:2]) + return identity if len(relative.parts) == 3 and relative.name == 'video.mp4' and _ID.fullmatch(identity) else None + + +def _source_identity(root, folder, report): + """The saved video this run started from: the preview it upscales, or the footage it continues.""" + source = (report.get('upscaled_preview') or {}).get('output') if isinstance(report.get('upscaled_preview'), dict) else None + if not source: + try: + with (folder / 'comfy-request.json').open('rb') as stream: + data = stream.read(1024 * 1024 + 1) + command = json.loads(data).get('command') if len(data) <= 1024 * 1024 else None + if isinstance(command, list) and '--continue-from' in command: + source = command[command.index('--continue-from') + 1] + except (OSError, ValueError, AttributeError, IndexError): + pass + return _identity_of(root, source) if isinstance(source, str) else None + + +def _prompt_digest(folder): + try: + with os.fdopen(os.open(folder / 'prompt.txt', os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) + | getattr(os, 'O_BINARY', 0)), 'rb') as stream: + data = stream.read(_PROMPT_BYTES + 1) + text = data.decode('utf-8').strip() if len(data) <= _PROMPT_BYTES else '' + except (OSError, ValueError): + return None + return hashlib.sha256(text.encode('utf-8')).hexdigest() if text else None + + +def _tasks(root, rows, folders, reports): + """Name each row's task: the earliest video connected to it by a preview and its upscale, a continuation, + one agent job, or the same prompt text on the same day. A video with no such link is its own task.""" + parent = {row['id']: row['id'] for row in rows} + def find(item): + while parent[item] != item: + parent[item] = parent[parent[item]] + item = parent[item] + return item + seen = {} + for row in rows: + identity = row['id'] + tokens = ['id:' + identity] + source = _source_identity(root, folders[identity], reports[identity]) + if source: + tokens.append('id:' + source) + if row.get('agent', {}).get('job'): + tokens.append('job:' + row['agent']['job']) + digest = _prompt_digest(folders[identity]) + if digest: + tokens.append('prompt:%s:%s' % (identity.split('/', 1)[0], digest)) + for token in tokens: + if token in seen: + parent[find(identity)] = find(seen[token]) + else: + seen[token] = identity + groups = {} + for row in rows: + groups.setdefault(find(row['id']), []).append(row) + for members in groups.values(): + earliest = min(members, key=lambda row: (row['created_at'], row['id']))['id'] + for row in members: + row['task'] = earliest + + def _favorite(path): """A missing or damaged optional sidecar does not hide a saved video.""" saved = path.with_suffix('.favorite.json') @@ -133,7 +205,7 @@ def list_videos(output_directory, *, before=None, limit=24, favorite=None): candidates.append((key, path, info)) except (OSError, ValueError): continue - rows, unreadable = [], [] + rows, unreadable, folders, reports = [], [], {}, {} for key, path, info in sorted(candidates, key=lambda item: item[0], reverse=True): relative = Path('FreeVideo') / key[1] / 'video.mp4' if relative.as_posix() in hidden: @@ -171,6 +243,7 @@ def list_videos(output_directory, *, before=None, limit=24, favorite=None): agent = None if agent: summary['agent'] = agent + folders[key[1]], reports[key[1]] = path.parent, report rows.append(dict(summary, id=key[1], bytes=info.st_size, favorite=is_favorite, created_at=datetime.fromtimestamp(info.st_mtime, timezone.utc).isoformat(), cursor=str(key[0]) + ':' + key[1])) @@ -182,6 +255,7 @@ def list_videos(output_directory, *, before=None, limit=24, favorite=None): break more = len(rows) > limit rows = rows[:limit] + _tasks(root, rows, folders, reports) next_cursor = rows[-1]['cursor'] if more else None for row in rows: row.pop('cursor') @@ -207,6 +281,23 @@ def prompt_text(output_directory, identity): return data.decode('utf-8').strip() if len(data) <= _PROMPT_BYTES else None +def prompt_seed(output_directory, identity): + """The seed a saved video was generated with, or None (an export, an import, an old or damaged receipt).""" + path = _video(output_directory, identity) + root = Path(output_directory).resolve() / 'FreeVideo' + saved = path.with_name('comfy-request.json') + try: + if saved.is_symlink() or not saved.resolve().is_relative_to(root): + return None + with os.fdopen(os.open(saved, os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_BINARY', 0)), 'rb') as stream: + data = stream.read(4 * 1024**2 + 1) + state = json.loads(data) if len(data) <= 4 * 1024**2 else None + except (OSError, ValueError): + return None + seed = state.get('seed') if isinstance(state, dict) and state.get('status') == 'complete' else None + return seed if type(seed) is int and 0 <= seed < 2**64 else None + + def thumbnail(output_directory, identity): path = _video(output_directory, identity) saved = path.with_suffix('.thumbnail.jpg') @@ -1238,7 +1329,9 @@ async def prompt(request): text = await asyncio.to_thread(prompt_text, folder_paths.get_output_directory(), request.query.get('id')) except (OSError, ValueError, UnicodeDecodeError): raise web.HTTPNotFound(text='Prompt unavailable') from None - return web.json_response(dict(prompt=text), headers={'Cache-Control': 'no-store'}) + # The seed is optional: without it, reusing the settings fills only the prompt and the size. + seed = await asyncio.to_thread(prompt_seed, folder_paths.get_output_directory(), request.query.get('id')) + return web.json_response(dict(prompt=text, **({} if seed is None else {'seed': seed})), headers={'Cache-Control': 'no-store'}) @server.routes.post('/freevideo/library/favorite') async def favorite(request): diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index 93a12312..97331251 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -263,6 +263,11 @@ def execute(cls, text, width, height, seconds, seed, first=None, last=None, note('video', path(comfy_bridge.installation_root())) except Exception as error: logging.warning('[FreeVideo] Could not record the video for the picture worker: %s', error) + try: # nor splitting a clip's sound, which starts again after the video + from .stems.service import stop_for_generation as stop_stems_for_generation + stop_stems_for_generation(PromptServer.instance) + except Exception as error: + logging.warning('[FreeVideo] Could not stop sound separation before the video: %s', error) prewarm = {} result_reused = [False] result_report = [None] @@ -422,7 +427,11 @@ def busy(): gpu = comfy_bridge.installation()[1].get('gpu_uuid') except (OSError, ValueError, KeyError, TypeError): gpu = None - kept = OWNER.finished(next_part=next_part, gpu=gpu, warm=True, + try: + admission = json.loads(output.with_suffix('.engine.json').read_text(encoding='utf-8')).get('resident_admission') + except (OSError, ValueError, AttributeError): + admission = None + kept = OWNER.finished(next_part=next_part, gpu=gpu, warm=True, decisions=admission, agent=server is not None and str(getattr(server, 'client_id', '') or '').startswith('agent:')) from .comfy_environment import executor_device if target and not result_reused[0] and not executor_device(): diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.py index fd6174d4..e1226ea5 100644 --- a/freevideo_engine/comfy_progress.py +++ b/freevideo_engine/comfy_progress.py @@ -428,6 +428,19 @@ async def feedback_info(request): info = await asyncio.to_thread(feedback.collect, request.query.get('report_id')) return web.json_response(info, headers={'Cache-Control': 'no-store'}) + @server.routes.get('/freevideo/stems-support') + async def stems_support(request): + # The editor page asks once whether to offer sound separation; the agent feature list needs a token the page never holds. + def offered(): + from .agent.editor_runtime import available + from .agent.routes import stems_supported + from .comfy_bridge import installation + try: + return bool(available() and stems_supported(installation()[1])) + except (OSError, ValueError): + return False + return web.json_response({'available': await asyncio.to_thread(offered)}, headers={'Cache-Control': 'no-store'}) + @server.routes.get('/freevideo/progress') async def progress(request): return web.json_response({'progress': STATE.snapshot()}, diff --git a/freevideo_engine/comfy_service.py b/freevideo_engine/comfy_service.py index 11602648..5e9e48b1 100644 --- a/freevideo_engine/comfy_service.py +++ b/freevideo_engine/comfy_service.py @@ -29,6 +29,8 @@ from .monitoring import save from . import engine_binding +DIAGNOSE = 'If it keeps failing, run ./freevideo diagnose and send the report.' + DEFAULT_PORT = 8188 RECORD = 'comfyui-cli.json' STATE = 'service.json' @@ -151,10 +153,10 @@ def install(root, source, machine, port, out=say): out('Installing ComfyUI for FreeVideo. This happens once; the next start takes seconds.') if host_setup(root, source, machine, comfy, download=not (comfy / 'main.py').is_file()): raise RuntimeError('ComfyUI could not be installed. Run the same command again to continue; ' - 'downloaded files are kept.') + 'downloaded files are kept. ' + DIAGNOSE) python = managed_python(Path(root), comfy) if python is None: - raise RuntimeError('ComfyUI was installed, but its environment is incomplete. Run the same command again.') + raise RuntimeError('ComfyUI was installed, but its environment is incomplete. Run the same command again. ' + DIAGNOSE) else: add_node_packages(root, source, machine, comfy, out) if switching: @@ -434,7 +436,7 @@ def wait_until_open(root, process, *, timeout=300, out=say): state_file = folder(root) / STATE deadline = time.monotonic() + timeout shown = time.monotonic() - interactive = sys.stdout.isatty() + interval = 15 if sys.stdout.isatty() else 30 # A log file gets a sign of life too, less often than a terminal. out('Starting ComfyUI ...') while time.monotonic() < deadline: state = read_json(state_file) or {} @@ -445,12 +447,12 @@ def wait_until_open(root, process, *, timeout=300, out=say): raise RuntimeError(state.get('error') or 'ComfyUI could not start.') if process.poll() is not None: raise RuntimeError(state.get('error') if mine and state.get('error') else - 'The FreeVideo service stopped while starting. See %s.' % (folder(root) / 'service.log')) - if interactive and time.monotonic() - shown >= 15: + 'The FreeVideo service stopped while starting. See %s. %s' % (folder(root) / 'service.log', DIAGNOSE)) + if time.monotonic() - shown >= interval: shown = time.monotonic() out(' still starting (%d s) ...' % (timeout - (deadline - time.monotonic()))) time.sleep(.3) - raise RuntimeError('ComfyUI is still starting after %d s. See %s.' % (timeout, folder(root) / 'service.log')) + raise RuntimeError('ComfyUI is still starting after %d s. See %s. %s' % (timeout, folder(root) / 'service.log', DIAGNOSE)) def respawn(root, state, *, port=None): @@ -700,18 +702,22 @@ def open_command(root, *, port=None, browser=True, restart=False, out=say): def wait_for_running(root, timeout=300, absent=0., out=say): """Wait while the service starts; `absent` allows for one that a service manager starts shortly.""" - started = time.monotonic() + started = shown = time.monotonic() + interval = 15 if sys.stdout.isatty() else 30 while time.monotonic() - started < timeout: state = running(root) if state is not None and state.get('status') == 'open': return state + if state is not None and time.monotonic() - shown >= interval: + shown = time.monotonic() + out(' still starting (%d s) ...' % (shown - started)) if state is None and time.monotonic() - started >= absent: previous = read_json(folder(root) / STATE) or {} if previous.get('status') == 'failed': raise RuntimeError(previous.get('error') or 'ComfyUI could not start.') return None time.sleep(.3) - raise RuntimeError('ComfyUI is still starting. Check freevideo status.') + raise RuntimeError('ComfyUI is still starting. Check freevideo status. ' + DIAGNOSE) def status_command(root, out=say): diff --git a/freevideo_engine/diagnostics.py b/freevideo_engine/diagnostics.py index 48df0e79..bd34e4d4 100644 --- a/freevideo_engine/diagnostics.py +++ b/freevideo_engine/diagnostics.py @@ -237,7 +237,7 @@ def web_files(root, machine): return dict(error='No FreeVideo source is recorded') -def collect(root, config, run, output, *, complete=False, extra_files=(), notes=''): +def collect(root, config, run, output, *, complete=False, extra_files=(), notes='', progress=None): root, config = resolved(root.expanduser()), io_path(config.expanduser().absolute()) run = io_path(run) if run is not None else None output = io_path(output) @@ -270,6 +270,8 @@ def add(name, raw, truncated=False): manifest['files'].append({'file': name, 'bytes': len(data), 'truncated': truncated, 'sha256': hashlib.sha256(data).hexdigest()}) manifest['payload_bytes'] += len(data) + if progress is not None: + progress(len(manifest['files'])) def add_path(name, path): if Path(path).name.lower() in SENSITIVE_FILES: diff --git a/freevideo_engine/editor_audio.py b/freevideo_engine/editor_audio.py index ab605d8f..30af9fd9 100644 --- a/freevideo_engine/editor_audio.py +++ b/freevideo_engine/editor_audio.py @@ -4,6 +4,8 @@ from .editor_speed import changed, audio_changed, source_sample_start +STEM_TRACKS = ('dialogue', 'music', 'ambience') + def clip_fields(raw, kind): result = {} @@ -29,6 +31,17 @@ def clip_fields(raw, kind): if kind != 'audio' or type(raw['duck']) is not bool: raise ValueError('Invalid audio ducking') result['duck'] = raw['duck'] + if 'stem_of' in raw: + value = raw['stem_of'] + if (kind != 'audio' or not isinstance(value, str) or not re.fullmatch(r'c[1-9]\d{0,8}', value) + or value == raw['id'] or raw.get('link')): + raise ValueError('Invalid separated sound') + result['stem_of'] = value + if 'stem_volume' in raw: + value = raw['stem_volume'] + if kind != 'audio' or type(value) not in (int, float) or not 0 <= value <= 2: + raise ValueError('Invalid separated sound') + result['stem_volume'] = value return result @@ -39,6 +52,10 @@ def track_fields(raw): if type(raw[name]) is not bool: raise ValueError('Invalid audio track flag') result[name] = raw[name] + if 'stem' in raw: + if raw['stem'] not in STEM_TRACKS: + raise ValueError('Invalid audio track flag') + result['stem'] = raw['stem'] return result diff --git a/freevideo_engine/editor_export.py b/freevideo_engine/editor_export.py index 01a0659c..b85ba971 100644 --- a/freevideo_engine/editor_export.py +++ b/freevideo_engine/editor_export.py @@ -113,7 +113,7 @@ def _layout(project): project = validate(project) raw_tracks = [t for t in project['tracks'] if not is_text_track(t)] if not isinstance(raw_tracks, list) or [t.get('id') for t in raw_tracks if isinstance(t, dict)] not in ( - ['v1', 'a1'], ['v1', 'a1', 'a2']): + ['v1', 'a1'], ['v1', 'a1', 'a2'], ['v1', 'a1', 'a3'], ['v1', 'a1', 'a2', 'a3'], ['v1', 'a1', 'a2', 'a3', 'a4', 'a5', 'a6']): raise ExportError() tracks, seen = {}, set() for raw in raw_tracks: @@ -777,7 +777,7 @@ def save_lead(lead): if progress: progress(written / total * .99) mixed = np.zeros((2, SAMPLES_PER_FRAME), dtype=np.float32) - for track in ('a1', 'a2'): + for track in ('a1', 'a2', 'a3', 'a4', 'a5', 'a6'): clip = active.get(track) if track not in audio_tracks: continue diff --git a/freevideo_engine/editor_text.py b/freevideo_engine/editor_text.py index 8e768924..a38a0c7c 100644 --- a/freevideo_engine/editor_text.py +++ b/freevideo_engine/editor_text.py @@ -91,7 +91,7 @@ def validate_text_project(raw, validate_media): if row['clips']: result['tracks'].append(row) style = clean_style(raw.get('subtitle_style',{}),'subtitle') if style and any(t['kind'] == 'subtitle' for t in result['tracks']): result['subtitle_style'] = style - result['tracks'].sort(key=lambda t: ['v1','a1','a2','t1','s1'].index(t['id'])) + result['tracks'].sort(key=lambda t: ['v1','a1','a2','a3','a4','a5','a6','t1','s1'].index(t['id'])) return result diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py index 4ceba4ec..5e07be14 100644 --- a/freevideo_engine/encode_worker.py +++ b/freevideo_engine/encode_worker.py @@ -568,25 +568,69 @@ def attempt(index, failure): comfy.model_management.unload_all_models() gc.collect() comfy.model_management.soft_empty_cache() - from diffusers.modular_pipelines.minimax_h3.encoders import encode_vae_condition - from .vae_weights import load_video_encoder - from src.inference.render import PIXEL_MEAN, PIXEL_STD - vae_started = time.perf_counter() - if request.get('metrics'): - save(request['metrics'], {**loading, 'success': False, 'phase': 'keyframe_vae', - 'load_seconds': load_seconds, 'prompt_encode_seconds': prompt_encode_seconds}) - # Only the encoder is read; the decoder in the shared shards is skipped. - vae = (resident.input_video_encoder(request['base'], dict(width=width, height=height)) - if resident is not None else load_video_encoder(request['base'])[0]) - torch.cuda.synchronize() - keyframe_metrics['keyframe_vae_load_seconds'] = time.perf_counter() - vae_started - vae_started = time.perf_counter() - with torch.no_grad(): - conditions = [encode_vae_condition(vae, - torch.from_numpy(np.array(image)).to('cuda').permute(2, 0, 1)[None, :, None], - PIXEL_MEAN, PIXEL_STD, 42).cpu() for image in images] - torch.cuda.synchronize() - keyframe_metrics['keyframe_vae_encode_seconds'] = time.perf_counter() - vae_started + from .input_cache import digest + from .media_encoding import _latent_cache, _cached_latent, _remember_latent, make_latent_cache_key + # Only a request that names a cache root caches; profiling runs stay uncached. + cache = None + if request.get('reference_latent_cache'): + try: + cache = _latent_cache(request['reference_latent_cache'], request['base'], 'cuda') + except Exception: + cache = None + conditions = [None, None] + keys = [] + files = [keyframes['first'], keyframes['last']] + working_dir = Path(request['output']).parent / 'media' + working_dir.mkdir(parents=True, exist_ok=True) + for idx, img_file in enumerate(files): + c_hash = digest(img_file) + k = make_latent_cache_key( + content_hash=c_hash, + width=width, + height=height, + frames=1, + crop_mode='put_on_canvas_first' if idx == 0 else 'put_on_canvas_last', # stretched vs cropped + base=request['base'], + device='cuda', + dtype='float32', + ) + keys.append(k) + if cache is not None: + latent = _cached_latent(cache, k, working_dir) + if latent is not None: + conditions[idx] = latent + if any(c is None for c in conditions): + from diffusers.modular_pipelines.minimax_h3.encoders import encode_vae_condition + from .vae_weights import load_video_encoder + from src.inference.render import PIXEL_MEAN, PIXEL_STD + vae_started = time.perf_counter() + if request.get('metrics'): + save(request['metrics'], {**loading, 'success': False, 'phase': 'keyframe_vae', + 'load_seconds': load_seconds, 'prompt_encode_seconds': prompt_encode_seconds}) + # Only the encoder is read; the decoder in the shared shards is skipped. + vae = (resident.input_video_encoder(request['base'], dict(width=width, height=height)) + if resident is not None else load_video_encoder(request['base'])[0]) + torch.cuda.synchronize() + keyframe_metrics['keyframe_vae_load_seconds'] = time.perf_counter() - vae_started + vae_started = time.perf_counter() + with torch.no_grad(): + for idx, (img, cond) in enumerate(zip(images, conditions)): + if cond is None: + cond = encode_vae_condition(vae, + torch.from_numpy(np.array(img)).to('cuda').permute(2, 0, 1)[None, :, None], + PIXEL_MEAN, PIXEL_STD, 42).cpu() + conditions[idx] = cond + if cache is not None: + _remember_latent(cache, keys[idx], cond, working_dir) + torch.cuda.synchronize() + keyframe_metrics['keyframe_vae_encode_seconds'] = time.perf_counter() - vae_started + del vae + gc.collect() + comfy.model_management.soft_empty_cache() + else: + print(json.dumps({'event': 'keyframe_vae_cache_hit'}), flush=True) + keyframe_metrics['keyframe_vae_load_seconds'] = 0. + keyframe_metrics['keyframe_vae_encode_seconds'] = 0. value.update(keyframe_anchors=['first', 'last'], condition_latents=conditions, keyframe_files=[keyframes['first'], keyframes['last']], width=width, height=height) from .keyframes import validate_conditioning diff --git a/freevideo_engine/geometry.py b/freevideo_engine/geometry.py index a1014d64..e51d49fe 100644 --- a/freevideo_engine/geometry.py +++ b/freevideo_engine/geometry.py @@ -24,7 +24,7 @@ def geometry(width=1344, height=768, frames=None, seconds=None): alignment='Pinned H3 VAE: round up to 17*n+5 frames; no resampling or hidden resolution reduction.') -def sampler_for_canvas(sampler, width, height): +def sampler_for_canvas(sampler, width, height, scheduler=None): """Bind canvas constants in an isolated globals dict, preserving upstream code. Engine.sample supplies no_grad. Never edit the upstream file or mutate its @@ -32,7 +32,10 @@ def sampler_for_canvas(sampler, width, height): The upstream sampler builds its schedulers by name; it gets the engine's, whose step grid is the one the modulation tables were made with. """ - from .step_grid import scheduler + from .step_grid import scheduler as engine_scheduler + # A caller may supply a factory that adds a start step or an observer to the + # same engine scheduler; the default is that scheduler itself. + scheduler = scheduler or engine_scheduler original = inspect.unwrap(sampler) bound = types.FunctionType(original.__code__, dict(original.__globals__, LATENT_H=height // 16, LATENT_W=width // 16, diff --git a/freevideo_engine/image_gen/image_models.json b/freevideo_engine/image_gen/image_models.json index 86f7f861..269cae28 100644 --- a/freevideo_engine/image_gen/image_models.json +++ b/freevideo_engine/image_gen/image_models.json @@ -685,6 +685,27 @@ } ] } + ], + "stems": [ + { + "file": "stems/bandit_plus_cocktail.pt", + "component": "stems", + "bytes": 148724733, + "sha256": "02ad94875ae060e03428af0e667a5a5cbfa1a6c2355f07f32c499be8d615ba1c", + "repo": "BowenXue/FreeVideo-models", + "revision": "2ed0b3b98ea92dec01699dc97d96e15cfe208d69", + "model_id": "bandit-plus-cocktail", + "sources": [ + { + "id": "official", + "url": "https://huggingface.co/BowenXue/FreeVideo-models/resolve/2ed0b3b98ea92dec01699dc97d96e15cfe208d69/stems/bandit_plus_cocktail.pt" + }, + { + "id": "modelscope", + "url": "https://www.modelscope.cn/models/BowenXue/FreeVideo-models/resolve/master/stems/bandit_plus_cocktail.pt" + } + ] + } ] } } diff --git a/freevideo_engine/image_gen/image_pack.py b/freevideo_engine/image_gen/image_pack.py index 4fdd0c33..2308bbbe 100644 --- a/freevideo_engine/image_gen/image_pack.py +++ b/freevideo_engine/image_gen/image_pack.py @@ -140,7 +140,7 @@ def validate_catalog(spec): raise PackError('invalid_catalog') image_rows = spec['groups']['image'] # A group of its own holds only its own component; the image group holds neither. - own = {group: spec['groups'].get(group, []) for group in ('segment', 'music')} + own = {group: spec['groups'].get(group, []) for group in ('segment', 'music', 'stems')} found = image_rows + [row for found in own.values() for row in found] if (not image_rows or len({r['file'] for r in found}) != len(found) or set(spec['groups']) - {'image', *own} or any(row['component'] in own for row in image_rows) @@ -150,7 +150,7 @@ def validate_catalog(spec): _relative(row['file']) if (type(row['bytes']) is not int or row['bytes'] <= 0 or not re.fullmatch('[0-9a-f]{64}', row['sha256']) - or row['component'] not in ('dit', 'text_encoder', 'vae', 'text_encoder_config', 'processor', 'segment', 'music')): + or row['component'] not in ('dit', 'text_encoder', 'vae', 'text_encoder_config', 'processor', 'segment', 'music', 'stems')): raise PackError('invalid_catalog') ids = [s['id'] for s in row['sources']] required = {'official', 'modelscope'} @@ -233,8 +233,13 @@ def _owned_intact(path, stamp): b'freevideo_engine/licenses/Qwen3-Embedding-Apache-2.0.txt.\nhttps://huggingface.co/Qwen/Qwen3-Embedding-0.6B\n' b'Silero VAD, Copyright (c) 2020-present Silero Team. Licensed under the MIT License; see ' b'freevideo_engine/licenses/Silero-VAD-MIT.txt.\nhttps://github.com/snakers4/silero-vad\n') +STEMS_NOTICE_TEXT = (b'Bandit Plus (cocktail-party separation model). Weights: Watcharasupat et al., CC BY-NC 4.0; see ' + b'freevideo_engine/licenses/Bandit-CC-BY-NC-4.0.txt.\nhttps://zenodo.org/records/10160698\n' + b'Code: Copyright (c) the Bandit authors, Apache-2.0; see freevideo_engine/licenses/Bandit-Apache-2.0.txt.\n' + b'https://github.com/kwatcharasupat/source-separation\n') NOTICES = {'image': ('NOTICE.txt', NOTICE_TEXT), 'segment': ('SAM2-NOTICE.txt', SEGMENT_NOTICE_TEXT), - 'music': ('MUSIC-NOTICE.txt', MUSIC_NOTICE_TEXT)} + 'music': ('MUSIC-NOTICE.txt', MUSIC_NOTICE_TEXT), + 'stems': ('STEMS-NOTICE.txt', STEMS_NOTICE_TEXT)} def _notice(): @@ -929,7 +934,7 @@ def main(argv=None): commands = parser.add_subparsers(dest='command', required=True) download = commands.add_parser('download') download.add_argument('--background', action='store_true') - download.add_argument('--group', choices=('image', 'segment'), default='image') + download.add_argument('--group', choices=('image', 'segment', 'music', 'stems'), default='image') download.add_argument('--root', type=Path) download.add_argument('--engine-root', type=Path) download.add_argument('--status-file', type=Path, required=True) diff --git a/freevideo_engine/input_cache.py b/freevideo_engine/input_cache.py index a7c0c4f1..29a7014b 100644 --- a/freevideo_engine/input_cache.py +++ b/freevideo_engine/input_cache.py @@ -121,7 +121,13 @@ def put(self, key, source, metrics, *, move=False): if digest(temporary) != checksum: raise ValueError('Input changed during cache copy; partial retained') from .file_ops import publish - publish(temporary, target) + try: + publish(temporary, target) + except FileExistsError: + if not target.exists() or digest(target) != checksum: + raise + if not move: + temporary.unlink(missing_ok=True) receipt = self.receipt(key) if receipt.exists(): # Keep old receipts too, including malformed ones, for diagnostics. diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml index f1a5bdf4..06beb9d5 100644 --- a/freevideo_engine/launcher/Main.qml +++ b/freevideo_engine/launcher/Main.qml @@ -206,7 +206,7 @@ ApplicationWindow { return (u.blockers && u.blockers.length ? upgradeBlocker(u.blockers[0]) + t(" ", "") : u.error ? u.error.split("\n")[0] + "\n" : "") + (u.step === "release" ? t("The faster model is in use; the old model files are kept for now.", "已在用更快的模型，旧模型文件暂时保留。") : u.step === "switch" && !u.kept ? t("The previous model's files are all kept. Select \"Retry\" on the setup page to restore it.", "原来的模型文件都还在，在安装页点“重试”可恢复。") - : t("Your previous model is unchanged and still works.", "原来的模型没有变化，可以继续使用。")) + : t("Your previous model is unchanged and still works. Select \"Try again\", or report a problem.", "原来的模型没有变化，可以继续使用。可以点“重试”，或提交问题。")) return "" } function updateHeadline() { @@ -485,7 +485,7 @@ ApplicationWindow { objectName: "reportStatus" visible: s.report.status !== "idle"; Layout.fillWidth: true; font.pixelSize: theme.micro color: s.report.status === "error" ? theme.danger : theme.muted - text: s.report.status === "running" ? t("Exporting full logs…", "正在导出完整日志…") : + text: s.report.status === "running" ? t("Exporting full logs…", "正在导出完整日志…") + (s.report.files > 0 ? t(" · " + s.report.files + " files packed", " · 已打包 " + s.report.files + " 个文件") : "") : s.report.status === "error" ? t("Export failed: ", "导出失败：") + s.report.error : t("Report saved locally: ", "报告已保存到本机：") + s.report.path + (s.report.collection_errors ? t("\nReport saved with some diagnostic information missing.", "\n报告已保存，部分诊断信息未能收集。") : "") @@ -1481,7 +1481,7 @@ ApplicationWindow { } FText { visible: s.report.status !== "idle"; Layout.fillWidth: true; font.pixelSize: theme.micro; color: theme.muted - text: s.report.status === "running" ? t("Exporting full logs…", "正在导出完整日志…") : + text: s.report.status === "running" ? t("Exporting full logs…", "正在导出完整日志…") + (s.report.files > 0 ? t(" · " + s.report.files + " files packed", " · 已打包 " + s.report.files + " 个文件") : "") : s.report.status === "error" ? s.report.error : t("Saved: ", "已保存：") + s.report.path } } diff --git a/freevideo_engine/launcher_copy.py b/freevideo_engine/launcher_copy.py index cc581f30..940e711f 100644 --- a/freevideo_engine/launcher_copy.py +++ b/freevideo_engine/launcher_copy.py @@ -43,7 +43,7 @@ ('Preparing installation', '准备安装'), ('Checking the environment', '正在检查运行环境'), ('Testing download sources', '正在测试下载源'), ('Preparing download tools', '正在准备下载工具'), ('GitHub mirror', 'GitHub 镜像'), ('attempt', '正在连接'), - ('fallback', '正在切换下载源'), ('route-retry', '正在切换连接'), + ('fallback', '正在切换下载源'), ('switching', '正在切换下载源'), ('route-retry', '正在切换连接'), ('verifying', '正在检查'), ('no-progress', '下载无进展'), ('slow-transfer', '持续低速'), ('resume-refused', '当前下载源不支持续传'), ('Download PyTorch + CUDA', '下载运行组件'), @@ -78,14 +78,14 @@ ('Install ComfyUI packages', '安装 ComfyUI 组件'), ('Check ComfyUI dependencies', '检查 ComfyUI 环境'), ('Finish ComfyUI setup', '完成 ComfyUI 安装'), ('ComfyUI ready', 'ComfyUI 已就绪'), ('Prepare ComfyUI', '准备 ComfyUI'), ('Update ComfyUI packages', '更新 ComfyUI 组件'), - ('Install FreeVideo workflow', '安装 FreeVideo 工作流'), + ('Install FreeVideo workflow', '安装 FreeVideo 工作流'), ('Connecting workflow nodes', '正在连接工作流节点'), ('Open ComfyUI', '打开 ComfyUI'), ('Starting ComfyUI', '启动 ComfyUI'), ('Loading nodes and the web interface', '正在加载界面'), ('Still loading. This may take a few more minutes.', '仍在加载，可能还需要几分钟'), ('Checking configuration', '检查配置'), ('Check installation', '检查安装'), ('Verify bundled files', '校验离线包'), ('Check local files', '检查本地文件'), ('All bundled files verified', '离线包校验完成'), - ('Detecting your GPU and testing acceleration', '检测显卡与加速组件'), + ('Detecting the GPU', '正在检测显卡'), ('Testing acceleration', '正在测试加速组件'), ('Start ComfyUI', '启动 ComfyUI'), ('Open FreeVideo workflow', '打开 FreeVideo 工作流'), ('Ready', '已就绪'), ('Verified and ready', '已就绪'), ('Stopped; files retained', '已暂停，下载进度已保留'), diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index 38c0f2b7..7eca332b 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -224,8 +224,10 @@ def __init__(self, source=None, *, controller=None, store=None, updater=None, sm try: self.bridge = create(launcher_root()) self.controller.update_bridge = self.bridge - except OSError: + except (OSError, ValueError) as error: + # Only updates started from the web page are lost; the launcher still opens. self.bridge = None + print('Launcher bridge unavailable: %s' % type(error).__name__, file=sys.stderr) if self.form['environment_method'] == 'manual' and self.form['offline_runtime'] and not controller: root = io_path(self.form['offline_runtime']) if (root / 'portable.json').is_file(): @@ -1182,7 +1184,7 @@ def diagnostic_error(self): raw = self._raw_error() for state in (self.cleaner.state, self.importer.state, self.upgrade.state): value = state.get('error') or '' - if alias_failure(value) and value not in raw: + if (alias_failure(value) or (state is self.upgrade.state and state.get('status') == 'failed')) and value not in raw: raw += ('\n' if raw else '') + value pack_report = self.image_pack_report() if pack_report: @@ -1203,7 +1205,13 @@ def export_report(self, output, overwrite=False): notes = self.diagnostic_error() logs = tuple(path for _, path in self.controller.terminal_sources()) replacements = [(self.token, '')] if self.token else [] - self.report = dict(status='running', path=str(output), error='') + started = time.monotonic() + self.report = dict(status='running', path=str(output), error='', files=0) + + def packed(count): + # The count is shown only when packing takes longer than a few seconds. + if self.report.get('status') == 'running': + self.report = dict(self.report, files=count if time.monotonic() - started > 5 else 0) def work(): target = Path(output) @@ -1212,7 +1220,7 @@ def work(): replacing = False try: result = collect(root, root / 'machine.json', None, partial, - complete=True, extra_files=logs, notes=notes) + complete=True, extra_files=logs, notes=notes, progress=packed) if partial != target: replacing = True os.replace(io_path(partial), io_path(target)) diff --git a/freevideo_engine/licenses/Bandit-Apache-2.0.txt b/freevideo_engine/licenses/Bandit-Apache-2.0.txt new file mode 100644 index 00000000..3b6e7a43 --- /dev/null +++ b/freevideo_engine/licenses/Bandit-Apache-2.0.txt @@ -0,0 +1,207 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, and + distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by the + copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all other + entities that control, are controlled by, or are under common control with + that entity. For the purposes of this definition, "control" means (i) the + power, direct or indirect, to cause the direction or management of such + entity, whether by contract or otherwise, or (ii) ownership of + fifty percent (50%) or more of the outstanding shares, or (iii) beneficial + ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity exercising + permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation source, + and configuration files. + + "Object" form shall mean any form resulting from mechanical transformation + or translation of a Source form, including but not limited to compiled + object code, generated documentation, and conversions to + other media types. + + "Work" shall mean the work of authorship, whether in Source or Object + form, made available under the License, as indicated by a copyright notice + that is included in or attached to the work (an example is provided in the + Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object form, + that is based on (or derived from) the Work and for which the editorial + revisions, annotations, elaborations, or other modifications represent, + as a whole, an original work of authorship. For the purposes of this + License, Derivative Works shall not include works that remain separable + from, or merely link (or bind by name) to the interfaces of, the Work and + Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including the original + version of the Work and any modifications or additions to that Work or + Derivative Works thereof, that is intentionally submitted to Licensor for + inclusion in the Work by the copyright owner or by an individual or + Legal Entity authorized to submit on behalf of the copyright owner. + For the purposes of this definition, "submitted" means any form of + electronic, verbal, or written communication sent to the Licensor or its + representatives, including but not limited to communication on electronic + mailing lists, source code control systems, and issue tracking systems + that are managed by, or on behalf of, the Licensor for the purpose of + discussing and improving the Work, but excluding communication that is + conspicuously marked or otherwise designated in writing by the copyright + owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity on + behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. + + Subject to the terms and conditions of this License, each Contributor + hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable copyright license to reproduce, prepare + Derivative Works of, publicly display, publicly perform, sublicense, + and distribute the Work and such Derivative Works in + Source or Object form. + +3. Grant of Patent License. + + Subject to the terms and conditions of this License, each Contributor + hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, + royalty-free, irrevocable (except as stated in this section) patent + license to make, have made, use, offer to sell, sell, import, and + otherwise transfer the Work, where such license applies only to those + patent claims licensable by such Contributor that are necessarily + infringed by their Contribution(s) alone or by combination of their + Contribution(s) with the Work to which such Contribution(s) was submitted. + If You institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work or a + Contribution incorporated within the Work constitutes direct or + contributory patent infringement, then any patent licenses granted to + You under this License for that Work shall terminate as of the date such + litigation is filed. + +4. Redistribution. + + You may reproduce and distribute copies of the Work or Derivative Works + thereof in any medium, with or without modifications, and in Source or + Object form, provided that You meet the following conditions: + + 1. You must give any other recipients of the Work or Derivative Works a + copy of this License; and + + 2. You must cause any modified files to carry prominent notices stating + that You changed the files; and + + 3. You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, patent, trademark, and attribution notices from + the Source form of the Work, excluding those notices that do not pertain + to any part of the Derivative Works; and + + 4. If the Work includes a "NOTICE" text file as part of its distribution, + then any Derivative Works that You distribute must include a readable copy + of the attribution notices contained within such NOTICE file, excluding + those notices that do not pertain to any part of the Derivative Works, + in at least one of the following places: within a NOTICE text file + distributed as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, within a + display generated by the Derivative Works, if and wherever such + third-party notices normally appear. The contents of the NOTICE file are + for informational purposes only and do not modify the License. + You may add Your own attribution notices within Derivative Works that You + distribute, alongside or as an addendum to the NOTICE text from the Work, + provided that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and may + provide additional or different license terms and conditions for use, + reproduction, or distribution of Your modifications, or for any such + Derivative Works as a whole, provided Your use, reproduction, and + distribution of the Work otherwise complies with the conditions + stated in this License. + +5. Submission of Contributions. + + Unless You explicitly state otherwise, any Contribution intentionally + submitted for inclusion in the Work by You to the Licensor shall be under + the terms and conditions of this License, without any additional + terms or conditions. Notwithstanding the above, nothing herein shall + supersede or modify the terms of any separate license agreement you may + have executed with Licensor regarding such Contributions. + +6. Trademarks. + + This License does not grant permission to use the trade names, trademarks, + service marks, or product names of the Licensor, except as required for + reasonable and customary use in describing the origin of the Work and + reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + + Unless required by applicable law or agreed to in writing, Licensor + provides the Work (and each Contributor provides its Contributions) + on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + either express or implied, including, without limitation, any warranties + or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS + FOR A PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any risks + associated with Your exercise of permissions under this License. + +8. Limitation of Liability. + + In no event and under no legal theory, whether in tort + (including negligence), contract, or otherwise, unless required by + applicable law (such as deliberate and grossly negligent acts) or agreed + to in writing, shall any Contributor be liable to You for damages, + including any direct, indirect, special, incidental, or consequential + damages of any character arising as a result of this License or out of + the use or inability to use the Work (including but not limited to damages + for loss of goodwill, work stoppage, computer failure or malfunction, + or any and all other commercial damages or losses), even if such + Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + + While redistributing the Work or Derivative Works thereof, You may choose + to offer, and charge a fee for, acceptance of support, warranty, + indemnity, or other liability obligations and/or rights consistent with + this License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf of any + other Contributor, and only if You agree to indemnify, defend, and hold + each Contributor harmless for any liability incurred by, or claims + asserted against, such Contributor by reason of your accepting any such + warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + + To apply the Apache License to your work, attach the following boilerplate + notice, with the fields enclosed by brackets "[]" replaced with your own + identifying information. (Don't include the brackets!) The text should be + enclosed in the appropriate comment syntax for the file format. We also + recommend that a file or class name and description of purpose be included + on the same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {{ year }} {{ organization }} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing + permissions and limitations under the License. diff --git a/freevideo_engine/licenses/Bandit-CC-BY-NC-4.0.txt b/freevideo_engine/licenses/Bandit-CC-BY-NC-4.0.txt new file mode 100644 index 00000000..b90ca830 --- /dev/null +++ b/freevideo_engine/licenses/Bandit-CC-BY-NC-4.0.txt @@ -0,0 +1,10 @@ +Bandit Plus cocktail-party separation weights (speech / music / effects). + +Credit: Karn N. Watcharasupat, Chih-Wei Wu, Iroro Orife, "Divide and Remaster / Banquet / Bandit: +Source Separation for the Cocktail Party Problem" (Bandit, Bandit Plus), and the Divide and Remaster +v3 dataset authors. Weights released at https://zenodo.org/records/10160698 +Code: https://github.com/kwatcharasupat/source-separation (Apache-2.0, see Bandit-Apache-2.0.txt). + +The weights are licensed under the Creative Commons Attribution-NonCommercial 4.0 International +License (CC BY-NC 4.0): https://creativecommons.org/licenses/by-nc/4.0/legalcode +FreeVideo converts the checkpoint to a plain state dictionary without changing any value. diff --git a/freevideo_engine/macos_runtime.py b/freevideo_engine/macos_runtime.py index 694e5a01..19a41851 100644 --- a/freevideo_engine/macos_runtime.py +++ b/freevideo_engine/macos_runtime.py @@ -495,7 +495,7 @@ def sample(self, *args, **kwargs): return result def _sample(self, conditioning, seed, *, width, height, frames, progress_offset=0, progress_total=None, - initial_latents=None, refine_steps=None, refine_schedule=None, continuation=None): + initial_latents=None, refine_steps=None, refine_schedule=None, continuation=None, step_state=None): import torch from .geometry import geometry, sampler_for_canvas from .system import system_memory @@ -531,9 +531,19 @@ def _sample(self, conditioning, seed, *, width, height, frames, progress_offset= from .reference_sampler import generate_latents active_steps = refine_steps if refining else self.steps canvas = geometry(width, height, frames=frames) - sampler = sampler_for_canvas(generate_latents, width, height) - progress = MacSamplingProgress(active_steps, len(self.model.transformer_blocks), - offset=progress_offset, total=progress_total) + # Only the plain first pass without a step cache saves its steps; a retry + # of it continues after the last saved one. + resumable = (step_state is not None and not refining and self.task == 't2va' + and continuation is None and not self.step_cache) + resumed = step_state.load() if resumable else None + start = step_state.start if resumed else 0 + sampler = sampler_for_canvas(generate_latents, width, height, + scheduler=step_state.scheduler if resumable else None) + progress = MacSamplingProgress(active_steps - start, len(self.model.transformer_blocks), + offset=progress_offset + start, + total=progress_total if progress_total is not None else active_steps + progress_offset) + if start: + print(json.dumps(dict(event='step_resume', completed_steps=start, total=active_steps)), flush=True) prefix = None class StepTimes(list): def append(owner, duration): @@ -541,6 +551,8 @@ def append(owner, duration): prefix.restore_video() super().append(duration) progress.complete(duration) + if resumable: + step_state.completed(duration) durations = StepTimes() previous_calls = dict(self.attention.backend_calls) previous_kernel_calls = {name: dict(counts) for name, counts in @@ -568,7 +580,7 @@ def host_guard(module, inputs): + 2 * continuation['audio'].shape[2]) self._plan_compute(canvas, prompt.shape[0]) previous_block_calls = self._block_metrics() - self.cursor.reset(self.steps - refine_steps if refining and refine_schedule is None else 0) + self.cursor.reset(self.steps - refine_steps if refining and refine_schedule is None else start) progress.start() resident = getattr(self, 'resident_bytes', 0) retention = (dict(resident_bytes=resident, working_reserve_bytes=self.reserve_bytes) @@ -589,7 +601,7 @@ def host_guard(module, inputs): audio_offset=canvas.get('reference_audio_tokens', 0)) as prefix, \ progress.layers(self.model.transformer_blocks), \ frame_preview.observing(latent_frames=canvas['latent_frames'], frame_h=height // 32, - frame_w=width // 32, step_offset=progress_offset, + frame_w=width // 32, step_offset=progress_offset + start, total=progress_total if progress_total is not None else active_steps): self.residency = residency from .step_cache import begin_pass, end_pass @@ -597,12 +609,16 @@ def host_guard(module, inputs): if not refining and self.step_cache: begin_pass(self.model, active_steps, self.step_cache) try: + if resumable: + step_state.attach(self.model, self.backend.device) video, audio = sampler(self.model, prompt, tags, canvas['frames'], self.steps, seed, self.backend.device, step_seconds=durations, **(dict(conditions=conditions) if conditions else {}), **(dict(initial_latents=initial_latents, refine_steps=refine_steps, refine_schedule=refine_schedule) if refining else {})) finally: + if resumable: + step_state.detach() step_cache = end_pass(self.model) if prefix is not None: video, audio = prefix.finish(video, audio) @@ -628,7 +644,8 @@ def host_guard(module, inputs): max_bytes=max(self.allocator_limits)), attention_calls={name: value - previous_calls[name] for name, value in self.attention.backend_calls.items()}, - step_seconds=list(durations), step_cache=step_cache, + step_seconds=list(step_state.previous_seconds if resumable else durations), step_cache=step_cache, + **(dict(resumed_from_step=start) if start else {}), block_seconds=progress.block_seconds, block_seconds_scope=BLOCK_SECONDS_SCOPE, conditions=progress.conditions, load_seconds=self.load_seconds, residency=self._residency_metrics(residency, previous_residency), diff --git a/freevideo_engine/macos_stages.py b/freevideo_engine/macos_stages.py index b5df0c92..43b1328d 100644 --- a/freevideo_engine/macos_stages.py +++ b/freevideo_engine/macos_stages.py @@ -12,7 +12,7 @@ import time import traceback -from . import processes +from . import processes, step_resume from .locking import LOCK_ENV, runtime_lock from .monitoring import save @@ -182,6 +182,9 @@ def interrupted(signum, frame): continue result['phase'] = stage save(path, result) + if stage == 'first-pass': + # Steps saved by an earlier request's pass must never be resumed. + step_resume.discard(artifacts) request = artifacts / (stage + '-request.json') receipt = artifacts / (stage + '-result.json') while True: @@ -198,6 +201,13 @@ def interrupted(signum, frame): pass_fds=(descriptor,), check=True) except subprocess.CalledProcessError: failed = read_receipt(receipt) + if failed.get('error_type') == 'ResumeRejected' and (artifacts / 'first-pass-step.pt').is_file(): + # The saved step did not reach the sampler. Start the pass + # over without spending the retry. + step_resume.discard(artifacts) + result.setdefault('resume_rejections', []).append(failed.get('error_message')) + receipt.unlink(missing_ok=True) + continue if not memory_exhausted(stage, failed): raise metal = METAL_FAILURE in str(failed.get('error_message', '')) @@ -246,6 +256,8 @@ def interrupted(signum, frame): if stage in ('first-pass', 'refinement'): # A later pass's failure is not a consecutive failed retry. metal_retry = False + if stage == 'first-pass': + step_resume.discard(artifacts) break if stage == 'first-pass' and retained_joined: continue @@ -440,6 +452,14 @@ def capture_first(video): # A memory retry keeps each pass in its own process. admission = (joined_admission(engine, value) if not refining and not retry and value.get('preview') is not True else dict(selected=False)) + # A first pass saves each step so that a retry can continue after the last one. + step_state = None + if not refining and not admission['selected']: + try: + step_state = step_resume.StepState(step_resume.step_path(artifacts), + first_pass_identity(value), sampling['base_steps']) + except (OSError, KeyError, ValueError): + step_state = None result['sampling_admission'] = admission if admission['selected']: result.update(joined_sampling=True, sampling_passes=[], latent_save_seconds=0.) @@ -471,6 +491,7 @@ def first_saved(video, audio, metrics): else: video, audio, measured = engine.sample(value['conditioning'], value['seed'] + (sampling['restart_seed_offset'] if refining else 0), + **(dict(step_state=step_state) if step_state is not None else {}), **sampling['second' if refining else 'first'], **(dict(continuation=chain.load(continuation, 'final' if refining else 'first')) if continuation is not None else {}), diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index c3a7a446..9cc4c84e 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -145,13 +145,19 @@ def prepare(media, canvas, directory): for anchor in ('first', 'last'): if not media.get(anchor): continue + c_hash = media[anchor].get('sha256') + if not c_hash and media[anchor].get('path'): + from .input_cache import digest + c_hash = digest(media[anchor]['path']) with Image.open(media[anchor]['path']) as source: image = ImageOps.fit(ImageOps.exif_transpose(source).convert('RGB'), (canvas['width'], canvas['height']), method=Image.Resampling.LANCZOS) target = directory / (anchor + '.npy') np.save(target, np.array(image)[None], allow_pickle=False) key_images.append(torch.from_numpy(np.array(image)).float().div_(255)[None]) - result.append({'kind': 'image', 'anchor': anchor, 'pixels': str(target), 'frames': 1}) + result.append({'kind': 'image', 'anchor': anchor, 'pixels': str(target), 'frames': 1, + 'content_hash': c_hash, 'source_path': media[anchor]['path'], + 'width': canvas['width'], 'height': canvas['height'], 'crop_mode': 'fit'}) for index, row in enumerate(media.get('references', [])): kind, path = row['kind'], row['path'] if row.get('use') == 'original': @@ -163,7 +169,11 @@ def prepare(media, canvas, directory): audio=model_audio(info, canvas, target))) ref_items.append({'type': 'audio'}) continue - ref = {'kind': kind} + c_hash = row.get('sha256') + if not c_hash and path: + from .input_cache import digest + c_hash = digest(path) + ref = {'kind': kind, 'content_hash': c_hash, 'source_path': path} if 'early_steps' in row: ref['early_steps'] = row['early_steps'] if row.get('guide'): @@ -174,9 +184,10 @@ def prepare(media, canvas, directory): if kind == 'image': with Image.open(path) as source: image = ImageOps.exif_transpose(source).convert('RGB') - image = image.resize(reference_size(*image.size, canvas), Image.Resampling.LANCZOS) + target_w, target_h = reference_size(*image.size, canvas) + image = image.resize((target_w, target_h), Image.Resampling.LANCZOS) np.save(target, np.array(image)[None], allow_pickle=False) - ref.update(pixels=str(target), frames=1) + ref.update(pixels=str(target), frames=1, width=target_w, height=target_h, crop_mode='resize') ref_items.append({'type': 'image', 'data': torch.from_numpy(np.array(image)).float().div_(255)[None]}) elif kind == 'video': import av @@ -213,7 +224,7 @@ def prepare(media, canvas, directory): if count < 22: raise ValueError('The reference video is too short; it needs to be at least about 1 s long.') pixels.flush() - ref.update(pixels=str(target), frames=count, normalized_fps=24) + ref.update(pixels=str(target), frames=count, normalized_fps=24, width=width, height=height, crop_mode='video_24fps') used = count / 24. # The clip's own sound is a reference too, unless the request turned it off. soundtrack, longer_audio, _ = (None, False, None) if row.get('sound') is False or row.get('guide') else \ @@ -233,6 +244,7 @@ def prepare(media, canvas, directory): ref['audio'], longer, length = _audio(path, target, limit / 24.) if ref['audio'] is None: raise ValueError('Reference audio file has no audio stream') + ref.update(width=0, height=0, frames=0, crop_mode='audio_32khz') ref_items.append({'type': 'audio'}) if longer: # Numbered per kind like the prompt's