FreeVideo 0.4.31

THIRD_PARTY_NOTICES.md

@@ -21,6 +21,7 @@ FreeVideo code uses [Apache-2.0](LICENSE). Its dependencies and models retain th
 | [Eigen](https://gitlab.com/libeigen/eigen) 5.0.1, compiled into sherpa-onnx and ONNX Runtime; source: https://gitlab.com/libeigen/eigen/-/tree/5.0.1 | MPL-2.0 |
 | [Silero VAD](https://github.com/snakers4/silero-vad) v5.0 voice-activity model, © 2020-present Silero Team (optional download with subtitles, license file downloaded with the model; the onnx-community export is downloaded with the music model) | [MIT](freevideo_engine/licenses/Silero-VAD-MIT.txt) |
 | [ACE-Step 1.5](https://github.com/ace-step/ACE-Step-1.5) XL turbo music model, © 2026 ACEStep (optional download for music; inference subset adapted in `freevideo_engine/music/acestep_model.py`) | [MIT](freevideo_engine/licenses/ACE-Step-MIT.txt) |
+| [Bandit Plus](https://zenodo.org/records/10160698) cocktail-party separation model, weights by Watcharasupat et al. (optional download for sound separation, attribution required, non-commercial); code from [Bandit](https://github.com/kwatcharasupat/source-separation), © the Bandit authors, reduced to an inference subset in `freevideo_engine/stems/bandit/` (modified) | [CC BY-NC 4.0](freevideo_engine/licenses/Bandit-CC-BY-NC-4.0.txt) (weights), [Apache-2.0](freevideo_engine/licenses/Bandit-Apache-2.0.txt) (code) |
 | [vector-quantize-pytorch](https://github.com/lucidrains/vector-quantize-pytorch), © 2020 Phil Wang (residual FSQ decoding adapted in `freevideo_engine/music/acestep_model.py`) | [MIT](freevideo_engine/licenses/Vector-Quantize-PyTorch-MIT.txt) |
 | [Qwen3-Embedding-0.6B](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B), © Alibaba Cloud (text encoder of the music model, downloaded with it) | [Apache-2.0](freevideo_engine/licenses/Qwen3-Embedding-Apache-2.0.txt) |
 | [NVIDIA NeMo stt_en_conformer_ctc_small](https://huggingface.co/nvidia/stt_en_conformer_ctc_small), © NVIDIA, exported to ONNX and quantized to int8 by the sherpa-onnx project, changed from the original; not endorsed by NVIDIA (optional download for English subtitles; [notice](freevideo_engine/licenses/NeMo-stt_en_conformer_ctc_small-NOTICE.txt)) | [CC-BY-4.0](https://creativecommons.org/licenses/by/4.0/) |

freevideo_engine/adaptive.py

@@ -785,6 +785,11 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry):
         output_frames = (count * canvas['fps'] + rate - 1) // rate
         if hidden or not 0 < output_frames <= expected:
             raise ValueError('Original audio exceeds the generated canvas')
+        if isinstance(original_audio.get('segment'), dict):
+            # A chain part keeps its whole picture grid; the join trims the tail.
+            output_frames = expected
+        if hidden or not 0 < output_frames <= expected:
+            raise ValueError('Original audio exceeds the generated canvas')
     if continuation is not None:
         from .continuation import OVERLAP_FRAMES
         if (not isinstance(continuation, dict) or continuation.get('overlap_frames') != OVERLAP_FRAMES

freevideo_engine/agent/errors.py

@@ -121,6 +121,10 @@
                               'submit the music job again with a new ref; if it fails again, write the score yourself.'),
     'music_failed': (409, True, 'Composing the music did not finish.',
                      'Read the project, then submit the music job again with a new ref; if it fails again, write the score yourself.'),
+    'stems_download_failed': (409, True, 'The sound separation model could not be downloaded.',
+                              'Tell the user the model needs a network connection, then submit the job again with a new ref.'),
+    'stems_failed': (409, True, 'Splitting the sound did not finish.',
+                     'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'),
     'clip_source_unusable': (409, False, 'A clip\'s source is missing, or cannot be read as far as the clip uses it.',
                              'Read the project and check each clip\'s source: it must exist and play, and in + frames must stay '
                              'within its real length. Correct the project, then export the new revision.'),
@@ -235,6 +239,8 @@ def recovery(code, *, kind=None, stage='request', resumable=True):
         return 'Tell the user to free disk space, then submit the picture again with a new ref.'
     if kind == 'music' and code == 'out_of_memory':
         return 'Close other GPU applications, then submit the music job again with a new ref.'
+    if kind == 'stems' and code == 'out_of_memory':
+        return 'Close other GPU applications, then submit the job again with a new ref.'
     if kind == 'image' and code == 'no_memory':
         return 'Lower the picture dimensions or use fewer reference pictures, then submit with a new ref.'
     if kind in RECOVERY and stage == 'terminal':

freevideo_engine/agent/guide.py

@@ -144,6 +144,7 @@
         'On 409 project_changed, recompute edits from error.project and its current_revision, then send a new request. On clip_busy wait for generation to finish.',
         'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.',
         'Music: POST /projects/{id}/music with the revision you read and a caption; wait for the job, then read the project: result.clip_ids are the new clips on a2. Run it after the cut is joined; a second run needs the earlier score removed from a2.',
+        'Split sound: POST /projects/{id}/stems with the revision you read and the clip_id of a sound clip; wait for the job. The clip\'s sound is split into dialogue, music and ambience on tracks a4, a5 and a6 over the clip\'s range, and the clip itself goes silent. result.parts lists each track with its name, clip_id, start, end and frames. Offered only where features lists projects.stems; a clip is split once, and deleting the three parts gives its sound back. To save the three tracks and the original mix as four wav files, POST /projects/{id}/stems/export with the revision and the split clip\'s clip_id; the answer lists folder and files at once.',
         'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.',
         'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.',
         'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.',
@@ -377,6 +378,9 @@
         '没有安装语音模型时返回 no_speech_model，改为自己写字幕行',
     ('POST', '/projects/{id}/music'): '为剪辑时间线作一段与其等长的配乐，放在第二条声音轨道的开头，音量低于视频声音并带淡入淡出。'
         '剪辑拼好后先读取 revision，并删去该轨道上原有的配乐；首次使用会下载音乐模型（约 20 GB）',
+    ('POST', '/projects/{id}/stems'): '把一个声音片段的声音分成人声、音乐和环境声，放在其下方的 a4、a5、a6 三条轨道上，原片段静音。'
+        '先读取 revision 与片段 ID，等待返回的任务完成，结果 parts 给出各轨道的名称、起止帧；仅在 features 含 projects.stems 时可用',
+    ('POST', '/projects/{id}/stems/export'): '把已拆开片段的人声、音乐、环境声和原混音保存为四个 wav 文件；先读取 revision 与被拆片段的 ID，返回立即给出文件夹和文件名',
     ('POST', '/images'): '像图片编辑页面一样打开图片；提交 source，会话会保留源图片副本和图层文档',
     ('GET', '/images/{id}'): '读取图片编辑会话的图层文档、选区和 revision；编辑前先读取 revision 和图层标识',
     ('PUT', '/images/{id}'): '替换整个图片文档；先读取会话并提交其 revision。'

freevideo_engine/agent/jobs.py

@@ -32,7 +32,7 @@
 ACTIVE = ('waiting', 'queued', 'running', 'cancelling')
 CAST_AGAIN = 'POST /freevideo/agent/v1/cast/%s/generate to make the reference pictures again.'
 DONE = ('succeeded', 'failed', 'cancelled', 'interrupted')
-GPU_KINDS = ('vlm', 'enhance', 'image', 'music')
+GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems')
 IMAGE_ERRORS = errors.IMAGE_ERRORS
 IMAGE_WAIT_ERRORS = ('busy', 'busy_video')
 IMAGE_WAIT_NOTICES = {
@@ -1485,6 +1485,9 @@ def run():
             elif row['kind'] == 'music':
                 from .music import run_music
                 result = run_music(self, row)
+            elif row['kind'] == 'stems':
+                from .stems import run_stems
+                result = run_stems(self, row)
             else:
                 result = self.run_assemble(row)
             return result

freevideo_engine/agent/projects.py

@@ -11,6 +11,7 @@
 import hashlib
 import json
 import logging
+from pathlib import Path
 import re
 import secrets
 import threading
@@ -413,6 +414,70 @@ def music(self, identity, value, *, jobs, ref=None, client=None):
             self._remember(ref, digest, result, client)
             return result
 
+    def stems(self, identity, value, *, jobs, ref=None, client=None):
+        """Queue a sound-splitting job for one sound clip of the revision the caller read."""
+        schema.validate(schema.project_schema('stems'), value)
+        with self.listen_lock:
+            digest, replay = self._receipt('projects:stems', dict(id=identity, body=value), ref, client)
+            if replay is not None:
+                return replay
+            stored = self._load(identity)['project']
+            if stored['revision'] != value['revision']:
+                raise ProjectError('project_changed', project=stored)
+            clip = _clips(stored).get(value['clip_id'])
+            if clip is None or not clip.get('source') or clip.get('stem_of'):
+                raise _invalid('/clip_id', 'a sound clip with media that is not itself a separated part', value['clip_id'])
+            job = jobs.submit(dict(kind='stems', editor_project=identity,
+                                   **{k: v for k, v in value.items() if k != 'ref'}), client=client)[0]
+            result = dict(job=jobs.view(job), jobs=[jobs.view(job)])
+            self._remember(ref, digest, result, client)
+            return result
+
+    def stems_export(self, identity, value, *, ref=None, client=None):
+        """Copy the four wav files of a split clip into the saved sounds; the clip must be one that was split."""
+        from ..stems.separate import SeparationError, export_tracks
+        schema.validate(schema.project_schema('stems_export'), value)
+        with self.listen_lock:
+            digest, replay = self._receipt('projects:stems_export', dict(id=identity, body=value), ref, client)
+            if replay is not None:
+                return replay
+            stored = self._load(identity)['project']
+            if stored['revision'] != value['revision']:
+                raise ProjectError('project_changed', project=stored)
+            parts = [clip for clip in _clips(stored).values() if clip.get('stem_of') == value['clip_id']]
+            if not parts:
+                raise _invalid('/clip_id', 'a clip that was split into dialogue, music and ambience', value['clip_id'])
+            folder = parts[0]['source']['id'].rsplit('/', 1)[0]
+            if not folder.startswith('freevideo-stems/') or folder.count('/') != 1:
+                raise _invalid('/clip_id', 'a clip split by this version', value['clip_id'])
+            name = folder.split('/', 1)[1]
+            try:
+                files = export_tracks(Path(self.root) / folder, Path(self.root) / 'freevideo-stems-saved' / name)
+            except SeparationError as error:
+                raise JobError('stems_failed', 'The separated sound cannot be saved (%s)' % error) from error
+            result = dict(editor_project=identity, revision=stored['revision'], clip_id=value['clip_id'],
+                          folder='freevideo-stems-saved/' + name, files=files)
+            self._remember(ref, digest, result, client)
+            return result
+
+    def lay_stems(self, identity, clip_id, sources):
+        """Put the three parts on sub-tracks under the clip and silence the clip; returns the save and the new clip IDs."""
+        before, refused = {}, {}
+        def change(project):
+            before['ids'] = set(_clips(project))
+            reply = editor_runtime.runtime().call('placeStems', project, [clip_id, {
+                name: dict(kind='generated', id=source.split(':', 1)[1]) for name, source in sources.items()}],
+                module='model', serialize=True)[0]
+            if not reply['ok']:
+                refused.update(reply['error'])
+                return dict(ok=True, project=project)
+            return dict(ok=True, project=reply['value'])
+        saved = self._update(identity, change)
+        if refused:
+            code = STATE_CODES.get(refused.get('code'), 'stems_failed')
+            raise JobError(code, 'The separated sound cannot be laid under the clip (%s)' % refused.get('code'))
+        return saved, sorted(set(_clips(saved)) - before['ids'])
+
     def lay_music(self, identity, source):
         """Put a finished score on a2 from frame 0 of the project as it is now; returns the save and the new clip IDs."""
         found = comfy_editor.sources(self.root, [source])[source]

freevideo_engine/agent/routes.py

@@ -172,7 +172,13 @@ def submit_batch(jobs, body, client):
     return jobs.submit_request(body, client=client)[0]
 
 
-def features(continuation, vlm):
+def stems_supported(machine):
+    """Sound separation is offered on NVIDIA machines with enough video memory, never on a Mac."""
+    from ..stems.service import supported
+    return supported(machine.get('vram_gib'), uuid=machine.get('gpu_uuid')) and machine.get('device_backend', 'cuda') == 'cuda'
+
+
+def features(continuation, vlm, stems=False):
     """What an agent can use here (ready says whether the engine can run now). A missing name is not offered."""
     from .editor_runtime import available
     rules = available()
@@ -183,7 +189,8 @@ def features(continuation, vlm):
             'queue.shared': True, 'queue.busy': True, 'chain.auto': bool(continuation),
             'enhance': bool(vlm.get('ready')), 'drama.film': True, 'images.edit': rules,
             'images.layers': rules, 'images.heal': rules, 'images.render': rules, 'projects.read': True,
-            **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen', 'music')}}
+            **{'projects.' + name: rules for name in ('insert', 'split', 'move', 'trim', 'remove', 'export', 'generate', 'listen', 'music')},
+            'projects.stems': bool(rules and stems)}
 
 
 def whitebox_support(jobs):
@@ -221,7 +228,7 @@ def capabilities(port, jobs):
     return dict(
         api_version=API_VERSION, freevideo_version=release or __version__, build=__version__, ready=ready, problem=problem,
         continuation=continuation,
-        features=features(continuation, vlm),
+        features=features(continuation, vlm, stems=stems_supported(machine)),
         whitebox=whitebox_support(jobs),
         device=dict(backend=machine.get('device_backend', 'cuda') if machine else None,
                     vram_limit_gib=machine.get('vram_gib'), ram_limit_gib=machine.get('ram_gib')),
@@ -233,6 +240,7 @@ def capabilities(port, jobs):
                'export': 'Render a saved revision of an editing project to a video',
                'listen': 'Listen to the sound of an editing project for subtitle lines and times',
                'music': 'Compose music for an editing project and lay it on its second sound track',
+               'stems': 'Split the sound of a clip into dialogue, music and ambience on sub-tracks',
                'image_edit': 'Render a saved revision of an image session to a picture'},
         limits=dict(width=[256, 4096, 32], height=[256, 4096, 32], seconds=[1.625, 60 if continuation else BLOCK_SECONDS],
                     seconds_tested=[4, 15],
@@ -507,7 +515,9 @@ async def respond(request):
                         raise JobError('page_session_required', 'Page session required')
                 header = request.headers.get('Authorization', '')
                 given = header[7:].strip() if header[:7].lower() == 'bearer ' else request.headers.get('X-FreeVideo-Agent', '')
-                if definition.get('auth', True) or given:
+                # The editor page may drive a few routes with its page session instead of the token.
+                by_page = bool(definition.get('page_ok')) and not given and valid_page_session(request)
+                if (definition.get('auth', True) and not by_page) or given:
                     try:
                         token = expected()
                     except (OSError, ValueError):
@@ -2215,7 +2225,7 @@ async def handler(request):
                 raise JobError('idempotency_conflict', 'Conflicting keys')
             args = [] if operation == 'create' else [identity]
             extra = dict(client=request.headers.get('X-FreeVideo-Client', '')[:80] or None)
-            if operation in ('generate', 'export', 'listen', 'music'):
+            if operation in ('generate', 'export', 'listen', 'music', 'stems'):
                 extra['jobs'] = jobs
             if operation == 'generate':
                 def capability(size):
@@ -2225,9 +2235,9 @@ def capability(size):
                         return None
                 extra['capability'] = capability
             result = await asyncio.to_thread(getattr(projects, operation), *args, value, ref=key or ref, **extra)
-            if operation in ('generate', 'export', 'listen', 'music'):
+            if operation in ('generate', 'export', 'listen', 'music', 'stems'):
                 await jobs.tick()
-            return reply(result, 201 if operation in ('create', 'generate', 'export', 'listen', 'music') else 200)
+            return reply(result, 201 if operation in ('create', 'generate', 'export', 'listen', 'music', 'stems') else 200)
         return guarded(handler)
 
     for operation, (method, suffix, _) in contract.PROJECT_OPERATIONS.items():