FreeVideo 0.4.32

MANIFEST.in

@@ -68,5 +68,11 @@ include web/assets/tour/c9-s1.webp
 include web/assets/tour/c10-person.webp
 include web/assets/tour/c10-s1.mp4
 include web/assets/tour/c10-s1.webp
+include web/assets/tour/c11-s1.mp4
+include web/assets/tour/c11-s1.webp
+include web/assets/tour/c12-s1.mp4
+include web/assets/tour/c12-s1.webp
+include web/assets/tour/c13-s1.mp4
+include web/assets/tour/c13-s1.webp
 include web/assets/tour/c10-source.webp
 include web/assets/tour/fox-sheet.webp

THIRD_PARTY_NOTICES.md

@@ -17,6 +17,7 @@ FreeVideo code uses [Apache-2.0](LICENSE). Its dependencies and models retain th
 | [FreeToken](https://github.com/FlashML-org/FreeToken) prompt VLM operators, [Qwen3-VL-4B-Instruct](https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct) optional weights | [Apache-2.0](freevideo_engine/licenses/FreeToken-Apache-2.0.txt) |
 | [Qwen-Image-2.1](https://huggingface.co/Comfy-Org/Qwen-Image-2.1) Turbo image generation models, including Qwen3-VL-8B and VAE (optional download; [notice](freevideo_engine/image_gen/licenses/NOTICE.txt) installed with the models), © 2026 Hangzhou Tongyi Laboratory Technology Co., Ltd. All Rights Reserved. | [Qwen RESEARCH LICENSE AGREEMENT](https://huggingface.co/Comfy-Org/Qwen-Image-2.1/blob/main/LICENSE) |
 | [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) 1.13.8 speech runtime, built by FreeVideo without text-to-speech, ALSA and PortAudio (modified build; `__init__.py` edited; source snapshot 11afbd00 kept by FreeVideo; optional download for automatic subtitles, [build note](freevideo_engine/licenses/sherpa-onnx-build-NOTE.txt)). Includes statically linked kaldi-native-fbank, kaldi-decoder, kaldifst, OpenFst, simple-sentencepiece (Apache-2.0), KissFFT (BSD-3-Clause), pybind11 (BSD-3-Clause), nlohmann/json (MIT), hclust-cpp and Darts-clone (BSD-2-Clause), ThreadPool (Zlib). The Windows build also contains msvcp140.dll and msvcp140_1.dll (Microsoft Visual C++ 2022 runtime 14.44.35211, © Microsoft Corporation), unmodified and redistributed as permitted by Microsoft for its redistributable files. | Apache-2.0 |
+| [CosyVoice](https://github.com/FunAudioLLM/CosyVoice) code adapted for dubbing and [Fun-CosyVoice3-0.5B-2512](https://modelscope.cn/models/FunAudioLLM/Fun-CosyVoice3-0.5B-2512) weights, © 2024 Alibaba Inc (optional download); includes MIT-licensed parts of Matcha-TTS, HiFi-GAN, x_transformers and Snake | [Apache-2.0](freevideo_engine/licenses/CosyVoice-Apache-2.0.txt); [MIT notices](freevideo_engine/licenses/CosyVoice-NOTICE.txt) |
 | [ONNX Runtime](https://github.com/microsoft/onnxruntime) 1.28.2 (CPU), © Microsoft Corporation; its third-party notices are shipped inside the sherpa-onnx wheel | MIT |
 | [Eigen](https://gitlab.com/libeigen/eigen) 5.0.1, compiled into sherpa-onnx and ONNX Runtime; source: https://gitlab.com/libeigen/eigen/-/tree/5.0.1 | MPL-2.0 |
 | [Silero VAD](https://github.com/snakers4/silero-vad) v5.0 voice-activity model, © 2020-present Silero Team (optional download with subtitles, license file downloaded with the model; the onnx-community export is downloaded with the music model) | [MIT](freevideo_engine/licenses/Silero-VAD-MIT.txt) |

__init__.py

@@ -31,6 +31,12 @@ async def comfy_entrypoint():
     except Exception as error:
         import logging
         logging.warning('[FreeVideo] Image generation routes not registered: %s', error)
+    try:
+        from .freevideo_engine.dubbing.service import register as dubbing_routes
+        dubbing_routes()
+    except Exception as error:
+        import logging
+        logging.warning('[FreeVideo] Dubbing routes not registered: %s', error)
     from .freevideo_engine.comfy_launcher_api import register as launcher_routes
     launcher_routes()
     from .freevideo_engine.comfy_updates import register as update_routes

constraints/unified.txt

@@ -17,6 +17,7 @@ cuda-toolkit==13.0.3.0
 cryptography==50.0.1
 einops==0.8.2
 filelock==3.32.3
+flatbuffers==25.12.19
 fsspec==2026.7.0
 h11==0.16.0
 hf-xet==1.6.1a0
@@ -50,10 +51,12 @@ nvidia-nvjitlink==13.3.33
 nvidia-nvshmem-cu13==3.4.5
 nvidia-nvtx==13.0.85
 omegaconf==2.3.1
+onnxruntime==1.31.1
 packaging==26.3
 peft==0.20.0
 pillow==12.3.0
 pip==26.2.1
+protobuf==7.36.2
 psutil==7.2.2
 pygments==2.21.0
 pycparser==3.0

freevideo_engine/agent/errors.py

@@ -125,6 +125,14 @@
                               'Tell the user the model needs a network connection, then submit the job again with a new ref.'),
     'stems_failed': (409, True, 'Splitting the sound did not finish.',
                      'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'),
+    'voice_failed': (409, True, 'Changing the voice did not finish.',
+                     'Read the project, then submit the job again with a new ref; if it fails again, leave the sound as it is.'),
+    'voice_model_missing': (409, False, 'The dubbing model is not ready on this machine.',
+                            'Tell the user to install the dubbing model first; do not retry.'),
+    'no_speech': (409, False, 'No speech was found in the selection.',
+                  'Do not retry; choose a stretch of sound with someone speaking.'),
+    'shorten_unavailable': (409, False, 'Lines cannot be shortened here yet.',
+                            'Write the shorter line yourself and send it with the other lines.'),
     'clip_source_unusable': (409, False, 'A clip\'s source is missing, or cannot be read as far as the clip uses it.',
                              'Read the project and check each clip\'s source: it must exist and play, and in + frames must stay '
                              'within its real length. Correct the project, then export the new revision.'),
@@ -239,7 +247,7 @@ def recovery(code, *, kind=None, stage='request', resumable=True):
         return 'Tell the user to free disk space, then submit the picture again with a new ref.'
     if kind == 'music' and code == 'out_of_memory':
         return 'Close other GPU applications, then submit the music job again with a new ref.'
-    if kind == 'stems' and code == 'out_of_memory':
+    if kind in ('stems', 'voice', 'voice_lines') and code == 'out_of_memory':
         return 'Close other GPU applications, then submit the job again with a new ref.'
     if kind == 'image' and code == 'no_memory':
         return 'Lower the picture dimensions or use fewer reference pictures, then submit with a new ref.'

freevideo_engine/agent/guide.py

@@ -145,6 +145,7 @@
         'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.',
         'Music: POST /projects/{id}/music with the revision you read and a caption; wait for the job, then read the project: result.clip_ids are the new clips on a2. Run it after the cut is joined; a second run needs the earlier score removed from a2.',
         'Split sound: POST /projects/{id}/stems with the revision you read and the clip_id of a sound clip; wait for the job. The clip\'s sound is split into dialogue, music and ambience on tracks a4, a5 and a6 over the clip\'s range, and the clip itself goes silent. result.parts lists each track with its name, clip_id, start, end and frames. Offered only where features lists projects.stems; a clip is split once, and deleting the three parts gives its sound back. To save the three tracks and the original mix as four wav files, POST /projects/{id}/stems/export with the revision and the split clip\'s clip_id; the answer lists folder and files at once.',
+        'Change a voice: POST /projects/{id}/voice/lines with the revision you read and a selection (clip_id, and start and end in seconds if only part) to write down the lines; read result.lines. Change the text of the lines to redo and POST /projects/{id}/voice/replace with the same selection and all lines (original_text unchanged); only the changed lines are spoken again, in the voice of the original, and the rest keep their sound. result.lines gives where each new line starts and ends; a line that cannot fit before the next one keeps its original voice with replaced false and warning too_long. POST /projects/{id}/voice/estimate with the lines answers at once with the seconds the changed lines take. Offered only where features lists projects.voice.',
         'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.',
         'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.',
         'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.',
@@ -380,6 +381,10 @@
         '剪辑拼好后先读取 revision，并删去该轨道上原有的配乐；首次使用会下载音乐模型（约 20 GB）',
     ('POST', '/projects/{id}/stems'): '把一个声音片段的声音分成人声、音乐和环境声，放在其下方的 a4、a5、a6 三条轨道上，原片段静音。'
         '先读取 revision 与片段 ID，等待返回的任务完成，结果 parts 给出各轨道的名称、起止帧；仅在 features 含 projects.stems 时可用',
+    ('POST', '/projects/{id}/voice/lines'): '把选中声音片段里说的话逐句写下来，每句带起止秒数；先读取 revision 与片段 ID，等待返回的任务完成，结果 lines 给出各句；仅在 features 含 projects.voice 时可用',
+    ('POST', '/projects/{id}/voice/replace'): '把改过文字的句子用原说话人的声音重新说一遍，放回原句的位置，没改的句子保持原声；提交 voice/lines 给出的全部句子，只改 text；结果 lines 给出每句新声的起止，放不进下一句之前的句子保留原声',
+    ('POST', '/projects/{id}/voice/shorten'): '按目标秒数改短一句的文字；只返回文字，不保存也不配音',
+    ('POST', '/projects/{id}/voice/estimate'): '估算改过的句子配出来的秒数；立即返回',
     ('POST', '/projects/{id}/stems/export'): '把已拆开片段的人声、音乐、环境声和原混音保存为四个 wav 文件；先读取 revision 与被拆片段的 ID，返回立即给出文件夹和文件名',
     ('POST', '/images'): '像图片编辑页面一样打开图片；提交 source，会话会保留源图片副本和图层文档',
     ('GET', '/images/{id}'): '读取图片编辑会话的图层文档、选区和 revision；编辑前先读取 revision 和图层标识',

freevideo_engine/agent/jobs.py

@@ -32,7 +32,7 @@
 ACTIVE = ('waiting', 'queued', 'running', 'cancelling')
 CAST_AGAIN = 'POST /freevideo/agent/v1/cast/%s/generate to make the reference pictures again.'
 DONE = ('succeeded', 'failed', 'cancelled', 'interrupted')
-GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems')
+GPU_KINDS = ('vlm', 'enhance', 'image', 'music', 'stems', 'voice')
 IMAGE_ERRORS = errors.IMAGE_ERRORS
 IMAGE_WAIT_ERRORS = ('busy', 'busy_video')
 IMAGE_WAIT_NOTICES = {
@@ -532,7 +532,7 @@ def load(self):
 
         def readable(row):
             return (isinstance(row, dict) and specs.JOB_ID.fullmatch(str(row.get('id')))
-                    and row.get('kind') in ('generate', 'assemble', 'export', 'listen', 'image_edit') + GPU_KINDS
+                    and row.get('kind') in ('generate', 'assemble', 'export', 'listen', 'voice_lines', 'image_edit') + GPU_KINDS
                     and row.get('status') in ACTIVE + DONE and isinstance(row.get('spec'), dict)
                     and type(row.get('created_at')) in (int, float) and math.isfinite(row['created_at'])
                     and (self._receipts is None or (type(row.get('_receipt_revision', 0)) is int
@@ -601,7 +601,7 @@ def readable(row):
             records = list(latest.values())
         for row in records:
             if readable(row):
-                if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'listen', 'image_edit') + GPU_KINDS:
+                if row.get('status') == 'running' and row['kind'] in ('assemble', 'export', 'listen', 'voice_lines', 'image_edit') + GPU_KINDS:
                     # This process restarted while the work ran in it; nothing continues it.
                     self.finish(row, 'failed', error=dict(code='interrupted',
                                 message='ComfyUI restarted while this job ran; submit it again'), save=False)
@@ -1488,6 +1488,12 @@ def run():
             elif row['kind'] == 'stems':
                 from .stems import run_stems
                 result = run_stems(self, row)
+            elif row['kind'] == 'voice_lines':
+                from .voice import run_voice_lines
+                result = run_voice_lines(self, row)
+            elif row['kind'] == 'voice':
+                from .voice import run_voice_replace
+                result = run_voice_replace(self, row)
             else:
                 result = self.run_assemble(row)
             return result