diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 60712dfe..85051f6a 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -69,8 +69,8 @@ Before a release that changes an AI integration, dependency, or default model, r cache-warming and timing protocol in the [verification records](docs/reference/verification.md). Do not release an applicable change until all selected checks pass. Published performance baselines must use the public API defaults; -label reduced settings as compatibility checks. The dub check fails if its timing -summary is missing or if one segment loses more than 3.0 seconds during synchronization. +label reduced settings as compatibility checks. The dub check fails on missing timing +measurements or translation/synthesis failures. It reports excessive speeds for listening review. ### Linting & type checking diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index abe377a5..2d28c653 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,28 @@ # Release Notes +## 0.61.2 + +- Dub sentences at source-word timestamps using per-speaker voice references. + Keep shorter speech at natural speed; report speedups above 1.1×. + Apply 5 ms fades at phrase boundaries without moving or cutting speech. +- Improve translation validation, long-text synthesis, failure reporting and + audio-only MP4 support. Select the dubbed track for default playback when retaining + original audio. Require `videopython-chatterbox>=0.1.7.post2`. +- Document a TranslateGemma 12B configuration tested on Polish/English speech. + The default translation model remains unchanged. +- Preserve source spacing when splitting phrases, reject invalid translation budgets + at construction, and keep Ollama cleanup failures from discarding completed work. +- Reduce CUDA speech-decoder overhead and avoid repeated translation-model reloads. +- Remove obsolete slowdown and truncation settings and result fields. See + [Update timing consumers](docs/how-to/dubbing.md#update-timing-consumers). +- Fast-paced multi-speaker dialogue, short interruptions and dense jargon remain + challenging. Speaker assignment and voice consistency can vary; numbers and + technical terms can be mistranslated or mispronounced. Longer generated speech + can require excessive acceleration. Review dubbed audio before publication. + No overall dubbing speedup is established. See the + [verification results](docs/reference/verification.md#final-dubbing-review-0612) + and [planned quality work](ROADMAP.md#improve-dubbing-quality). + ## 0.61.1 - Speaker diarization skips embeddings for chunk/speaker pairs with no speech and diff --git a/ROADMAP.md b/ROADMAP.md index a09db10f..56e4a524 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -61,6 +61,34 @@ Examine whether diarization must keep the channels when they are not correlated, them to constrain or to replace the speaker clustering. Keep the mono path for correlated stereo, which is the usual case. +### Improve dubbing quality + +Fast-paced dialogue with multiple speakers, short interruptions, numerical claims +and specialized vocabulary needs stronger quality checks across transcription, +translation, synthesis and timing. + +- Improve speaker assignment across interruptions and different recording lengths. +- Keep cloned voices consistent across phrases while retaining natural expression. +- Fit translated speech without extreme acceleration or loss of intelligibility. +- Preserve meaning, units and terminology, and pronounce numbers correctly. +- Check missing, added and repeated speech independently of generation success. + +Use varied recordings and language directions for validation, with listening review +alongside automated checks. Current limitations and measurements are in the +[verification record](docs/reference/verification.md#final-dubbing-review-0612). + +### Reduce speech synthesis time + +Speech generation dominates local dubbing time, even when transcription, diarization +and voice references are processed efficiently. Build on decoder graph replay with +further measured runtime improvements that reduce complete dubbing time. Keep only +changes whose benefit justifies their implementation and maintenance cost. + +Prioritize inference improvements with the existing model and voice settings. Measure +setup cost, complete dubbing time and memory use, while checking intelligibility, +speaker identity, expression and timing against the current output. Keep the injectable +speech backend contract usable by downstream projects. + ### Validate the release candidate Publish `1.0.0rc1` before the stable release. Use it from a clean consumer environment diff --git a/docs/how-to/dubbing.md b/docs/how-to/dubbing.md index 28d2b54c..303f9125 100644 --- a/docs/how-to/dubbing.md +++ b/docs/how-to/dubbing.md @@ -75,13 +75,20 @@ result = dubber.dub(video=video, target_lang="es", progress_callback=on_progress ## Fit it in less memory -The default pipeline keeps all four models resident. `low_memory=True` releases each -one after its stage — recommended for GPUs with ≤12 GB VRAM or hosts under 32 GB RAM: +`low_memory=True` releases the in-process transcription, separation and speech +synthesis models after their stages — recommended for GPUs with ≤12 GB VRAM or +hosts under 32 GB RAM: ```python dubber = VideoDubber(low_memory=True) ``` +Translation requests keep the Ollama model resident for five minutes between calls, +including when the server defaults to immediate eviction. `low_memory=True` +explicitly unloads it after translation, before speech synthesis loads. Standalone +`OllamaTranslator` users can override `keep_alive` (`None` uses the server policy) +and call `unload()` to release residency requested by the translator. + Combine it with `dub_file()` for the smallest footprint; see [Process hour-long videos](long-videos.md#dub-without-loading-frames). @@ -129,14 +136,15 @@ except GarbageTranscriptError as exc: ## Check the timing fit -Translated speech that does not fit the source's spoken gaps gets time-stretched or -truncated. High truncation rates are a translation-quality red flag worth surfacing: +The pipeline first uses available silence before the next turn. If complete speech +still cannot fit at the preferred 1.1× maximum, it goes faster instead of cutting +words from the end. Excessive speed can sound unnatural and warrants review: ```python ts = result.timing_summary if ts is not None: print(f"{ts.clean_count}/{ts.total_segments} clean") - print(f"{ts.truncated_count} truncated, worst {ts.max_truncation_seconds:.2f}s") + print(f"{ts.excessive_speed_count} above preferred speed; maximum {ts.max_speed_factor:.2f}×") print(f"mean speed factor {ts.mean_speed_factor:.3f}") ``` @@ -157,6 +165,12 @@ for seg in transcription.segments: dubber.dub_and_replace(video=video, target_lang="es", transcription=transcription) ``` +On CUDA, compatible local speech decoder operations use graph replay to reduce +launch overhead. Model weights, precision and generation settings stay unchanged. +Other input layouts use the original execution path, as does CPU synthesis; if +graph capture is unavailable, synthesis falls back automatically. Graph buffers +are released with the model. + | Supplied transcription | `enable_diarization` | Behavior | |---|---|---| | Has speaker labels | any | Supplied speakers are used; the flag is ignored | @@ -168,22 +182,96 @@ The diarize-on-supplied path needs word-level timings, so transcriptions loaded ## Pick the translation model -Translation goes through `OllamaTranslator`, a single Ollama text model. It sends -segments under a structured-output schema and reads back length-budgeted translations — -the prompt carries a per-segment character budget derived from the source duration and a -`low_confidence` hint sourced from Whisper's `avg_logprob`. Long sources are chunked to -fit the context window, with one parse-retry for segments the first pass misses. +Translation goes through `OllamaTranslator`, a single Ollama text model. Each request +translates one bounded source part, with neighboring text marked as context only. +Long turns split at sentence, clause or word boundaries, falling back to character +boundaries for text without spaces, and reassemble under their original segment +and speaker. A duration-derived character target encourages concise speech while +preserving meaning. Requests remain sequential to isolate segment identities; this +adds request overhead compared with batching. Invalid identities, +duplicate entries, empty responses and output-budget exhaustion trigger a retry. +If any part remains unavailable, the whole parent appears in `translation_failures`. + +When constructing `OllamaTranslator` directly, set `max_tokens` to at least 140 and +`n_ctx` to at least `max_tokens + 1020`. Defaults are 4096 and 8192 respectively. +The equivalent `options` keys, `num_predict` and `num_ctx`, override these values; +invalid effective budgets fail at construction. For example, a 4096-token context +can use `max_tokens=1024`. These are allocation estimates, not tokenizer guarantees. + +Unloading requests release of the model on the Ollama server. A failed release +request logs a warning and clears the local client without discarding translations +or masking an exception from the caller. Server memory may remain allocated. + +Select a model already downloaded in Ollama. For example, run +`ollama pull translategemma:12b`, then configure the dubber: ```python dubber = VideoDubber( - translator_model="qwen3.6:27b", + translator_model="translategemma:12b", translator_host="http://localhost:11434", + low_memory=True, ) ``` -Segments the model never returns land on `result.translation_failures` as indices, with -empty translated text. Any language pair is attempted — the pipeline does not reject a -target language up front. +Any language pair is attempted — the pipeline does not reject a target language up +front. An empty `translation_failures` list establishes response availability, not +semantic accuracy. Review meaning, numbers, names and speaker alignment before publishing. + +Local Chatterbox synthesis splits long translations into bounded calls, preserving +voice and expression settings, and joins their audio before synchronizing the parent +turn. Calls approaching the backend's speech-token ceiling retry with smaller text +units. Sampling excludes invalid vocoder token IDs while preserving end-of-speech. +Invalid token outputs are also rejected before GPU indexing and retried up +to three attempts. The required `videopython-chatterbox>=0.1.7.post2` fixes the empty alignment +reduction for short text in the backend itself. The duration check can flag a likely cap but cannot prove +every word was spoken. +Tiny adjacent fragments can join within one speaker when the gap is at most 150 ms; +groups contain at most four turns spanning at most ten seconds. The longer fragment +supplies the expression profile. Isolated groups shorter than 100 ms are reported +as synthesis failures. Original transcript entries remain available separately. + +Check `result.synthesis_failures` for original segment indices whose speech could not +be generated or timed. Also inspect `result.timing_summary`: fitting the complete +speech into the source window can require excessive speed. Verify the final audio, +including the ends of long turns, rather than relying on success counts alone. + +Speaker diarization turns are not necessarily good dubbing units. Before translation, +long turns are split into phrases at sentence ends, then clauses or pauses, using +validated word timestamps. Phrase text is sliced from the source transcript to retain +its internal spacing. Complete sentences take priority over earlier commas; +roughly eight-second phrases are preferred without creating tiny word fragments. +Missing, partial or inconsistent word alignment leaves the source segment intact; +we do not invent timestamps by dividing text proportionally. + +`source_transcription` remains unchanged. Translated phrases expose +`source_segment_index` to trace them to that transcript; translation and synthesis +failure lists still refer to original source segment indices. Speaker reference +extraction uses the full original turns, preserving voice identity across phrases. + +Each phrase starts at its source-word anchor. Shorter generated speech keeps its +natural speed and pauses until the next phrase, instead of filling the whole window +with a slowdown. Available gaps can absorb longer speech before acceleration. +Assembly applies a 5 ms fade at each phrase edge to reduce clicks at silence +boundaries. It keeps the sample count and source anchor unchanged. +See [Check the timing fit](#check-the-timing-fit) for speed limits and result checks. + +Dubbing prefers FFmpeg's [Rubber Band filter](https://ffmpeg.org/ffmpeg-filters.html#rubberband) +when stretching is necessary and the filter is available, with an explicit warning +and `atempo` fallback on builds without it. General `Audio.time_stretch()` calls +retain the `atempo` default; callers may select `method="rubberband"` explicitly. +Phrase alignment reduces accumulated timing drift; it is not phoneme-level lip sync, +and translation or TTS quality still requires listening review. + +## Update timing consumers + +Remove the `min_speed` argument from `TimingSynchronizer` calls. Shorter speech +keeps its natural speed. Use `max_speed` to set the preferred acceleration limit. + +Remove uses of `TimingAdjustment.was_truncated`, `truncation_seconds`, and +`excessive_slowdown`. Remove uses of `TimingSummary.truncated_count`, +`max_truncation_seconds`, `excessive_slowdown_count`, and `min_speed_factor`. +Regenerate saved timing summaries from the current pipeline. Use +`excessive_speed_count` and `max_speed_factor` to select output for listening review. ## Swap the TTS backend diff --git a/docs/reference/ai/dubbing.md b/docs/reference/ai/dubbing.md index 7d6a7be2..ad6ccc07 100644 --- a/docs/reference/ai/dubbing.md +++ b/docs/reference/ai/dubbing.md @@ -41,7 +41,8 @@ or the same knobs as flat kwargs — the constructor builds a `DubbingConfig` ei result = dubber.dub(video, target_lang="es") result.num_segments, result.source_lang, result.target_lang -result.translation_failures # indices the model never returned +result.translation_failures # original indices with missing/invalid translation parts +result.synthesis_failures # original indices without generated speech for segment in result.translated_segments: print(f"{segment.original_text!r} -> {segment.translated_text!r}") @@ -75,9 +76,14 @@ loud/quiet shape instead of using flat defaults everywhere. ## TimingSummary -Aggregate stats over the per-segment timing adjustments the synchronizer applied. High -truncation counts mean the translation produced text too long for the source's spoken -regions. +Aggregate stats over the per-segment timing adjustments. `excessive_speed_count` +counts turns exceeding the preferred maximum speed; `max_speed_factor` records the +fastest adjustment. The pipeline borrows following silence before speeding up and +preserves complete speech instead of clipping its tail. Small tempo-filter duration +errors are corrected by resampling the entire output, which can slightly shift pitch. +`clean_count` includes speed factors within 0.01 of 1.0; `stretched_count` includes +the remaining adjustments. See [Update timing consumers](../../how-to/dubbing.md#update-timing-consumers) +when migrating callers or saved results. ::: videopython.ai.dubbing.models.TimingSummary diff --git a/docs/reference/ai/understanding.md b/docs/reference/ai/understanding.md index 8cd3486e..d6e19a56 100644 --- a/docs/reference/ai/understanding.md +++ b/docs/reference/ai/understanding.md @@ -29,6 +29,12 @@ Diarization skips speaker embeddings for chunk/speaker pairs with no speech. Com pyannote embedding backends also share frame extraction across speakers in each chunk, then apply each speaker's original mask separately. Segmentation overlap, precision, and clustering settings are unchanged. Other embedding models keep the existing extraction path. +When frame extraction is shared, the backend's `embedding_batch_size` counts audio +chunks rather than chunk/speaker pairs. Pooling materializes one frame row per active +pair in addition to the shared chunk frames. Thus, a batch of 32 chunks with three +active speakers per chunk can hold 32 shared plus 96 pooled frame rows. This uses +more frame-tensor memory than the upstream batch of 32 pairs; it is not a fourfold +estimate of total GPU memory. The measured optimization retains this batching strategy. On a 30-minute five-speaker recording, sharing frame extraction reduced warm diarization time from 21.18s to 16.16s, with identical speaker labels and exact timestamps. Small diff --git a/docs/reference/verification.md b/docs/reference/verification.md index e40d135b..9570f4f4 100644 --- a/docs/reference/verification.md +++ b/docs/reference/verification.md @@ -39,6 +39,393 @@ OLLAMA_HOST=127.0.0.1:11434 uv run python scripts/verify_mcp_workflow.py \ ## AI model verification +### Final dubbing review, 0.61.2 + +On 2026-09-09, the listener approved the one-minute Polish-to-English demo with +TranslateGemma 12B and 5 ms phrase-boundary fades. The fades preserve phrase +anchors, sample counts and interior samples. The demo used two cloned voices, +background preservation, seed 1777 and `low_memory=True` on an RTX 2060 SUPER +with 8 GB VRAM. It reused the saved diarized transcription. + +Generation and export took 176.54 seconds. All ten phrases had speed factor 1.0, +with no reported translation or synthesis failures. The H.264 video and single +default AAC audio track fully decoded. Independent ASR recognized 141 words in +both raw and faded speech, matching the expected count. Both checks recognized +“health” as “hell”; equal word counts do not establish exact pronunciation. +The listener accepted the recording, rather than providing a phonetic audit. + +The model comparison also covered 72 timed phrases from seconds 750–1050 of an +English podcast, translated into Polish. Assistant text review favored the 12B +library configuration among the tested Qwen3.5 9B, Hy-MT2 7B and TranslateGemma +4B/12B configurations. Remaining errors include units, names, financial terms +and duplicated context. This is a local qualitative comparison, not an independent +human ranking. The default `qwen3.6:27b` was not compared and remains unchanged. + +The 12B library translation run took 535.34 seconds with no structural failures. +It used CPU offloading; these settings are a compatibility check, not a default-model +performance baseline. No new ten-minute performance comparison was completed. +Further tuning needs fresh validation material because the podcast cut was also +used for prompt diagnostics. The [dubbing guide](../how-to/dubbing.md#pick-the-translation-model) +shows the tested model configuration. + +The maintained real-model harness then passed `env`, `imports` (38 entrypoints), +`ollama` and `dub` with the same 12B override. This fresh run included transcription +and diarization, produced two voice samples and ten translated phrases, and reported +no translation/synthesis failures or excessive speeds (maximum 1.0×). The dub check +took 249.31 seconds; all four checks took 263.04 seconds. This run used the harness +seed behavior and did not reuse the approved demo's frozen transcription. + +```bash +OLLAMA_HOST=127.0.0.1:11435 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ + uv run python scripts/verify_ai_models.py \ + --only env,imports,ollama,dub --video cam1_1min.mp4 \ + --workdir verify-results/release-0.61.2 \ + --ollama-model translategemma:12b --low-memory \ + --source-lang pl --target-lang en --enable-diarization +``` + +Final release preparation passed pre-commit, lock validation, strict documentation +build, wheel/sdist builds and clean-wheel public-import/render/MCP smoke checks. +The functional code had passed the full 1,285-test suite. The final source cleanup +removed a module docstring; all 17 translator tests passed afterward. Package +contents contain no experiment scripts, media, model caches or private instructions. + +#### Fresh two-language sanity checks + +After the review fixes, both recordings ran through fresh transcription, diarization, +per-speaker cloning and background preservation with TranslateGemma 12B, seed 1777 +and `low_memory=True`. Both exports decoded and reported no translation or synthesis +failures. These checks show that generation success does not establish speech quality. + +| Input | Direction | Elapsed | Phrases | Flagged speedups | Maximum speed | +|---|---|---:|---:|---:|---:| +| One-minute conversation | Polish to English | 171.04 s | 10 | 0 | 1.04× | +| Five-minute podcast excerpt | English to Polish | 1,142.58 s | 67 | 49 | 8.4× | + +The conversation's translations matched the approved demo, but independent ASR +flagged possible extra speech around 37 seconds in raw and mixed audio. In the +podcast, ASR recognized a property-sale sentence in raw speech but missed it after +3.16× acceleration and in the final mix. The maximum 8.4× adjustment fit a 2.52-second +“Tak.” output into a 0.30-second source window. Raw Polish speech also had number +recognition errors. These differences require listening to distinguish synthesis +and intelligibility errors from recognition errors. + +The podcast text review found unit substitutions, duplicated neighboring content, +and changes to financial actions. The fresh excerpt produced three speaker labels; +an earlier full-recording run assigned four labels within that same interval. Neither +count is a verified count of people. The listener also reported inconsistent voices +and dramatic speedups. This remains a difficult quality case, not a clean quality +pass or evidence of a regression caused by the review fixes. The +[roadmap](https://github.com/BartWojtowicz/videopython/blob/main/ROADMAP.md#improve-dubbing-quality) tracks the broader improvement work. + +### Dubbing decoder optimization, 0.61.2 + +On 2026-09-08, full Polish-to-English dubbing of `cam1_10min.mp4` was measured +on an RTX 2060 SUPER (8 GB), Python 3.12.12, PyTorch 2.13.0+cu130, +videopython-chatterbox 0.1.7.post1 and pyannote-audio 4.0.7. Both variants include +the 0.61.1 diarization optimization. The isolated candidate adds only CUDA graph replay +for the existing decoder's feed-forward and normalization operations. + +The comparison calls `VideoDubber.dub_file` with diarization, per-speaker voice +cloning, background preservation, original-audio retention and `low_memory=True`. +It includes decoding, model loading, transcription, diarization, separation, +translation, synthesis, synchronization, mixing and final video output. Imports +and downloaded model files were warmed before timing. Local instrumentation saves +intermediate outputs in both variants and adds some overhead. + +Translation uses the same Ollama 0.33.3 `qwen3.5:4b` model and seed in both runs. +This smaller model makes the benchmark practical on this GPU; it does not replace +the library default or establish translation parity with that larger model. +The dedicated Ollama server uses `OLLAMA_KEEP_ALIVE=0`. Python, NumPy and PyTorch +seeds are reset identically before each synthesis call. + +| Measurement | Before | Decoder graphs | +|---|---:|---:| +| Full dubbing, including final video output | 656.77 s | 568.12 s | +| Speech synthesis calls, including model loading and reference preparation | 508.00 s | 417.86 s | +| Peak GPU memory (whole device, sampled every second) | 7,338 MiB | 7,806 MiB | +| Peak process RSS | 6,993,460 KiB | 6,992,940 KiB | + +Full time fell by 13.5%; synthesis time fell by 17.7%. The additional GPU memory +is a material tradeoff on small GPUs. Capture failures restore the original path. +Ninety decoder graphs remained active throughout all 21 synthesis calls. + +Both runs used the same two speakers, selected reference samples, translated text, +per-segment expression and seeds. All 21 synthesis calls succeeded with identical +sample counts. Maximum absolute waveform difference was 4.04e-6 on normalized +floating-point audio; outputs are not bit-identical. Translation had no failures, +and synchronization summaries matched, with no truncated segments. + +Input SHA-256: +`deeaa2055a9061ea04fdddafdcc846be0454c677cabc5e1d5c546b595d4e1e7b`. + +These are single full-pipeline runs on one recording, supported by repeated short +synthesis probes. They do not establish universal speedups or bitwise equivalence +on other hardware, recordings or dependency versions. Reference-cache and attention- +observer cleanup experiments improved full time by only 0.6% together and were +excluded from the release because their benefit did not justify the extra code. +Experimental benchmark helpers and intermediate media are kept outside the commit. + +### Dubbing reliability, 0.61.2 + +Follow-up checks on 2026-09-08 used the same GPU environment and explicitly selected +`qwen3.5:4b`; the library default remains unchanged. These checks include bounded +translation and synthesis and are separate from the decoder-only comparison above. + +| Check | Result | +|---|---| +| Full `cam1_10min.mp4` dubbing | 747.608 s; 21/21 turns generated, no translation or synthesis failures, no timing truncation | +| First 10 minutes of `all_in_30min.mp4` | Recovery generated 42/42 positive-duration turns; one zero-duration ASR fragment reported as a synthesis failure | +| Long Polish synthesis | 235 words, ten bounded calls, 105.44 s of raw audio; normalized ASR matched every source word, including the closing sentence | +| Export validation | Both final MP4 files fully decoded with FFmpeg; cam1 retained video, and the audio-only All-In input retained dubbed and original audio | + +The cam1 run preceded the final short-alignment and vocabulary guards. The final +long-speech probe exercised those guards. All-In's first full run failed on short +alignment and invalid vocoder tokens; recovery reused successful transcription, +translation and speech and regenerated failed or unfinished turns. Its final +127.093 s recovery/reassembly time is **not** a full-pipeline benchmark. A clean +full All-In run with every final guard has not been measured. + +This is a robustness improvement, not a translation or timing quality pass. +The tested 4B model still reversed meanings and mishandled idioms. All-In timing +adjustment truncated 33 of 42 generated turns; final ASR confirmed missing closing +speech. Raw long-turn synthesis retained its endings. The current reported +truncation-seconds metric also includes time-stretch savings, so it overstates +actual tail removal. These remain follow-up work. + +The implementation suite passed 1,265 tests with CUDA hidden, plus lint, typing +and strict documentation checks. Experimental scripts, intermediate transcripts, +and media remain local and are not shipped in the release. + +#### Review follow-up + +The initial reliability candidate was 13.8% slower than the 656.77 s baseline and +31.6% slower than the graph-only candidate. Saved stage timings attribute the +increase to both translation (40.00 → 115.24 s) and synthesis (417.86 → 518.37 s). +These are different generated workloads; the figures do not isolate individual +code changes or measure the final branch. + +A fresh full cam1 run with the review fixes completed in **702.922 s**: **7.0% +slower than the 656.77 s baseline**, and 6.0% faster than the initial reliability +candidate. Translation took 47.119 s; synthesis took 541.291 s across +51 bounded backend calls. All 21 turns generated successfully, with no translation +or synthesis failures, no timing truncation and no excessive speed. Peak sampled +whole-device memory was 7,161 MiB, versus 7,338 MiB in the baseline. + +These remain **4B-model compatibility measurements**, not default-model performance +baselines. The final cam1 output contains 378.64 s of raw speech and 1,326 translated +words, versus 332.48 s and 1,299 words in the baseline. Changed generated workloads +prevent attributing the net difference solely to execution overhead. Cam1 translation +still mishandles the idiom “sezon ogórkowy” as “pickling/cucumber season”. + +The fresh full ten-minute All-In run completed in **1,189.936 s**. All 41 usable +turns generated successfully across 81 bounded calls. Original indices 34 (20 ms) +and 37 (zero duration) were reported in `synthesis_failures`. There were no +structural translation failures or hard timing truncations. Translation took +88.473 s; synthesis took 957.863 s and produced 637.48 s of raw speech. Peak sampled +whole-device GPU memory was 7,157 MiB. This is a complete run, unlike the earlier +recovery/reassembly measurement. + +**All-In still fails quality.** Twenty-one of 41 turns exceeded the preferred speed; +mean speed was 2.568× and maximum speed was 16.840×. Inspecting the worst short +outputs found extra speech in raw TTS output, not merely silence: “Ale” generated +5.94 s, “Tak.” 4.74 s, and “to.” 3.44 s, with ASR detecting unrequested words. Faster +timing preserves the generated waveform but cannot repair hallucinated speech or +make those extreme speeds intelligible. The 4B translator still reverses “the market +is ripping” into a falling market and confuses valuation multiples with revenue growth. + +Both final MP4s fully decoded with every video/audio stream explicitly mapped. +Cam1 contains H.264 video plus dubbed/original AAC; All-In contains two AAC tracks. +Final-audio ASR retained both closing sentences, including All-In's previously lost +“trochę szalony”. Raw longest and closing turns were also transcribed. This checks +selected coverage risks, not every word or overall listening quality. + +The release verification script's `dub` check also passed on a 68.28 s cam1 extract +with `qwen3.5:4b`, CUDA, low-memory mode, diarization and cloning: three turns, two +speakers, no translation/synthesis failures, no truncation or excessive speed. The +script now honors the model override, exposes low-memory mode, reports excessive +speeds and fails on synthesis failures. This is an operational compatibility check; +it does not contradict the semantic failures above. + +The full suite passed **1,275 tests** with CUDA hidden; typing passed all 144 source +and test files. The final translator/Ollama checks passed another 27 tests after +test-typing cleanup. The live Ollama residency probe confirmed the model was loaded +between calls and absent after explicit unload. + +The review fixes bound tiny groups to four turns and ten seconds, reject isolated +sub-100 ms groups with original-index failure reporting, split unbroken text, and +restore soft spoken-length hints. Translation explicitly retains Ollama between +requests and unloads it at the low-memory stage boundary. Requests still isolate +one source part at a time; this trades batching throughput for segment ownership. + +Timing now borrows following silence and preserves the entire generated utterance +by allowing speeds above the preferred 1.3× maximum. Excessive speeds are reported +in the timing summary and logs. This avoids deliberate tail clipping, but does not +guarantee natural delivery or correct translation. Residual tempo-filter duration +errors are fitted by resampling the entire output, with a possible small pitch shift. + +Two fresh-process CUDA pool probes each generated “za”, “Ja” and a full Polish +sentence twice. Shared pools reserved 178 MiB less than separate pools on all six +calls, with matching sample counts and maximum waveform difference below 8e-7. +Timings varied and do not establish a pool-sharing speedup. A separate GPU probe +verified replay under `no_grad` after capture in inference mode. + +The short-alignment dependency fix is prepared as local Chatterbox commit `57fb312` +(version 0.1.7.post2). Seven direct PyTorch boundary tests and the GPU probes above +passed without the alignment monkeypatch. At that stage publication was deferred: +the review candidate retained `>=0.1.7.post1` and the compatibility guard. The full +cam1 and All-In measurements above used that configuration. + +#### Published Chatterbox integration + +On 2026-09-08, the published `videopython-chatterbox==0.1.7.post2` wheel was +installed and the dependency minimum and lock updated. The local alignment +monkeypatch and its two wrapper tests were removed; vocabulary masking and vocoder +token validation remain separate protections. + +Seven direct PyTorch tests passed against the installed dependency's alignment +implementation, covering text widths 1–6 and 20. Seeded CUDA voice cloning produced +“za” (0.96 s) and “Ja” (0.64 s), with the native alignment method unchanged after +model initialization. A 235-word Polish synthesis produced 105.44 s across ten +backend calls; all 235 normalized words matched independent ASR, including the +closing sentence. These examples do not establish that the short-utterance +hallucinations in the All-In record are fixed. + +The English TTS harness initially exhausted GPU memory when loading Whisper while +the synthesis model was still referenced. Explicitly unloading TTS before ASR +resolved this: the rerun passed with 0.818 word-set overlap and generated the cloned +voice sample successfully. The recognized sentence differed in “riverbank” versus +“river bank”; this overlap metric is not a complete semantic or cloning-quality audit. +The core suite passed all 1,273 tests (the two obsolete wrapper tests were removed). +Ruff, formatting, mypy, lock validation, wheel/sdist builds and strict documentation +build passed. The wheel contains the token protections, excludes the alignment +monkeypatch and requires the published post2 dependency. + +These are functional integration checks, not new full-pipeline benchmarks. The +cam1 and All-In performance and quality results above still describe the earlier +dependency configuration. + +#### Comparison against merged 0.61.1 + +On 2026-09-08, the 0.61.2 branch was rebased onto merged `main` at `3ca57b9`. +`git range-diff` confirmed all four release patches were unchanged; the candidate +tested here is `f9b9c37`. Separate processes imported each checkout, using the same +RTX 2060 SUPER, PyTorch 2.13.0+cu130, published Chatterbox 0.1.7.post2, reference WAVs, +expression settings and per-case seeds. Using post2 on both sides isolates the +videopython changes; this is not a comparison of the two historical lockfiles. + +| Matched check | Merged main | 0.61.2 candidate | +|---|---:|---:| +| 235-word Polish passage: generated duration | 30.22 s | 105.44 s | +| Passage: normalized ASR word edit distance | 216 | 0 | +| Known short-utterance hallucinations reproduced | 3/3 | 3/3 | +| Same 41 All-In raw utterances: timing reports tail removal | 31/41 | 0/41 | + +The long passage retained all 235 words on the candidate. Main's ASR returned 112 +words, including repeated invented phrases and an unrequested closing, so that +word count does not represent 112 correctly retained source words. The short cases +requested “Ale”, “Tak.” and “to.” with original All-In seeds 1793, 1805 and 1812. +Both versions produced 5.94 s, 4.74 s and 3.44 s, respectively, with the same +unrequested speech in ASR. Saved PCM samples differed by at most one 16-bit step. + +The timing comparison reused identical cached speech in the original turn windows, +without pipeline gap borrowing. The candidate reported 23 excessive speeds, peaking +at 35.63×; this is not the earlier full-pipeline maximum of 16.84×. Preserving all +samples at such speeds does not establish intelligibility. In a separate closing-turn +check (22.56 s fitted to 17 s), both versions retained the closing phrase in ASR; +word edit distances were 8 on main and 10 on the candidate. That example does not +show a transcription-quality improvement from faster timing. + +Paired translation checks used the existing `qwen3.5:4b`, seed 1777 and default +translator settings on cam1 segments 0–2 and All-In segments 30–33. Both versions +returned all seven segments without structural failures. Both reversed “the market +is ripping” into “rynek się wali” (the market is collapsing). For “sezon ogórkowy” +in the January pizzeria discussion, main returned “summer season” and the candidate +returned “pickling season”; neither preserved the quiet-business-period meaning. +These examples do not demonstrate better semantic translation on the candidate, +and do not evaluate the default 27B model. No model downloads were needed. + +These targeted checks establish improved long-speech fidelity and avoidance of hard +tail cuts, not uniformly better dubbing quality. They are not new full-video latency +benchmarks or listening tests. Post-rebase checks passed 160 dubbing/speech tests +and 17 translator tests. Raw WAVs, recognized text, settings and per-turn timing +records are retained locally under `.cache/dubbing/main-quality-comparison/`. + +#### Listener follow-up: early finishes on cam1_1min + +The listener reported stretch artifacts and a long silence near 38 seconds in both +one-minute dubs. Both had clamped all three turns to 0.8×. Capturing the candidate's +raw speech reproduced durations of 4.76 s, 25.08 s and 6.60 s against source windows +of 7.16 s, 39.64 s and 11.48 s. The second turn begins at 8.02 s: the old minimum +speed left roughly eight seconds of unused time before its 47.66 s boundary. + +That candidate treated the minimum as a preference and reported excessive slowdowns, +as it did for excessive speedups. A corrected mix reused identical raw +speech, translations, source audio and background; only timing changes. Rubber Band +at 0.665×, 0.633× and 0.575× fills all three windows. The second turn's ASR ending +moves from 23.82 s in the raw clip to 37.66 s in the corrected clip, or approximately +45.68 s on the video timeline. Some natural trailing silence remains before 47.66 s. + +This fixes the early-finish mechanism but does not establish artifact-free speech. +Normalized ASR word edit counts for raw versus corrected turns were 2/2, 3/9 and +0/0. The old 0.8× atempo version of the middle turn produced 7 edits while leaving +the long unused tail. At the full required duration, atempo produced 11 edits; +Rubber Band's default and long-window settings produced 10 and 11. The selected +smooth-transient setting produced 9. ASR is an imperfect proxy for perceptual +quality; the new listening sample still requires review. Existing translation +errors and short-input hallucinations are unaffected. + +Pitch/duration tests cover slowing and accelerating a 220 Hz signal, including +activity near the output ending. The focused audio/dubbing suite passed 220 tests; +lint and mypy passed. Both output streams fully decoded. Samples and comparison +records are retained under `.cache/dubbing/cam1_1min_listen/`, including +`cam1_1min-0.61.2-pacing-fixed-en.mp4`. + +#### Source-word phrase scheduling replaces paragraph slowdowns + +The listener rejected the full-window slowdown as unnatural. The next design keeps +speaker turns for diarization/reference extraction, but derives dubbing phrases from +validated word timestamps before translation. Sentence boundaries take priority over +commas; pauses and clauses help bound longer runs. Tiny tails are kept with their +neighbors. Original source segments are preserved and phrase failures map back to +them through `source_segment_index`. Missing or inconsistent word timing retains +the original segment rather than inventing proportional timestamps. + +On `cam1_1min.mp4`, three original turns became ten phrases; the middle turn became +six sentences anchored at 8.02, 13.34, 19.30, 26.44, 33.64 and 39.68 seconds. Shorter +generated phrases keep their natural pace; available gaps absorb overruns before +acceleration. The user's preferred range is now 0.9–1.1×. Larger necessary speedups +remain reported to preserve complete generated speech. Forced slowdowns are removed. + +The new full-pipeline review video used the same existing 4B translator, post2 +Chatterbox, seed policy, diarization, speaker cloning and background preservation. +All ten phrases ran at exactly 1.0×: zero stretches, truncations, excessive speeds, +translation failures or synthesis failures. A separate check under the new 0.9/1.1 +defaults confirmed every synchronized waveform was sample-identical to raw TTS. +Both exported streams fully decoded. This removes time-stretch artifacts from this +sample by avoiding time stretching entirely, rather than selecting another filter. + +Final ASR returned 145 words against 145 translated words, with three word edits: +the negation in “can't say” became “can say”, “health” became “hell”, and an extra +“Umm” appeared. These require listening review; successful scheduling is not proof +of exact spoken meaning. The 4B translation still mistranslates the quiet-season +idiom. Late phrases remain anchored throughout the final part of the video, with +speech recognized through approximately 59 seconds. + +Validation passed 184 focused dubbing, phrase, translation and speech tests, plus +lint, typing and strict documentation checks. Artifacts are retained under +`.cache/dubbing/cam1_1min_phrases/`; the review output is +`cam1_1min-phrases-en.mp4`. This is a pacing/quality check, not a controlled new +full-video performance benchmark or phoneme-level lip-sync claim. + +The listener accepted this phrase-based version for 0.61.2 with the remaining +sentence pauses. A subsequent placement experiment was discarded: release code +keeps the original phrase anchors and introduces no accumulated start-time shifts. +Final local release checks passed 1,283 tests, Ruff/formatting, mypy, lock validation +and strict documentation build. Wheel and sdist builds passed; a clean wheel install +passed public-import, render and MCP smoke checks. The wheel requires published +Chatterbox post2 and excludes the discarded placement experiment. + ### Diarization optimization, 0.61.1 On 2026-09-08, the diarization embedding path was measured on an NVIDIA GeForce @@ -186,8 +573,10 @@ input established the failure limit. | Baseline 3 | 10/17 | 1.085 | 2.000 s | | Complete model run | 7/17 | 1.102 | 2.400 s | -The dub verification fails if the timing summary is missing or if one segment loses -more than 3.0 seconds during synchronization. +These historical runs used the former truncation threshold. The current dub +verification fails on missing timing measurements or translation/synthesis failures. +It reports excessive speeds for listening review; synchronization preserves complete +generated speech. ## 4K effects performance diff --git a/pyproject.toml b/pyproject.toml index 82038640..c4db5fa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "videopython" -version = "0.61.1" +version = "0.61.2" description = "LLM-friendly Python video editing with JSON plans, local AI, and MCP tools." authors = [ { name = "Bartosz Wójtowicz", email = "bartoszwojtowicz@outlook.com" }, @@ -93,9 +93,9 @@ ai = [ # Audio source separation (understanding/separation.py) "demucs>=4.0.0", # Voice cloning TTS (generation/audio.py). Our fork of chatterbox-tts with - # corrected metadata; import name is still `chatterbox`. Not upstream — its - # `==` pins make [ai] unsatisfiable. See DEVELOPMENT.md "Dependencies". - "videopython-chatterbox>=0.1.7.post1", + # corrected metadata and short-text alignment fix; import name is `chatterbox`. + # Upstream's `==` pins make [ai] unsatisfiable. See DEVELOPMENT.md "Dependencies". + "videopython-chatterbox>=0.1.7.post2", # Local media generation (generation/*). >=0.35 ships QwenImagePipeline + Wan2.2. "diffusers>=0.35.0", # Required, despite looking like diffusers' problem: the Wan2.2 i2v pipeline diff --git a/scripts/verify_ai_models.py b/scripts/verify_ai_models.py index b15f8216..1b4edf7d 100644 --- a/scripts/verify_ai_models.py +++ b/scripts/verify_ai_models.py @@ -28,6 +28,7 @@ class Context: source_lang: str target_lang: str enable_diarization: bool + low_memory: bool = False @dataclass @@ -506,6 +507,7 @@ def check_tts(ctx: Context) -> Outcome: cloned_note = "tts_cloned.wav written" except Exception as exc: # noqa: BLE001 - reported, not fatal to the check cloned_note = f"cloning raised {type(exc).__name__}: {exc}" + tts.unload() free_weights() heard = AudioToText().transcribe(audio) @@ -655,22 +657,9 @@ def check_detection(ctx: Context) -> Outcome: ) -_MAX_DUB_TRUNCATION_SECONDS = 3.0 - - @check("dub", "Full dub end-to-end, translated and audible", needs_video=True, needs_cuda=True) def check_dub(ctx: Context) -> Outcome: - """The integration that has broken most often, asserted on its output. - - Checks, in order of how badly each has bitten: - - 1. ``translation_failures`` is empty. This is the exact signal the 0.53.0 - reasoning-model bug produced -- every segment failed and was dubbed with - empty text, while the pipeline reported success. - 2. The dubbed track is not silent. A pipeline that "succeeds" into silence - has happened via the separator writing stems to the wrong directory. - 3. The worst timing truncation does not exceed the manual-run baseline. - """ + """Check translation, synthesis, requested speaker cloning, and audible output.""" import numpy as np from videopython.ai.dubbing import VideoDubber @@ -678,7 +667,12 @@ def check_dub(ctx: Context) -> Outcome: assert ctx.video is not None # guaranteed by needs_video video = Video.from_path(str(ctx.video)) - dubber = VideoDubber(device=ctx.device) if ctx.device else VideoDubber() + dubbing_options: dict[str, Any] = {"low_memory": ctx.low_memory} + if ctx.device: + dubbing_options["device"] = ctx.device + if ctx.ollama_model: + dubbing_options["translator_model"] = ctx.ollama_model + dubber = VideoDubber(**dubbing_options) result = dubber.dub( video, target_lang=ctx.target_lang, @@ -710,6 +704,9 @@ def check_dub(ctx: Context) -> Outcome: measurements: dict[str, Any] = { "segments": segments, "translation_failures": len(failures), + "synthesis_failures": result.synthesis_failures, + "low_memory": ctx.low_memory, + "translator_model": ctx.ollama_model or "default", "dubbed_peak_amplitude": round(peak, 6), "diarization": ctx.enable_diarization, "speakers": sorted(result.source_transcription.speakers), @@ -718,9 +715,9 @@ def check_dub(ctx: Context) -> Outcome: } if timing is not None: measurements |= { - "truncated": f"{timing.truncated_count}/{timing.total_segments}", "mean_speed_factor": round(timing.mean_speed_factor, 3), - "max_truncation_seconds": round(timing.max_truncation_seconds, 3), + "excessive_speed_count": timing.excessive_speed_count, + "max_speed_factor": round(timing.max_speed_factor, 3), } if failures: @@ -731,28 +728,24 @@ def check_dub(ctx: Context) -> Outcome: ) if ctx.enable_diarization and not result.source_transcription.speakers: return Outcome(passed=False, detail="diarization returned no speakers", measurements=measurements) + if result.synthesis_failures: + return Outcome( + passed=False, + detail=f"speech failed for original segments {result.synthesis_failures}", + measurements=measurements, + ) if ctx.enable_diarization and not result.voice_samples: return Outcome(passed=False, detail="voice cloning produced no speaker samples", measurements=measurements) if peak <= 1e-4: return Outcome(passed=False, detail="dubbed track is silent", measurements=measurements) if timing is None: return Outcome(passed=False, detail="timing summary is missing", measurements=measurements) - if timing.max_truncation_seconds > _MAX_DUB_TRUNCATION_SECONDS: - return Outcome( - passed=False, - detail=( - f"worst timing truncation {timing.max_truncation_seconds:.3f}s exceeds " - f"{_MAX_DUB_TRUNCATION_SECONDS:.1f}s" - ), - measurements=measurements, - ) - free_weights() return Outcome( passed=True, detail=( f"{segments} segments {ctx.source_lang}->{ctx.target_lang}, all translated, audible, " - f"worst truncation {timing.max_truncation_seconds:.3f}s" + f"{timing.excessive_speed_count} above preferred speed, maximum {timing.max_speed_factor:.3f}x" ), measurements=measurements, ) @@ -794,6 +787,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--workdir", type=Path, default=Path("./verify-out"), help="where checks write artifacts") parser.add_argument("--device", help="torch device override (default: auto)") parser.add_argument("--ollama-model", help="override the Ollama model under test") + parser.add_argument("--low-memory", action="store_true", help="unload models between dubbing stages") parser.add_argument("--source-lang", default="pl") parser.add_argument("--target-lang", default="es") parser.add_argument( @@ -830,6 +824,7 @@ def main(argv: list[str] | None = None) -> int: source_lang=args.source_lang, target_lang=args.target_lang, enable_diarization=args.enable_diarization, + low_memory=args.low_memory, ) results: list[tuple[Check, str, str, dict[str, Any]]] = [] diff --git a/src/tests/ai/test_dubbing.py b/src/tests/ai/test_dubbing.py index c1261053..d6d25955 100644 --- a/src/tests/ai/test_dubbing.py +++ b/src/tests/ai/test_dubbing.py @@ -140,23 +140,19 @@ def test_initialization_defaults(self): """Test default initialization values.""" sync = TimingSynchronizer() - assert sync.min_speed == 0.8 - assert sync.max_speed == 1.3 + assert sync.max_speed == 1.1 def test_initialization_custom(self): """Test custom initialization values.""" - sync = TimingSynchronizer(min_speed=0.5, max_speed=2.0) + sync = TimingSynchronizer(max_speed=2.0) - assert sync.min_speed == 0.5 assert sync.max_speed == 2.0 def test_initialization_invalid(self): """Test that invalid parameters raise errors.""" - with pytest.raises(ValueError, match="min_speed must be positive"): - TimingSynchronizer(min_speed=0) - - with pytest.raises(ValueError, match="max_speed must be greater than min_speed"): - TimingSynchronizer(min_speed=1.0, max_speed=0.5) + for speed in (0.5, float("inf"), float("nan")): + with pytest.raises(ValueError, match="max_speed must be finite and at least 1.0"): + TimingSynchronizer(max_speed=speed) def test_synchronize_segment_no_change(self, sample_audio): """Test synchronizing when no change is needed.""" @@ -167,7 +163,6 @@ def test_synchronize_segment_no_change(self, sample_audio): assert abs(result.metadata.duration_seconds - target_duration) < 0.1 assert adjustment.speed_factor == 1.0 - assert not adjustment.was_truncated def test_synchronize_segment_speed_up(self, sample_audio): """Test synchronizing by speeding up audio.""" @@ -181,27 +176,34 @@ def test_synchronize_segment_speed_up(self, sample_audio): assert adjustment.speed_factor > 1.0 assert adjustment.original_duration > adjustment.actual_duration - def test_synchronize_segment_slow_down(self, sample_audio): - """Test synchronizing by slowing down audio.""" + def test_synchronize_segment_keeps_natural_pace(self, sample_audio): + """Shorter speech is not slowed merely to fill its time window.""" sync = TimingSynchronizer() - target_duration = sample_audio.metadata.duration_seconds * 1.2 # Need 20% slowdown + target_duration = sample_audio.metadata.duration_seconds * 1.2 result, adjustment = sync.synchronize_segment(sample_audio, target_duration, segment_index=5) - # Result should be longer (but may still be shorter than target if at min_speed limit) - assert adjustment.speed_factor < 1.0 + np.testing.assert_array_equal(result.data, sample_audio.data) + assert adjustment.speed_factor == 1.0 assert adjustment.segment_index == 5 - def test_synchronize_segment_truncation(self, sample_audio): - """Test that audio is truncated when even max speed isn't enough.""" + def test_short_generated_speech_is_not_stretched_to_fill_turn(self, sample_audio): + target = sample_audio.metadata.duration_seconds * 2 + result, adjustment = TimingSynchronizer().synchronize_segment(sample_audio, target) + np.testing.assert_array_equal(result.data, sample_audio.data) + assert adjustment.speed_factor == 1.0 + + def test_synchronize_segment_preserves_speech_above_preferred_speed(self, sample_audio): + """Keep the full utterance when the preferred speed cannot fit it.""" sync = TimingSynchronizer(max_speed=1.3) target_duration = sample_audio.metadata.duration_seconds / 2.0 # Need 50% shorter result, adjustment = sync.synchronize_segment(sample_audio, target_duration) - # Should be truncated to target duration + # Fit by speeding up, without slicing the tail assert abs(result.metadata.duration_seconds - target_duration) < 0.1 - assert adjustment.was_truncated + assert adjustment.excessive_speed + assert adjustment.speed_factor >= 2.0 def test_synchronize_segments(self, sample_audio): """Test synchronizing multiple segments.""" @@ -291,6 +293,24 @@ def test_assemble_with_timing_no_unnecessary_normalization(self, sample_audio): # sample_audio peaks at 0.5; with no overlap, output peak should match assert abs(peak - 0.5) < 0.01 + def test_assembly_smooths_edges_without_moving_or_cutting_speech(self, sample_audio): + sample_audio.data += 0.2 + original = sample_audio.data.copy() + sync = TimingSynchronizer() + fitted, _ = sync.synchronize_segment(sample_audio, 2.0) + result = sync.assemble_with_timing([fitted, fitted], [0.25, 2.5], 5.0) + + sr = sample_audio.metadata.sample_rate + fade = round(0.005 * sr) + assert result.metadata.duration_seconds == 5.0 + for start in (sr // 4, 5 * sr // 2): + placed = result.data[start : start + len(original)] + assert placed[0] == placed[-1] == 0 + assert np.any(placed[:fade]) and np.any(placed[-fade:]) + np.testing.assert_array_equal(placed[fade:-fade], original[fade:-fade]) + assert not np.any(result.data[9 * sr // 4 : 5 * sr // 2]) + np.testing.assert_array_equal(sample_audio.data, original) + class TestDubbingResult: """Tests for DubbingResult data model.""" @@ -373,100 +393,23 @@ def test_empty_adjustments(self): assert summary.total_segments == 0 assert summary.clean_count == 0 assert summary.stretched_count == 0 - assert summary.truncated_count == 0 assert summary.mean_speed_factor == 1.0 - assert summary.max_truncation_seconds == 0.0 - def test_classifies_clean_stretched_truncated(self): - """Mixed adjustments are bucketed into clean / stretched / truncated.""" + def test_classifies_clean_and_accelerated_segments(self): from videopython.ai.dubbing.models import TimingAdjustment, TimingSummary adjustments = [ - # Clean: speed factor within tolerance, not truncated. - TimingAdjustment( - segment_index=0, - original_duration=2.0, - target_duration=2.0, - actual_duration=2.0, - speed_factor=1.0, - was_truncated=False, - ), - # Stretched up. - TimingAdjustment( - segment_index=1, - original_duration=2.0, - target_duration=2.0, - actual_duration=1.9, - speed_factor=1.05, - was_truncated=False, - ), - # Stretched down. - TimingAdjustment( - segment_index=2, - original_duration=2.0, - target_duration=2.5, - actual_duration=2.5, - speed_factor=0.85, - was_truncated=False, - ), - # Stretched at max speed but not truncated. - TimingAdjustment( - segment_index=3, - original_duration=2.6, - target_duration=2.0, - actual_duration=2.0, - speed_factor=1.3, - was_truncated=False, - ), - # Truncated: clamped at max speed but still too long. - TimingAdjustment( - segment_index=4, - original_duration=4.0, - target_duration=2.0, - actual_duration=2.0, - speed_factor=1.3, - was_truncated=True, - ), + TimingAdjustment(0, 2.0, 2.0, 2.0, 1.0, False), + TimingAdjustment(1, 2.1, 2.0, 2.0, 1.05, False), + TimingAdjustment(2, 2.6, 2.0, 2.0, 1.3, True), ] - summary = TimingSummary.from_adjustments(adjustments) - - assert summary.total_segments == 5 + assert summary.total_segments == 3 assert summary.clean_count == 1 - assert summary.stretched_count == 3 - assert summary.truncated_count == 1 - # Mean of 1.0, 1.05, 0.85, 1.3, 1.3 = 1.10. - assert summary.mean_speed_factor == pytest.approx(1.10) - # Worst-case truncation: 4.0 - 2.0 = 2.0s. - assert summary.max_truncation_seconds == pytest.approx(2.0) - - def test_max_truncation_picks_worst_case(self): - """max_truncation_seconds is the largest (original - actual) across truncated segments.""" - from videopython.ai.dubbing.models import TimingAdjustment, TimingSummary - - adjustments = [ - TimingAdjustment( - segment_index=0, - original_duration=3.0, - target_duration=2.0, - actual_duration=2.0, - speed_factor=1.3, - was_truncated=True, - ), - TimingAdjustment( - segment_index=1, - original_duration=10.0, - target_duration=5.0, - actual_duration=5.0, - speed_factor=1.3, - was_truncated=True, - ), - ] - - summary = TimingSummary.from_adjustments(adjustments) - - assert summary.truncated_count == 2 - assert summary.max_truncation_seconds == pytest.approx(5.0) + assert summary.stretched_count == 2 + assert summary.mean_speed_factor == pytest.approx((1.0 + 1.05 + 1.3) / 3) + assert summary.excessive_speed_count == 1 + assert summary.max_speed_factor == 1.3 def test_round_trip_to_dict(self): """from_dict(to_dict(s)) must equal the original summary.""" @@ -475,10 +418,10 @@ def test_round_trip_to_dict(self): summary = TimingSummary( total_segments=10, clean_count=5, - stretched_count=3, - truncated_count=2, + stretched_count=5, mean_speed_factor=1.12, - max_truncation_seconds=0.7, + excessive_speed_count=2, + max_speed_factor=1.3, ) restored = TimingSummary.model_validate(summary.model_dump()) @@ -499,9 +442,9 @@ def test_dubbing_result_carries_summary(self, sample_audio, sample_segment): total_segments=1, clean_count=1, stretched_count=0, - truncated_count=0, mean_speed_factor=1.0, - max_truncation_seconds=0.0, + excessive_speed_count=0, + max_speed_factor=1.0, ) result = DubbingResult( @@ -517,7 +460,7 @@ def test_dubbing_result_carries_summary(self, sample_audio, sample_segment): assert result.timing_summary.total_segments == 1 def test_dubbing_result_default_timing_summary_none(self, sample_audio, sample_segment): - """DubbingResult constructed without a TimingSummary keeps the field as None (back-compat).""" + """DubbingResult constructed without a TimingSummary has no timing measurements.""" from videopython.base.transcription import Transcription translated = TranslatedSegment( @@ -1663,7 +1606,7 @@ def _make_two_speaker_transcription(self): TranscriptionSegment( start=1.0, end=2.0, - text="hi back", + text="hello back there", words=[ TranscriptionWord(start=1.0, end=1.5, word="hi", speaker="A"), TranscriptionWord(start=1.5, end=2.0, word="back", speaker="A"), @@ -1756,6 +1699,8 @@ def generate_audio(self, text, voice_sample=None, voice_sample_path=None, **_kwa # time_stretch on the fake-saved WAVs and fail. We're only checking # encode counts and TTS argument plumbing here. class FakeSynchronizer: + max_speed = 1.3 + def synchronize_segments(self, segments, durations): return segments, [] @@ -1850,6 +1795,8 @@ def generate_audio(self, text, voice_sample=None, voice_sample_path=None, **_kwa monkeypatch.setattr(LocalDubbingPipeline, "_init_tts", fake_init_tts) class FakeSynchronizer: + max_speed = 1.3 + def synchronize_segments(self, segments, durations): return segments, [] @@ -1931,7 +1878,7 @@ def fake_run(cmd, *, stdin=None): # Map: video, dubbed audio (input 1), subtitles from input 0. map_indices = [i for i, v in enumerate(cmd) if v == "-map"] assert len(map_indices) == 3 - assert cmd[map_indices[0] + 1] == "0:v:0" + assert cmd[map_indices[0] + 1] == "0:v:0?" assert cmd[map_indices[1] + 1] == "1:a:0" assert cmd[map_indices[2] + 1] == "0:s?" assert "-c:v" in cmd and cmd[cmd.index("-c:v") + 1] == "copy" @@ -1962,7 +1909,7 @@ def fake_run(cmd, *, stdin=None): cmd = captured["cmd"] map_args = [cmd[i + 1] for i, v in enumerate(cmd) if v == "-map"] - assert map_args == ["0:v:0", "1:a:0", "0:a?", "0:s?"] + assert map_args == ["0:v:0?", "1:a:0", "0:a?", "0:s?"] class TestVideoDubberDubFile: @@ -2972,3 +2919,156 @@ def test_expressiveness_normal_in_band(self): # ratio = 1.0 sits squarely in the normal band → no-knobs profile result = expressiveness_for(self._audio_with_amplitude(1.0), baseline_rms=1.0) assert result == Expressiveness() + + +@pytest.mark.parametrize("fragment_duration", [0, 0.05]) +def test_synthesis_failures_use_original_indices_after_joining(sample_audio, monkeypatch, fragment_duration): + from videopython.ai.dubbing.pipeline import LocalDubbingPipeline + from videopython.base.transcription import Transcription, TranscriptionSegment + + segments = [ + TranscriptionSegment(start=0, end=0.2, text="And", words=[], speaker="A"), + TranscriptionSegment(start=0.2, end=1, text="some longer speech", words=[], speaker="A"), + TranscriptionSegment(start=1, end=1 + fragment_duration, text="Yes", words=[], speaker="B"), + ] + translated = [ + TranslatedSegment(original_segment=s, translated_text=s.text, source_lang="en", target_lang="pl") + for s in segments + ] + monkeypatch.setattr(LocalDubbingPipeline, "_translate", lambda *args: (translated, [])) + attempted = [] + + def fail(self, **kwargs): + attempted.append(kwargs["segment"]) + return None + + monkeypatch.setattr(LocalDubbingPipeline, "_tts_segment_audio", fail) + result = LocalDubbingPipeline().process( + source_audio=sample_audio, + transcription=Transcription(segments=segments, language="en"), + target_lang="pl", + preserve_background=False, + voice_clone=False, + ) + assert result.synthesis_failures == [0, 1, 2] + assert len(attempted) == 1 + assert attempted[0].translated_text == "And some longer speech" + assert [s.translated_text for s in result.translated_segments] == [s.text for s in segments] + + +@pytest.mark.parametrize("channels", [1, 2]) +def test_timing_retains_last_samples_when_tempo_output_overruns(monkeypatch, channels): + # Simulate atempo's duration error; an end marker must survive the final fit. + data = np.zeros((1010,) if channels == 1 else (1010, 2), dtype=np.float32) + data[-10:] = 0.7 + audio = Audio(data, AudioMetadata(1000, channels, 2, 1.01, 1010)) + monkeypatch.setattr(Audio, "time_stretch", lambda self, speed, **kwargs: self) + fitted, adjustment = TimingSynchronizer().synchronize_segment(audio, 0.5) + assert len(fitted.data) == 500 + np.testing.assert_allclose(fitted.data[-1], 0.7) + assert adjustment.excessive_speed + + +def test_pipeline_uses_following_gap_before_excessive_speed(monkeypatch): + from videopython.ai.dubbing.pipeline import LocalDubbingPipeline + from videopython.base.transcription import Transcription + + source = Audio.create_silent(4, stereo=False) + segments = [ + TranscriptionSegment(start=0, end=1, text="first full turn", words=[], speaker="A"), + TranscriptionSegment(start=3, end=4, text="next full turn", words=[], speaker="B"), + ] + translated = [ + TranslatedSegment(original_segment=s, translated_text=s.text, source_lang="en", target_lang="pl") + for s in segments + ] + monkeypatch.setattr(LocalDubbingPipeline, "_translate", lambda *args: (translated, [])) + monkeypatch.setattr( + LocalDubbingPipeline, + "_tts_segment_audio", + lambda self, **kw: Audio.create_silent(2 if kw["speaker"] == "A" else 1, stereo=False), + ) + durations = [] + original = TimingSynchronizer.synchronize_segments + + def record(self, audio_segments, target_durations): + durations.extend(target_durations) + return original(self, audio_segments, target_durations) + + monkeypatch.setattr(TimingSynchronizer, "synchronize_segments", record) + result = LocalDubbingPipeline().process( + source_audio=source, + transcription=Transcription(segments=segments, language="en"), + target_lang="pl", + preserve_background=False, + voice_clone=False, + ) + assert durations == pytest.approx([2, 1]) + assert result.timing_summary.excessive_speed_count == 0 + + +def test_small_timing_overrun_reports_only_the_applied_speed(): + from videopython.ai.dubbing.models import TimingSummary + + audio = Audio.create_silent(1.008, sample_rate=24000, stereo=False) + fitted, adjustment = TimingSynchronizer().synchronize_segment(audio, 1.0) + assert fitted.metadata.duration_seconds == 1.0 + assert adjustment.speed_factor == pytest.approx(len(audio.data) / len(fitted.data)) + summary = TimingSummary.from_adjustments([adjustment]) + assert summary.clean_count == 1 + assert summary.stretched_count == 0 + assert summary.mean_speed_factor == pytest.approx(1.008) + assert summary.max_speed_factor == pytest.approx(1.008) + + +def test_dub_is_only_default_audio_when_original_tracks_are_retained(tmp_path): + import json + import subprocess + + from videopython.ai.dubbing.remux import replace_audio_stream_from_audio + + source = tmp_path / "original.m4a" + destination = tmp_path / "dubbed.m4a" + subprocess.run( + [ + "ffmpeg", + "-v", + "error", + "-f", + "lavfi", + "-i", + "sine=duration=0.2", + "-map", + "0:a", + "-map", + "0:a", + "-c:a", + "aac", + "-disposition:a", + "default", + str(source), + ], + check=True, + capture_output=True, + ) + replace_audio_stream_from_audio( + source, Audio.create_silent(0.2, stereo=False), destination, keep_original_audio=True + ) + streams = json.loads( + subprocess.check_output( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "a", + "-show_entries", + "stream_disposition=default", + "-of", + "json", + str(destination), + ], + text=True, + ) + )["streams"] + assert [stream["disposition"]["default"] for stream in streams] == [1, 0, 0] diff --git a/src/tests/ai/test_dubbing_phrases.py b/src/tests/ai/test_dubbing_phrases.py new file mode 100644 index 00000000..e6eb2730 --- /dev/null +++ b/src/tests/ai/test_dubbing_phrases.py @@ -0,0 +1,101 @@ +import pytest + +from videopython.ai.dubbing._phrases import timed_phrases +from videopython.base.transcription import TranscriptionSegment, TranscriptionWord + + +def make_segment(text, start=0, speaker="A"): + words = [ + TranscriptionWord(word=w, start=start + i * 0.5, end=start + (i + 1) * 0.5, speaker=speaker) + for i, w in enumerate(text.split()) + ] + return TranscriptionSegment.from_words(words, speaker=speaker, avg_logprob=-0.2) + + +def test_sentences_win_over_commas_and_preserve_all_words(): + source = make_segment("Well, this is our first sentence. Now this is our second sentence.") + before = source.model_dump() + phrases, parents = timed_phrases([source]) + assert [s.text for s in phrases] == ["Well, this is our first sentence.", "Now this is our second sentence."] + assert parents == [0, 0] + assert [w for s in phrases for w in s.words] == source.words + assert [(s.start, s.end) for s in phrases] == [(0, 3), (3, 6)] + assert all(s.speaker == "A" and s.avg_logprob == -0.2 for s in phrases) + assert source.model_dump() == before + + +def test_long_speaker_turn_has_multiple_real_timing_anchors(): + source = make_segment(" ".join(["one two three four five six."] * 10)) + phrases, parents = timed_phrases([source]) + assert len(phrases) == 10 + assert parents == [0] * 10 + assert phrases[-1].start == 27 + assert phrases[-1].end == source.end + assert all(s.end - s.start <= 8 for s in phrases) + + +def test_partial_or_invalid_word_alignment_keeps_original_text(): + source = make_segment("These are the full words. And these are more words.") + for changes in ({"words": source.words[:-1]}, {"words": []}, {"start": 2}, {"text": source.text + " missing"}): + segment = source.model_copy(update=changes) + phrases, parents = timed_phrases([segment]) + assert phrases == [segment] + assert parents == [0] + + +def test_does_not_create_tiny_tail_or_mix_speakers(): + a = make_segment("This is a full sentence. Yes.") + b = make_segment("Here is another complete sentence. Here is one more sentence.", start=10, speaker="B") + phrases, parents = timed_phrases([a, b]) + assert phrases[0] is a + assert parents == [0, 1, 1] + assert [s.speaker for s in phrases] == ["A", "B", "B"] + + +def test_translation_failures_map_to_source_turns(): + from unittest.mock import Mock + + from videopython.ai.dubbing.config import DubbingConfig + from videopython.ai.dubbing.models import TranslatedSegment + from videopython.ai.dubbing.pipeline import LocalDubbingPipeline + from videopython.base.transcription import Transcription + + source = make_segment("This is the first sentence. Here is the second sentence.") + pipeline = LocalDubbingPipeline(DubbingConfig()) + translator = Mock() + translator.translation_failures = [1] + translator.translate_segments.side_effect = lambda **kw: [ + TranslatedSegment(original_segment=s, translated_text="hello", source_lang="en", target_lang="pl") + for s in kw["segments"] + ] + pipeline._translator = translator + translated, failures = pipeline._translate(Transcription(segments=[source]), "en", "pl", lambda *args: None) + assert failures == [0] + assert len(translated) == 2 + assert [s.source_segment_index for s in translated] == [0, 0] + assert source.text == "This is the first sentence. Here is the second sentence." + + +def test_common_title_does_not_split_a_name(): + source = make_segment("Here we meet Dr. Smith who is speaking. This is the next sentence.") + phrases, _ = timed_phrases([source]) + assert phrases[0].text == "Here we meet Dr. Smith who is speaking." + + +@pytest.mark.parametrize( + "tokens, first, second", + [ + (["私", "は", "今日", "走る。", "明日", "も", "外を", "走る。"], "私は今日走る。", "明日も外を走る。"), + (["我", "今天", "在", "工作。", "明天", "我", "也要", "工作。"], "我今天在工作。", "明天我也要工作。"), + (["ราคา", "นี้", "คือ", "3,000.", "ราคา", "นั้น", "คือ", "5,000."], "ราคานี้คือ3,000.", "ราคานั้นคือ5,000."), + (["It", "costs", "3", ",000.", "That", "costs", "5", ",000."], "It costs 3,000.", "That costs 5,000."), + ], +) +def test_phrase_text_preserves_source_spacing(tokens, first, second): + words = [TranscriptionWord(word=w, start=i * 0.5, end=(i + 1) * 0.5) for i, w in enumerate(tokens)] + source = TranscriptionSegment(start=0, end=4, text=first + "\n" + second, words=words) + phrases, parents = timed_phrases([source]) + assert [phrase.text for phrase in phrases] == [first, second] + assert [word for phrase in phrases for word in phrase.words] == words + assert [(phrase.start, phrase.end) for phrase in phrases] == [(0, 2), (2, 4)] + assert parents == [0, 0] diff --git a/src/tests/ai/test_speech_chunks.py b/src/tests/ai/test_speech_chunks.py new file mode 100644 index 00000000..fa70827c --- /dev/null +++ b/src/tests/ai/test_speech_chunks.py @@ -0,0 +1,212 @@ +import sys +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np +import pytest + +from videopython.ai._text_chunks import split_text +from videopython.ai.generation.audio import TextToSpeech + + +class _Tensor(np.ndarray): + """Only the tensor operations used at the backend boundary.""" + + def cpu(self): + return self + + def float(self): + return self.astype(np.float32) + + def numpy(self): + return np.asarray(self) + + def numel(self): + return self.size + + +torch = SimpleNamespace( + tensor=lambda value: np.asarray(value).view(_Tensor), + ones=lambda *shape: np.ones(shape).view(_Tensor), + zeros=lambda *shape: np.zeros(shape).view(_Tensor), +) + + +@pytest.fixture(autouse=True) +def speech_tensor_runtime(monkeypatch): + monkeypatch.setitem(sys.modules, "torch", torch) + + +@pytest.mark.parametrize( + "text", + [ + "First sentence. " * 90, + "A long clause, and more words; " * 90, + "word " * 300, + " Some\n whitespace.\tMore words! ", + ], +) +def test_split_covers_text_once(text): + parts = split_text(text, 80) + assert " ".join(parts) == " ".join(text.split()) + assert all(len(part) <= 80 for part in parts) + + +def test_punctuation_at_budget_edge_cannot_exceed_limit(): + text = "word " * 40 + ". More speech follows." + parts = split_text(text, 200) + assert all(len(part) <= 200 for part in parts) + assert " ".join(parts) == text + + +@pytest.mark.parametrize("text", ["漢字" * 140, "ภาษาไทย" * 40, "https://example.com/" + "a" * 240]) +def test_unbroken_text_is_bounded_without_dropping_characters(text): + parts = split_text(text, 80) + assert "".join(parts) == text + assert all(0 < len(part) <= 80 for part in parts) + + +def test_unbroken_speech_reaches_backend_in_bounded_parts(): + tts = TextToSpeech(language="zh") + tts._model = Mock() + tts._model.generate.return_value = torch.ones(1, 240) + text = "漢字" * 140 + tts.generate_audio(text) + calls = tts._model.generate.call_args_list + assert "".join(c.kwargs["text"] for c in calls) == text + assert all(len(c.kwargs["text"]) <= 200 for c in calls) + + +def test_long_speech_preserves_settings_and_order(): + tts = TextToSpeech(language="pl") + tts._model = Mock() + tts._model.generate.side_effect = lambda **kw: torch.ones(1, 240) * len(kw["text"]) + text = "Pierwsze zdanie. Drugie zdanie, z dalszym wyjaśnieniem. " * 20 + audio = tts.generate_audio(text, voice_sample_path="speaker.wav", exaggeration=0.7, cfg_weight=0.4, temperature=0.6) + calls = tts._model.generate.call_args_list + assert " ".join(c.kwargs["text"] for c in calls) == text.strip() + assert len(calls) > 1 + for i, call in enumerate(calls): + assert call.kwargs["audio_prompt_path"] == "speaker.wav" + assert call.kwargs["language_id"] == "pl" + assert call.kwargs["exaggeration"] == 0.7 + assert call.kwargs["cfg_weight"] == 0.4 + assert call.kwargs["temperature"] == 0.6 + np.testing.assert_array_equal(audio.data[i * 240 : (i + 1) * 240], len(call.kwargs["text"])) + + +def test_cap_suspect_audio_is_replaced_with_smaller_calls(): + tts = TextToSpeech(language="en") + tts._model = Mock() + tts._model.generate.side_effect = [torch.zeros(1, 40 * 24000), torch.ones(1, 240), torch.ones(1, 240) * 2] + text = "hello " * 30 + audio = tts.generate_audio(text) + calls = tts._model.generate.call_args_list + assert " ".join(c.kwargs["text"] for c in calls[1:]) == text.strip() + assert len(audio.data) == 480 + assert np.all(audio.data[:240] == 1) + assert np.all(audio.data[240:] == 2) + + +def test_subchunk_failure_does_not_return_partial_audio(): + tts = TextToSpeech() + tts._model = Mock() + tts._model.generate.side_effect = [torch.ones(1, 240), RuntimeError("failed")] + with pytest.raises(RuntimeError, match="failed"): + tts.generate_audio("sentence with words. " * 30) + + +def test_short_text_uses_one_unchanged_call(): + tts = TextToSpeech() + tts._model = Mock() + tts._model.generate.return_value = torch.ones(1, 240) + tts.generate_audio("Hello, world!") + tts._model.generate.assert_called_once_with(text="Hello, world!", language_id="en", audio_prompt_path=None) + + +def test_tiny_groups_respect_speaker_changes_and_gaps(): + from videopython.ai.dubbing._synthesis import synthesis_groups + from videopython.ai.dubbing.models import TranslatedSegment + from videopython.base.transcription import TranscriptionSegment + + def seg(start, end, text, speaker): + original = TranscriptionSegment(start=start, end=end, text=text, words=[], speaker=speaker) + return TranslatedSegment(original_segment=original, translated_text=text, source_lang="en", target_lang="pl") + + segments = [ + seg(0, 0.2, "I", "A"), + seg(0.2, 3, "dalsza część zdania", "A"), + seg(3, 3.2, "I", "B"), + seg(4, 4.2, "I", "B"), + seg(4.2, 6, "kolejne zdanie", "B"), + ] + assert synthesis_groups(segments) == [[0, 1], [2], [3, 4]] + assert segments[0].translated_text == "I" + + +def test_chained_tiny_groups_have_bounded_span_and_members(): + from videopython.ai.dubbing._synthesis import synthesis_groups + from videopython.ai.dubbing.models import TranslatedSegment + from videopython.base.transcription import TranscriptionSegment + + segments = [ + TranslatedSegment( + original_segment=TranscriptionSegment(start=i * 0.9, end=(i + 1) * 0.9, text="a", words=[], speaker="A"), + translated_text="a", + source_lang="en", + target_lang="pl", + ) + for i in range(60) + ] + groups = synthesis_groups(segments) + assert [i for group in groups for i in group] == list(range(60)) + assert max(map(len, groups)) <= 4 + assert all(segments[g[-1]].end - segments[g[0]].start <= 10 for g in groups) + segments[1].end = 30 + assert synthesis_groups(segments)[0] == [0] + + +def test_invalid_speech_tokens_never_reach_vocoder(): + from videopython.ai.generation._speech_tokens import InvalidSpeechTokens, guard_speech_tokens + + vocoder = Mock(return_value="audio") + guarded = guard_speech_tokens(vocoder, 6561) + for tokens in (torch.tensor([6727]), torch.tensor([-1]), torch.tensor([])): + with pytest.raises(InvalidSpeechTokens): + guarded(speech_tokens=tokens) + vocoder.assert_not_called() + valid = torch.tensor([0, 6560]) + assert guarded(speech_tokens=valid) == "audio" + vocoder.assert_called_once_with(speech_tokens=valid) + + +def test_invalid_token_retry_is_bounded_and_keeps_settings(): + from videopython.ai.generation._speech_tokens import InvalidSpeechTokens + + tts = TextToSpeech(language="pl") + tts._model = Mock() + tts._model.generate.side_effect = [InvalidSpeechTokens("bad"), torch.ones(1, 240)] + tts.generate_audio("za", voice_sample_path="speaker.wav", exaggeration=0.7) + assert tts._model.generate.call_args_list[0] == tts._model.generate.call_args_list[1] + tts._model.generate.reset_mock() + tts._model.generate.side_effect = InvalidSpeechTokens("bad") + with pytest.raises(InvalidSpeechTokens): + tts.generate_audio("za") + assert tts._model.generate.call_count == 3 + + +def test_sampling_excludes_invalid_ids_but_preserves_eos_and_valid_logits(): + from videopython.ai.generation._speech_tokens import restrict_speech_vocabulary + + head = Mock(out_features=10) + original = torch.tensor([[0.1 * i for i in range(10)]]) + before = original.copy() + restrict_speech_vocabulary(head, 3, 4) + hook = head.register_forward_hook.call_args.args[0] + masked = hook(head, (), original) + assert masked is original + np.testing.assert_array_equal(masked[:, :3], before[:, :3]) + np.testing.assert_array_equal(masked[:, 4], before[:, 4]) + assert (masked[:, 3] == -(2**15)).all() + assert (masked[:, 5:] == -(2**15)).all() + assert np.isfinite(masked + 0.5 * (masked - masked)).all() diff --git a/src/tests/ai/test_speech_graphs.py b/src/tests/ai/test_speech_graphs.py new file mode 100644 index 00000000..6725b472 --- /dev/null +++ b/src/tests/ai/test_speech_graphs.py @@ -0,0 +1,165 @@ +import sys +from contextlib import nullcontext +from types import SimpleNamespace + +import numpy as np +import pytest + +from videopython.ai.generation._speech_graphs import SpeechGraphs +from videopython.ai.generation.audio import TextToSpeech + + +class LlamaMLP: + training = False + + def __init__(self): + self.calls = 0 + self.child = SimpleNamespace(_forward_hooks={}, _forward_pre_hooks={}) + + def modules(self): + return [self, self.child] + + def forward(self, x): + self.calls += 1 + return x.data * 2 + + +class Input: + def __init__(self, value=1, shape=(2, 1, 4), device="cuda", dtype="float32", contiguous=True): + self.data = np.full(shape, value, dtype=np.float32) + self.shape = shape + self.ndim = len(shape) + self.device = SimpleNamespace(type=device) + self.dtype = dtype + self.contiguous = contiguous + + def is_contiguous(self): + return self.contiguous + + def stride(self): + return tuple(s // self.data.itemsize for s in self.data.strides) + + +@pytest.fixture +def runtime(monkeypatch): + torch = SimpleNamespace( + float32="float32", + is_grad_enabled=lambda: False, + inference_mode=nullcontext, + cuda=SimpleNamespace(empty_cache=lambda: None), + ) + monkeypatch.setitem(sys.modules, "torch", torch) + module = LlamaMLP() + owner = SpeechGraphs( + SimpleNamespace(t3=SimpleNamespace(tfmr=SimpleNamespace(layers=[SimpleNamespace(mlp=module)]))) + ) + wrapper = owner._wrappers[0] + captures = [] + + def capture(x): + captures.append(x.shape) + static = np.empty_like(x.data) + output = np.empty_like(x.data) + buffer = SimpleNamespace(copy_=lambda value: np.copyto(static, value.data)) + graph = SimpleNamespace(replay=lambda: np.multiply(static, 2, out=output)) + return graph, buffer, output + + monkeypatch.setattr(wrapper, "_capture", capture) + return module, owner, captures, torch + + +def test_replay_copies_new_token_values(runtime): + module, _, captures, _ = runtime + x = Input(1) + assert np.all(module.forward(x) == 2) + x.data.fill(7) + assert np.all(module.forward(x) == 14) + assert len(captures) == 1 + assert module.calls == 0 + + +@pytest.mark.parametrize( + "kwargs", + [{"shape": (2, 8, 4)}, {"shape": (1, 1, 4)}, {"device": "cpu"}, {"dtype": "float16"}, {"contiguous": False}], +) +def test_other_layouts_use_original_forward(runtime, kwargs): + module, _, captures, _ = runtime + assert np.all(module.forward(Input(**kwargs)) == 2) + assert not captures + assert module.calls == 1 + + +def test_training_and_gradients_bypass_capture(runtime): + module, _, captures, torch = runtime + module.training = True + module.forward(Input()) + module.training = False + torch.is_grad_enabled = lambda: True + module.forward(Input()) + assert module.calls == 2 + assert not captures + + +def test_nested_hooks_added_after_capture_still_run_eagerly(runtime): + module, _, captures, _ = runtime + module.forward(Input()) + module.child._forward_hooks[1] = object() + module.forward(Input()) + assert module.calls == 1 + assert len(captures) == 1 + + +def test_changed_shape_does_not_replay_wrong_graph(runtime): + module, _, captures, _ = runtime + module.forward(Input()) + assert module.forward(Input(shape=(2, 1, 8))).shape == (2, 1, 8) + assert module.calls == 1 + assert len(captures) == 1 + + +def test_capture_failure_restores_forwards_and_retries(runtime, monkeypatch): + module, owner, _, _ = runtime + + def fail(x): + raise RuntimeError("capture unsupported") + + monkeypatch.setattr(owner._wrappers[0], "_capture", fail) + assert np.all(module.forward(Input(3)) == 6) + assert owner.disabled + assert not owner._wrappers + assert "forward" not in vars(module) + owner.close() + + +def test_close_releases_buffers_and_restores_instance_override(runtime): + module, owner, _, _ = runtime + wrapper = owner._wrappers[0] + module.forward(Input()) + assert wrapper.state is not None + owner.close() + assert wrapper.state is None + assert not owner._wrappers + assert "forward" not in vars(module) + previous = module.forward + module.forward = previous + owner = SpeechGraphs( + SimpleNamespace(t3=SimpleNamespace(tfmr=SimpleNamespace(layers=[SimpleNamespace(mlp=module)]))) + ) + owner.close() + assert module.forward is previous + + +def test_tts_unload_closes_graphs_before_releasing_model(monkeypatch): + import videopython.ai._predictor as predictor + + tts = TextToSpeech() + closed = [] + tts._speech_graphs = SimpleNamespace(close=lambda: closed.append(True)) + + def release(device): + assert closed == [True] + assert tts._speech_graphs is None + assert tts._model is None + + monkeypatch.setattr(predictor, "release_device_memory", release) + tts.unload() diff --git a/src/tests/ai/test_translator.py b/src/tests/ai/test_translator.py index a3886a3a..dd7f9e4b 100644 --- a/src/tests/ai/test_translator.py +++ b/src/tests/ai/test_translator.py @@ -5,15 +5,15 @@ import json from types import SimpleNamespace from typing import Any +from unittest.mock import Mock, patch + +import pytest from videopython.ai.dubbing.translation import ( LANGUAGE_NAMES, OllamaTranslator, _build_system_prompt, - _build_user_prompt, - _chunk_segment_indices, _parse_translations, - _target_chars_for, ) from videopython.base.transcription import TranscriptionSegment @@ -30,6 +30,7 @@ def __init__(self, contents: list[str], capabilities: list[str] | None = None) - self.capabilities = ["completion", "thinking"] if capabilities is None else capabilities self.calls = 0 self.chat_kwargs: list[dict[str, Any]] = [] + self.generate = Mock() def show(self, model: str) -> SimpleNamespace: return SimpleNamespace(capabilities=self.capabilities) @@ -59,11 +60,6 @@ def _translator_with(contents: list[str]) -> tuple[OllamaTranslator, _FakeOllama # --------------------------------------------------------------------------- helpers -def test_target_chars_uses_language_rate() -> None: - assert _target_chars_for(1.0, "en") == int(14.0 * 1.15) - assert _target_chars_for(0.0, "en") == 1 # minimum 1 - - def test_build_system_prompt_names_languages() -> None: prompt = _build_system_prompt("en", "es") assert "English" in prompt @@ -71,17 +67,26 @@ def test_build_system_prompt_names_languages() -> None: assert "translations" in prompt # describes the JSON object shape -def test_build_user_prompt_marks_low_confidence() -> None: - prompt = _build_user_prompt([_seg("hello", avg_logprob=-2.0), _seg("world", avg_logprob=0.0)], "es") - assert '"low_confidence": true' in prompt - assert "target_chars" in prompt - - -def test_chunk_segment_indices_splits_on_budget() -> None: - segs = [_seg("a" * 100) for _ in range(10)] - chunks = _chunk_segment_indices(segs, n_ctx=600, max_tokens=100) - assert len(chunks) > 1 - assert sum(len(c) for c in chunks) == 10 # every segment placed exactly once +def test_unbroken_translation_is_bounded_with_soft_timing_and_full_progress(): + translator = OllamaTranslator(model="m", n_ctx=1200, max_tokens=140) + entries = [] + + def generate(**kwargs): + entry = json.loads(kwargs["text"].split("\nTarget:\n")[1]) + entries.append(entry) + return {"translations": [{"i": entry["i"], "translated": "translated"}]} + + translator._client.generate_json = Mock(side_effect=generate) + progress = [] + text = "漢字" * 140 + result = translator.translate_segments([_seg(text, 0, 20)], "en", "zh", progress.append) + assert "".join(e["text"] for e in entries) == text + assert all(len(e["text"]) <= 40 for e in entries) + assert sum(e["target_chars"] for e in entries) == 280 + assert progress == sorted(progress) + assert progress[-2:] == [0.95, 1.0] + assert translator.translation_failures == [] + assert result[0].translated_text == " ".join(["translated"] * len(entries)) def test_parse_translations() -> None: @@ -94,25 +99,31 @@ def test_parse_translations() -> None: def test_translate_segments_happy_path() -> None: - content = json.dumps({"translations": [{"i": 0, "translated": "hola"}, {"i": 1, "translated": "mundo"}]}) - translator, fake = _translator_with([content]) + translator, fake = _translator_with( + [ + json.dumps({"translations": [{"i": 0, "translated": "hola"}]}), + json.dumps({"translations": [{"i": 1, "translated": "mundo"}]}), + ] + ) out = translator.translate_segments([_seg("hello"), _seg("world")], target_lang="es", source_lang="en") assert [s.translated_text for s in out] == ["hola", "mundo"] assert translator.translation_failures == [] - assert fake.calls == 1 + assert fake.calls == 2 + assert all(call["keep_alive"] == "5m" for call in fake.chat_kwargs) def test_translate_segments_retries_missing() -> None: translator, fake = _translator_with( [ json.dumps({"translations": [{"i": 0, "translated": "hola"}]}), # i=1 missing - json.dumps({"translations": [{"i": 0, "translated": "mundo"}]}), # retry batch idx 0 -> orig 1 + json.dumps({"translations": []}), + json.dumps({"translations": [{"i": 1, "translated": "mundo"}]}), # stable identity on retry ] ) out = translator.translate_segments([_seg("hello"), _seg("world")], target_lang="es") assert [s.translated_text for s in out] == ["hola", "mundo"] assert translator.translation_failures == [] - assert fake.calls == 2 + assert fake.calls == 3 def test_translate_segments_records_failures() -> None: @@ -131,8 +142,10 @@ def test_non_translatable_segments_skipped() -> None: def test_unload_and_languages() -> None: - translator, _ = _translator_with(["{}"]) + translator, fake = _translator_with(["{}"]) translator.unload() # idempotent + translator.unload() + fake.generate.assert_called_once_with(model="m", keep_alive=0) assert OllamaTranslator.get_supported_languages() == LANGUAGE_NAMES @@ -168,10 +181,10 @@ def chat( def test_translate_segments_multiple_chunks() -> None: - translator = OllamaTranslator(model="m", n_ctx=1000, max_tokens=100) # small ctx forces splitting + translator = OllamaTranslator(model="m", n_ctx=2000, max_tokens=300) # small ctx forces splitting fake = _EchoOllama() translator._client._client = fake - segs = [_seg("w" * 200, start=float(i), end=float(i) + 1) for i in range(12)] + segs = [_seg("word " * 100, start=float(i), end=float(i) + 1) for i in range(12)] out = translator.translate_segments(segs, target_lang="es") @@ -192,7 +205,7 @@ def test_translate_segments_progress_milestones() -> None: translator, _ = _translator_with([content]) ticks: list[float] = [] translator.translate_segments([_seg("hello")], target_lang="es", progress_callback=ticks.append) - assert any(abs(t - 0.5) < 1e-9 for t in ticks) # first pass reaches 0.5 + assert ticks[-2] == 0.95 # requests span the whole translation window assert ticks[-1] == 1.0 @@ -210,3 +223,84 @@ def test_translation_disables_reasoning_on_thinking_model() -> None: assert out[0].translated_text == "hola" assert translator.translation_failures == [] assert fake.chat_kwargs[0]["think"] is False + + +def test_rejects_ambiguous_or_invalid_results() -> None: + assert _parse_translations({"translations": [{"i": 0, "translated": "a"}, {"i": 0, "translated": "b"}]}) == {} + invalid_values: list[Any] = [None, 42, [], "", " "] + for value in invalid_values: + assert _parse_translations({"translations": [{"i": 0, "translated": value}]}) == {} + for index in (True, "0", 0.5, -1): + assert _parse_translations({"translations": [{"i": index, "translated": "a"}]}) == {} + + +def test_one_letter_speech_is_translated() -> None: + translator, _ = _translator_with([json.dumps({"translations": [{"i": 0, "translated": "and"}]})]) + assert translator.translate_segments([_seg("I")], target_lang="en", source_lang="pl")[0].translated_text == "and" + + +def test_failed_part_invalidates_whole_parent() -> None: + translator, _ = _translator_with( + [ + json.dumps({"translations": [{"i": 0, "translated": "first"}]}), + json.dumps({"translations": []}), + ] + ) + source = _seg("A sentence. " * 100) + out = translator.translate_segments([source], target_lang="pl") + assert out[0].translated_text == "" + assert out[0].original_segment is source + assert translator.translation_failures == [0] + + +def test_complete_json_at_output_limit_is_rejected() -> None: + import pytest + + from videopython.ai._ollama import OllamaError + + translator, fake = _translator_with(["{}"]) + response = SimpleNamespace(message=SimpleNamespace(content="{}"), done_reason="length") + with patch.object(fake, "chat", return_value=response), pytest.raises(OllamaError, match="exhausted"): + translator._client.generate_json(system="test", text="test", schema={}) + + +def test_extra_invalid_identity_invalidates_response() -> None: + assert ( + _parse_translations( + { + "translations": [ + {"i": 0, "translated": "valid"}, + {"i": -1, "translated": "invalid"}, + ] + } + ) + == {} + ) + + +@pytest.mark.parametrize("kwargs", [{"n_ctx": 4096}, {"max_tokens": 139}, {"options": {"num_ctx": 4096}}]) +def test_invalid_effective_budgets_fail_at_construction(kwargs): + with pytest.raises(ValueError, match=r"n_ctx.*max_tokens"): + OllamaTranslator(**kwargs) + + +def test_options_can_supply_valid_smaller_budgets(): + translator = OllamaTranslator(n_ctx=4096, options={"num_predict": 1024}) + assert translator.n_ctx == 4096 + assert translator.max_tokens == 1024 + + +def test_failed_unload_preserves_translation_and_context_error(caplog): + translator, fake = _translator_with(['{"translations": [{"i": 0, "translated": "hola"}]}']) + fake.generate.side_effect = ConnectionError("server restarted") + with translator: + result = translator.translate_segments([_seg("hello")], "es", "en") + assert result[0].translated_text == "hola" + assert translator._client._client is None + assert "server memory may remain allocated" in caplog.text + translator.unload() + fake.generate.assert_called_once_with(model="m", keep_alive=0) + + translator._client._client = fake + with pytest.raises(ValueError, match="original failure"), translator: + raise ValueError("original failure") diff --git a/src/tests/audio/test_audio.py b/src/tests/audio/test_audio.py index 145cc9a2..754ecf01 100644 --- a/src/tests/audio/test_audio.py +++ b/src/tests/audio/test_audio.py @@ -1471,3 +1471,29 @@ def test_from_path_reports_ffmpeg_failure(): with pytest.raises((AudioLoadError, FFmpegProbeError)): Audio.from_path(broken_path) + + +@pytest.mark.parametrize("speed", [0.6, 1.5]) +def test_rubberband_preserves_pitch_and_fills_duration(speed): + from videopython.ai.dubbing.timing import _speech_stretch_method + + if _speech_stretch_method() != "rubberband": + pytest.skip("FFmpeg lacks the optional rubberband filter") + sample_rate = 24000 + t = np.arange(sample_rate * 2) / sample_rate + data = (0.2 * np.sin(2 * np.pi * 220 * t)).astype(np.float32) + audio = Audio(data, AudioMetadata(sample_rate, 1, 2, 2.0, len(data))) + stretched = audio.time_stretch(speed, method="rubberband") + assert stretched.metadata.duration_seconds == pytest.approx(2 / speed, abs=0.06) + middle = stretched.data[len(stretched.data) // 4 : 3 * len(stretched.data) // 4] + spectrum = np.abs(np.fft.rfft(middle)) + frequency = np.fft.rfftfreq(len(middle), 1 / sample_rate)[np.argmax(spectrum)] + assert frequency == pytest.approx(220, abs=2) + assert np.sqrt(np.mean(stretched.data[-2400:] ** 2)) > 0.05 + + +def test_time_stretch_rejects_nonfinite_speed(): + audio = Audio.create_silent(1, stereo=False) + for speed in (float("inf"), float("nan")): + with pytest.raises(ValueError, match="finite"): + audio.time_stretch(speed) diff --git a/src/videopython/ai/_ollama.py b/src/videopython/ai/_ollama.py index 1088827c..d3ab87d1 100644 --- a/src/videopython/ai/_ollama.py +++ b/src/videopython/ai/_ollama.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging import re from typing import Any @@ -12,6 +13,8 @@ from videopython.ai.errors import AiError from videopython.ai.keyframe import encode_png_b64 +logger = logging.getLogger(__name__) + class OllamaError(AiError, RuntimeError): """Ollama returned unusable output (non-JSON or an unexpected shape).""" @@ -76,9 +79,17 @@ class OllamaStructuredClient: reported by Ollama. An explicit ``num_ctx`` in ``options`` always wins. """ - def __init__(self, model: str, *, host: str | None = None, options: dict[str, Any] | None = None) -> None: + def __init__( + self, + model: str, + *, + host: str | None = None, + options: dict[str, Any] | None = None, + keep_alive: str | int | None = None, + ) -> None: self.model = model self.host = host + self.keep_alive = keep_alive self.options: dict[str, Any] = {"temperature": 0.0, **(options or {})} self._client: Any = None self._thinking_capable: bool | None = None @@ -121,6 +132,8 @@ def generate_json( kwargs: dict[str, Any] = {} if self._supports_thinking(): kwargs["think"] = False + if self.keep_alive is not None: + kwargs["keep_alive"] = self.keep_alive client = self._get_client() try: response = client.chat(model=self.model, messages=messages, format=schema, options=options, **kwargs) @@ -130,6 +143,8 @@ def generate_json( raise options = {**options, "num_ctx": retry_num_ctx} response = client.chat(model=self.model, messages=messages, format=schema, options=options, **kwargs) + if getattr(response, "done_reason", None) == "length": + raise OllamaError("Ollama exhausted its output budget; refusing an incomplete response") content = response.message.content try: data = json.loads(content) @@ -140,4 +155,13 @@ def generate_json( return data def unload(self) -> None: - self._client = None + try: + if self._client is not None and self.keep_alive is not None: + # Release explicit residency before the low-memory pipeline + # loads its next GPU model. Merely dropping the HTTP client + # leaves the Ollama runner resident on the server. + self._client.generate(model=self.model, keep_alive=0) + except Exception as exc: + logger.warning("Could not unload Ollama model %s; server memory may remain allocated: %s", self.model, exc) + finally: + self._client = None diff --git a/src/videopython/ai/_text_chunks.py b/src/videopython/ai/_text_chunks.py new file mode 100644 index 00000000..ce209be6 --- /dev/null +++ b/src/videopython/ai/_text_chunks.py @@ -0,0 +1,32 @@ +import re + + +def split_text(text: str, max_chars: int) -> list[str]: + """Prefer sentence, then clause, then word boundaries; retain every character. + + Whitespace is normalized. Text without usable boundaries (including scripts + without spaces and long URLs) is split at the character budget as a last resort. + """ + if max_chars < 1: + raise ValueError("max_chars must be positive") + remaining = " ".join(text.split()) + chunks: list[str] = [] + while len(remaining) > max_chars: + window = remaining[: max_chars + 1] + boundary = 0 + for pattern in (r"(?:[.!?](?:\s|$)|[。！？])", r"(?:[,;:](?:\s|$)|[，；：])", r"\s"): + matches = [ + match + for match in re.finditer(pattern, window) + if match.start() + (0 if pattern == r"\s" else 1) <= max_chars + ] + if matches: + boundary = matches[-1].start() + (0 if pattern == r"\s" else 1) + break + if not boundary: + boundary = max_chars + chunks.append(remaining[:boundary].strip()) + remaining = remaining[boundary:].strip() + if remaining: + chunks.append(remaining) + return chunks diff --git a/src/videopython/ai/dubbing/_phrases.py b/src/videopython/ai/dubbing/_phrases.py new file mode 100644 index 00000000..aa603d7f --- /dev/null +++ b/src/videopython/ai/dubbing/_phrases.py @@ -0,0 +1,95 @@ +import math +import re + +from videopython.base.transcription import TranscriptionSegment + +_MAX_SECONDS = 8.0 +_MIN_WORDS = 4 +_ABBREVIATIONS = {"mr.", "mrs.", "ms.", "dr.", "prof.", "e.g.", "i.e.", "np.", "itd.", "itp."} + + +def timed_phrases(segments: list[TranscriptionSegment]) -> tuple[list[TranscriptionSegment], list[int]]: + """Return phrases and their source indices, preserving words and speaker ownership. + + Prefer sentence ends, then pauses/clauses. Bound long runs at word boundaries, + avoiding tiny fragments which are poor translation and voice-cloning prompts. + Missing, partial or inconsistent word timing leaves the source segment intact; + text alone cannot establish trustworthy phrase timestamps. + """ + phrases: list[TranscriptionSegment] = [] + parents: list[int] = [] + for parent, segment in enumerate(segments): + words = segment.words + valid = ( + len(words) >= 2 * _MIN_WORDS + and "".join(segment.text.split()) == "".join("".join(w.word.split()) for w in words) + and all( + math.isfinite(w.start) + and math.isfinite(w.end) + and segment.start <= w.start <= w.end <= segment.end + and w.speaker in (None, segment.speaker) + for w in words + ) + and all(a.start <= b.start and a.end <= b.end for a, b in zip(words, words[1:])) + ) + cuts: list[int] = [] + if valid: + start = 0 + while len(words) - start >= 2 * _MIN_WORDS: + strong: list[int] = [] + weak: list[int] = [] + bounded: list[int] = [] + for end in range(start + _MIN_WORDS, len(words) - _MIN_WORDS + 1): + left, right = words[end - 1], words[end] + span = left.end - words[start].start + if span > _MAX_SECONDS: + break + if span < 1.5 or left.end > right.start: + continue + ending = left.word.rstrip().rstrip("\"'”’)]}") + bounded.append(end) + if ending.lower() not in _ABBREVIATIONS and ending.endswith((".", "!", "?", "…", "。", "！", "？")): + strong.append(end) + elif span >= 2.5 and ( + ending.endswith((",", ";", ":", "，", "；", "：")) or right.start - left.end >= 0.4 + ): + weak.append(end) + # A complete sentence beats an earlier comma: do not detach + # subordinate clauses when the sentence fits the phrase budget. + if strong: + cut = strong[0] + elif words[-1].end - words[start].start <= _MAX_SECONDS: + break + elif weak or bounded: + cut = (weak or bounded)[-1] + else: + break + cuts.append(cut) + start = cut + if not cuts: + phrases.append(segment) + parents.append(parent) + continue + # Align cuts to the source text so token boundaries do not add spaces. + text_offsets = [0, *(match.end() for match in re.finditer(r"\S", segment.text))] + text_start = 0 + text_chars = 0 + start = 0 + for end in [*cuts, len(words)]: + phrase_words = words[start:end] + text_chars += sum(len("".join(word.word.split())) for word in phrase_words) + text_end = text_offsets[text_chars] + phrases.append( + segment.model_copy( + update={ + "start": phrase_words[0].start, + "end": phrase_words[-1].end, + "text": segment.text[text_start:text_end].strip(), + "words": list(phrase_words), + } + ) + ) + parents.append(parent) + start = end + text_start = text_end + return phrases, parents diff --git a/src/videopython/ai/dubbing/_synthesis.py b/src/videopython/ai/dubbing/_synthesis.py new file mode 100644 index 00000000..a6865507 --- /dev/null +++ b/src/videopython/ai/dubbing/_synthesis.py @@ -0,0 +1,29 @@ +from videopython.ai.dubbing.models import TranslatedSegment + +MAX_GROUP_SECONDS = 10.0 +MAX_GROUP_MEMBERS = 4 +MIN_SYNTHESIS_SECONDS = 0.1 + + +def synthesis_groups(segments: list[TranslatedSegment]) -> list[list[int]]: + groups: list[list[int]] = [] + for i, segment in enumerate(segments): + if groups: + previous = segments[groups[-1][-1]] + tiny = ( + min(previous.duration, segment.duration) < 1.0 + or min(len(previous.translated_text.strip()), len(segment.translated_text.strip())) < 10 + ) + if ( + tiny + and len(groups[-1]) < MAX_GROUP_MEMBERS + and segment.end - segments[groups[-1][0]].start <= MAX_GROUP_SECONDS + and previous.speaker == segment.speaker + and 0 <= segment.start - previous.end <= 0.15 + and previous.translated_text.strip() + and segment.translated_text.strip() + ): + groups[-1].append(i) + continue + groups.append([i]) + return groups diff --git a/src/videopython/ai/dubbing/models.py b/src/videopython/ai/dubbing/models.py index 41f55679..620fb006 100644 --- a/src/videopython/ai/dubbing/models.py +++ b/src/videopython/ai/dubbing/models.py @@ -21,7 +21,7 @@ class TimingAdjustment: target_duration: Target duration to fit into. actual_duration: Actual duration after adjustment. speed_factor: Speed factor applied (> 1 means sped up). - was_truncated: Whether the segment had to be truncated. + excessive_speed: Whether fitting exceeded the preferred speed maximum. """ segment_index: int @@ -29,7 +29,7 @@ class TimingAdjustment: target_duration: float actual_duration: float speed_factor: float - was_truncated: bool + excessive_speed: bool # Speed factors within this band of 1.0 are treated as a "clean" timing @@ -85,6 +85,7 @@ class TranslatedSegment(BaseModel): speaker: Speaker identifier if available. start: Start time in seconds. end: End time in seconds. + source_segment_index: Original transcript index when this is a dubbing phrase. """ original_segment: TranscriptionSegment @@ -94,6 +95,7 @@ class TranslatedSegment(BaseModel): speaker: str | None = None start: float = 0.0 end: float = 0.0 + source_segment_index: int | None = None @model_validator(mode="after") def _default_timing_from_segment(self) -> TranslatedSegment: @@ -142,20 +144,14 @@ def has_detailed_separation(self) -> bool: class TimingSummary(BaseModel): - """Aggregate stats over per-segment timing adjustments. - - Surfaces how aggressively the timing synchronizer had to compress or - truncate dubbed segments to fit the source's spoken regions. High - truncation rates indicate translation produced text too long for the - source duration. - """ + """Summarize speed changes and count adjustments above the preferred maximum.""" total_segments: int clean_count: int stretched_count: int - truncated_count: int mean_speed_factor: float - max_truncation_seconds: float + excessive_speed_count: int + max_speed_factor: float @classmethod def from_adjustments(cls, adjustments: list[TimingAdjustment]) -> TimingSummary: @@ -166,35 +162,20 @@ def from_adjustments(cls, adjustments: list[TimingAdjustment]) -> TimingSummary: total_segments=0, clean_count=0, stretched_count=0, - truncated_count=0, mean_speed_factor=1.0, - max_truncation_seconds=0.0, + excessive_speed_count=0, + max_speed_factor=1.0, ) - clean = 0 - stretched = 0 - truncated = 0 - speed_sum = 0.0 - max_truncation = 0.0 - for adj in adjustments: - speed_sum += adj.speed_factor - if adj.was_truncated: - truncated += 1 - truncation = adj.original_duration - adj.actual_duration - if truncation > max_truncation: - max_truncation = truncation - elif abs(adj.speed_factor - 1.0) <= CLEAN_SPEED_TOLERANCE: - clean += 1 - else: - stretched += 1 + clean = sum(abs(adj.speed_factor - 1.0) <= CLEAN_SPEED_TOLERANCE for adj in adjustments) return cls( total_segments=total, clean_count=clean, - stretched_count=stretched, - truncated_count=truncated, - mean_speed_factor=speed_sum / total, - max_truncation_seconds=max_truncation, + stretched_count=total - clean, + mean_speed_factor=sum(adj.speed_factor for adj in adjustments) / total, + excessive_speed_count=sum(adj.excessive_speed for adj in adjustments), + max_speed_factor=max(adj.speed_factor for adj in adjustments), ) @@ -215,6 +196,8 @@ class DubbingResult(BaseModel): translation_failures: Indices of segments the translator could not translate (missing after its parse-retry pass); those segments are dubbed with empty text. + synthesis_failures: Original segment indices whose speech failed to + generate or had less than 100 ms after fragment joining. """ model_config = ConfigDict(arbitrary_types_allowed=True) @@ -229,6 +212,8 @@ class DubbingResult(BaseModel): timing_summary: TimingSummary | None = None transcript_quality: TranscriptQuality | None = None translation_failures: list[int] = Field(default_factory=list) + # Original segment indices, including every member of a failed joined turn. + synthesis_failures: list[int] = Field(default_factory=list) @property def num_segments(self) -> int: diff --git a/src/videopython/ai/dubbing/pipeline.py b/src/videopython/ai/dubbing/pipeline.py index 65867e44..305f9e2b 100644 --- a/src/videopython/ai/dubbing/pipeline.py +++ b/src/videopython/ai/dubbing/pipeline.py @@ -160,22 +160,25 @@ def _translate( self._init_translator(source_lang=source_lang, target_lang=target_lang) # Translation stage spans 0.35 → 0.50 of overall pipeline progress. - # MarianMT runs sequentially over 8-segment batches; on a 15-min - # source that's minutes of silent dwell on 0.35 without per-batch - # ticks. Map the [0,1] translation fraction onto that 15% window. + # Map completed bounded translation requests onto the stage's window. def _on_translation_progress(fraction: float) -> None: clamped = max(0.0, min(1.0, fraction)) report_progress(f"Translating text ({int(clamped * 100)}%)", 0.35 + 0.15 * clamped) + from videopython.ai.dubbing._phrases import timed_phrases + + phrases, parents = timed_phrases(transcription.segments) translated_segments = self._translator.translate_segments( - segments=transcription.segments, + segments=phrases, target_lang=target_lang, source_lang=source_lang, progress_callback=_on_translation_progress, ) - # Capture per-segment failures (always empty for Marian) before + # Capture per-segment failures before # _maybe_unload nukes the backend in low_memory mode. - translation_failures = list(self._translator.translation_failures) + translation_failures = sorted({parents[i] for i in self._translator.translation_failures}) + for segment, parent in zip(translated_segments, parents): + segment.source_segment_index = parent self._maybe_unload("_translator") return translated_segments, translation_failures @@ -428,6 +431,9 @@ def report_progress(stage: str, progress: float) -> None: report_progress("Generating dubbed speech", 0.50) + from videopython.ai.dubbing._synthesis import MIN_SYNTHESIS_SECONDS, synthesis_groups + + synthesis_failures: list[int] = [] dubbed_segments: list[Audio] = [] target_durations: list[float] = [] start_times: list[float] = [] @@ -436,8 +442,21 @@ def report_progress(stage: str, progress: float) -> None: # The dict is loop-scoped state so the finally block can clean up. speaker_wav_paths: dict[str, Path] = {} try: - for i, segment in enumerate(translated_segments): - if segment.duration < 0.1: + for indices in synthesis_groups(translated_segments): + i = max(indices, key=lambda index: translated_segments[index].duration) + segment = translated_segments[indices[0]] + if len(indices) > 1: + segment = segment.model_copy( + update={ + "translated_text": " ".join( + translated_segments[index].translated_text for index in indices + ), + "end": translated_segments[indices[-1]].end, + } + ) + if segment.duration < MIN_SYNTHESIS_SECONDS: + if segment.translated_text.strip(): + synthesis_failures.extend(indices) continue # Translation filter (translation.py:_is_translatable_text) # leaves translated_text="" for punctuation-only or empty @@ -446,8 +465,8 @@ def report_progress(stage: str, progress: float) -> None: if not segment.translated_text.strip(): continue - progress = 0.50 + (0.30 * (i / len(translated_segments))) - report_progress(f"Generating speech ({i + 1}/{len(translated_segments)})", progress) + progress = 0.50 + (0.30 * (indices[0] / len(translated_segments))) + report_progress(f"Generating speech ({indices[-1] + 1}/{len(translated_segments)})", progress) speaker = segment.speaker or "speaker_0" dubbed_audio = self._tts_segment_audio( @@ -460,10 +479,21 @@ def report_progress(stage: str, progress: float) -> None: expressiveness=expressiveness_per_segment[i], ) if dubbed_audio is None: + synthesis_failures.extend(indices) continue dubbed_segments.append(dubbed_audio) - target_durations.append(segment.duration) + # Borrow only unused time before the next original turn, even if + # that turn failed translation/synthesis. Never cross a speaker. + next_index = indices[-1] + 1 + next_start = ( + translated_segments[next_index].start + if next_index < len(translated_segments) + else source_audio.metadata.duration_seconds + ) + available = max(segment.duration, next_start - segment.start) + needed = dubbed_audio.metadata.duration_seconds + target_durations.append(max(segment.duration, min(available, needed))) start_times.append(segment.start) finally: for path in speaker_wav_paths.values(): @@ -504,6 +534,14 @@ def report_progress(stage: str, progress: float) -> None: timing_summary=timing_summary, transcript_quality=transcript_quality, translation_failures=translation_failures, + synthesis_failures=sorted( + { + translated_segments[i].source_segment_index + if translated_segments[i].source_segment_index is not None + else i + for i in synthesis_failures + } + ), ) def revoice( diff --git a/src/videopython/ai/dubbing/remux.py b/src/videopython/ai/dubbing/remux.py index 9b7eaf92..526086d9 100644 --- a/src/videopython/ai/dubbing/remux.py +++ b/src/videopython/ai/dubbing/remux.py @@ -25,14 +25,9 @@ class RemuxError(AiError, RuntimeError): def _build_stream_maps(keep_original_audio: bool) -> list[str]: - """ffmpeg ``-map`` flags for the video + audio + subtitle streams. - - Convention: dubbed audio (input 1) is the *first* audio track so default - playback uses it; original audio (input 0) tags onto the back when - ``keep_original_audio=True`` for editorial A/B. Subtitles from input 0 - are carried with ``?`` so sources without subs don't fail the mux. - """ - maps = ["-map", "0:v:0", "-map", "1:a:0"] + """Map the dubbed audio first, followed by optional original audio and subtitles.""" + # MP4/M4A inputs can contain audio only; still produce the dubbed audio. + maps = ["-map", "0:v:0?", "-map", "1:a:0"] if keep_original_audio: maps += ["-map", "0:a?"] maps += ["-map", "0:s?"] @@ -96,6 +91,10 @@ def replace_audio_stream_from_audio( "-i", "-", *_build_stream_maps(keep_original_audio), + "-disposition:a", + "0", + "-disposition:a:0", + "default", "-c:v", "copy", "-c:a", diff --git a/src/videopython/ai/dubbing/timing.py b/src/videopython/ai/dubbing/timing.py index a3d1a447..e3a14e52 100644 --- a/src/videopython/ai/dubbing/timing.py +++ b/src/videopython/ai/dubbing/timing.py @@ -2,41 +2,43 @@ from __future__ import annotations +import logging +import subprocess +from dataclasses import replace +from functools import lru_cache +from typing import Literal + import numpy as np from videopython.ai.dubbing.models import TimingAdjustment from videopython.audio import Audio, AudioMetadata +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=1) +def _speech_stretch_method() -> Literal["atempo", "rubberband"]: + """Prefer Rubber Band when installed; retain compatibility with core ffmpeg.""" + result = subprocess.run(["ffmpeg", "-hide_banner", "-filters"], capture_output=True, text=True, check=True) + if any(len(fields := line.split()) > 1 and fields[1] == "rubberband" for line in result.stdout.splitlines()): + return "rubberband" + logger.warning("FFmpeg has no rubberband filter; dubbing falls back to atempo time stretching") + return "atempo" + class TimingSynchronizer: """Synchronizes dubbed audio segments to match original timing. Adjusts the speed of dubbed audio segments to fit within the timing - constraints of the original speech while maintaining natural-sounding speech. + constraints of the original speech, preserving complete utterances. Speeds + beyond the preferred range are reported because they can sound unnatural. """ - # Speed limits for natural-sounding speech - MIN_SPEED: float = 0.8 # Slowest allowed (20% slower) - MAX_SPEED: float = 1.3 # Fastest allowed (30% faster) - - def __init__( - self, - min_speed: float | None = None, - max_speed: float | None = None, - ): - """Initialize the timing synchronizer. - - Args: - min_speed: Minimum speed factor (default: 0.8). - max_speed: Maximum speed factor (default: 1.3). - """ - self.min_speed = min_speed if min_speed is not None else self.MIN_SPEED - self.max_speed = max_speed if max_speed is not None else self.MAX_SPEED - - if self.min_speed <= 0: - raise ValueError("min_speed must be positive") - if self.max_speed <= self.min_speed: - raise ValueError("max_speed must be greater than min_speed") + def __init__(self, max_speed: float = 1.1): + """Set the preferred maximum speed; larger speeds preserve speech when needed.""" + if not np.isfinite(max_speed) or max_speed < 1.0: + raise ValueError("max_speed must be finite and at least 1.0") + self.max_speed = max_speed def synchronize_segment( self, @@ -64,41 +66,62 @@ def synchronize_segment( target_duration=target_duration, actual_duration=original_duration, speed_factor=1.0, - was_truncated=False, + excessive_speed=False, ) # Calculate required speed factor required_speed = original_duration / target_duration - # Clamp to acceptable range - clamped_speed = max(self.min_speed, min(self.max_speed, required_speed)) - - # Check if we need to truncate - was_truncated = False - if required_speed > self.max_speed: - # Even at max speed, audio is too long - will need truncation - was_truncated = True + # Source-word phrase anchors carry the delivery rhythm. Do not stretch + # a shorter translation to fill silence; only accelerate an overrun. + speed_factor = max(1.0, required_speed) # Apply time stretch - if abs(clamped_speed - 1.0) > 0.01: - synchronized_audio = audio.time_stretch(clamped_speed) + if abs(speed_factor - 1.0) > 0.01: + synchronized_audio = audio.time_stretch(speed_factor, method=_speech_stretch_method()) else: synchronized_audio = audio + speed_factor = 1.0 - # Truncate if still too long + # atempo has a small sample-count error. Fit its *whole* output to the + # window instead of slicing off the ending. This residual resampling can + # shift pitch slightly; the main duration change above preserves pitch. actual_duration = synchronized_audio.metadata.duration_seconds if actual_duration > target_duration: - synchronized_audio = synchronized_audio.slice(0, target_duration) - actual_duration = target_duration - was_truncated = True + frames = max(1, int(target_duration * synchronized_audio.metadata.sample_rate)) + data = synchronized_audio.data + positions = np.linspace(0, len(data) - 1, frames) + source_positions = np.arange(len(data)) + fitted = ( + np.interp(positions, source_positions, data) + if data.ndim == 1 + else np.column_stack([np.interp(positions, source_positions, channel) for channel in data.T]) + ) + speed_factor *= len(data) / frames + metadata = replace( + synchronized_audio.metadata, + frame_count=frames, + duration_seconds=frames / synchronized_audio.metadata.sample_rate, + ) + synchronized_audio = Audio(fitted.astype(np.float32), metadata) + actual_duration = metadata.duration_seconds + + excessive_speed = speed_factor > self.max_speed + 0.01 + if excessive_speed: + logger.warning( + "Dubbed turn %d requires %.2fx speed (preferred maximum %.2fx)", + segment_index, + speed_factor, + self.max_speed, + ) return synchronized_audio, TimingAdjustment( segment_index=segment_index, original_duration=original_duration, target_duration=target_duration, actual_duration=actual_duration, - speed_factor=clamped_speed, - was_truncated=was_truncated, + speed_factor=speed_factor, + excessive_speed=excessive_speed, ) def synchronize_segments( @@ -193,7 +216,15 @@ def assemble_with_timing( output = np.zeros(end_sample, dtype=np.float32) for start_sample, seg_data in normalized: stop = start_sample + len(seg_data) - output[start_sample:stop] += seg_data + # Fade only the edges, without copying or cutting the whole phrase. + fade_samples = min(round(0.005 * sample_rate), len(seg_data) // 2) + if fade_samples: + ramp = np.linspace(0, 1, fade_samples, dtype=np.float32) + output[start_sample : start_sample + fade_samples] += seg_data[:fade_samples] * ramp + output[start_sample + fade_samples : stop - fade_samples] += seg_data[fade_samples:-fade_samples] + output[stop - fade_samples : stop] += seg_data[-fade_samples:] * ramp[::-1] + else: + output[start_sample:stop] += seg_data # Single post-mix peak guard, equivalent to Audio.overlay's per-call # rescale collapsed into one pass. For non-overlapping dub segments diff --git a/src/videopython/ai/dubbing/translation.py b/src/videopython/ai/dubbing/translation.py index a68abbc5..e6a8a392 100644 --- a/src/videopython/ai/dubbing/translation.py +++ b/src/videopython/ai/dubbing/translation.py @@ -1,39 +1,25 @@ -"""Context-aware dub translation via a local Ollama text model. - -``OllamaTranslator`` is the single translation backend: it sends the -transcription segments to a local Ollama model under a structured-output schema -and reads back length-budgeted, context-aware translations. The pipeline always -uses it (the old Marian / llama-cpp backends were removed in the Ollama -consolidation). -""" - from __future__ import annotations import json -import logging +from copy import deepcopy from typing import TYPE_CHECKING, Any, Callable from videopython.ai._ollama import OllamaError, OllamaStructuredClient from videopython.ai._predictor import ManagedPredictor +from videopython.ai._text_chunks import split_text from videopython.base.transcription import TranscriptionSegment if TYPE_CHECKING: from videopython.ai.dubbing.models import TranslatedSegment -logger = logging.getLogger(__name__) - # Default Ollama text model for translation; override via the `model` arg (and # `ollama pull` it first). Any instruct model that supports structured output works. DEFAULT_TRANSLATION_MODEL = "qwen3.6:27b" def _is_translatable_text(text: str) -> bool: - """Return True if text has enough content to be worth translating. - - Whisper routinely emits punctuation-only or single-character segments - (" .", "...", "?", "♪"). Require at least 2 alphanumeric characters. - """ - return sum(1 for c in text if c.isalnum()) >= 2 + """Ignore punctuation/music markers, but retain single-letter spoken words.""" + return any(c.isalnum() for c in text) LANGUAGE_NAMES = { @@ -73,8 +59,10 @@ def _is_translatable_text(text: str) -> bool: } -# Average characters per second of natural speech, for the per-segment -# ``target_chars`` budget. The prompt treats it as a ±15% target, not a cap. +# Conservative character/token estimate without a language-specific tokenizer. +_CHARS_PER_TOKEN = 2.0 + +# Soft spoken-length hints, never a reason to discard source meaning. _SPEECH_CHARS_PER_SEC: dict[str, float] = { "en": 14.0, "es": 14.0, "pt": 13.5, "it": 13.5, "fr": 13.0, "de": 12.0, "pl": 12.5, "nl": 12.5, "ru": 12.0, "uk": 12.0, "cs": 12.0, "sk": 12.0, @@ -83,22 +71,15 @@ def _is_translatable_text(text: str) -> bool: "th": 9.0, "vi": 11.0, "ar": 10.0, "he": 10.0, "hi": 11.0, "ta": 10.0, "id": 12.0, "ms": 12.0, "tr": 12.0, "el": 12.0, } # fmt: skip -_SPEECH_CHARS_DEFAULT = 12.0 - -# avg_logprob below this marks a transcription window we don't trust. -_LOW_LOGPROB_HINT_THRESHOLD = -1.0 -# Conservative chars/token for sizing chunks without a tokenizer (low end so any -# source language stays safe), plus prompt-envelope and per-segment overheads. -_CHARS_PER_TOKEN = 2.0 -_PROMPT_OVERHEAD_TOKENS = 300 -_SEGMENT_ENVELOPE_CHARS = 40 _TRANSLATION_SCHEMA: dict[str, Any] = { "type": "object", "properties": { "translations": { "type": "array", + "minItems": 1, + "maxItems": 1, "items": { "type": "object", "properties": {"i": {"type": "integer"}, "translated": {"type": "string"}}, @@ -112,88 +93,56 @@ def _is_translatable_text(text: str) -> bool: } -def _chunk_segment_indices(segments: list[TranscriptionSegment], n_ctx: int, max_tokens: int) -> list[list[int]]: - """Group positions in ``segments`` into batches that fit one model call. - - Each batch keeps ``prompt_tokens + max_tokens <= n_ctx``, approximated from - character length via ``_CHARS_PER_TOKEN``. A segment whose own serialized - form exceeds the budget gets its own chunk. - """ - prompt_token_budget = n_ctx - max_tokens - _PROMPT_OVERHEAD_TOKENS - if prompt_token_budget <= 0: - return [[i] for i in range(len(segments))] - char_budget = int(prompt_token_budget * _CHARS_PER_TOKEN) - - chunks: list[list[int]] = [] - current: list[int] = [] - current_chars = 0 - for i, seg in enumerate(segments): - seg_chars = len(seg.text) + _SEGMENT_ENVELOPE_CHARS - if current and current_chars + seg_chars > char_budget: - chunks.append(current) - current = [] - current_chars = 0 - current.append(i) - current_chars += seg_chars - if current: - chunks.append(current) - return chunks - - -def _target_chars_for(duration_seconds: float, target_lang: str) -> int: - """Character-count budget for a segment of ``duration_seconds`` in ``target_lang``.""" - rate = _SPEECH_CHARS_PER_SEC.get(target_lang, _SPEECH_CHARS_DEFAULT) - return max(1, int(duration_seconds * rate * 1.15)) - - def _build_system_prompt(source_lang: str, target_lang: str) -> str: src_name = LANGUAGE_NAMES.get(source_lang, source_lang) tgt_name = LANGUAGE_NAMES.get(target_lang, target_lang) return ( f"You are a professional dub translator. Translate from {src_name} to {tgt_name}.\n" - "Preserve register and proper nouns. Match each segment's syllable count so the\n" - "dub fits the original timing -- translation is for spoken audio, not subtitles.\n" - "Aim for `target_chars` characters per segment (+/-15%).\n" - "If a segment is non-speech filler keep it as filler; do not invent content.\n" - "If a segment carries `low_confidence`, translate conservatively.\n" + "Translate ONLY the target text into natural spoken language. Context is for understanding only: " + "never translate or borrow content from context. Preserve every claim, negation, number, unit, " + "proper name, and speaker perspective. Interpret idioms by their meaning in context, not literally. " + "If low_confidence is set, translate conservatively without inventing missing words. " + "Use correct financial and technical terminology. Do not summarize, explain, embellish or " + "complete unfinished fragments. Semantic fidelity takes priority over timing or length. " + "Prefer concise phrasing when equally faithful, but never omit meaning to fit timing.\n" + "Aim for target_chars characters (+/-15%) using concise spoken phrasing. This is a soft " + "timing target, not a cap: retain every claim even when it requires more characters.\n" "\n" 'Return a JSON object {"translations": [{"i": , "translated": ""}, ...]} ' "with exactly one entry per input segment." ) -def _build_user_prompt(segments: list[TranscriptionSegment], target_lang: str) -> str: - lines: list[str] = [] - for idx, seg in enumerate(segments): - entry: dict[str, Any] = { - "i": idx, - "text": seg.text, - "target_chars": _target_chars_for(seg.end - seg.start, target_lang), - } - if seg.avg_logprob is not None and seg.avg_logprob < _LOW_LOGPROB_HINT_THRESHOLD: - entry["low_confidence"] = True - lines.append(json.dumps(entry, ensure_ascii=False)) - return "Input segments:\n" + "\n".join(lines) + f"\n\nTranslate all {len(segments)} segments." - - def _parse_translations(data: dict[str, Any]) -> dict[int, str]: """Extract ``{i: translated_text}`` from the model's ``{"translations": [...]}``.""" out: dict[int, str] = {} - for obj in data.get("translations", []): - if isinstance(obj, dict) and "i" in obj and "translated" in obj: - try: - out[int(obj["i"])] = str(obj["translated"]) - except (TypeError, ValueError): - continue - return out + invalid: set[int] = set() + entries = data.get("translations") + if not isinstance(entries, list): + return {} + for obj in entries: + if not isinstance(obj, dict) or type(obj.get("i")) is not int: + return {} + index = obj["i"] + value = obj.get("translated") + if index in out or index in invalid: + out.pop(index, None) + invalid.add(index) + elif not isinstance(value, str) or not value.strip() or index < 0: + invalid.add(index) + else: + out[index] = " ".join(value.split()) + return {} if invalid else out class OllamaTranslator(ManagedPredictor): """Dub translation via a local Ollama text model. The model must support Ollama's structured-output ``format``; ``ollama pull - `` first. ``n_ctx`` sizes the per-call chunking (long sources are - split across calls); ``options`` are extra Ollama generation options. + `` first. Long text is split into bounded requests. ``n_ctx`` reserves + room for the prompt, source text and ``max_tokens`` output budget. ``options`` + can override these as ``num_ctx`` and ``num_predict``; effective budgets are + validated at construction. """ def __init__( @@ -205,56 +154,26 @@ def __init__( max_tokens: int = 4096, temperature: float = 0.1, options: dict[str, Any] | None = None, + keep_alive: str | int | None = "5m", ) -> None: - self.n_ctx = n_ctx - self.max_tokens = max_tokens client_options = {"temperature": temperature, "num_ctx": n_ctx, "num_predict": max_tokens, **(options or {})} - self._client = OllamaStructuredClient(model=model, host=host, options=client_options) - self._failures_last_call: list[int] = [] - - def _translate_chunk( - self, segments: list[TranscriptionSegment], target_lang: str, source_lang: str - ) -> dict[int, str]: - """One model call. Empty dict on unusable output (caller retries / records failure).""" - try: - data = self._client.generate_json( - system=_build_system_prompt(source_lang, target_lang), - text=_build_user_prompt(segments, target_lang), - schema=_TRANSLATION_SCHEMA, + self.n_ctx = int(client_options["num_ctx"]) + self.max_tokens = int(client_options["num_predict"]) + # Reserve space for target-language expansion and the JSON envelope. + self._part_chars = min( + 800, + int((self.n_ctx - self.max_tokens - 1000) * _CHARS_PER_TOKEN), + int((self.max_tokens - 100) * _CHARS_PER_TOKEN / 2), + ) + if self._part_chars < 40: + raise ValueError( + f"Translation requires max_tokens (num_predict) >= 140 and n_ctx (num_ctx) " + f">= max_tokens + 1020; got n_ctx={self.n_ctx}, max_tokens={self.max_tokens}" ) - except OllamaError: - return {} - return _parse_translations(data) - - def _translate_chunked( - self, - segments: list[TranscriptionSegment], - target_lang: str, - source_lang: str, - progress_callback: Callable[[float], None] | None = None, - progress_start: float = 0.0, - progress_end: float = 1.0, - ) -> dict[int, str]: - """Translate across one or more calls, each kept under ``n_ctx``.""" - results: dict[int, str] = {} - if not segments: - if progress_callback is not None: - progress_callback(progress_end) - return results - - chunks = _chunk_segment_indices(segments, self.n_ctx, self.max_tokens) - if len(chunks) > 1: - logger.info("OllamaTranslator: splitting %d segments into %d chunks", len(segments), len(chunks)) - for chunk_num, chunk_positions in enumerate(chunks): - chunk_result = self._translate_chunk([segments[p] for p in chunk_positions], target_lang, source_lang) - for local_idx, text in chunk_result.items(): - # Drop out-of-range model indices; those segments stay "missing" and get retried. - if 0 <= local_idx < len(chunk_positions): - results[chunk_positions[local_idx]] = text - if progress_callback is not None: - fraction = (chunk_num + 1) / len(chunks) - progress_callback(progress_start + (progress_end - progress_start) * fraction) - return results + # Keep the model resident between bounded requests; low-memory pipelines + # explicitly unload it at the end of translation before loading TTS. + self._client = OllamaStructuredClient(model=model, host=host, options=client_options, keep_alive=keep_alive) + self._failures_last_call: list[int] = [] def translate_segments( self, @@ -263,42 +182,66 @@ def translate_segments( source_lang: str | None = None, progress_callback: Callable[[float], None] | None = None, ) -> list[TranslatedSegment]: - """Translate segments with a parse-retry pass; unrecovered ones land in - ``translation_failures`` with empty text. Progress ramps 0 -> 0.5 (first - pass), 0.9 (after retry), 1.0 (done).""" + """Translate bounded source parts independently, retrying invalid replies. + + A failed part leaves its entire parent empty in ``translation_failures``. + """ from videopython.ai.dubbing.models import TranslatedSegment effective_source = source_lang or "en" self._failures_last_call = [] - translatable_indices = [i for i, seg in enumerate(segments) if _is_translatable_text(seg.text)] - translatable_segments = [segments[i] for i in translatable_indices] - - results = self._translate_chunked( - translatable_segments, target_lang, effective_source, progress_callback, 0.0, 0.5 - ) - - missing_local = [li for li in range(len(translatable_segments)) if li not in results] - if missing_local: - logger.info("OllamaTranslator: retrying %d/%d segments", len(missing_local), len(translatable_segments)) - retry = self._translate_chunked( - [translatable_segments[li] for li in missing_local], - target_lang, - effective_source, - progress_callback, - 0.5, - 0.9, + units: list[tuple[int, int, str]] = [] + for parent, segment in enumerate(segments): + if _is_translatable_text(segment.text): + for part, text in enumerate(split_text(segment.text, self._part_chars)): + units.append((parent, part, text)) + translated_parts: dict[int, list[str]] = {} + failed: set[int] = set() + for identity, (parent, part, text) in enumerate(units): + before = units[identity - 1][2][-240:] if identity else "" + after = units[identity + 1][2][:240] if identity + 1 < len(units) else "" + entry: dict[str, Any] = {"i": identity, "parent": parent, "part": part, "text": text} + source_chars = len(" ".join(segments[parent].text.split())) + part_duration = max(0.0, segments[parent].end - segments[parent].start) * len(text) / source_chars + entry["target_chars"] = max(1, round(part_duration * _SPEECH_CHARS_PER_SEC.get(target_lang, 12.0))) + logprob = segments[parent].avg_logprob + if logprob is not None and logprob < -1.0: + entry["low_confidence"] = True + prompt = ( + "Context only (do not translate): " + + json.dumps({"before": before, "after": after}, ensure_ascii=False) + + "\nTarget:\n" + + json.dumps(entry, ensure_ascii=False) ) - for retry_local, text in retry.items(): - results[missing_local[retry_local]] = text - if progress_callback is not None: - progress_callback(0.9) - - for li in range(len(translatable_segments)): - if li not in results: - self._failures_last_call.append(translatable_indices[li]) - - translation_for_orig = {translatable_indices[li]: text for li, text in results.items()} + translated = None + schema = deepcopy(_TRANSLATION_SCHEMA) + schema["properties"]["translations"]["items"]["properties"]["i"]["const"] = identity + for _attempt in range(2): + try: + data = self._client.generate_json( + system=_build_system_prompt(effective_source, target_lang), + text=prompt + + ("\nReturn exactly the requested identity and all target text." if _attempt else ""), + schema=schema, + ) + parsed = _parse_translations(data) + if set(parsed) == {identity} and len(parsed[identity]) <= max(80, 6 * len(text)): + translated = parsed[identity] + break + except OllamaError: + pass + if translated is None: + failed.add(parent) + else: + translated_parts.setdefault(parent, []).append(translated) + if progress_callback is not None: + progress_callback(0.95 * (identity + 1) / len(units)) + # Never publish an incomplete parent when just one of its parts failed. + self._failures_last_call = sorted(failed) + translation_for_orig = { + parent: " ".join(parts) for parent, parts in translated_parts.items() if parent not in failed + } translated_segments = [ TranslatedSegment( original_segment=seg, diff --git a/src/videopython/ai/generation/_speech_graphs.py b/src/videopython/ai/generation/_speech_graphs.py new file mode 100644 index 00000000..31d3e3a3 --- /dev/null +++ b/src/videopython/ai/generation/_speech_graphs.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import logging +from typing import Any + +logger = logging.getLogger(__name__) +_MISSING = object() + + +class _TokenGraph: + """One graph for a decoder module's two-row, single-token forward pass. + + Prefill, CPU, training and other layouts retain their original forward path. + The output buffer is consumed inside the decoder layer before its next call; + this wrapper is deliberately limited to Chatterbox's MLP and RMSNorm modules. + """ + + def __init__(self, module: Any, owner: SpeechGraphs): + self.module = module + self.owner = owner + self.original = module.forward + self.previous = vars(module).get("forward", _MISSING) + self.children = tuple(module.modules())[1:] + self.state: tuple[Any, Any, Any] | None = None + self.signature: Any = None + + def _capture(self, x: Any) -> tuple[Any, Any, Any]: + import torch + + static = x.clone() + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for _ in range(2): + self.original(static) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + if self.owner.pool is None: + self.owner.pool = torch.cuda.graph_pool_handle() + # Decoder layers execute serially in capture order. Each output remains + # alive in its wrapper until unload, so later captures cannot reuse it. + with torch.cuda.graph(graph, pool=self.owner.pool): + output = self.original(static) + return graph, static, output + + def __call__(self, x: Any) -> Any: + import torch + + if ( + self.owner.disabled + or self.module.training + or torch.is_grad_enabled() + or x.device.type != "cuda" + or x.dtype != torch.float32 + or x.ndim != 3 + or x.shape[:2] != (2, 1) + or not x.is_contiguous() + or any(child._forward_hooks or child._forward_pre_hooks for child in self.children) + ): + return self.original(x) + signature = (tuple(x.shape), x.dtype, x.device, tuple(x.stride())) + if self.state is None: + try: + state = self._capture(x) + except RuntimeError: + # Release other graphs too, so a failed optimization cannot + # keep their private pools resident while falling back. + self.owner.close() + logger.warning("CUDA speech graph capture unavailable; using the original forward path.") + else: + self.state = state + self.signature = signature + # Leave the exception scope before retrying: its traceback can + # otherwise keep failed-capture tensors and their pool alive. + if self.state is None: + torch.cuda.empty_cache() + return self.original(x) + if signature != self.signature: + return self.original(x) + graph, static, output = self.state + # Captured buffers may be inference tensors even when a later caller + # uses no_grad. Mutations must retain the inference context in that case. + with torch.inference_mode(): + static.copy_(x) + graph.replay() + return output + + +class SpeechGraphs: + """Own decoder graphs for one loaded model and restore forwards on unload.""" + + def __init__(self, model: Any): + self.disabled = False + self.pool: Any = None + self._wrappers: list[_TokenGraph] = [] + transformer = getattr(getattr(model, "t3", None), "tfmr", None) + for layer in getattr(transformer, "layers", ()): + for name in ("mlp", "input_layernorm", "post_attention_layernorm"): + module = getattr(layer, name, None) + if module is None or type(module).__name__ not in {"LlamaMLP", "LlamaRMSNorm"}: + continue + wrapper = _TokenGraph(module, self) + self._wrappers.append(wrapper) + module.forward = wrapper + + def close(self) -> None: + self.disabled = True + for wrapper in self._wrappers: + if wrapper.previous is _MISSING: + del wrapper.module.forward + else: + wrapper.module.forward = wrapper.previous + wrapper.state = None + self._wrappers.clear() + self.pool = None diff --git a/src/videopython/ai/generation/_speech_tokens.py b/src/videopython/ai/generation/_speech_tokens.py new file mode 100644 index 00000000..a05dd2d8 --- /dev/null +++ b/src/videopython/ai/generation/_speech_tokens.py @@ -0,0 +1,35 @@ +from functools import wraps +from typing import Any, Callable + + +class InvalidSpeechTokens(RuntimeError): + """A generation cannot safely be decoded by the speech vocoder.""" + + +def restrict_speech_vocabulary(head: Any, vocab_size: int, eos_id: int) -> None: + """Keep sampling inside the vocoder vocabulary, allowing EOS to end speech.""" + if not 0 < vocab_size <= eos_id < head.out_features: + raise ValueError("Unexpected Chatterbox speech vocabulary configuration") + + def restrict(_module: Any, _inputs: Any, output: Any) -> Any: + scores = output + # Classifier-free guidance combines two heads later. A finite mask + # avoids inf - inf => NaN while underflowing to zero probability. + scores[..., vocab_size:eos_id] = -(2**15) + scores[..., eos_id + 1 :] = -(2**15) + return scores + + head.register_forward_hook(restrict) + + +def guard_speech_tokens(inference: Callable[..., Any], vocab_size: int) -> Callable[..., Any]: + """Reject invalid generated tokens before an embedding lookup poisons CUDA.""" + + @wraps(inference) + def guarded(*args: Any, **kwargs: Any) -> Any: + tokens = kwargs.get("speech_tokens", args[0] if args else None) + if tokens is None or not tokens.numel() or not bool(((tokens >= 0) & (tokens < vocab_size)).all()): + raise InvalidSpeechTokens("Chatterbox returned empty or out-of-range speech tokens") + return inference(*args, **kwargs) + + return guarded diff --git a/src/videopython/ai/generation/audio.py b/src/videopython/ai/generation/audio.py index 1c73c05e..b2db5429 100644 --- a/src/videopython/ai/generation/audio.py +++ b/src/videopython/ai/generation/audio.py @@ -7,6 +7,7 @@ from videopython.ai._device import log_device_initialization, select_device from videopython.ai._predictor import ManagedPredictor from videopython.ai._revisions import pinned +from videopython.ai._text_chunks import split_text from videopython.audio import Audio, AudioMetadata if TYPE_CHECKING: @@ -33,11 +34,22 @@ def __init__( self.device = device self.language = language self._model: Any = None + self._speech_graphs: Any = None + + def unload(self) -> None: + if self._speech_graphs is not None: + self._speech_graphs.close() + self._speech_graphs = None + super().unload() def _init_local(self) -> None: from videopython.ai._optional import require ChatterboxMultilingualTTS = require("chatterbox.mtl_tts", feature="TextToSpeech").ChatterboxMultilingualTTS + from videopython.ai.generation._speech_tokens import ( + guard_speech_tokens, + restrict_speech_vocabulary, + ) requested_device = self.device device = select_device(self.device, mps_allowed=False) @@ -45,6 +57,18 @@ def _init_local(self) -> None: # No repo id to key a revision on: Chatterbox resolves its own repo + # revision internally, so there is nothing to pass revision= to. self._model = ChatterboxMultilingualTTS.from_pretrained(device=device) + self._model.s3gen.inference = guard_speech_tokens( + self._model.s3gen.inference, self._model.s3gen.flow.input_embedding.num_embeddings + ) + restrict_speech_vocabulary( + self._model.t3.speech_head, + self._model.s3gen.flow.input_embedding.num_embeddings, + self._model.t3.hp.stop_speech_token, + ) + if device == "cuda": + from videopython.ai.generation._speech_graphs import SpeechGraphs + + self._speech_graphs = SpeechGraphs(self._model) self.device = device log_device_initialization( "TextToSpeech", @@ -88,6 +112,9 @@ def generate_audio( import numpy as np + parts = split_text(text, 200) + if not parts: + raise ValueError("Speech text must not be empty") if self._model is None: self._init_local() @@ -116,16 +143,39 @@ def generate_audio( knobs["temperature"] = temperature try: - wav = self._model.generate( - text=text, - language_id=self.language, - audio_prompt_path=str(speaker_wav_path) if speaker_wav_path else None, - **knobs, - ) - - audio_data = wav.cpu().float().numpy().squeeze() - if audio_data.ndim == 0: - audio_data = np.array([audio_data], dtype=np.float32) + # Chatterbox caps generation at 1,000 speech tokens (~40 seconds). + # Keep individual calls well below that cap, then synchronize the + # complete parent utterance once in the dubbing pipeline. + def synthesize(part: str, budget: int) -> list[np.ndarray]: + from videopython.ai.generation._speech_tokens import InvalidSpeechTokens + + for attempt in range(3): + try: + wav = self._model.generate( + text=part, + language_id=self.language, + audio_prompt_path=str(speaker_wav_path) if speaker_wav_path else None, + **knobs, + ) + break + except InvalidSpeechTokens: + if attempt == 2: + raise + data = wav.cpu().float().numpy().reshape(-1) + if not len(data) or not np.isfinite(data).all(): + raise ValueError("Speech generation returned empty or non-finite audio") + # Duration is a conservative cap warning, not proof of coverage. + # Discard suspect audio and regenerate smaller text units. + if len(data) >= 38 * self.SAMPLE_RATE: + smaller = split_text(part, max(1, budget // 2)) + if len(smaller) < 2 or budget <= 40: + raise RuntimeError("Speech generation may have reached its token limit") + return [audio for chunk in smaller for audio in synthesize(chunk, budget // 2)] + return [data] + + arrays = [audio for part in parts for audio in synthesize(part, 200)] + # Keep natural leading/trailing pauses; do not crossfade phonemes. + audio_data = np.concatenate(arrays) if len(arrays) > 1 else arrays[0] metadata = AudioMetadata( sample_rate=self.SAMPLE_RATE, diff --git a/src/videopython/ai/understanding/_pyannote_patches.py b/src/videopython/ai/understanding/_pyannote_patches.py index 670d25a9..a393490d 100644 --- a/src/videopython/ai/understanding/_pyannote_patches.py +++ b/src/videopython/ai/understanding/_pyannote_patches.py @@ -70,16 +70,10 @@ def _get_embeddings( exclude_overlap: bool = False, hook: Any = None, ) -> np.ndarray: - """Extract embeddings, skipping (chunk, speaker) pairs that cannot affect the result. - - pyannote runs the embedding model on every pair, but a pair whose mask is - empty is discarded twice downstream: the clustering filter drops it, and the - pipeline force-assigns it to the throw-away cluster. Its embedding is the - silent-mask constant regardless of the audio, so reusing that constant keeps - the output bit-exact. For compatible deterministic pyannote backends, - also share frame extraction between speakers of the same chunk. Masks only - enter the network after frame extraction. Different batch shapes can still - introduce small floating-point differences in the embeddings. + """Extract active pair embeddings; reuse the zero-mask constant for inactive pairs. + + Compatible backends share frame extraction across speakers. Different batch + shapes can introduce small floating-point differences in the embeddings. """ import torch diff --git a/src/videopython/audio/audio.py b/src/videopython/audio/audio.py index 4c54598b..5424aa7c 100644 --- a/src/videopython/audio/audio.py +++ b/src/videopython/audio/audio.py @@ -6,7 +6,7 @@ import wave from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal import numpy as np import soxr @@ -667,16 +667,18 @@ def fit_to_duration(self, target_duration: float) -> "Audio": return self.concat(silence) return self - def time_stretch(self, speed: float) -> Audio: + def time_stretch(self, speed: float, *, method: Literal["atempo", "rubberband"] = "atempo") -> Audio: """ Time-stretch audio by a speed factor (pitch-preserving). - Uses ffmpeg's atempo filter for high-quality time stretching. - For speeds outside the 0.5-2.0 range, multiple atempo filters are chained. + Uses ffmpeg's atempo filter by default. Rubber Band is available for + speech when ffmpeg was built with librubberband. Args: speed: Speed multiplier. 2.0 = twice as fast (half duration), 0.5 = half speed (double duration). + method: Time-stretch filter. Rubber Band preserves formants and uses + smooth transients for speech; requires ffmpeg's rubberband filter. Returns: Audio: New Audio object with time-stretched audio. @@ -685,8 +687,10 @@ def time_stretch(self, speed: float) -> Audio: ValueError: If speed is not positive. AudioLoadError: If ffmpeg fails. """ - if speed <= 0: - raise ValueError("Speed must be positive") + if not np.isfinite(speed) or speed <= 0: + raise ValueError("Speed must be positive and finite") + if method not in ("atempo", "rubberband"): + raise ValueError(f"Unknown time-stretch method: {method}") if abs(speed - 1.0) < 0.001: # No change needed @@ -696,6 +700,16 @@ def time_stretch(self, speed: float) -> Audio: # decomposition the streaming filter graph uses (single source of truth). filters = atempo_chain(speed) filter_str = ",".join(filters) if filters else "anull" + if method == "rubberband": + # Prefer a single pass; decompose only outside Rubber Band's range. + factors = [speed] if 0.01 <= speed <= 100 else [float(stage.split("=")[1]) for stage in filters] + filter_str = ( + ",".join( + f"rubberband=tempo={factor}:pitch=1:transients=smooth:formant=preserved:pitchq=quality" + for factor in factors + ) + or "anull" + ) # Save current audio to temp WAV, process with ffmpeg, read back with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as input_file: diff --git a/uv.lock b/uv.lock index f4189747..442f6257 100644 --- a/uv.lock +++ b/uv.lock @@ -5445,7 +5445,7 @@ wheels = [ [[package]] name = "videopython" -version = "0.61.1" +version = "0.61.2" source = { editable = "." } dependencies = [ { name = "numpy" }, @@ -5545,7 +5545,7 @@ requires-dist = [ { name = "transformers", marker = "extra == 'mcp'", specifier = ">=5.2.0" }, { name = "transnetv2-pytorch", marker = "extra == 'ai'", specifier = ">=1.0.5" }, { name = "transnetv2-pytorch", marker = "extra == 'mcp'", specifier = ">=1.0.5" }, - { name = "videopython-chatterbox", marker = "extra == 'ai'", specifier = ">=0.1.7.post1" }, + { name = "videopython-chatterbox", marker = "extra == 'ai'", specifier = ">=0.1.7.post2" }, ] provides-extras = ["ai", "mcp"] @@ -5571,7 +5571,7 @@ dev = [ [[package]] name = "videopython-chatterbox" -version = "0.1.7.post1" +version = "0.1.7.post2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "conformer" }, @@ -5590,9 +5590,9 @@ dependencies = [ { name = "torchaudio" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/96/8d/a4f718a91cac10d5a8f6e31df2c58bf494ccd2b3d5f8ca00e11b7e8129a2/videopython_chatterbox-0.1.7.post1.tar.gz", hash = "sha256:15f4694174d03b09b5614a00179db0169b45eeb9c17183150cbf5f43be8a44f3", size = 74663, upload-time = "2026-08-02T17:45:08.388Z" } +sdist = { url = "https://files.pythonhosted.org/packages/64/98/cfc94fb87ef600e06c31ba9865b1cd56692bf39e0fa65cc85f3ca8e5680b/videopython_chatterbox-0.1.7.post2.tar.gz", hash = "sha256:4d2a5221cbe407ffbe0b369c57b340fd6bc1958092d923b4756cdfc50835a56b", size = 75096, upload-time = "2026-09-08T16:15:29.415Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/65/e4/7d9af958a4d08e76fc13c87cccaf57103366e6b04241950f7a6b9d7b75a5/videopython_chatterbox-0.1.7.post1-py3-none-any.whl", hash = "sha256:9d41d2727d845147eeacf611fd5fd730b1c8ac33a411b004a4e840d6d782d095", size = 107473, upload-time = "2026-08-02T17:45:06.758Z" }, + { url = "https://files.pythonhosted.org/packages/48/37/ad055353e911a502a3251bdbc31452fcc537fcab91bd9c621ace7dadaba1/videopython_chatterbox-0.1.7.post2-py3-none-any.whl", hash = "sha256:df9d37fcd637b67ac67670ef23c976b180b6de7bc158017a136130aa95a11db8", size = 107909, upload-time = "2026-09-08T16:15:27.86Z" }, ] [[package]]