From 63211592fa694e364c09c3d8707ed16619dfcb34 Mon Sep 17 00:00:00 2001 From: yousan Date: Sun, 20 Sep 2026 16:22:39 +0900 Subject: [PATCH] =?UTF-8?q?fix(timing):=20C++=20=E3=81=AE=20timing=20walk?= =?UTF-8?q?=20=E3=82=92=E6=9C=AC=E7=95=AA=E6=A4=9C=E6=9F=BB=E5=8F=AF?= =?UTF-8?q?=E8=83=BD=E3=81=AB=E3=81=97=20parity=20gate=20=E3=81=AE?= =?UTF-8?q?=E7=A9=BA=E6=8C=AF=E3=82=8A=E3=82=92=E5=A1=9E=E3=81=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #653 (durations が ceil 前) に着手する前に、それを検出できるはずの 仕組みが 3 か所とも空振りしていることが分かったので先に塞ぐ。 1. C++ の parity テストが本番コードを 1 行も通っていなかった。 test_phoneme_timing_parity.cpp は golden fixture を読みながら spec アルゴリズムを自身の翻訳単位で再実装しており、コメントにもそう 書いてあった (piper.cpp が onnxruntime を引き込むためリンクできない)。 他 5 ランタイムは本番関数を呼んでいるので、C++ だけが出荷される 挙動について何も検査していなかった。cursor walk を ORT 非依存の timing_helpers.hpp に切り出し、本番実体を直接検査する test_timing_helpers.cpp (16 ケース) を新設した。変異 5 種が 5/5 検出 されることを実測確認している。 fixture をそのまま本番に流すことはできない。fixture は token ベース (`^`, `k`, ..., `$`)、本番は id ベースで PAD/BOS/EOS を skip するため で、これ自体が #697 の内容。糊塗せずテストに記録した。 2. timing-parity.yml の paths: が守る対象を含んでいなかった。 per-runtime-presence job は「parity テストを削除したら fail する — that is intentional」と宣言しているのに、守る 6 ファイルが 1 つも trigger に無く、削除するだけの PR では workflow が起動しなかった。 他 5 ランタイムの timing 実装も未登録だった。5 件 → 18 件に拡張。 同型の穴が他 7 workflow にもあることを走査で確認し #704 に起票。 3. golden fixture の 7 ケースが全て整数 durations だった。実 ONNX 出力 は常に小数なのに代表しておらず、ceil(整数) == 整数 なので frame 量子化 を変えても fixture は 1 バイトも変わらない。実測: Go に math.Ceil を 入れる変異は旧 fixture では ok で素通りし、新 fixture では FAIL する。 小数 2 ケースと 60 エントリの長文ケースを追加し、後から整数化・短縮 されたら --check が fail する anti-vacuity guard も入れた。 併せて cursor の accumulator を float から double にした。格納は float の まま (PhonemeInfo は ABI 公開されており幅を変えると ABI gate が検出する) なので丸めは 1 エントリ 1 回になり累積が消える。1153 frame で float32 は float64 基準から 2.1e-3 ms ずれており、parity テストが宣言する許容誤差 1e-6 ms を大きく超えていた — レプリカが double だったため露見していな かった。 C++ 51/51、6 ランタイムの parity テスト全通過、音声出力は不変。 --- .github/workflows/timing-parity.yml | 42 +- CHANGELOG.md | 5 + scripts/regenerate_timing_fixture.py | 99 +++ src/cpp/piper.cpp | 132 +--- src/cpp/tests/CMakeLists.txt | 19 + src/cpp/tests/test_timing_helpers.cpp | 333 ++++++++++ src/cpp/timing_helpers.hpp | 222 +++++++ .../phoneme_timing/golden_matrix.json | 582 ++++++++++++++++++ 8 files changed, 1332 insertions(+), 102 deletions(-) create mode 100644 src/cpp/tests/test_timing_helpers.cpp create mode 100644 src/cpp/timing_helpers.hpp diff --git a/.github/workflows/timing-parity.yml b/.github/workflows/timing-parity.yml index 11367c0bb..edde55f77 100644 --- a/.github/workflows/timing-parity.yml +++ b/.github/workflows/timing-parity.yml @@ -31,6 +31,25 @@ on: - 'scripts/regenerate_timing_fixture.py' - 'docs/spec/phoneme-timing-contract.toml' - '.github/workflows/timing-parity.yml' + # The five other runtime implementations. Without these a PR that + # changes only, say, timing.rs never runs the drift check. + - 'src/rust/piper-core/src/timing.rs' + - 'src/go/piperplus/timing.go' + - 'src/wasm/openjtalk-web/src/timing.js' + - 'src/csharp/PiperPlus.Core/Inference/TimingWriter.cs' + - 'src/cpp/timing_helpers.hpp' + - 'src/cpp/piper.cpp' + # The per-runtime parity tests that `per-runtime-presence` below exists + # to stop anyone deleting. They were NOT listed, so a PR that deleted + # one skipped this workflow entirely and the guard never ran -- the one + # change it was written to catch was the one it could not see. + - 'src/python_run/tests/test_phoneme_timing_parity.py' + - 'src/rust/piper-core/tests/test_phoneme_timing_parity.rs' + - 'src/go/piperplus/timing_parity_test.go' + - 'src/cpp/tests/test_phoneme_timing_parity.cpp' + - 'src/cpp/tests/test_timing_helpers.cpp' + - 'src/csharp/PiperPlus.Core.Tests/TimingWriterParityTests.cs' + - 'src/wasm/openjtalk-web/test/js/test-phoneme-timing-parity.js' push: branches: [ dev ] paths: @@ -38,6 +57,26 @@ on: - 'tests/fixtures/phoneme_timing/**' - 'scripts/regenerate_timing_fixture.py' - 'docs/spec/phoneme-timing-contract.toml' + - '.github/workflows/timing-parity.yml' + # The five other runtime implementations. Without these a PR that + # changes only, say, timing.rs never runs the drift check. + - 'src/rust/piper-core/src/timing.rs' + - 'src/go/piperplus/timing.go' + - 'src/wasm/openjtalk-web/src/timing.js' + - 'src/csharp/PiperPlus.Core/Inference/TimingWriter.cs' + - 'src/cpp/timing_helpers.hpp' + - 'src/cpp/piper.cpp' + # The per-runtime parity tests that `per-runtime-presence` below exists + # to stop anyone deleting. They were NOT listed, so a PR that deleted + # one skipped this workflow entirely and the guard never ran -- the one + # change it was written to catch was the one it could not see. + - 'src/python_run/tests/test_phoneme_timing_parity.py' + - 'src/rust/piper-core/tests/test_phoneme_timing_parity.rs' + - 'src/go/piperplus/timing_parity_test.go' + - 'src/cpp/tests/test_phoneme_timing_parity.cpp' + - 'src/cpp/tests/test_timing_helpers.cpp' + - 'src/csharp/PiperPlus.Core.Tests/TimingWriterParityTests.cs' + - 'src/wasm/openjtalk-web/test/js/test-phoneme-timing-parity.js' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} @@ -72,7 +111,7 @@ jobs: steps: - uses: actions/checkout@v6.1.0 - - name: Assert all 6 per-runtime parity test files exist + - name: Assert all per-runtime parity test files exist run: | set -e missing=0 @@ -88,6 +127,7 @@ jobs: check src/rust/piper-core/tests/test_phoneme_timing_parity.rs check src/go/piperplus/timing_parity_test.go check src/cpp/tests/test_phoneme_timing_parity.cpp + check src/cpp/tests/test_timing_helpers.cpp check src/csharp/PiperPlus.Core.Tests/TimingWriterParityTests.cs check src/wasm/openjtalk-web/test/js/test-phoneme-timing-parity.js if [ "$missing" -ne 0 ]; then diff --git a/CHANGELOG.md b/CHANGELOG.md index 17194a6d6..ac06f9336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- C++: phoneme timing の cursor walk を `src/cpp/timing_helpers.hpp` に切り出し、**本番コードに初めてテストを通した**。`test_phoneme_timing_parity.cpp` は cross-runtime golden fixture を読みながら、spec のアルゴリズムを**自身の翻訳単位で再実装**していた (コメントに `To stay self-contained (no onnxruntime/espeak link), this test re-implements the spec algorithm` と明記)。他 5 ランタイムはいずれも本番関数を呼んでいる (Python `durations_to_timing` / Rust `durations_to_timing` / Go `DurationsToTiming` / JS `durationsToTiming` / C# `TimingWriter.CalculateTiming`) ため、**C++ だけが出荷される挙動について何も検査していなかった**。新設した `test_timing_helpers.cpp` (16 ケース) が本番実体を直接検査し、変異 5 種 (PAD/BOS/EOS skip の削除 / skip 時に cursor を進めない / accumulator を float に戻す / 促音 overlap 比率の変更 / frame の truncate を round に) がすべて検出されることを実測確認した +- C++: timing cursor の accumulator を `float` から `double` にした。値の格納は `float` のまま (`PhonemeInfo` は `SynthesisResult` 経由で shared library から公開されており、幅を変えると `ABI diff (libpiper_plus.so head vs base)` gate が検出する struct レイアウト変更になる) なので、丸めは **1 エントリにつき 1 回**になり、発話全体にわたる累積がなくなる。実測: ceil 後相当の 1153 frame で float32 累積は float64 基準から **2.1e-3 ms** ずれており、同 parity テストが宣言する許容誤差 `kAbsTolMs = 1e-6` を大きく超えていた (レプリカが `double` だったため露見していなかった) +- ci: `timing-parity.yml` の `paths:` を 5 件から 18 件に拡張した。`per-runtime-presence` job は「parity テストを削除したら fail する — that is intentional」と宣言しているのに、**守る対象 6 ファイルが 1 つも trigger に入っておらず**、parity テストを削除するだけの PR では workflow 自体が起動しなかった。他 5 ランタイムの timing 実装も未登録で、`timing.rs` だけを変更する PR では drift check が走らなかった。同型の穴が他 7 workflow にもあることを走査で確認し #704 に起票した +- tests: timing golden fixture に判別力のあるケースを 3 件追加した (`fractional_durations` / `fractional_just_above_integer` / `long_utterance_accumulation`)。従来の 7 ケースは**全て整数 durations** (最大 10 frame / 最長 11 音素) で、実際の ONNX `durations` 出力が常に小数であるにもかかわらず代表していなかった。`ceil(整数) == 整数` なので、frame 量子化を変えても fixture は 1 バイトも変わらない。**実測**: Go に `math.Ceil` を入れる変異は旧 fixture では `ok` で素通りし、新 fixture では `--- FAIL` になる。Rust でも同様に検出される。併せて `scripts/regenerate_timing_fixture.py` に「小数ケース 2 件以上 / 最長ケース 50 エントリ以上」の anti-vacuity guard を入れ、ケースが後から整数化・短縮されたら `--check` が fail するようにした (変異 2 種で 2/2 検出を確認) + - Rust / Go: phoneme timing の音素名を実際の音素に解決するようにした (#656)。 両ランタイムの CLI は `ph_0`, `ph_1`, ... (Rust) / `p0`, `p1`, ... (Go) という**連番プレースホルダ**を出力しており、 timing 出力からどの音素がどの区間かを識別できず lip-sync / 字幕用途で使えなかった。 `docs/spec/phoneme-timing-contract.toml` の `[reverse_map]` (first-wins 衝突解決 / PUA `U+XXXX` fallback / 明示マッピング優先) を canonical (`src/python_run/piper_plus/timing.py`) と JS mirror に合わせて実装した — Rust `build_phoneme_id_reverse_map` + `phoneme_ids_to_tokens`、 Go `BuildPhonemeIDReverseMap` + `PhonemeIDsToTokens`。 逆引きマップに無い ID は `` 形式で出し、 どの ID が未知なのかが分かるようにした (`ph_N` は何も伝えない)。 **反復順**: Python の dict と JS の object は挿入順を保つため first-wins の「最初」が一意だが、 Rust の `HashMap` と Go の `map` は反復順が非決定的なため**キーをソートしてから**反復する。 ID 衝突の無いマップでは順序が結果に影響しないため canonical と完全一致する (in-tree fixture は 173 キーで衝突 0 件); 衝突のあるマップでは異なる勝者を選び得ることをコメントに明記した - Rust / Go: 合成に実際に渡した phoneme ID 列を結果として返すようにした (`SynthesisResult::phoneme_ids` / `SynthesisResult.PhonemeIDs`)。 両ランタイムは `durations` を original 長へ**先頭から** `truncate` していたが、 padded layout は `[BOS, frontPad.., body.., backPad.., EOS]` なので body が `frontPad` 個ずれ、 index 対応で音素名を引き当てると**発話音素が pad の duration を受け取る** (#689 と同型)。 `truncate` をやめ padding 後の長さのまま返し、 対応付け用の ID 列を併せて返す形にした。 呼び出し元が `phonemize_to_ids(text)` を再実行して対応付けるのは誤りで、 Rust では Strategy C が 10 文字以下のテキストを SSML でラップするため合成に使う音素列がテキスト由来のものと別物になる (実測: `"Sol"` で ID 11 個に対し durations 111 個)。 CLI は長さが一致しない場合のみプレースホルダにフォールバックし警告を出す — 誤った音素名を出すほうが害が大きいため - C#: 逆引きマップの spec 違反 2 件を修正した (#656)。 (1) `reverse.TryAdd(ids[0], display)` が**キーの最初の ID しか登録せず**、 1 音素が複数 ID を持つモデルで残りの ID が `"?"` にフォールバックしていた — contract の例は `{"b": [6, 7]}` → `{6: 'b', 7: 'b'}` を要求する。 (2) `OpenJTalkToPiperMapping.CharToToken` に無い PUA 文字が**生の PUA 文字のまま**出力されており、 `[reverse_map.pua_handling]` が要求する `U+XXXX` 形式になっていなかった (他 5 ランタイムはすべて `U+XXXX`)。 修正前に回帰テスト 2 件が fail することを実測確認済み diff --git a/scripts/regenerate_timing_fixture.py b/scripts/regenerate_timing_fixture.py index fe43503d8..9e0c3e519 100755 --- a/scripts/regenerate_timing_fixture.py +++ b/scripts/regenerate_timing_fixture.py @@ -98,6 +98,51 @@ "hop_length": 256, }, }, + { + "name": "fractional_durations", + "description": ( + "Non-integer frame counts — what a real ONNX `durations` output " + "actually looks like. Every other case here is integer-valued, " + "and ceil(int) == int, so an implementation that forgot to apply " + "the frame quantisation the decoder uses would match all of them " + "(issue #653). This case is the one that can tell them apart." + ), + "inputs": { + "durations": [2.5, 3.25, 1.75, 4.125], + "phoneme_tokens": ["a", "b", "c", "d"], + "sample_rate": 22050, + "hop_length": 256, + }, + }, + { + "name": "fractional_just_above_integer", + "description": ( + "Durations a hair above an integer. Distinguishes truncation from " + "rounding from ceiling: trunc(3.0001)=3, round(3.0001)=3, " + "ceil(3.0001)=4. A single .5 case cannot separate round from ceil." + ), + "inputs": { + "durations": [3.0001, 7.0001], + "phoneme_tokens": ["x", "y"], + "sample_rate": 22050, + "hop_length": 256, + }, + }, + { + "name": "long_utterance_accumulation", + "description": ( + "60 phonemes of fractional duration. The other cases top out at " + "11 entries / 50 frames, where a float32 cursor drifts only ~5e-5 " + "ms from float64 and slips under every runtime's tolerance. This " + "length makes the accumulator width observable." + ), + "inputs": { + "durations": [round(2.0 + (i % 7) * 0.375, 6) for i in range(60)], + "phoneme_tokens": [f"p{i % 10}" for i in range(60)], + "sample_rate": 22050, + "hop_length": 256, + }, + }, { "name": "all_zero_durations", "description": "All-zero durations → contiguous zero-length boundaries", @@ -140,6 +185,49 @@ def build_fixture() -> dict: } +# Minimum discriminating power the case list must retain. +# +# Every original case used integer frame counts, and `ceil(int) == int`, so a +# runtime that applied the decoder's frame quantisation (or failed to) matched +# all of them. Measured: applying `math.Ceil` to the Go implementation passed +# the integer-only fixture and fails the current one. The long case exists for +# the same reason at a different axis -- 11 entries is too short for a float32 +# cursor to drift past any runtime's tolerance. +# +# These are asserted rather than left to reviewer vigilance because the failure +# mode is silent: a "simplified" case list still produces a green parity matrix. +MIN_FRACTIONAL_CASES = 2 +MIN_LONG_CASE_ENTRIES = 50 + + +def _validate_discriminating_power(cases: list[dict]) -> list[str]: + """Return human-readable reasons the case list cannot detect known defects.""" + problems: list[str] = [] + + fractional = [ + c + for c in cases + if any(float(d) != int(float(d)) for d in c["inputs"]["durations"]) + ] + if len(fractional) < MIN_FRACTIONAL_CASES: + problems.append( + f"only {len(fractional)} case(s) use non-integer durations " + f"(need >= {MIN_FRACTIONAL_CASES}). Real ONNX `durations` output is " + "always fractional; with integer-only cases a frame-quantisation " + "change is invisible to every runtime's parity test (issue #653)." + ) + + longest = max((len(c["inputs"]["durations"]) for c in cases), default=0) + if longest < MIN_LONG_CASE_ENTRIES: + problems.append( + f"longest case has {longest} entries (need >= {MIN_LONG_CASE_ENTRIES}). " + "Short cases keep cursor accumulation error below every runtime's " + "tolerance, so a narrower accumulator goes undetected." + ) + + return problems + + def _serialize(fixture: dict) -> str: return json.dumps(fixture, indent=2, ensure_ascii=False) + "\n" @@ -155,6 +243,17 @@ def main() -> int: ) args = parser.parse_args() + problems = _validate_discriminating_power(CASES) + if problems: + print( + "ERROR: the fixture case list cannot detect the defects it is " + "meant to pin:", + file=sys.stderr, + ) + for problem in problems: + print(f" - {problem}", file=sys.stderr) + return 1 + serialized = _serialize(build_fixture()) rel = FIXTURE_PATH.relative_to(REPO_ROOT) diff --git a/src/cpp/piper.cpp b/src/cpp/piper.cpp index fcc3249aa..a926259bb 100644 --- a/src/cpp/piper.cpp +++ b/src/cpp/piper.cpp @@ -38,6 +38,7 @@ #include "phoneme_parser.hpp" #include "phoneme_timing_concat.hpp" #include "trim_helpers.hpp" +#include "timing_helpers.hpp" #include "language_detector.hpp" #include "spanish_phonemize.hpp" #include "french_phonemize.hpp" @@ -97,27 +98,14 @@ constexpr int MAX_INTRA_THREADS = 4; // VITS predicts an inflated EOS under the padded context that emits an // audible artifact otherwise. 0 = drop the entire EOS region. -// PUA to multi-char phoneme mapping for display -static const std::unordered_map puaToPhoneme = { - {0xE000, "a:"}, {0xE001, "i:"}, {0xE002, "u:"}, {0xE003, "e:"}, {0xE004, "o:"}, - {0xE005, "cl"}, {0xE006, "ky"}, {0xE007, "kw"}, {0xE008, "gy"}, {0xE009, "gw"}, - {0xE00A, "ty"}, {0xE00B, "dy"}, {0xE00C, "py"}, {0xE00D, "by"}, {0xE00E, "ch"}, - {0xE00F, "ts"}, {0xE010, "sh"}, {0xE011, "zy"}, {0xE012, "hy"}, {0xE013, "ny"}, - {0xE014, "my"}, {0xE015, "ry"}, - // Question type markers (Issue #204) - {0xE016, "?!"}, {0xE017, "?."}, {0xE018, "?~"}, - // N phoneme variants (Issue #207) - {0xE019, "N_m"}, {0xE01A, "N_n"}, {0xE01B, "N_ng"}, {0xE01C, "N_uvular"}, - // Multilingual phoneme tokens - {0xE01D, "rr"}, {0xE01E, "y_vowel"} -}; // Convert phoneme to readable string for logging static std::string phonemeToString(Phoneme ph) { // Check if it's a PUA character if (ph >= 0xE000 && ph <= 0xF8FF) { - auto it = puaToPhoneme.find(ph); - if (it != puaToPhoneme.end()) { + const auto &pua = puaToPhonemeMap(); + auto it = pua.find(ph); + if (it != pua.end()) { return it->second; } } @@ -132,15 +120,14 @@ const std::string instanceName{"piper"}; std::string getVersion() { return VERSION; } -// True if the string is a single UTF-8 codepoint -bool isSingleCodepoint(std::string s) { - return utf8::distance(s.begin(), s.end()) == 1; -} +// True if the string is a single UTF-8 codepoint. +// Exported via piper.hpp; the implementation lives in timing_helpers.hpp so +// the timing walk can use it without linking this translation unit. +bool isSingleCodepoint(std::string s) { return isSingleCodepointUtf8(s); } // Get the first UTF-8 codepoint of a string Phoneme getCodepoint(std::string s) { - utf8::iterator character_iter(s.begin(), s.begin(), s.end()); - return *character_iter; + return static_cast(firstCodepointUtf8(s)); } // Load JSON config information for phonemization @@ -333,8 +320,6 @@ void parseModelConfig(json &configRoot, ModelConfig &modelConfig) { } /* parseModelConfig */ // Constants for phoneme timing -static const std::string UNKNOWN_PHONEME = "?"; -static const float JAPANESE_CL_OVERLAP_RATIO = 0.3f; static const int DEFAULT_HOP_SIZE = 256; // Resolve hop_size from the voice config, falling back to DEFAULT_HOP_SIZE @@ -365,7 +350,12 @@ static void appendUnitTimings(SynthesisResult &out, const SynthesisResult &unit, out.hasTimingInfo = true; } -// Helper function to extract phoneme timings from duration information +// Extract phoneme timings from duration information. +// +// The walk itself lives in timing_helpers.hpp so that +// tests/test_phoneme_timing_parity.cpp can exercise the real arithmetic -- +// this translation unit pulls in onnxruntime, which the parity test cannot +// link, so it used to check a hand-written copy of the algorithm instead. std::vector extractTimingsFromDurations( const std::vector& durations, const std::vector& phonemeIds, @@ -374,85 +364,25 @@ std::vector extractTimingsFromDurations( int sampleRate, PhonemeType phonemeType ) { + const std::vector entries = computePhonemeTimings( + durations, phonemeIds, idMap, hopSize, sampleRate, + usesOpenJTalk(phonemeType)); + + // TimingEntry and PhonemeInfo are layout-identical but distinct types: + // PhonemeInfo is part of the exported SynthesisResult, so it stays in + // piper.hpp (which the ABI gate diffs) rather than being replaced by the + // helper's type. std::vector timings; - - // Build reverse map from phoneme ID to UTF-8 string. - // idMap key is Phoneme (char32_t); encode it properly so isSingleCodepoint() - // and the utf8-checked functions never see invalid byte sequences. - std::unordered_map phonemeIdToStringMap; - for (const auto& [phonemeChar, ids] : idMap) { - if (!ids.empty()) { - std::string phonemeUtf8; - utf8::append(static_cast(phonemeChar), - std::back_inserter(phonemeUtf8)); - phonemeIdToStringMap[ids[0]] = std::move(phonemeUtf8); - } - } - - float frameLength = static_cast(hopSize) / sampleRate; - float currentTime = 0.0f; - int currentFrame = 0; - - for (size_t i = 0; i < phonemeIds.size() && i < durations.size(); ++i) { - PhonemeId id = phonemeIds[i]; - float duration = durations[i]; // Duration in frames - - // Skip special tokens (PAD, BOS, EOS) - if (id == 0 || id == 1 || id == 2) { - currentFrame += static_cast(duration); - currentTime += duration * frameLength; - continue; - } - - // Get phoneme string - std::string phonemeStr = UNKNOWN_PHONEME; - auto it = phonemeIdToStringMap.find(id); - if (it != phonemeIdToStringMap.end()) { - phonemeStr = it->second; - } else { - // Try to decode single character - if (id > 2 && id < 128) { - phonemeStr = std::string(1, static_cast(id)); - } - } - + timings.reserve(entries.size()); + for (const auto& e : entries) { PhonemeInfo info; - info.phoneme = phonemeStr; - info.start_time = currentTime; - info.start_frame = currentFrame; - - currentFrame += static_cast(duration); - currentTime += duration * frameLength; - - info.end_time = currentTime; - info.end_frame = currentFrame; - - timings.push_back(info); - } - - // Adjust timings for Japanese if needed - if (usesOpenJTalk(phonemeType)) { - for (size_t i = 0; i < timings.size(); ++i) { - // Convert PUA mapped phonemes back to original - if (isSingleCodepoint(timings[i].phoneme)) { - // Get the first codepoint (handles multi-byte UTF-8, e.g. PUA U+E000+) - Phoneme ph = getCodepoint(timings[i].phoneme); - auto it = puaToPhoneme.find(ph); - if (it != puaToPhoneme.end()) { - timings[i].phoneme = it->second; - } - } - - // Adjust timing for specific phonemes like 'cl' (促音) - if (timings[i].phoneme == "cl" && i > 0) { - // Overlap with previous phoneme - float overlap = (timings[i].end_time - timings[i].start_time) * JAPANESE_CL_OVERLAP_RATIO; - timings[i-1].end_time += overlap; - timings[i].start_time += overlap; - } - } + info.phoneme = e.phoneme; + info.start_time = e.start_time; + info.end_time = e.end_time; + info.start_frame = e.start_frame; + info.end_frame = e.end_frame; + timings.push_back(std::move(info)); } - return timings; } diff --git a/src/cpp/tests/CMakeLists.txt b/src/cpp/tests/CMakeLists.txt index 35721e997..46a022dde 100644 --- a/src/cpp/tests/CMakeLists.txt +++ b/src/cpp/tests/CMakeLists.txt @@ -956,6 +956,25 @@ set_tests_properties(test_phoneme_timing_parity PROPERTIES WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} ) +# Direct tests for the PRODUCTION phoneme timing cursor walk +# (src/cpp/timing_helpers.hpp, called by piper.cpp:extractTimingsFromDurations). +# The parity test above reads the cross-runtime fixture but re-implements the +# spec algorithm, so until this target existed the C++ timing arithmetic had +# no coverage at all. The header depends only on phoneme_ids.hpp + utf8, so +# this links gtest alone -- no onnxruntime. +add_executable(test_timing_helpers test_timing_helpers.cpp) +target_include_directories(test_timing_helpers PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/../ + ${CMAKE_CURRENT_SOURCE_DIR}/../../ +) +target_link_libraries(test_timing_helpers + gtest_main +) +add_test(NAME test_timing_helpers COMMAND test_timing_helpers) +set_tests_properties(test_timing_helpers PROPERTIES + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) + # Cross-runtime speaker encoder mel parity test. # Reads test/fixtures/speaker_encoder_golden.json (generated from the Python # torchaudio reference) and asserts the spec mel pipeline (Hann + slaney mel diff --git a/src/cpp/tests/test_timing_helpers.cpp b/src/cpp/tests/test_timing_helpers.cpp new file mode 100644 index 000000000..c070dc1dc --- /dev/null +++ b/src/cpp/tests/test_timing_helpers.cpp @@ -0,0 +1,333 @@ +// Direct tests for the PRODUCTION phoneme timing cursor walk +// (src/cpp/timing_helpers.hpp, called by piper.cpp:extractTimingsFromDurations). +// +// Why this file exists: until it did, the C++ timing arithmetic had no test at +// all. test_phoneme_timing_parity.cpp reads the cross-runtime golden fixture +// but re-implements the spec algorithm in its own translation unit -- it said +// so in a comment -- because piper.cpp pulls in onnxruntime and the parity test +// cannot link it. Every other runtime's parity test calls its production +// function, so C++ was the only column of the matrix asserting nothing about +// shipped behaviour. +// +// The fixture STILL cannot drive production directly, and that is a finding +// rather than an oversight: the fixture is token-based ("^", "k", "o", ..., +// "$") while production is id-based and skips PAD/BOS/EOS (ids 0/1/2) without +// emitting entries. The fixture's own `basic_konnichiwa` case contains "^" and +// "$", which a real model maps to ids 1 and 2. Python canonical emits entries +// for them; C++ production does not. That 4-vs-2 split across runtimes is +// issue #697 and is deliberately NOT papered over here. +// +// So: this file pins what production actually does, at the level of individual +// arithmetic decisions, so that changing any of them is a visible diff. + +#include + +#include +#include +#include + +#include "timing_helpers.hpp" + +namespace { + +using piper::computePhonemeTimings; +using piper::PhonemeId; +using piper::PhonemeIdMap; +using piper::TimingEntry; + +constexpr int kHop = 256; +constexpr int kRate = 22050; + +// Seconds per frame at the canonical hop/rate. +double frameSeconds() { + return static_cast(kHop) / static_cast(kRate); +} + +// A map with one single-codepoint phoneme per id. Ids start at 3 so nothing +// collides with PAD/BOS/EOS. +PhonemeIdMap simpleIdMap() { + PhonemeIdMap m; + m[U'a'] = {3}; + m[U'k'] = {4}; + m[U'o'] = {5}; + return m; +} + +} // namespace + +// --------------------------------------------------------------------------- +// PAD / BOS / EOS handling +// --------------------------------------------------------------------------- + +TEST(TimingHelpers, SkipsPadBosEosButStillAdvancesTheCursor) { + // ids 1 (BOS) and 2 (EOS) bracket one real phoneme. Only the real one is + // emitted, and it must start AFTER the BOS duration -- if the cursor did not + // advance, start_time would be 0. + const std::vector durations = {3.0f, 5.0f, 4.0f}; + const std::vector ids = {1, 3, 2}; + + const auto out = computePhonemeTimings(durations, ids, simpleIdMap(), kHop, + kRate, /*applyJapanese=*/false); + + ASSERT_EQ(out.size(), 1u) << "BOS/EOS must not produce entries"; + EXPECT_EQ(out[0].phoneme, "a"); + EXPECT_NEAR(out[0].start_time, 3.0 * frameSeconds(), 1e-6); + EXPECT_NEAR(out[0].end_time, 8.0 * frameSeconds(), 1e-6); + EXPECT_EQ(out[0].start_frame, 3); + EXPECT_EQ(out[0].end_frame, 8); +} + +TEST(TimingHelpers, PadIdZeroIsSkippedToo) { + // Strategy A interspersed padding uses id 0. It occupies real audio frames, + // so the cursor advances, but it is not a phoneme anyone can lip-sync to. + const std::vector durations = {2.0f, 6.0f}; + const std::vector ids = {0, 4}; + + const auto out = computePhonemeTimings(durations, ids, simpleIdMap(), kHop, + kRate, false); + + ASSERT_EQ(out.size(), 1u); + EXPECT_EQ(out[0].phoneme, "k"); + EXPECT_NEAR(out[0].start_time, 2.0 * frameSeconds(), 1e-6); +} + +// --------------------------------------------------------------------------- +// Duration arithmetic -- the values issue #653 will change +// --------------------------------------------------------------------------- + +TEST(TimingHelpers, FractionalDurationsAreUsedRaw) { + // PINS PRE-#653 BEHAVIOUR. The ONNX `durations` output is exported before + // `torch.ceil`, but the decoder allocates ceil(d_i) frames to phoneme i, so + // these times are systematically short (measured: 73-76% of real audio). + // When #653 lands, this expectation must change to the ceil'd values -- + // that is the point of pinning it. Integer-only fixtures cannot catch the + // difference because ceil(int) == int, which is exactly why the shared + // golden matrix (all 7 cases integer) misses it. + const std::vector durations = {2.5f, 3.5f}; + const std::vector ids = {3, 4}; + + const auto out = computePhonemeTimings(durations, ids, simpleIdMap(), kHop, + kRate, false); + + ASSERT_EQ(out.size(), 2u); + EXPECT_NEAR(out[0].end_time, 2.5 * frameSeconds(), 1e-6) + << "raw 2.5 frames, not ceil(2.5)=3"; + EXPECT_NEAR(out[1].end_time, 6.0 * frameSeconds(), 1e-6) + << "raw 2.5+3.5, not ceil(2.5)+ceil(3.5)=7"; +} + +TEST(TimingHelpers, FrameIndicesTruncateWhileTimesDoNot) { + // PINS A KNOWN INTERNAL INCONSISTENCY (issue #653): start_frame/end_frame + // accumulate `static_cast(duration)` (truncation) while + // start_time/end_time accumulate the raw value. For 2.7 frames the two + // disagree: frame 2, time 2.7. Both should end up on the same ceil basis. + const std::vector durations = {2.7f}; + const std::vector ids = {3}; + + const auto out = computePhonemeTimings(durations, ids, simpleIdMap(), kHop, + kRate, false); + + ASSERT_EQ(out.size(), 1u); + EXPECT_EQ(out[0].end_frame, 2) << "truncating cast"; + EXPECT_NEAR(out[0].end_time, 2.7 * frameSeconds(), 1e-6) << "raw float"; + // State the disagreement explicitly so a fix cannot land silently. + EXPECT_NE(static_cast(out[0].end_frame) * frameSeconds(), + static_cast(out[0].end_time)); +} + +TEST(TimingHelpers, NegativeDurationsAreNotClamped) { + // PINS A DIVERGENCE. docs/spec/phoneme-timing-contract.toml + // [calculation.negative_handling] says negative durations clamp to 0 with a + // warning, and Python/Rust/Go/JS/C# all do. C++ does not: it adds the + // negative value, moving the cursor BACKWARDS. Recorded here rather than + // fixed under cover of an extraction. + const std::vector durations = {5.0f, -2.0f, 5.0f}; + const std::vector ids = {3, 4, 5}; + + const auto out = computePhonemeTimings(durations, ids, simpleIdMap(), kHop, + kRate, false); + + ASSERT_EQ(out.size(), 3u); + EXPECT_NEAR(out[1].end_time, 3.0 * frameSeconds(), 1e-6) + << "5 + (-2) = 3: the cursor went backwards instead of clamping"; + EXPECT_LT(out[1].end_time, out[1].start_time) + << "end before start -- the observable symptom"; +} + +// --------------------------------------------------------------------------- +// Accumulator width +// --------------------------------------------------------------------------- + +TEST(TimingHelpers, AccumulatorIsDoubleNotFloat) { + // The cursor used to be a float. Accumulating ~1150 frames of float32 drifts + // ~2e-3 ms from the float64 reference that Python/Rust/Go/JS produce, which + // is above the 1e-6 ms tolerance the cross-runtime parity test declares. + // Storage stays float (PhonemeInfo is ABI-exported), so only the final + // rounding remains -- one rounding per entry, not compounded. + const std::size_t n = 400; + std::vector durations(n, 3.0f); + std::vector ids(n, 3); + + const auto out = computePhonemeTimings(durations, ids, simpleIdMap(), kHop, + kRate, false); + ASSERT_EQ(out.size(), n); + + // Reference computed the same way Python does: one float64 accumulator. + double reference = 0.0; + for (std::size_t i = 0; i < n; ++i) { + reference += 3.0 * frameSeconds(); + } + + // A float32 accumulator over this many steps lands well outside 1e-6 s. + EXPECT_NEAR(out[n - 1].end_time, reference, 1e-6) + << "float32 accumulation would miss this"; +} + +// --------------------------------------------------------------------------- +// Phoneme name resolution +// --------------------------------------------------------------------------- + +TEST(TimingHelpers, UnknownIdFallsBackToQuestionMark) { + const std::vector durations = {4.0f}; + const std::vector ids = {9999}; + + const auto out = + computePhonemeTimings(durations, ids, simpleIdMap(), kHop, kRate, false); + + ASSERT_EQ(out.size(), 1u); + EXPECT_EQ(out[0].phoneme, "?"); +} + +TEST(TimingHelpers, PrintableAsciiIdDecodesToItsOwnCharacter) { + // An id in (2, 128) with no map entry is treated as its own ASCII char. + const std::vector durations = {4.0f}; + const std::vector ids = {static_cast('z')}; + + const auto out = + computePhonemeTimings(durations, ids, simpleIdMap(), kHop, kRate, false); + + ASSERT_EQ(out.size(), 1u); + EXPECT_EQ(out[0].phoneme, "z"); +} + +TEST(TimingHelpers, AsciiFallbackMasksUnmappedIdsWithControlCharacters) { + // PINS A SHARP EDGE. The "id > 2 && id < 128 -> that ASCII char" fallback + // runs BEFORE the "?" fallback, so an unmapped id in that range yields a + // control character rather than an obvious unknown marker. Id 7 becomes BEL. + // Harmless for shipped models (ids 3..127 are all mapped) but it silently + // produces an unprintable phoneme name for any that are not. + const std::vector durations = {4.0f}; + const std::vector ids = {7}; + + const auto out = + computePhonemeTimings(durations, ids, simpleIdMap(), kHop, kRate, false); + + ASSERT_EQ(out.size(), 1u); + EXPECT_EQ(out[0].phoneme, std::string(1, '\a')) + << "not \"?\": the ASCII branch wins for 2 < id < 128"; +} + +TEST(TimingHelpers, OnlyTheFirstIdOfAMultiIdKeyResolves) { + // PINS A KNOWN DEFECT. buildPhonemeIdToStringMap registers ids[0] only, so a + // phoneme carrying several ids resolves for the first and falls back to "?" + // for the rest. The identical bug was fixed in C# under issue #656; C++ is + // tracked separately. Pinned so the fix is a deliberate, visible change. + // Ids are >= 128 on purpose: for 2 < id < 128 the walk falls back to + // decoding the id as its own ASCII character, which would mask the defect + // behind a plausible-looking (but wrong) single-char name. Id 7 renders as + // BEL, not "?" -- verified by running this test with {6, 7}. + PhonemeIdMap m; + m[U'b'] = {200, 201}; + const std::vector durations = {4.0f, 4.0f}; + const std::vector ids = {200, 201}; + + const auto out = computePhonemeTimings(durations, ids, m, kHop, kRate, false); + + ASSERT_EQ(out.size(), 2u); + EXPECT_EQ(out[0].phoneme, "b"); + EXPECT_EQ(out[1].phoneme, "?") << "second id of the same key is unresolved"; +} + +TEST(TimingHelpers, PuaRenameOnlyAppliesWhenJapaneseAdjustmentsAreOn) { + PhonemeIdMap m; + m[static_cast(0xE019)] = {10}; // N_m + const std::vector durations = {4.0f}; + const std::vector ids = {10}; + + const auto off = computePhonemeTimings(durations, ids, m, kHop, kRate, false); + ASSERT_EQ(off.size(), 1u); + EXPECT_NE(off[0].phoneme, "N_m") << "raw PUA codepoint when the pass is off"; + + const auto on = computePhonemeTimings(durations, ids, m, kHop, kRate, true); + ASSERT_EQ(on.size(), 1u); + EXPECT_EQ(on[0].phoneme, "N_m"); +} + +TEST(TimingHelpers, GeminateOverlapMovesTheBoundaryBackwards) { + // 'cl' (促音) bleeds 30% of its own span into the previous phoneme: the + // previous entry's end and the geminate's start both move later by that + // amount, so the boundary between them shifts. + PhonemeIdMap m; + m[U'a'] = {3}; + m[static_cast(0xE005)] = {11}; // cl + const std::vector durations = {10.0f, 10.0f}; + const std::vector ids = {3, 11}; + + const auto plain = computePhonemeTimings(durations, ids, m, kHop, kRate, false); + const auto adjusted = computePhonemeTimings(durations, ids, m, kHop, kRate, true); + ASSERT_EQ(plain.size(), 2u); + ASSERT_EQ(adjusted.size(), 2u); + + const double span = 10.0 * frameSeconds(); + const double overlap = span * 0.3; + EXPECT_NEAR(adjusted[0].end_time, plain[0].end_time + overlap, 1e-6); + EXPECT_NEAR(adjusted[1].start_time, plain[1].start_time + overlap, 1e-6); + // The geminate's END is untouched, so its span shrinks. + EXPECT_NEAR(adjusted[1].end_time, plain[1].end_time, 1e-6); +} + +// --------------------------------------------------------------------------- +// Degenerate inputs +// --------------------------------------------------------------------------- + +TEST(TimingHelpers, NonPositiveRateOrHopReturnsEmpty) { + const std::vector durations = {4.0f}; + const std::vector ids = {3}; + EXPECT_TRUE( + computePhonemeTimings(durations, ids, simpleIdMap(), kHop, 0, false).empty()); + EXPECT_TRUE( + computePhonemeTimings(durations, ids, simpleIdMap(), 0, kRate, false).empty()); +} + +TEST(TimingHelpers, WalkStopsAtTheShorterOfIdsAndDurations) { + // Strategy A can leave the two out of step; the walk must not read past + // either end. + const std::vector durations = {4.0f, 4.0f, 4.0f}; + const std::vector ids = {3, 4}; + + const auto out = + computePhonemeTimings(durations, ids, simpleIdMap(), kHop, kRate, false); + EXPECT_EQ(out.size(), 2u); +} + +TEST(TimingHelpers, EmptyInputProducesNoEntries) { + EXPECT_TRUE(computePhonemeTimings({}, {}, simpleIdMap(), kHop, kRate, false) + .empty()); +} + +// --------------------------------------------------------------------------- +// Anti-vacuity: the suite must fail if the helper stops doing anything. +// --------------------------------------------------------------------------- + +TEST(TimingHelpersGate, HelperProducesNonTrivialOutput) { + // A stub returning {} would satisfy several assertions above by accident + // (the ASSERT_EQ size checks catch it, but only if they run). This is a + // bare TEST so no fixture SetUp can skip it. + const std::vector durations = {1.0f, 2.0f, 3.0f}; + const std::vector ids = {3, 4, 5}; + const auto out = + computePhonemeTimings(durations, ids, simpleIdMap(), kHop, kRate, false); + ASSERT_EQ(out.size(), 3u); + EXPECT_GT(out[2].end_time, 0.0f); + EXPECT_GT(out[2].end_time, out[0].end_time); +} diff --git a/src/cpp/timing_helpers.hpp b/src/cpp/timing_helpers.hpp new file mode 100644 index 000000000..3a0d09046 --- /dev/null +++ b/src/cpp/timing_helpers.hpp @@ -0,0 +1,222 @@ +// Phoneme timing cursor walk, shared by src/cpp/piper.cpp and +// src/cpp/tests/test_phoneme_timing_parity.cpp. +// +// This lived inside piper.cpp, whose translation unit pulls in onnxruntime via +// piper.hpp. Linking it from a test therefore meant linking ORT, so +// test_phoneme_timing_parity.cpp took the other route and re-implemented the +// spec algorithm in its own file -- it said so in a comment: +// +// To stay self-contained (no onnxruntime/espeak link), this test +// re-implements the spec algorithm +// +// Every other runtime's parity test calls its production function (Python +// durations_to_timing, Rust durations_to_timing, Go DurationsToTiming, JS +// durationsToTiming, C# TimingWriter.CalculateTiming). C++ was the only one +// checking a copy, so the C++ column of the cross-runtime parity matrix +// asserted nothing about shipped behaviour. A change to piper.cpp's cursor +// walk -- or a failure to make one, which is what issue #653 is -- could not +// be detected here. +// +// Extracting it makes the existing parity fixture test the real thing. The +// header depends only on phoneme_ids.hpp (which is ORT-free) plus the standard +// library, so the test links no model runtime. +// +// The PUA display table and the two UTF-8 helpers come along because the walk +// resolves phoneme names as it goes; they are the same definitions piper.cpp +// used, moved rather than copied. + +#ifndef PIPER_PLUS_TIMING_HELPERS_HPP +#define PIPER_PLUS_TIMING_HELPERS_HPP + +#include +#include +#include +#include +#include +#include + +#include "phoneme_ids.hpp" +#include "utf8.h" + +namespace piper { + +// One phoneme's span. Times are seconds, frames are hop-sized indices. +// +// The fields are float because piper.hpp's PhonemeInfo is float and this +// struct has to stay layout-compatible with it: PhonemeInfo is reachable from +// SynthesisResult, which the shared library exports, and widening it would +// change the struct layout that the `ABI diff (libpiper_plus.so head vs base)` +// gate compares via abi-dumper's DWARF records. +// +// The ACCUMULATOR is double regardless (see computePhonemeTimings). Only the +// per-entry store rounds to float, so the error is one rounding per entry +// instead of one per entry compounded across the whole utterance. +struct TimingEntry { + std::string phoneme; + float start_time = 0.0f; + float end_time = 0.0f; + int start_frame = 0; + int end_frame = 0; +}; + +// Fallback display for an id with no entry in the model's phoneme_id_map. +inline const std::string kUnknownPhoneme = "?"; + +// Fraction of a geminate's own span that bleeds into the preceding phoneme. +inline constexpr double kJapaneseClOverlapRatio = 0.3; + +// PUA to multi-char phoneme mapping for display. +inline const std::unordered_map &puaToPhonemeMap() { + static const std::unordered_map kMap = { + {0xE000, "a:"}, {0xE001, "i:"}, {0xE002, "u:"}, {0xE003, "e:"}, + {0xE004, "o:"}, {0xE005, "cl"}, {0xE006, "ky"}, {0xE007, "kw"}, + {0xE008, "gy"}, {0xE009, "gw"}, {0xE00A, "ty"}, {0xE00B, "dy"}, + {0xE00C, "py"}, {0xE00D, "by"}, {0xE00E, "ch"}, {0xE00F, "ts"}, + {0xE010, "sh"}, {0xE011, "zy"}, {0xE012, "hy"}, {0xE013, "ny"}, + {0xE014, "my"}, {0xE015, "ry"}, + // Question type markers (Issue #204) + {0xE016, "?!"}, {0xE017, "?."}, {0xE018, "?~"}, + // N phoneme variants (Issue #207) + {0xE019, "N_m"}, {0xE01A, "N_n"}, {0xE01B, "N_ng"}, {0xE01C, "N_uvular"}, + // Multilingual phoneme tokens + {0xE01D, "rr"}, {0xE01E, "y_vowel"}}; + return kMap; +} + +// True if the string is a single UTF-8 codepoint. +inline bool isSingleCodepointUtf8(const std::string &s) { + return utf8::distance(s.begin(), s.end()) == 1; +} + +// First UTF-8 codepoint of a string. Undefined for an empty string. +inline char32_t firstCodepointUtf8(const std::string &s) { + utf8::iterator it(s.begin(), s.begin(), s.end()); + return static_cast(*it); +} + +// Build "phoneme id -> display string" from the model's phoneme_id_map. +// +// NOTE: only ids[0] of each key is registered, so a phoneme mapped to several +// ids resolves for its first id and falls back to "?" for the rest. That is +// the behaviour piper.cpp shipped; the same defect was fixed in C# under +// issue #656 and is tracked for C++ separately rather than being changed here +// under cover of an extraction. +inline std::unordered_map +buildPhonemeIdToStringMap(const PhonemeIdMap &idMap) { + std::unordered_map out; + for (const auto &[phonemeChar, ids] : idMap) { + if (ids.empty()) { + continue; + } + std::string phonemeUtf8; + utf8::append(static_cast(phonemeChar), + std::back_inserter(phonemeUtf8)); + out[ids[0]] = std::move(phonemeUtf8); + } + return out; +} + +// PUA display rename + geminate overlap. Split out so a test can drive it +// directly instead of having to construct an OpenJTalk voice. +inline void applyJapanesePhonemeAdjustments(std::vector &timings) { + const auto &pua = puaToPhonemeMap(); + for (std::size_t i = 0; i < timings.size(); ++i) { + // Convert PUA mapped phonemes back to original. + if (isSingleCodepointUtf8(timings[i].phoneme)) { + auto it = pua.find(firstCodepointUtf8(timings[i].phoneme)); + if (it != pua.end()) { + timings[i].phoneme = it->second; + } + } + + // 促音 (geminate) overlaps backwards into the previous phoneme. + if (timings[i].phoneme == "cl" && i > 0) { + const double overlap = + (static_cast(timings[i].end_time) - timings[i].start_time) * + kJapaneseClOverlapRatio; + timings[i - 1].end_time = + static_cast(timings[i - 1].end_time + overlap); + timings[i].start_time = static_cast(timings[i].start_time + overlap); + } + } +} + +// Walk the durations, emitting one entry per non-special phoneme. +// +// PAD (0), BOS (1) and EOS (2) advance the cursor without producing an entry, +// per docs/spec/phoneme-timing-contract.toml [concatenation] +// forbidden_offset_sources: "the cursor walk advances over pad/bos/eos ids +// without emitting entries for them". +// +// `durations` are frame counts straight from the ONNX `durations` output. +// They are pre-`torch.ceil`, while the decoder allocates `ceil(d_i)` frames to +// phoneme i -- that discrepancy is issue #653 and is NOT addressed here; this +// extraction deliberately preserves the current arithmetic so the parity +// fixture keeps its present values and the change that fixes #653 shows up as +// a fixture diff rather than being folded into a refactor. +// +// `applyJapaneseAdjustments` controls the PUA rename + geminate overlap pass, +// which piper.cpp runs only for OpenJTalk phoneme types. +inline std::vector computePhonemeTimings( + const std::vector &durations, + const std::vector &phonemeIds, const PhonemeIdMap &idMap, + int hopSize, int sampleRate, bool applyJapaneseAdjustments) { + std::vector timings; + if (sampleRate <= 0 || hopSize <= 0) { + return timings; + } + + const std::unordered_map idToString = + buildPhonemeIdToStringMap(idMap); + + // double accumulator, float storage -- see the TimingEntry comment. + const double frameLength = + static_cast(hopSize) / static_cast(sampleRate); + double currentTime = 0.0; + long long currentFrame = 0; + + const std::size_t n = std::min(phonemeIds.size(), durations.size()); + for (std::size_t i = 0; i < n; ++i) { + const PhonemeId id = phonemeIds[i]; + const double duration = static_cast(durations[i]); + + // Skip special tokens (PAD, BOS, EOS): advance the cursor only. + if (id == 0 || id == 1 || id == 2) { + currentFrame += static_cast(duration); + currentTime += duration * frameLength; + continue; + } + + std::string phonemeStr = kUnknownPhoneme; + auto it = idToString.find(id); + if (it != idToString.end()) { + phonemeStr = it->second; + } else if (id > 2 && id < 128) { + // Printable ASCII ids double as their own character. + phonemeStr = std::string(1, static_cast(id)); + } + + TimingEntry info; + info.phoneme = std::move(phonemeStr); + info.start_time = static_cast(currentTime); + info.start_frame = static_cast(currentFrame); + + currentFrame += static_cast(duration); + currentTime += duration * frameLength; + + info.end_time = static_cast(currentTime); + info.end_frame = static_cast(currentFrame); + + timings.push_back(std::move(info)); + } + + if (applyJapaneseAdjustments) { + applyJapanesePhonemeAdjustments(timings); + } + + return timings; +} + +} // namespace piper + +#endif // PIPER_PLUS_TIMING_HELPERS_HPP diff --git a/tests/fixtures/phoneme_timing/golden_matrix.json b/tests/fixtures/phoneme_timing/golden_matrix.json index e7523e998..5badd2490 100644 --- a/tests/fixtures/phoneme_timing/golden_matrix.json +++ b/tests/fixtures/phoneme_timing/golden_matrix.json @@ -268,6 +268,588 @@ "sample_rate": 22050 } }, + { + "name": "fractional_durations", + "description": "Non-integer frame counts — what a real ONNX `durations` output actually looks like. Every other case here is integer-valued, and ceil(int) == int, so an implementation that forgot to apply the frame quantisation the decoder uses would match all of them (issue #653). This case is the one that can tell them apart.", + "inputs": { + "durations": [ + 2.5, + 3.25, + 1.75, + 4.125 + ], + "phoneme_tokens": [ + "a", + "b", + "c", + "d" + ], + "sample_rate": 22050, + "hop_length": 256 + }, + "expected": { + "phonemes": [ + { + "phoneme": "a", + "start_ms": 0.0, + "end_ms": 29.024943310657598, + "duration_ms": 29.024943310657598 + }, + { + "phoneme": "b", + "start_ms": 29.024943310657598, + "end_ms": 66.75736961451247, + "duration_ms": 37.73242630385487 + }, + { + "phoneme": "c", + "start_ms": 66.75736961451247, + "end_ms": 87.07482993197279, + "duration_ms": 20.317460317460316 + }, + { + "phoneme": "d", + "start_ms": 87.07482993197279, + "end_ms": 134.96598639455783, + "duration_ms": 47.89115646258503 + } + ], + "total_duration_ms": 134.96598639455783, + "sample_rate": 22050 + } + }, + { + "name": "fractional_just_above_integer", + "description": "Durations a hair above an integer. Distinguishes truncation from rounding from ceiling: trunc(3.0001)=3, round(3.0001)=3, ceil(3.0001)=4. A single .5 case cannot separate round from ceil.", + "inputs": { + "durations": [ + 3.0001, + 7.0001 + ], + "phoneme_tokens": [ + "x", + "y" + ], + "sample_rate": 22050, + "hop_length": 256 + }, + "expected": { + "phonemes": [ + { + "phoneme": "x", + "start_ms": 0.0, + "end_ms": 34.831092970521546, + "duration_ms": 34.831092970521546 + }, + { + "phoneme": "y", + "start_ms": 34.831092970521546, + "end_ms": 116.10209523809525, + "duration_ms": 81.27100226757369 + } + ], + "total_duration_ms": 116.10209523809525, + "sample_rate": 22050 + } + }, + { + "name": "long_utterance_accumulation", + "description": "60 phonemes of fractional duration. The other cases top out at 11 entries / 50 frames, where a float32 cursor drifts only ~5e-5 ms from float64 and slips under every runtime's tolerance. This length makes the accumulator width observable.", + "inputs": { + "durations": [ + 2.0, + 2.375, + 2.75, + 3.125, + 3.5, + 3.875, + 4.25, + 2.0, + 2.375, + 2.75, + 3.125, + 3.5, + 3.875, + 4.25, + 2.0, + 2.375, + 2.75, + 3.125, + 3.5, + 3.875, + 4.25, + 2.0, + 2.375, + 2.75, + 3.125, + 3.5, + 3.875, + 4.25, + 2.0, + 2.375, + 2.75, + 3.125, + 3.5, + 3.875, + 4.25, + 2.0, + 2.375, + 2.75, + 3.125, + 3.5, + 3.875, + 4.25, + 2.0, + 2.375, + 2.75, + 3.125, + 3.5, + 3.875, + 4.25, + 2.0, + 2.375, + 2.75, + 3.125, + 3.5, + 3.875, + 4.25, + 2.0, + 2.375, + 2.75, + 3.125 + ], + "phoneme_tokens": [ + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9", + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9", + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9", + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9", + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9", + "p0", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", + "p7", + "p8", + "p9" + ], + "sample_rate": 22050, + "hop_length": 256 + }, + "expected": { + "phonemes": [ + { + "phoneme": "p0", + "start_ms": 0.0, + "end_ms": 23.219954648526077, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p1", + "start_ms": 23.219954648526077, + "end_ms": 50.7936507936508, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p2", + "start_ms": 50.7936507936508, + "end_ms": 82.72108843537416, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p3", + "start_ms": 82.72108843537416, + "end_ms": 119.00226757369614, + "duration_ms": 36.281179138321995 + }, + { + "phoneme": "p4", + "start_ms": 119.00226757369614, + "end_ms": 159.63718820861678, + "duration_ms": 40.63492063492063 + }, + { + "phoneme": "p5", + "start_ms": 159.63718820861678, + "end_ms": 204.62585034013605, + "duration_ms": 44.98866213151928 + }, + { + "phoneme": "p6", + "start_ms": 204.62585034013605, + "end_ms": 253.96825396825398, + "duration_ms": 49.342403628117914 + }, + { + "phoneme": "p7", + "start_ms": 253.96825396825398, + "end_ms": 277.18820861678006, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p8", + "start_ms": 277.18820861678006, + "end_ms": 304.76190476190476, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p9", + "start_ms": 304.76190476190476, + "end_ms": 336.68934240362813, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p0", + "start_ms": 336.68934240362813, + "end_ms": 372.9705215419501, + "duration_ms": 36.281179138321995 + }, + { + "phoneme": "p1", + "start_ms": 372.9705215419501, + "end_ms": 413.6054421768707, + "duration_ms": 40.63492063492063 + }, + { + "phoneme": "p2", + "start_ms": 413.6054421768707, + "end_ms": 458.59410430839, + "duration_ms": 44.98866213151928 + }, + { + "phoneme": "p3", + "start_ms": 458.59410430839, + "end_ms": 507.9365079365079, + "duration_ms": 49.342403628117914 + }, + { + "phoneme": "p4", + "start_ms": 507.9365079365079, + "end_ms": 531.156462585034, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p5", + "start_ms": 531.156462585034, + "end_ms": 558.7301587301587, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p6", + "start_ms": 558.7301587301587, + "end_ms": 590.657596371882, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p7", + "start_ms": 590.657596371882, + "end_ms": 626.9387755102041, + "duration_ms": 36.281179138321995 + }, + { + "phoneme": "p8", + "start_ms": 626.9387755102041, + "end_ms": 667.5736961451247, + "duration_ms": 40.63492063492063 + }, + { + "phoneme": "p9", + "start_ms": 667.5736961451247, + "end_ms": 712.562358276644, + "duration_ms": 44.98866213151928 + }, + { + "phoneme": "p0", + "start_ms": 712.562358276644, + "end_ms": 761.9047619047619, + "duration_ms": 49.342403628117914 + }, + { + "phoneme": "p1", + "start_ms": 761.9047619047619, + "end_ms": 785.124716553288, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p2", + "start_ms": 785.124716553288, + "end_ms": 812.6984126984127, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p3", + "start_ms": 812.6984126984127, + "end_ms": 844.625850340136, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p4", + "start_ms": 844.625850340136, + "end_ms": 880.9070294784581, + "duration_ms": 36.281179138321995 + }, + { + "phoneme": "p5", + "start_ms": 880.9070294784581, + "end_ms": 921.5419501133787, + "duration_ms": 40.63492063492063 + }, + { + "phoneme": "p6", + "start_ms": 921.5419501133787, + "end_ms": 966.530612244898, + "duration_ms": 44.98866213151928 + }, + { + "phoneme": "p7", + "start_ms": 966.530612244898, + "end_ms": 1015.8730158730159, + "duration_ms": 49.342403628117914 + }, + { + "phoneme": "p8", + "start_ms": 1015.8730158730159, + "end_ms": 1039.092970521542, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p9", + "start_ms": 1039.092970521542, + "end_ms": 1066.6666666666667, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p0", + "start_ms": 1066.6666666666667, + "end_ms": 1098.5941043083901, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p1", + "start_ms": 1098.5941043083901, + "end_ms": 1134.875283446712, + "duration_ms": 36.281179138321995 + }, + { + "phoneme": "p2", + "start_ms": 1134.875283446712, + "end_ms": 1175.5102040816328, + "duration_ms": 40.63492063492063 + }, + { + "phoneme": "p3", + "start_ms": 1175.5102040816328, + "end_ms": 1220.498866213152, + "duration_ms": 44.98866213151928 + }, + { + "phoneme": "p4", + "start_ms": 1220.498866213152, + "end_ms": 1269.8412698412699, + "duration_ms": 49.342403628117914 + }, + { + "phoneme": "p5", + "start_ms": 1269.8412698412699, + "end_ms": 1293.061224489796, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p6", + "start_ms": 1293.061224489796, + "end_ms": 1320.6349206349207, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p7", + "start_ms": 1320.6349206349207, + "end_ms": 1352.562358276644, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p8", + "start_ms": 1352.562358276644, + "end_ms": 1388.843537414966, + "duration_ms": 36.281179138321995 + }, + { + "phoneme": "p9", + "start_ms": 1388.843537414966, + "end_ms": 1429.4784580498867, + "duration_ms": 40.63492063492063 + }, + { + "phoneme": "p0", + "start_ms": 1429.4784580498867, + "end_ms": 1474.467120181406, + "duration_ms": 44.98866213151928 + }, + { + "phoneme": "p1", + "start_ms": 1474.467120181406, + "end_ms": 1523.8095238095239, + "duration_ms": 49.342403628117914 + }, + { + "phoneme": "p2", + "start_ms": 1523.8095238095239, + "end_ms": 1547.0294784580499, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p3", + "start_ms": 1547.0294784580499, + "end_ms": 1574.6031746031747, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p4", + "start_ms": 1574.6031746031747, + "end_ms": 1606.530612244898, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p5", + "start_ms": 1606.530612244898, + "end_ms": 1642.81179138322, + "duration_ms": 36.281179138321995 + }, + { + "phoneme": "p6", + "start_ms": 1642.81179138322, + "end_ms": 1683.4467120181407, + "duration_ms": 40.63492063492063 + }, + { + "phoneme": "p7", + "start_ms": 1683.4467120181407, + "end_ms": 1728.43537414966, + "duration_ms": 44.98866213151928 + }, + { + "phoneme": "p8", + "start_ms": 1728.43537414966, + "end_ms": 1777.7777777777778, + "duration_ms": 49.342403628117914 + }, + { + "phoneme": "p9", + "start_ms": 1777.7777777777778, + "end_ms": 1800.9977324263039, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p0", + "start_ms": 1800.9977324263039, + "end_ms": 1828.5714285714287, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p1", + "start_ms": 1828.5714285714287, + "end_ms": 1860.498866213152, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p2", + "start_ms": 1860.498866213152, + "end_ms": 1896.780045351474, + "duration_ms": 36.281179138321995 + }, + { + "phoneme": "p3", + "start_ms": 1896.780045351474, + "end_ms": 1937.4149659863947, + "duration_ms": 40.63492063492063 + }, + { + "phoneme": "p4", + "start_ms": 1937.4149659863947, + "end_ms": 1982.403628117914, + "duration_ms": 44.98866213151928 + }, + { + "phoneme": "p5", + "start_ms": 1982.403628117914, + "end_ms": 2031.7460317460318, + "duration_ms": 49.342403628117914 + }, + { + "phoneme": "p6", + "start_ms": 2031.7460317460318, + "end_ms": 2054.965986394558, + "duration_ms": 23.219954648526077 + }, + { + "phoneme": "p7", + "start_ms": 2054.965986394558, + "end_ms": 2082.539682539683, + "duration_ms": 27.573696145124718 + }, + { + "phoneme": "p8", + "start_ms": 2082.539682539683, + "end_ms": 2114.4671201814062, + "duration_ms": 31.927437641723355 + }, + { + "phoneme": "p9", + "start_ms": 2114.4671201814062, + "end_ms": 2150.748299319728, + "duration_ms": 36.281179138321995 + } + ], + "total_duration_ms": 2150.748299319728, + "sample_rate": 22050 + } + }, { "name": "all_zero_durations", "description": "All-zero durations → contiguous zero-length boundaries",