From 3c972c59741e12059e7cf6c458d8393ddfd4aa06 Mon Sep 17 00:00:00 2001 From: mirek190 Date: Wed, 15 Jul 2026 21:23:12 +0100 Subject: [PATCH 1/8] TTS OutieTTS 1.0 + cloning audio --- CMakeLists.txt | 6 + README.md | 6 +- docs/gguf.md | 1 + docs/tts.md | 111 ++++ include/engine/models/outetts/assets.h | 68 +++ include/engine/models/outetts/dac.h | 41 ++ include/engine/models/outetts/llama.h | 51 ++ include/engine/models/outetts/loader.h | 27 + include/engine/models/outetts/session.h | 37 ++ include/engine/models/outetts/tokenizer.h | 52 ++ include/engine/models/qwen3_asr/assets.h | 2 + model_specs/outetts.json | 65 ++ src/framework/runtime/registry.cpp | 2 + src/models/outetts/assets.cpp | 181 ++++++ src/models/outetts/dac.cpp | 701 ++++++++++++++++++++++ src/models/outetts/llama.cpp | 507 ++++++++++++++++ src/models/outetts/loader.cpp | 135 +++++ src/models/outetts/session.cpp | 509 ++++++++++++++++ src/models/outetts/tokenizer.cpp | 198 ++++++ src/models/qwen3_asr/assets.cpp | 32 +- tools/convert_outetts_dac.py | 55 ++ tools/model_manager.py | 78 +++ 22 files changed, 2857 insertions(+), 8 deletions(-) create mode 100644 include/engine/models/outetts/assets.h create mode 100644 include/engine/models/outetts/dac.h create mode 100644 include/engine/models/outetts/llama.h create mode 100644 include/engine/models/outetts/loader.h create mode 100644 include/engine/models/outetts/session.h create mode 100644 include/engine/models/outetts/tokenizer.h create mode 100644 model_specs/outetts.json create mode 100644 src/models/outetts/assets.cpp create mode 100644 src/models/outetts/dac.cpp create mode 100644 src/models/outetts/llama.cpp create mode 100644 src/models/outetts/loader.cpp create mode 100644 src/models/outetts/session.cpp create mode 100644 src/models/outetts/tokenizer.cpp create mode 100644 tools/convert_outetts_dac.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 05bb3e944..3339855db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -257,6 +257,12 @@ add_library(engine_runtime STATIC src/models/omnivoice/assets.cpp src/models/omnivoice/tokenizer_text.cpp src/models/omnivoice/audio_tokenizer.cpp + src/models/outetts/assets.cpp + src/models/outetts/dac.cpp + src/models/outetts/llama.cpp + src/models/outetts/loader.cpp + src/models/outetts/session.cpp + src/models/outetts/tokenizer.cpp src/models/omnivoice/prompt_builder.cpp src/models/omnivoice/generator.cpp src/models/omnivoice/postprocess.cpp diff --git a/README.md b/README.md index 3cacbca01..c9831ff6c 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ audio.cpp would not be moving this quickly without generous contributors bringin | **miocodec** | audio codec, voice conversion backend | lang agnostic | MioCodec v2, 25 Hz, 44.1 kHz | | **miotts** | TTS, voice cloning | en, ja | MioTTS-1.7B | | **omnivoice** | TTS, voice cloning, voice design | 646+ langs | OmniVoice, Qwen3-0.6B based | +| **outetts** | TTS, voice cloning | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | Llama-OuteTTS-1.0-1B | | **pocket_tts** | TTS, voice cloning | en, de, it, pt, es | PocketTTS-100M | | **nemotron_asr** | ASR | 100+ ASR prompt codes incl. auto | Nemotron 3.5 ASR Streaming 0.6B | | **qwen3_asr** | ASR | zh, en, yue, ar, de, fr, es, pt, id, it, ko, ru, th, vi, ja, tr, hi, ms, nl, sv, da, fi, pl, cs, fil, fa, el, ro, hu, mk | Qwen3-ASR-0.6B, Qwen3-ASR-1.7B-hf | @@ -386,6 +387,7 @@ Recommended top-level install packages: | `moss_tts_local_v1_5` | MOSS-TTS-Local Transformer v1.5 | No | | `nemotron_asr` | Nemotron ASR | **Yes** | | `omnivoice` | OmniVoice | **Yes** | +| `outetts_1_0_1b` | OuteTTS 1.0 1B with IBM DAC codec and Qwen3-aligned voice cloning | No | | `parakeet_tdt_0_6b_v3` | Parakeet TDT 0.6B v3 | **Yes** | | `pocket_tts` | PocketTTS | **Yes** | | `qwen3_asr_0_6b` | Qwen3 ASR 0.6B | **Yes** | @@ -639,10 +641,10 @@ The framework also has a reusable GGUF tensor source and a streaming converter. container reader is shared by all model families; a family still has to list a `.gguf` checkpoint as one of its accepted assets because model configuration and tensor naming remain architecture-specific. Qwen3 ASR, Qwen3 Forced Aligner, Qwen3 TTS, Nemotron -3.5 ASR, VibeVoice-ASR, Higgs Audio STT, Hviske ASR, and Citrinet ASR currently accept +3.5 ASR, VibeVoice-ASR, Higgs Audio STT, Hviske ASR, Citrinet ASR, and OuteTTS currently accept `model.gguf` (including `speech_tokenizer/model.gguf` for TTS). The converter recursively embeds sidecar files up to 64 MiB by default using binary-safe metadata, including nested tokenizer models, -and Qwen3 ASR, Nemotron ASR, VibeVoice-ASR, Higgs Audio STT, Hviske ASR, and Citrinet ASR +and Qwen3 ASR, Nemotron ASR, VibeVoice-ASR, Higgs Audio STT, Hviske ASR, Citrinet ASR, and OuteTTS can load the resulting `model.gguf` as a standalone file. The converter embeds the selected package spec in new GGUF files. Standalone conversion with embedded sidecars is the default and fails if required package resources are missing. Pass `--no-sidecars` only to explicitly diff --git a/docs/gguf.md b/docs/gguf.md index d66840f11..68911b34c 100644 --- a/docs/gguf.md +++ b/docs/gguf.md @@ -273,6 +273,7 @@ Status labels: | `moss_tts_nano` | Done | Pass | --- | Pass | No (similarity drift, frame drift, text large drift) | | `nemotron_asr` | Done | Pass | --- | Pass | Pass (minor filler drift) | | `omnivoice` | Done | Pass | --- | No (runtime assert, no audio) | No (runtime assert, no audio) | +| `outetts` | Done | Pass (TTS + clone) | --- | --- | Pass (TTS + clone) | | `pocket_tts` | No | --- | --- | --- | --- | | `qwen3_asr` | Done | Pass | --- | Pass | Pass | | `qwen3_forced_aligner` | Done | Pass | --- | Pass | Pass | diff --git a/docs/tts.md b/docs/tts.md index d0ff8eee8..6037b4f98 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -492,6 +492,117 @@ audiocpp_cli --task clon --family irodori_tts --model /path/to/Irodori-TTS-500M- | `--session-option irodori_tts.rf_weight_context_mb=` | MB | `768` | RF sampler weight context size. | | `--session-option irodori_tts.codec_weight_context_mb=` | MB | `512` | DACVAE codec weight context size. | +## OuteTTS + +OuteTTS 1.0 1B generates 24 kHz speech with a Llama text/audio-token model and the +IBM DAC 1.5 kbps codec. The integration supports both no-reference generation and +native voice cloning. For cloning, the DAC encoder turns a reference WAV into the two +codec-token streams used to condition the language model; no separate encoder model or +speaker-profile file is required. + +| Field | Value | +|---|---| +| Family | `outetts` | +| Model directory | `models/Llama-OuteTTS-1.0-1B` | +| Task | `tts`, `clon` | +| Modes | `offline` | +| Languages | `en`, `ar`, `zh`, `nl`, `fr`, `de`, `it`, `ja`, `ko`, `lt`, `ru`, `es`, plus moderate-data `pt`, `be`, `bn`, `ka`, `hu`, `lv`, `fa`, `pl`, `sw`, `ta`, `uk` | +| Voice input | Optional reference WAV plus its transcript | +| Output | mono 24 kHz WAV | + +Install both the language model and its DAC dependency: + +```bash +python tools/model_manager.py install outetts_1_0_1b --models-dir models +``` + +Run the safetensors package: + +```bash +audiocpp_cli --task tts --family outetts \ + --model models/Llama-OuteTTS-1.0-1B \ + --backend cuda --text "Hello from OuteTTS." \ + --max-tokens 1024 --out out.wav +``` + +Clone a voice with either the safetensors package or standalone GGUF. The reference +transcript must match the spoken reference audio. Around ten seconds of clean speech is +recommended; the maximum accepted reference length is twenty seconds. The standalone +GGUF described below contains Qwen3 Forced Aligner and activates it automatically for +accurate per-word codec conditioning: + +```bash +audiocpp_cli --task clon --family outetts \ + --model models/Llama-OuteTTS-1.0-1B-Q8_0/model.gguf \ + --backend cuda \ + --voice-ref reference.wav \ + --reference-text "The exact words spoken in reference.wav." \ + --request-option reference_language=en \ + --text "This sentence uses the cloned voice." \ + --max-tokens 1024 --out cloned.wav +``` + +`--task tts` with the same `--voice-ref` and `--reference-text` options also enables +speaker conditioning, which is useful for clients that expose one TTS route. +Safetensors packages and older OuteTTS GGUFs do not contain the aligner. For those +models, pass `outetts.aligner_model_path`; cloning fails clearly instead of using +unreliable estimated word boundaries. + +The installer places `DAC.speech.v1.0` and `Qwen3-ForcedAligner-0.6B` beside the +OuteTTS directory. It converts the official DAC checkpoint to a safe tensor source. +To do that conversion manually: + +```bash +python tools/convert_outetts_dac.py \ + models/DAC.speech.v1.0/weights_24khz_1.5kbps_v1.0.pth \ + models/DAC.speech.v1.0/model.safetensors +``` + +Pack the language model, DAC, Qwen3 Forced Aligner, and sidecars into one standalone +Q8 GGUF: + +```bash +audiocpp_gguf \ + --input model_weights=models/Llama-OuteTTS-1.0-1B/model.safetensors \ + --input dac_weights=models/DAC.speech.v1.0/model.safetensors \ + --input aligner_weights=models/Qwen3-ForcedAligner-0.6B/model.safetensors \ + --root models/Llama-OuteTTS-1.0-1B \ + --sidecar models/DAC.speech.v1.0/config.json=dac/config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/config.json=aligner/config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/generation_config.json=aligner/generation_config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/preprocessor_config.json=aligner/preprocessor_config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/tokenizer_config.json=aligner/tokenizer_config.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/vocab.json=aligner/vocab.json \ + --sidecar models/Qwen3-ForcedAligner-0.6B/merges.txt=aligner/merges.txt \ + --output models/Llama-OuteTTS-1.0-1B-Q8_0/model.gguf \ + --type q8_0 +``` + +The resulting GGUF contains all three tensor groups, the model specification, and all +required sidecars. It does not need the original directories. Normal TTS does not run +the embedded aligner; it is initialized only when reference cloning is requested: + +```bash +audiocpp_cli --task tts --family outetts \ + --model models/Llama-OuteTTS-1.0-1B-Q8_0/model.gguf \ + --backend cuda --text "Hello from the standalone GGUF." \ + --max-tokens 1024 --out out.wav +``` + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--max-tokens` | integer | `1024` | Maximum audio-token generation length. Too small a value truncates speech. | +| `--temperature` | float | `0.4` for cloning; model default for TTS | Sampling temperature. Voice cloning follows the official OuteTTS default without changing temperature between words. | +| `--top-k` | integer | `40` | Top-k sampling limit. | +| `--top-p` | float | `0.9` | Nucleus sampling limit. | +| `--request-option min_p=` | float | `0.05` | Minimum probability relative to the most likely token. | +| `--repetition-penalty` | float | `1.1` | Repetition penalty over the latest 64 tokens. | +| `--request-option seed=` | integer | native clone: `4099`; quantized clone: `42` | Deterministic sampling seed. The defaults were separately verified for the native and Q8 cloning paths. | +| `--reference-text` | text | none | Exact transcript of `--voice-ref`; required for voice cloning. | +| `--request-option reference_language=` | language code | `en` | Language used by the optional reference aligner. | +| `--session-option outetts.weight_type=native|f32|f16|bf16|q8_0` | enum | `native` | Language-model weight storage type. For CUDA voice cloning, quantized weights remain compact in the GGUF but are expanded to F32 in VRAM to avoid generation divergence over long reference-codec prompts. Normal TTS and CPU cloning keep the selected type. | +| `--session-option outetts.aligner_model_path=` | model path | embedded aligner | Optional external Qwen3 Forced Aligner override, required only for safetensors packages and older GGUFs without the embedded aligner. | + ## Supertonic Supertonic 3 is a preset-voice multilingual TTS model. It does not use external speaker references in the current integration. diff --git a/include/engine/models/outetts/assets.h b/include/engine/models/outetts/assets.h new file mode 100644 index 000000000..edd22bf02 --- /dev/null +++ b/include/engine/models/outetts/assets.h @@ -0,0 +1,68 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include + +namespace engine::models::qwen3_asr { +struct Qwen3ASRAssets; +} + +namespace engine::models::outetts { + +struct OuteTTSLlama3RopeConfig { + float factor = 32.0F; + float low_freq_factor = 1.0F; + float high_freq_factor = 4.0F; + int64_t original_max_position_embeddings = 8192; +}; + +struct OuteTTSConfig { + int64_t bos_token_id = 0; + int64_t eos_token_id = 0; + int64_t pad_token_id = 0; + int64_t hidden_size = 0; + int64_t intermediate_size = 0; + int64_t max_position_embeddings = 0; + int64_t num_attention_heads = 0; + int64_t num_hidden_layers = 0; + int64_t num_key_value_heads = 0; + int64_t head_dim = 0; + int64_t vocab_size = 0; + float rms_norm_eps = 1.0e-5F; + float rope_theta = 500000.0F; + OuteTTSLlama3RopeConfig rope_scaling; + int64_t sample_rate = 24000; + int64_t hop_length = 320; + int64_t codebook_size = 1024; + int64_t codebooks = 2; + int64_t dac_latent_dim = 1024; + int64_t dac_decoder_dim = 1536; +}; + +struct OuteTTSGenerationConfig { + float temperature = 0.4F; + float repetition_penalty = 1.1F; + int64_t repetition_window = 64; + int64_t top_k = 40; + float top_p = 0.9F; + float min_p = 0.05F; + int64_t max_length = 8192; +}; + +struct OuteTTSAssets { + assets::ResourceBundle resources; + OuteTTSConfig config; + OuteTTSGenerationConfig generation; + std::shared_ptr model_weights; + std::shared_ptr dac_weights; + std::shared_ptr + embedded_aligner; +}; + +std::shared_ptr load_outetts_assets(const std::filesystem::path & model_path); + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/dac.h b/include/engine/models/outetts/dac.h new file mode 100644 index 000000000..57ba009d7 --- /dev/null +++ b/include/engine/models/outetts/dac.h @@ -0,0 +1,41 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/runtime/session.h" +#include "engine/models/outetts/assets.h" + +#include +#include +#include +#include + +namespace engine::models::outetts { + +class OuteTTSDacDecoder final { +public: + struct EncodedReference { + std::vector samples; + std::vector codebook1; + std::vector codebook2; + }; + + OuteTTSDacDecoder(std::shared_ptr assets, + core::ExecutionContext &execution_context, + size_t weight_context_bytes = 1024ull * 1024ull * 1024ull, + size_t graph_context_bytes = 1536ull * 1024ull * 1024ull, + assets::TensorStorageType weight_storage_type = + assets::TensorStorageType::F32); + ~OuteTTSDacDecoder(); + + runtime::AudioBuffer decode(const std::vector &codebook1, + const std::vector &codebook2); + + EncodedReference encode_reference(const runtime::AudioBuffer &audio); + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/llama.h b/include/engine/models/outetts/llama.h new file mode 100644 index 000000000..77503750f --- /dev/null +++ b/include/engine/models/outetts/llama.h @@ -0,0 +1,51 @@ +#pragma once + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/models/outetts/assets.h" + +#include +#include +#include +#include + +namespace engine::models::outetts { + +struct OuteTTSGenerateOptions { + int64_t max_new_tokens = 2048; + float temperature = 0.4F; + float repetition_penalty = 1.1F; + int64_t repetition_window = 64; + int64_t top_k = 40; + float top_p = 0.9F; + float min_p = 0.05F; + uint32_t seed = 0; +}; + +class OuteTTSLlamaRuntime final { +public: + OuteTTSLlamaRuntime( + std::shared_ptr assets, + core::BackendType backend_type, + int device, + int threads, + size_t weight_context_bytes = 4ull * 1024ull * 1024ull * 1024ull, + size_t constant_context_bytes = 256ull * 1024ull * 1024ull, + assets::TensorStorageType weight_storage_type = assets::TensorStorageType::Native); + ~OuteTTSLlamaRuntime(); + + OuteTTSLlamaRuntime(const OuteTTSLlamaRuntime &) = delete; + OuteTTSLlamaRuntime & operator=(const OuteTTSLlamaRuntime &) = delete; + + std::vector generate( + const std::vector & prompt, + const OuteTTSGenerateOptions & options, + int32_t eos_id, + int32_t audio_end_id) const; + +private: + struct Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/loader.h b/include/engine/models/outetts/loader.h new file mode 100644 index 000000000..cef666f00 --- /dev/null +++ b/include/engine/models/outetts/loader.h @@ -0,0 +1,27 @@ +#pragma once + +#include "engine/framework/runtime/model.h" +#include "engine/models/outetts/assets.h" + +#include + +namespace engine::models::outetts { + +class OuteTTSLoadedModel final : public runtime::ILoadedVoiceModel { +public: + explicit OuteTTSLoadedModel(std::shared_ptr assets); + const runtime::ModelMetadata & metadata() const noexcept override; + const runtime::CapabilitySet & capabilities() const noexcept override; + std::unique_ptr create_task_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options) const override; + +private: + std::shared_ptr assets_; + runtime::ModelMetadata metadata_; + runtime::CapabilitySet capabilities_; +}; + +std::shared_ptr make_outetts_loader(); + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/session.h b/include/engine/models/outetts/session.h new file mode 100644 index 000000000..33c7bc10b --- /dev/null +++ b/include/engine/models/outetts/session.h @@ -0,0 +1,37 @@ +#pragma once + +#include "engine/framework/runtime/session_base.h" +#include "engine/models/outetts/assets.h" +#include "engine/models/outetts/dac.h" +#include "engine/models/outetts/llama.h" +#include "engine/models/outetts/tokenizer.h" + +#include +#include + +namespace engine::models::outetts { + +class OuteTTSSession final : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession { +public: + OuteTTSSession(runtime::TaskSpec task, runtime::SessionOptions options, + std::shared_ptr assets); + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest &request) override; + runtime::TaskResult run(const runtime::TaskRequest &request) override; + +private: + OuteTTSLlamaRuntime &llama(bool voice_cloning); + + runtime::TaskSpec task_; + std::shared_ptr assets_; + OuteTTSTokenizer tokenizer_; + std::unique_ptr llama_; + std::unique_ptr clone_llama_; + OuteTTSDacDecoder dac_; + std::optional voice_profile_; +}; + +} // namespace engine::models::outetts diff --git a/include/engine/models/outetts/tokenizer.h b/include/engine/models/outetts/tokenizer.h new file mode 100644 index 000000000..3ea848e62 --- /dev/null +++ b/include/engine/models/outetts/tokenizer.h @@ -0,0 +1,52 @@ +#pragma once + +#include "engine/models/outetts/assets.h" + +#include +#include +#include +#include +#include + +namespace engine::models::outetts { + +struct OuteTTSVoiceFeatures { + int energy = 0; + int spectral_centroid = 0; + int pitch = 0; +}; + +struct OuteTTSVoiceWord { + std::string text; + double duration = 0.0; + OuteTTSVoiceFeatures features; + std::vector codebook1; + std::vector codebook2; +}; + +struct OuteTTSVoiceProfile { + std::string text; + OuteTTSVoiceFeatures global_features; + std::vector words; +}; + +class OuteTTSTokenizer { +public: + explicit OuteTTSTokenizer(std::shared_ptr assets); + + std::vector build_prompt(const std::string &text) const; + std::vector + build_clone_prompt(const std::string &text, + const OuteTTSVoiceProfile &profile) const; + bool is_stop_token(int32_t token) const noexcept; + int32_t eos_id() const noexcept; + int32_t audio_end_id() const noexcept; + bool append_audio_code(int32_t token, std::vector &codebook1, + std::vector &codebook2) const; + +private: + struct Impl; + std::shared_ptr impl_; +}; + +} // namespace engine::models::outetts diff --git a/include/engine/models/qwen3_asr/assets.h b/include/engine/models/qwen3_asr/assets.h index bf7d1c436..2ba86525c 100644 --- a/include/engine/models/qwen3_asr/assets.h +++ b/include/engine/models/qwen3_asr/assets.h @@ -84,5 +84,7 @@ std::shared_ptr load_qwen3_asr_assets(const std::filesyste std::shared_ptr load_qwen3_asr_assets( const std::filesystem::path & model_path, std::string_view package_family); +std::shared_ptr load_qwen3_asr_assets( + assets::ResourceBundle resources); } // namespace engine::models::qwen3_asr diff --git a/model_specs/outetts.json b/model_specs/outetts.json new file mode 100644 index 000000000..05f859f33 --- /dev/null +++ b/model_specs/outetts.json @@ -0,0 +1,65 @@ +{ + "family": "outetts", + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer": "model:tokenizer.json", + "tokenizer_config": "model:tokenizer_config.json", + "special_tokens_map": "model:special_tokens_map.json", + "dac_config": "model:dac/config.json" + }, + "optional_files": { + "aligner_config": "model:aligner/config.json", + "aligner_generation_config": "model:aligner/generation_config.json", + "aligner_tokenizer_config": "model:aligner/tokenizer_config.json", + "aligner_preprocessor_config": "model:aligner/preprocessor_config.json", + "aligner_processor_config": "model:aligner/processor_config.json", + "aligner_chat_template": "model:aligner/chat_template.json", + "aligner_chat_template_jinja": "model:aligner/chat_template.jinja", + "aligner_vocab": "model:aligner/vocab.json", + "aligner_merges": "model:aligner/merges.txt", + "aligner_tokenizer_json": "model:aligner/tokenizer.json" + }, + "tensors": { + "model_weights": { + "source": "weights:", + "prefix": "model_weights" + }, + "dac_weights": { + "source": "weights:", + "prefix": "dac_weights" + }, + "aligner_weights": { + "source": "weights:", + "prefix": "aligner_weights" + } + } + }, + { + "format": "safetensors", + "roots": { + "model": ".", + "dac": "../DAC.speech.v1.0" + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer": "model:tokenizer.json", + "tokenizer_config": "model:tokenizer_config.json", + "special_tokens_map": "model:special_tokens_map.json", + "dac_config": "dac:config.json" + }, + "tensors": { + "model_weights": "model:model.safetensors", + "dac_weights": "dac:model.safetensors" + } + } + ] +} diff --git a/src/framework/runtime/registry.cpp b/src/framework/runtime/registry.cpp index ed05b1fb5..2c28ed1c9 100644 --- a/src/framework/runtime/registry.cpp +++ b/src/framework/runtime/registry.cpp @@ -24,6 +24,7 @@ #include "engine/models/moss/moss_tts_nano/loader.h" #include "engine/models/nemotron_asr/loader.h" #include "engine/models/omnivoice/loader.h" +#include "engine/models/outetts/loader.h" #include "engine/models/pocket_tts/loader.h" #include "engine/models/qwen3_asr/loader.h" #include "engine/models/qwen3_forced_aligner/loader.h" @@ -248,6 +249,7 @@ ModelRegistry make_default_registry(const std::optional & engine::models::demucs::make_htdemucs_loader(), engine::models::roformer::make_mel_band_roformer_loader(), engine::models::omnivoice::make_omnivoice_loader(), + engine::models::outetts::make_outetts_loader(), engine::models::miocodec::make_miocodec_loader(), engine::models::miotts::make_miotts_loader(), engine::models::moss_tts_local::make_moss_tts_local_loader(), diff --git a/src/models/outetts/assets.cpp b/src/models/outetts/assets.cpp new file mode 100644 index 000000000..e7b2a3293 --- /dev/null +++ b/src/models/outetts/assets.cpp @@ -0,0 +1,181 @@ +#include "engine/models/outetts/assets.h" + +#include "engine/framework/assets/model_package.h" +#include "engine/framework/io/config.h" +#include "engine/framework/io/json.h" +#include "engine/models/qwen3_asr/assets.h" + +#include + +namespace engine::models::outetts { +namespace json = engine::io::json; +namespace { + +std::filesystem::path spec_path() { + return assets::default_model_package_spec_path("outetts"); +} + +OuteTTSConfig parse_config(const assets::ResourceBundle &resources) { + const auto root = resources.parse_json("config"); + if (json::require_string(root, "model_type") != "llama") { + throw std::runtime_error("OuteTTS expects a LlamaForCausalLM checkpoint"); + } + OuteTTSConfig out; + out.bos_token_id = json::require_i64(root, "bos_token_id"); + out.eos_token_id = json::require_i64(root, "eos_token_id"); + out.hidden_size = json::require_i64(root, "hidden_size"); + out.intermediate_size = json::require_i64(root, "intermediate_size"); + out.max_position_embeddings = + json::require_i64(root, "max_position_embeddings"); + out.num_attention_heads = json::require_i64(root, "num_attention_heads"); + out.num_hidden_layers = json::require_i64(root, "num_hidden_layers"); + out.num_key_value_heads = json::require_i64(root, "num_key_value_heads"); + out.head_dim = json::optional_i64(root, "head_dim", + out.hidden_size / out.num_attention_heads); + out.vocab_size = json::require_i64(root, "vocab_size"); + out.rms_norm_eps = json::optional_f32(root, "rms_norm_eps", out.rms_norm_eps); + out.rope_theta = json::optional_f32(root, "rope_theta", out.rope_theta); + const auto &rope = root.require("rope_scaling"); + if (json::require_string(rope, "rope_type") != "llama3") { + throw std::runtime_error("OuteTTS expects llama3 rope scaling"); + } + out.rope_scaling.factor = json::require_f32(rope, "factor"); + out.rope_scaling.low_freq_factor = json::require_f32(rope, "low_freq_factor"); + out.rope_scaling.high_freq_factor = + json::require_f32(rope, "high_freq_factor"); + out.rope_scaling.original_max_position_embeddings = + json::require_i64(rope, "original_max_position_embeddings"); + + const auto generation = resources.parse_json("generation_config"); + out.pad_token_id = json::require_i64(generation, "pad_token_id"); + engine::io::require_positive(out.hidden_size, "OuteTTS hidden_size"); + engine::io::require_positive(out.intermediate_size, + "OuteTTS intermediate_size"); + engine::io::require_positive(out.num_hidden_layers, "OuteTTS layer count"); + engine::io::require_positive(out.num_attention_heads, + "OuteTTS attention heads"); + engine::io::require_positive(out.num_key_value_heads, "OuteTTS KV heads"); + engine::io::require_divisible(out.num_attention_heads, + out.num_key_value_heads, + "OuteTTS attention heads"); + if (out.num_attention_heads * out.head_dim != out.hidden_size) { + throw std::runtime_error( + "OuteTTS hidden size does not match attention heads times head_dim"); + } + return out; +} + +OuteTTSGenerationConfig +parse_generation(const assets::ResourceBundle &resources) { + const auto root = resources.parse_json("generation_config"); + OuteTTSGenerationConfig out; + out.temperature = json::optional_f32(root, "temperature", out.temperature); + out.repetition_penalty = + json::optional_f32(root, "repetition_penalty", out.repetition_penalty); + out.top_k = json::optional_i64(root, "top_k", out.top_k); + out.top_p = json::optional_f32(root, "top_p", out.top_p); + out.min_p = json::optional_f32(root, "min_p", out.min_p); + return out; +} + +void validate_anchors(const OuteTTSAssets &assets) { + const auto &c = assets.config; + const auto &model = *assets.model_weights; + assets::require_tensor_shape(model, "model.embed_tokens.weight", + {c.vocab_size, c.hidden_size}); + assets::require_tensor_shape(model, "model.norm.weight", {c.hidden_size}); + assets::require_tensor_shape( + model, "model.layers.0.self_attn.q_proj.weight", + {c.num_attention_heads * c.head_dim, c.hidden_size}); + assets::require_tensor_shape( + model, "model.layers.0.self_attn.k_proj.weight", + {c.num_key_value_heads * c.head_dim, c.hidden_size}); + assets::require_tensor_shape(model, "model.layers.0.mlp.gate_proj.weight", + {c.intermediate_size, c.hidden_size}); + + const auto &dac = *assets.dac_weights; + assets::require_tensor_shape(dac, "quantizer.quantizers.0.codebook.weight", + {c.codebook_size, 8}); + assets::require_tensor_shape(dac, "quantizer.quantizers.1.codebook.weight", + {c.codebook_size, 8}); + assets::require_tensor_shape(dac, "quantizer.quantizers.0.in_proj.weight_v", + {8, c.dac_latent_dim, 1}); + assets::require_tensor_shape(dac, "quantizer.quantizers.1.in_proj.weight_v", + {8, c.dac_latent_dim, 1}); + assets::require_tensor_shape(dac, "encoder.block.0.weight_v", {64, 1, 7}); + assets::require_tensor_shape(dac, "encoder.block.6.weight_v", + {c.dac_latent_dim, c.dac_latent_dim, 3}); + assets::require_tensor_shape(dac, "decoder.model.0.weight_v", + {c.dac_decoder_dim, c.dac_latent_dim, 7}); + assets::require_tensor_shape(dac, "decoder.model.6.weight_v", {1, 96, 7}); +} + +std::shared_ptr +load_embedded_aligner(const assets::ResourceBundle &resources) { + if (!resources.has_file("aligner_config")) + return nullptr; + for (const char *id : {"aligner_generation_config", + "aligner_tokenizer_config"}) { + if (!resources.has_file(id)) { + throw std::runtime_error( + std::string("OuteTTS embedded aligner is missing resource: ") + id); + } + } + if (!resources.has_file("aligner_preprocessor_config") && + !resources.has_file("aligner_processor_config")) { + throw std::runtime_error( + "OuteTTS embedded aligner is missing its processor configuration"); + } + if (!(resources.has_file("aligner_tokenizer_json") || + (resources.has_file("aligner_vocab") && + resources.has_file("aligner_merges")))) { + throw std::runtime_error( + "OuteTTS embedded aligner is missing tokenizer resources"); + } + + const auto source = resources.open_tensor_source("aligner_weights"); + if (source->tensors().empty()) { + throw std::runtime_error( + "OuteTTS embedded aligner has no aligner_weights tensors"); + } + assets::ResourceBundle aligner(resources.model_root()); + const auto add_required = [&](const char *target, const char *source_id) { + aligner.add_file(target, resources.require_file(source_id)); + }; + const auto add_optional = [&](const char *target, const char *source_id) { + if (resources.has_file(source_id)) + aligner.add_file(target, resources.require_file(source_id)); + }; + add_required("config", "aligner_config"); + add_required("generation_config", "aligner_generation_config"); + add_required("tokenizer_config", "aligner_tokenizer_config"); + add_optional("preprocessor_config", "aligner_preprocessor_config"); + add_optional("processor_config", "aligner_processor_config"); + add_optional("chat_template", "aligner_chat_template"); + add_optional("chat_template_jinja", "aligner_chat_template_jinja"); + add_optional("vocab", "aligner_vocab"); + add_optional("merges", "aligner_merges"); + add_optional("tokenizer_json", "aligner_tokenizer_json"); + aligner.add_tensor_source("weights", resources.require_file("aligner_weights"), + "aligner_weights"); + return engine::models::qwen3_asr::load_qwen3_asr_assets( + std::move(aligner)); +} + +} // namespace + +std::shared_ptr +load_outetts_assets(const std::filesystem::path &model_path) { + auto out = std::make_shared(); + out->resources = + assets::load_resource_bundle_from_package_spec(model_path, spec_path()); + out->config = parse_config(out->resources); + out->generation = parse_generation(out->resources); + out->model_weights = out->resources.open_tensor_source("model_weights"); + out->dac_weights = out->resources.open_tensor_source("dac_weights"); + out->embedded_aligner = load_embedded_aligner(out->resources); + validate_anchors(*out); + return out; +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/dac.cpp b/src/models/outetts/dac.cpp new file mode 100644 index 000000000..2f29880d1 --- /dev/null +++ b/src/models/outetts/dac.cpp @@ -0,0 +1,701 @@ +#include "engine/models/outetts/dac.h" + +#include "engine/framework/audio/conversion.h" +#include "engine/framework/audio/resampling.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/structural_modules.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace engine::models::outetts { +namespace { + +struct GgmlContextDeleter { + void operator()(ggml_context *ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +struct SnakeWeights { + core::TensorValue alpha; +}; +struct ConvWeights { + modules::Conv1dWeights value; + int64_t in_channels = 0; + int64_t out_channels = 0; + int64_t kernel = 0; +}; +struct ConvTransposeWeights { + modules::ConvTranspose1dWeights value; + int64_t in_channels = 0; + int64_t out_channels = 0; + int64_t kernel = 0; +}; +struct ResidualWeights { + SnakeWeights snake1; + ConvWeights conv1; + SnakeWeights snake2; + ConvWeights conv2; +}; +struct DecoderBlockWeights { + SnakeWeights snake; + ConvTransposeWeights up; + std::vector residuals; + int stride = 1; +}; +struct EncoderBlockWeights { + std::vector residuals; + SnakeWeights snake; + ConvWeights down; + int stride = 1; +}; +struct HostQuantizerWeights { + std::vector codebook; + std::vector in_weight; + std::vector in_bias; + std::vector out_weight; + std::vector out_bias; +}; +struct QuantizerWeights { + core::TensorValue codebook; + ConvWeights out_proj; +}; +struct DacWeights { + std::shared_ptr store; + std::vector quantizers; + ConvWeights first; + std::vector blocks; + SnakeWeights final_snake; + ConvWeights final_conv; + ConvWeights encoder_first; + std::vector encoder_blocks; + SnakeWeights encoder_final_snake; + ConvWeights encoder_final_conv; + std::vector host_quantizers; +}; + +std::vector fold_weight_norm(const std::vector &v, + const std::vector &g, int64_t dim0, + int64_t dim1, int64_t kernel) { + std::vector out(v.size()); + for (int64_t i = 0; i < dim0; ++i) { + const size_t base = static_cast(i * dim1 * kernel); + double norm2 = 0.0; + for (int64_t j = 0; j < dim1 * kernel; ++j) { + const double x = v[base + static_cast(j)]; + norm2 += x * x; + } + const float scale = + g[static_cast(i)] / static_cast(std::sqrt(norm2)); + for (int64_t j = 0; j < dim1 * kernel; ++j) { + out[base + static_cast(j)] = + v[base + static_cast(j)] * scale; + } + } + return out; +} + +ConvWeights load_conv(core::BackendWeightStore &store, + const assets::TensorSource &source, + const std::string &prefix, int64_t out_channels, + int64_t in_channels, int64_t kernel, + assets::TensorStorageType storage_type) { + const auto v = source.require_f32(prefix + ".weight_v", + {out_channels, in_channels, kernel}); + const auto g = source.require_f32(prefix + ".weight_g", {out_channels, 1, 1}); + ConvWeights out; + out.in_channels = in_channels; + out.out_channels = out_channels; + out.kernel = kernel; + out.value.weight = store.make_from_f32( + core::TensorShape::from_dims({out_channels, in_channels, kernel}), + storage_type, fold_weight_norm(v, g, out_channels, in_channels, kernel)); + out.value.bias = + store.load_f32_tensor(source, prefix + ".bias", {out_channels}); + return out; +} + +ConvTransposeWeights load_conv_transpose( + core::BackendWeightStore &store, const assets::TensorSource &source, + const std::string &prefix, int64_t in_channels, int64_t out_channels, + int64_t kernel, assets::TensorStorageType storage_type) { + const auto v = source.require_f32(prefix + ".weight_v", + {in_channels, out_channels, kernel}); + const auto g = source.require_f32(prefix + ".weight_g", {in_channels, 1, 1}); + ConvTransposeWeights out; + out.in_channels = in_channels; + out.out_channels = out_channels; + out.kernel = kernel; + out.value.weight = store.make_from_f32( + core::TensorShape::from_dims({in_channels, out_channels, kernel}), + storage_type, fold_weight_norm(v, g, in_channels, out_channels, kernel)); + out.value.bias = + store.load_f32_tensor(source, prefix + ".bias", {out_channels}); + return out; +} + +SnakeWeights load_snake(core::BackendWeightStore &store, + const assets::TensorSource &source, + const std::string &name, int64_t channels) { + return {store.make_from_f32(core::TensorShape::from_dims({channels}), + assets::TensorStorageType::F32, + source.require_f32(name, {1, channels, 1}))}; +} + +ResidualWeights load_residual(core::BackendWeightStore &store, + const assets::TensorSource &source, + const std::string &prefix, int64_t channels, + assets::TensorStorageType storage_type) { + ResidualWeights out; + out.snake1 = load_snake(store, source, prefix + ".block.0.alpha", channels); + out.conv1 = load_conv(store, source, prefix + ".block.1", channels, channels, + 7, storage_type); + out.snake2 = load_snake(store, source, prefix + ".block.2.alpha", channels); + out.conv2 = load_conv(store, source, prefix + ".block.3", channels, channels, + 1, storage_type); + return out; +} + +DacWeights load_weights(const OuteTTSAssets &assets, + core::ExecutionContext &execution, size_t context_bytes, + assets::TensorStorageType storage_type) { + const auto &source = *assets.dac_weights; + DacWeights out; + out.store = std::make_shared( + execution.backend(), execution.backend_type(), "outetts.dac.weights", + context_bytes); + for (int i = 0; i < 2; ++i) { + const std::string p = "quantizer.quantizers." + std::to_string(i); + QuantizerWeights q; + q.codebook = + out.store->load_tensor(source, p + ".codebook.weight", + assets::TensorStorageType::F32, {1024, 8}); + q.out_proj = load_conv(*out.store, source, p + ".out_proj", 1024, 8, 1, + storage_type); + out.quantizers.push_back(std::move(q)); + } + out.first = load_conv(*out.store, source, "decoder.model.0", 1536, 1024, 7, + storage_type); + const int strides[] = {8, 5, 4, 2}; + int64_t in_channels = 1536; + for (int stage = 0; stage < 4; ++stage) { + const int64_t out_channels = in_channels / 2; + const std::string p = "decoder.model." + std::to_string(stage + 1); + DecoderBlockWeights block; + block.stride = strides[stage]; + block.snake = + load_snake(*out.store, source, p + ".block.0.alpha", in_channels); + block.up = + load_conv_transpose(*out.store, source, p + ".block.1", in_channels, + out_channels, 2 * strides[stage], storage_type); + for (int residual = 0; residual < 3; ++residual) { + block.residuals.push_back(load_residual( + *out.store, source, p + ".block." + std::to_string(residual + 2), + out_channels, storage_type)); + } + out.blocks.push_back(std::move(block)); + in_channels = out_channels; + } + out.final_snake = load_snake(*out.store, source, "decoder.model.5.alpha", 96); + out.final_conv = + load_conv(*out.store, source, "decoder.model.6", 1, 96, 7, storage_type); + + out.encoder_first = + load_conv(*out.store, source, "encoder.block.0", 64, 1, 7, storage_type); + const int encoder_strides[] = {2, 4, 5, 8}; + int64_t encoder_channels = 64; + for (int stage = 0; stage < 4; ++stage) { + const int64_t out_channels = encoder_channels * 2; + const std::string p = "encoder.block." + std::to_string(stage + 1); + EncoderBlockWeights block; + block.stride = encoder_strides[stage]; + for (int residual_index = 0; residual_index < 3; ++residual_index) { + block.residuals.push_back(load_residual( + *out.store, source, p + ".block." + std::to_string(residual_index), + encoder_channels, storage_type)); + } + block.snake = + load_snake(*out.store, source, p + ".block.3.alpha", encoder_channels); + block.down = + load_conv(*out.store, source, p + ".block.4", out_channels, + encoder_channels, 2 * encoder_strides[stage], storage_type); + out.encoder_blocks.push_back(std::move(block)); + encoder_channels = out_channels; + } + out.encoder_final_snake = + load_snake(*out.store, source, "encoder.block.5.alpha", 1024); + out.encoder_final_conv = load_conv(*out.store, source, "encoder.block.6", + 1024, 1024, 3, storage_type); + + for (int i = 0; i < 2; ++i) { + const std::string p = "quantizer.quantizers." + std::to_string(i); + HostQuantizerWeights q; + q.codebook = source.require_f32(p + ".codebook.weight", {1024, 8}); + q.in_weight = fold_weight_norm( + source.require_f32(p + ".in_proj.weight_v", {8, 1024, 1}), + source.require_f32(p + ".in_proj.weight_g", {8, 1, 1}), 8, 1024, 1); + q.in_bias = source.require_f32(p + ".in_proj.bias", {8}); + q.out_weight = fold_weight_norm( + source.require_f32(p + ".out_proj.weight_v", {1024, 8, 1}), + source.require_f32(p + ".out_proj.weight_g", {1024, 1, 1}), 1024, 8, 1); + q.out_bias = source.require_f32(p + ".out_proj.bias", {1024}); + out.host_quantizers.push_back(std::move(q)); + } + out.store->upload(); + return out; +} + +core::TensorValue snake(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const SnakeWeights &weights) { + const int64_t channels = input.shape.dims[1]; + auto alpha = core::reshape_tensor( + ctx, weights.alpha, core::TensorShape::from_dims({1, channels, 1})); + alpha = core::wrap_tensor(ggml_repeat(ctx.ggml, alpha.tensor, input.tensor), + input.shape, GGML_TYPE_F32); + auto ax = core::wrap_tensor(ggml_mul(ctx.ggml, input.tensor, alpha.tensor), + input.shape, GGML_TYPE_F32); + auto s = core::wrap_tensor(ggml_sin(ctx.ggml, ax.tensor), input.shape, + GGML_TYPE_F32); + auto s2 = core::wrap_tensor(ggml_mul(ctx.ggml, s.tensor, s.tensor), + input.shape, GGML_TYPE_F32); + auto denom = + core::wrap_tensor(ggml_scale_bias(ctx.ggml, alpha.tensor, 1.0F, 1.0e-9F), + input.shape, GGML_TYPE_F32); + auto frac = core::wrap_tensor(ggml_div(ctx.ggml, s2.tensor, denom.tensor), + input.shape, GGML_TYPE_F32); + return core::wrap_tensor(ggml_add(ctx.ggml, input.tensor, frac.tensor), + input.shape, GGML_TYPE_F32); +} + +core::TensorValue conv(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const ConvWeights &weights, int padding, + int dilation = 1) { + return modules::Conv1dModule({ + weights.in_channels, + weights.out_channels, + weights.kernel, + 1, + padding, + dilation, + true, + }) + .build(ctx, input, weights.value); +} + +core::TensorValue strided_conv(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const ConvWeights &weights, int stride, + int padding) { + return modules::Conv1dModule({ + weights.in_channels, + weights.out_channels, + weights.kernel, + stride, + padding, + 1, + true, + }) + .build(ctx, input, weights.value); +} + +core::TensorValue residual(core::ModuleBuildContext &ctx, + const core::TensorValue &input, + const ResidualWeights &weights, int dilation) { + auto x = snake(ctx, input, weights.snake1); + x = conv(ctx, x, weights.conv1, 3 * dilation, dilation); + x = snake(ctx, x, weights.snake2); + x = conv(ctx, x, weights.conv2, 0); + return modules::AddModule{}.build(ctx, input, x); +} + +std::vector load_reference_audio(const runtime::AudioBuffer &input) { + if (input.sample_rate <= 0 || input.channels <= 0 || input.samples.empty()) { + throw std::runtime_error( + "OuteTTS voice cloning requires non-empty reference audio"); + } + auto samples = audio::mixdown_interleaved_to_mono_average( + input.samples, input.channels, audio::MonoMixAccumulation::Float64); + if (input.sample_rate != 24000) { + audio::SoxrResampleOptions options; + options.output_length_policy = audio::SoxrOutputLengthPolicy::ActualOutput; + options.reject_empty_output = true; + options.warning_context = "OuteTTS reference audio"; + options.fallback_description = "linear reference-audio resampling"; + samples = audio::resample_mono_soxr_or_linear(samples, input.sample_rate, + 24000, options); + } + constexpr size_t max_samples = 20u * 24000u; + if (samples.size() > max_samples) { + throw std::runtime_error( + "OuteTTS reference audio is longer than the supported 20 seconds"); + } + return samples; +} + +std::vector prepare_reference_audio(const std::vector &input) { + auto samples = input; + double sum_sq = 0.0; + for (const float sample : samples) + sum_sq += static_cast(sample) * sample; + const float rms = static_cast( + std::sqrt(sum_sq / static_cast(samples.size()))); + if (rms > 1.0e-6F) { + const float scale = std::pow(10.0F, -18.0F / 20.0F) / rms; + for (float &sample : samples) + sample *= scale; + } + float peak = 0.0F; + for (const float sample : samples) + peak = std::max(peak, std::fabs(sample)); + const float peak_limit = std::pow(10.0F, -1.0F / 20.0F); + if (peak > peak_limit) { + const float scale = peak_limit / peak; + for (float &sample : samples) + sample *= scale; + } + samples.resize(((samples.size() + 319u) / 320u) * 320u, 0.0F); + return samples; +} + +std::pair, std::vector> +quantize_reference(const std::vector &frame_major, + const std::vector &quantizers) { + constexpr int hidden_size = 1024; + constexpr int codebook_size = 1024; + constexpr int codebook_dim = 8; + if (frame_major.empty() || frame_major.size() % hidden_size != 0 || + quantizers.size() != 2) { + throw std::runtime_error( + "OuteTTS DAC encoder produced invalid acoustic latents"); + } + const size_t frames = frame_major.size() / hidden_size; + std::vector residual_values = frame_major; + std::vector outputs[2] = { + std::vector(frames), + std::vector(frames), + }; + for (size_t q_index = 0; q_index < quantizers.size(); ++q_index) { + const auto &q = quantizers[q_index]; +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int64_t frame = 0; frame < static_cast(frames); ++frame) { + float projected[codebook_dim]; + float projected_norm_sq = 0.0F; + const size_t frame_offset = static_cast(frame) * hidden_size; + for (int d = 0; d < codebook_dim; ++d) { + float value = q.in_bias[static_cast(d)]; + const size_t weight_offset = static_cast(d) * hidden_size; + for (int h = 0; h < hidden_size; ++h) { + value += q.in_weight[weight_offset + static_cast(h)] * + residual_values[frame_offset + static_cast(h)]; + } + projected[d] = value; + projected_norm_sq += value * value; + } + const float projected_norm = std::sqrt(projected_norm_sq) + 1.0e-12F; + int best_code = 0; + float best_distance = std::numeric_limits::infinity(); + for (int code = 0; code < codebook_size; ++code) { + const size_t code_offset = static_cast(code) * codebook_dim; + float embedding_norm_sq = 0.0F; + for (int d = 0; d < codebook_dim; ++d) { + const float value = q.codebook[code_offset + static_cast(d)]; + embedding_norm_sq += value * value; + } + const float embedding_norm = std::sqrt(embedding_norm_sq) + 1.0e-12F; + float distance = 0.0F; + for (int d = 0; d < codebook_dim; ++d) { + const float difference = + projected[d] / projected_norm - + q.codebook[code_offset + static_cast(d)] / embedding_norm; + distance += difference * difference; + } + if (distance < best_distance) { + best_distance = distance; + best_code = code; + } + } + outputs[q_index][static_cast(frame)] = best_code; + } +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int64_t frame = 0; frame < static_cast(frames); ++frame) { + const int code = outputs[q_index][static_cast(frame)]; + const size_t code_offset = static_cast(code) * codebook_dim; + const size_t frame_offset = static_cast(frame) * hidden_size; + for (int out = 0; out < hidden_size; ++out) { + float value = q.out_bias[static_cast(out)]; + const size_t weight_offset = static_cast(out) * codebook_dim; + for (int d = 0; d < codebook_dim; ++d) { + value += q.out_weight[weight_offset + static_cast(d)] * + q.codebook[code_offset + static_cast(d)]; + } + residual_values[frame_offset + static_cast(out)] -= value; + } + } + } + return {std::move(outputs[0]), std::move(outputs[1])}; +} + +void normalize_decoded_audio(std::vector &audio) { + if (audio.empty()) { + return; + } + double sum_sq = 0.0; + for (const float sample : audio) { + sum_sq += static_cast(sample) * static_cast(sample); + } + const float rms = + static_cast(std::sqrt(sum_sq / static_cast(audio.size()))); + if (rms > 1.0e-6F) { + const float target_rms = std::pow(10.0F, -18.0F / 20.0F); + const float scale = target_rms / rms; + for (float &sample : audio) { + sample *= scale; + } + } + float peak = 0.0F; + for (const float sample : audio) { + peak = std::max(peak, std::fabs(sample)); + } + const float peak_limit = std::pow(10.0F, -1.0F / 20.0F); + if (peak > peak_limit && peak > 1.0e-6F) { + const float scale = peak_limit / peak; + for (float &sample : audio) { + sample *= scale; + } + } +} + +} // namespace + +struct OuteTTSDacDecoder::Impl { + Impl(std::shared_ptr assets_in, + core::ExecutionContext &execution_in, size_t weight_context_bytes, + size_t graph_context_bytes_in, assets::TensorStorageType storage_type) + : assets(std::move(assets_in)), execution(execution_in), + graph_context_bytes(graph_context_bytes_in), + weights(load_weights(*assets, execution, weight_context_bytes, + storage_type)) {} + + OuteTTSDacDecoder::EncodedReference + encode_reference(const runtime::AudioBuffer &input_audio) { + auto feature_samples = load_reference_audio(input_audio); + auto samples = prepare_reference_audio(feature_samples); + const int64_t sample_count = static_cast(samples.size()); + ggml_init_params params{graph_context_bytes, nullptr, true}; + std::unique_ptr ctx(ggml_init(params)); + if (!ctx) { + throw std::runtime_error( + "failed to create OuteTTS DAC encoder graph context"); + } + core::ModuleBuildContext build{ctx.get(), "outetts.dac.encode"}; + auto *pcm = + ggml_new_tensor_3d(ctx.get(), GGML_TYPE_F32, sample_count, 1, 1); + auto x = core::wrap_tensor( + pcm, core::TensorShape::from_dims({1, 1, sample_count}), GGML_TYPE_F32); + x = conv(build, x, weights.encoder_first, 3); + for (const auto &block : weights.encoder_blocks) { + x = residual(build, x, block.residuals[0], 1); + x = residual(build, x, block.residuals[1], 3); + x = residual(build, x, block.residuals[2], 9); + x = snake(build, x, block.snake); + x = strided_conv(build, x, block.down, block.stride, + (block.stride + 1) / 2); + } + x = snake(build, x, weights.encoder_final_snake); + x = conv(build, x, weights.encoder_final_conv, 1); + x = core::ensure_backend_addressable_layout(build, x); + ggml_set_output(x.tensor); + auto *graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, x.tensor); + ggml_gallocr_t allocator = ggml_gallocr_new( + ggml_backend_get_default_buffer_type(execution.backend())); + if (allocator == nullptr || !ggml_gallocr_reserve(allocator, graph) || + !ggml_gallocr_alloc_graph(allocator, graph)) { + if (allocator != nullptr) + ggml_gallocr_free(allocator); + throw std::runtime_error("failed to allocate OuteTTS DAC encoder graph"); + } + ggml_backend_tensor_set(pcm, samples.data(), 0, + samples.size() * sizeof(float)); + core::set_backend_threads(execution.backend(), + std::max(1, execution.config().threads)); + const auto status = core::compute_backend_graph(execution.backend(), graph); + ggml_backend_synchronize(execution.backend()); + if (status != GGML_STATUS_SUCCESS) { + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + throw std::runtime_error("OuteTTS DAC encoder graph compute failed"); + } + const int64_t frames = x.shape.dims[2]; + if (x.shape.rank != 3 || x.shape.dims[0] != 1 || x.shape.dims[1] != 1024 || + frames <= 0) { + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + throw std::runtime_error( + "OuteTTS DAC encoder produced an unexpected shape"); + } + std::vector channel_major(static_cast(frames) * 1024u); + ggml_backend_tensor_get(x.tensor, channel_major.data(), 0, + channel_major.size() * sizeof(float)); + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + + std::vector frame_major(channel_major.size()); +#ifdef _OPENMP +#pragma omp parallel for +#endif + for (int64_t frame = 0; frame < frames; ++frame) { + for (int64_t channel = 0; channel < 1024; ++channel) { + frame_major[static_cast(frame * 1024 + channel)] = + channel_major[static_cast(channel * frames + frame)]; + } + } + auto codes = quantize_reference(frame_major, weights.host_quantizers); + OuteTTSDacDecoder::EncodedReference result; + // OuteTTS derives conditioning features from the resampled source audio, + // while the DAC encoder receives a separately loudness-normalized copy. + result.samples = std::move(feature_samples); + result.codebook1 = std::move(codes.first); + result.codebook2 = std::move(codes.second); + return result; + } + + runtime::AudioBuffer decode(const std::vector &c1, + const std::vector &c2) { + const int64_t frames = static_cast(std::min(c1.size(), c2.size())); + if (frames <= 0) { + throw std::runtime_error( + "OuteTTS DAC requires at least one complete code pair"); + } + ggml_init_params params{graph_context_bytes, nullptr, true}; + std::unique_ptr ctx(ggml_init(params)); + if (!ctx) { + throw std::runtime_error("failed to create OuteTTS DAC graph context"); + } + core::ModuleBuildContext build{ctx.get(), "outetts.dac.decode"}; + auto *ids1 = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_I32, frames, 1); + auto *ids2 = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_I32, frames, 1); + auto id_value1 = core::wrap_tensor( + ids1, core::TensorShape::from_dims({1, frames}), GGML_TYPE_I32); + auto id_value2 = core::wrap_tensor( + ids2, core::TensorShape::from_dims({1, frames}), GGML_TYPE_I32); + auto e1 = modules::EmbeddingModule({1024, 8}).build( + build, id_value1, weights.quantizers[0].codebook); + auto e2 = modules::EmbeddingModule({1024, 8}).build( + build, id_value2, weights.quantizers[1].codebook); + e1 = modules::TransposeModule({{0, 2, 1}, 3}).build(build, e1); + e2 = modules::TransposeModule({{0, 2, 1}, 3}).build(build, e2); + auto z1 = conv(build, core::ensure_backend_addressable_layout(build, e1), + weights.quantizers[0].out_proj, 0); + auto z2 = conv(build, core::ensure_backend_addressable_layout(build, e2), + weights.quantizers[1].out_proj, 0); + auto x = modules::AddModule{}.build(build, z1, z2); + x = conv(build, x, weights.first, 3); + for (const auto &block : weights.blocks) { + x = snake(build, x, block.snake); + auto upsampled = modules::ConvTranspose1dModule({ + block.up.in_channels, + block.up.out_channels, + block.up.kernel, + block.stride, + 0, + 1, + true, + }) + .build(build, x, block.up.value); + const int padding = static_cast(std::ceil(block.stride / 2.0)); + const int64_t target_frames = upsampled.shape.dims[2] - 2 * padding; + x = modules::SliceModule({2, padding, target_frames}) + .build(build, upsampled); + x = residual(build, x, block.residuals[0], 1); + x = residual(build, x, block.residuals[1], 3); + x = residual(build, x, block.residuals[2], 9); + } + x = snake(build, x, weights.final_snake); + x = conv(build, x, weights.final_conv, 3); + x = modules::TanhModule{}.build(build, x); + x = core::ensure_backend_addressable_layout(build, x); + ggml_set_output(x.tensor); + auto *graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, x.tensor); + ggml_gallocr_t allocator = ggml_gallocr_new( + ggml_backend_get_default_buffer_type(execution.backend())); + if (allocator == nullptr || !ggml_gallocr_reserve(allocator, graph) || + !ggml_gallocr_alloc_graph(allocator, graph)) { + if (allocator != nullptr) + ggml_gallocr_free(allocator); + throw std::runtime_error("failed to allocate OuteTTS DAC graph"); + } + ggml_backend_tensor_set(ids1, c1.data(), 0, + static_cast(frames) * sizeof(int32_t)); + ggml_backend_tensor_set(ids2, c2.data(), 0, + static_cast(frames) * sizeof(int32_t)); + core::set_backend_threads(execution.backend(), + std::max(1, execution.config().threads)); + const auto status = core::compute_backend_graph(execution.backend(), graph); + ggml_backend_synchronize(execution.backend()); + if (status != GGML_STATUS_SUCCESS) { + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + throw std::runtime_error("OuteTTS DAC graph compute failed"); + } + runtime::AudioBuffer audio; + audio.sample_rate = 24000; + audio.channels = 1; + audio.samples.resize(static_cast(x.shape.dims[2])); + ggml_backend_tensor_get(x.tensor, audio.samples.data(), 0, + audio.samples.size() * sizeof(float)); + normalize_decoded_audio(audio.samples); + core::release_backend_graph_resources(execution.backend(), graph); + ggml_gallocr_free(allocator); + return audio; + } + + std::shared_ptr assets; + core::ExecutionContext &execution; + size_t graph_context_bytes; + DacWeights weights; +}; + +OuteTTSDacDecoder::OuteTTSDacDecoder( + std::shared_ptr assets, + core::ExecutionContext &execution_context, size_t weight_context_bytes, + size_t graph_context_bytes, assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique(std::move(assets), execution_context, + weight_context_bytes, graph_context_bytes, + weight_storage_type)) {} + +OuteTTSDacDecoder::~OuteTTSDacDecoder() = default; + +runtime::AudioBuffer +OuteTTSDacDecoder::decode(const std::vector &codebook1, + const std::vector &codebook2) { + return impl_->decode(codebook1, codebook2); +} + +OuteTTSDacDecoder::EncodedReference +OuteTTSDacDecoder::encode_reference(const runtime::AudioBuffer &audio) { + return impl_->encode_reference(audio); +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/llama.cpp b/src/models/outetts/llama.cpp new file mode 100644 index 000000000..541b12dc4 --- /dev/null +++ b/src/models/outetts/llama.cpp @@ -0,0 +1,507 @@ +#include "engine/models/outetts/llama.h" + +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/attention/qwen_causal_decoder.h" +#include "engine/framework/modules/lookup_modules.h" +#include "engine/framework/modules/weight_binding.h" +#include "../common/constant_tensor_cache.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::outetts { +namespace { + +namespace binding = modules::binding; + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +struct LayerWeights { + assets::TensorDataF32 input_norm; + modules::AttentionWeights attention; + assets::TensorDataF32 post_norm; + modules::LinearWeights gate; + modules::LinearWeights up; + modules::LinearWeights down; +}; + +struct ModelWeights { + std::shared_ptr store; + core::TensorValue embedding; + core::TensorValue lm_head; + core::TensorValue rope_factors; + std::vector layers; + assets::TensorDataF32 norm; +}; + +struct PrefillOutput { + std::vector logits; + runtime::TransformerKVState state; +}; + +std::vector llama3_rope_factors(const OuteTTSConfig & config) { + constexpr double pi = 3.14159265358979323846; + const auto & scaling = config.rope_scaling; + const double low_wavelength = static_cast(scaling.original_max_position_embeddings) / + static_cast(scaling.low_freq_factor); + const double high_wavelength = static_cast(scaling.original_max_position_embeddings) / + static_cast(scaling.high_freq_factor); + std::vector factors(static_cast(config.head_dim / 2), 1.0F); + for (int64_t i = 0; i < config.head_dim / 2; ++i) { + const double inv_freq = 1.0 / std::pow( + static_cast(config.rope_theta), + static_cast(2 * i) / static_cast(config.head_dim)); + const double wavelength = 2.0 * pi / inv_freq; + double scaled = inv_freq; + if (wavelength > low_wavelength) { + scaled = inv_freq / static_cast(scaling.factor); + } else if (wavelength >= high_wavelength) { + const double smooth = + (static_cast(scaling.original_max_position_embeddings) / wavelength - + static_cast(scaling.low_freq_factor)) / + (static_cast(scaling.high_freq_factor) - + static_cast(scaling.low_freq_factor)); + scaled = (1.0 - smooth) * inv_freq / static_cast(scaling.factor) + smooth * inv_freq; + } + // ggml divides its base inverse frequency by this tensor. This matches + // llama.cpp's rope_freqs.weight representation for Llama-3 scaling. + factors[static_cast(i)] = static_cast(inv_freq / scaled); + } + return factors; +} + +ModelWeights load_weights( + const OuteTTSAssets & assets, + ggml_backend_t backend, + core::BackendType backend_type, + size_t context_bytes, + assets::TensorStorageType storage_type) { + const auto & c = assets.config; + const auto & source = *assets.model_weights; + ModelWeights out; + out.store = std::make_shared( + backend, backend_type, "outetts.llama.weights", context_bytes); + out.embedding = out.store->load_tensor( + source, "model.embed_tokens.weight", storage_type, {c.vocab_size, c.hidden_size}); + out.lm_head = out.embedding; + out.rope_factors = out.store->make_from_f32( + core::TensorShape::from_dims({c.head_dim / 2}), + assets::TensorStorageType::F32, + llama3_rope_factors(c)); + out.layers.reserve(static_cast(c.num_hidden_layers)); + for (int64_t i = 0; i < c.num_hidden_layers; ++i) { + const std::string p = "model.layers." + std::to_string(i); + LayerWeights layer; + layer.input_norm = source.require_f32_tensor(p + ".input_layernorm.weight", {c.hidden_size}); + layer.attention.q_weight = out.store->load_tensor( + source, p + ".self_attn.q_proj.weight", storage_type, + {c.num_attention_heads * c.head_dim, c.hidden_size}); + layer.attention.k_weight = out.store->load_tensor( + source, p + ".self_attn.k_proj.weight", storage_type, + {c.num_key_value_heads * c.head_dim, c.hidden_size}); + layer.attention.v_weight = out.store->load_tensor( + source, p + ".self_attn.v_proj.weight", storage_type, + {c.num_key_value_heads * c.head_dim, c.hidden_size}); + layer.attention.out_weight = out.store->load_tensor( + source, p + ".self_attn.o_proj.weight", storage_type, + {c.hidden_size, c.num_attention_heads * c.head_dim}); + layer.post_norm = source.require_f32_tensor(p + ".post_attention_layernorm.weight", {c.hidden_size}); + layer.gate.weight = out.store->load_tensor( + source, p + ".mlp.gate_proj.weight", storage_type, {c.intermediate_size, c.hidden_size}); + layer.up.weight = out.store->load_tensor( + source, p + ".mlp.up_proj.weight", storage_type, {c.intermediate_size, c.hidden_size}); + layer.down.weight = out.store->load_tensor( + source, p + ".mlp.down_proj.weight", storage_type, {c.hidden_size, c.intermediate_size}); + out.layers.push_back(std::move(layer)); + } + out.norm = source.require_f32_tensor("model.norm.weight", {c.hidden_size}); + out.store->upload(); + return out; +} + +modules::QwenCausalDecoderConfig decoder_config(const OuteTTSConfig & c) { + modules::QwenCausalDecoderConfig out; + out.stack.hidden_size = c.hidden_size; + out.stack.intermediate_size = c.intermediate_size; + out.stack.num_attention_heads = c.num_attention_heads; + out.stack.num_key_value_heads = c.num_key_value_heads; + out.stack.head_dim = c.head_dim; + out.stack.layers = c.num_hidden_layers; + out.stack.rms_norm_eps = c.rms_norm_eps; + out.stack.rope_theta = c.rope_theta; + // Hugging Face Llama weights use split-half rotary pairs. llama.cpp's + // dedicated Llama converter permutes Q/K and then uses NORMAL RoPE, while + // audio.cpp preserves the source tensor layout in safetensors and GGUF. + out.stack.rope_type = GGML_ROPE_TYPE_NEOX; + out.stack.use_qk_norm = false; + out.stack.runtime.attention.prefill_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.attention.static_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.static_cache.update_mode = modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; + out.stack.runtime.static_cache.transpose_context = true; + out.logits_size = c.vocab_size; + return out; +} + +modules::QwenCausalDecoderWeights graph_weights( + const ModelWeights & weights, + common::ConstantTensorCache & constants) { + modules::QwenCausalDecoderWeights out; + out.stack.layers.reserve(weights.layers.size()); + for (const auto & source : weights.layers) { + modules::QwenDecoderLayerWeights layer; + layer.input_norm = binding::norm_data(constants, source.input_norm); + layer.self_attention = source.attention; + layer.post_norm = binding::norm_data(constants, source.post_norm); + layer.mlp.gate_proj = source.gate; + layer.mlp.up_proj = source.up; + layer.mlp.down_proj = source.down; + layer.rope_frequency_factors = weights.rope_factors; + out.stack.layers.push_back(std::move(layer)); + } + out.final_norm = binding::norm_data(constants, weights.norm); + out.lm_head.weight = weights.lm_head; + return out; +} + +void apply_repetition_penalty( + std::vector & logits, + const std::vector & ids, + int64_t window, + float penalty) { + if (penalty == 1.0F || window == 0) { + return; + } + const size_t begin = ids.size() > static_cast(window) ? ids.size() - static_cast(window) : 0; + std::unordered_set seen(ids.begin() + static_cast(begin), ids.end()); + for (const int32_t id : seen) { + if (id < 0 || static_cast(id) >= logits.size()) { + continue; + } + float & value = logits[static_cast(id)]; + value = value <= 0.0F ? value * penalty : value / penalty; + } +} + +int32_t sample_token(std::vector logits, const OuteTTSGenerateOptions & o, std::mt19937 & rng) { + if (!(o.temperature > 0.0F) || !std::isfinite(o.temperature)) { + return static_cast(std::max_element(logits.begin(), logits.end()) - logits.begin()); + } + std::vector order(logits.size()); + std::iota(order.begin(), order.end(), 0); + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return logits[a] > logits[b]; }); + if (o.top_k > 0 && static_cast(o.top_k) < order.size()) { + order.resize(static_cast(o.top_k)); + } + const float max_logit = logits[order.front()]; + std::vector probabilities(order.size(), 0.0F); + double sum = 0.0; + for (size_t i = 0; i < order.size(); ++i) { + probabilities[i] = std::exp((logits[order[i]] - max_logit) / o.temperature); + sum += probabilities[i]; + } + for (float & probability : probabilities) { + probability = static_cast(probability / sum); + } + const float max_probability = probabilities.front(); + float cumulative = 0.0F; + size_t kept = 0; + for (const float probability : probabilities) { + if (probability < max_probability * o.min_p && kept > 0) { + break; + } + cumulative += probability; + ++kept; + if (o.top_p < 1.0F && cumulative >= o.top_p) { + break; + } + } + order.resize(std::max(1, kept)); + probabilities.resize(order.size()); + std::discrete_distribution distribution(probabilities.begin(), probabilities.end()); + return static_cast(order[static_cast(distribution(rng))]); +} + +class CachedStepGraph { +public: + CachedStepGraph( + const OuteTTSConfig & config, + const ModelWeights & weights, + common::ConstantTensorCache & constants, + ggml_backend_t backend, + int threads, + int64_t capacity) + : config_(config), weights_(&weights), constants_(&constants), backend_(backend), threads_(threads), capacity_(capacity) { + ggml_init_params params{1536ull * 1024ull * 1024ull, nullptr, true}; + ctx_.reset(ggml_init(params)); + if (!ctx_) throw std::runtime_error("failed to create OuteTTS cached-step context"); + core::ModuleBuildContext build{ctx_.get(), "outetts.llama.cached_step"}; + input_id_ = ggml_new_tensor_2d(ctx_.get(), GGML_TYPE_I32, 1, 1); + auto id = core::wrap_tensor(input_id_, core::TensorShape::from_dims({1, 1}), GGML_TYPE_I32); + auto x = modules::EmbeddingModule({config_.vocab_size, config_.hidden_size}).build(build, id, weights_->embedding); + positions_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, 1); + cache_slot_ = ggml_new_tensor_1d(ctx_.get(), GGML_TYPE_I32, 1); + mask_ = ggml_new_tensor_4d(ctx_.get(), GGML_TYPE_F16, capacity_, 1, 1, 1); + auto position = core::wrap_tensor(positions_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + auto slot = core::wrap_tensor(cache_slot_, core::TensorShape::from_dims({1}), GGML_TYPE_I32); + auto mask = core::wrap_tensor(mask_, core::TensorShape::from_dims({1, 1, 1, capacity_}), GGML_TYPE_F16); + graph_ = ggml_new_graph_custom(ctx_.get(), 65536, false); + constants_->begin_graph(); + auto output = modules::QwenCausalDecoderModule(decoder_config(config_)).build_static_cache_tail( + build, graph_, x, position, graph_weights(*weights_, *constants_), capacity_, mask, slot); + cache_ = std::move(output.cache); + logits_ = output.logits.tensor; + ggml_set_output(logits_); + ggml_build_forward_expand(graph_, logits_); + constants_->finish_graph(); + constants_->ensure_uploaded(); + buffer_ = ggml_backend_alloc_ctx_tensors(ctx_.get(), backend_); + if (buffer_ == nullptr) throw std::runtime_error("failed to allocate OuteTTS cached-step graph"); + mask_values_.assign(static_cast(capacity_), ggml_fp32_to_fp16(-std::numeric_limits::infinity())); + } + + ~CachedStepGraph() { + core::release_backend_graph_resources(backend_, graph_); + if (buffer_ != nullptr) ggml_backend_buffer_free(buffer_); + } + + void import_state(const runtime::TransformerKVState & state) { cache_.import_state(state); } + + std::vector run(int32_t token) { + if (cache_.valid_steps() >= capacity_) throw std::runtime_error("OuteTTS cached-step capacity exceeded"); + ggml_backend_tensor_set(input_id_, &token, 0, sizeof(token)); + const int32_t position = static_cast(cache_.current_end()); + const int32_t slot = static_cast(cache_.valid_steps()); + ggml_backend_tensor_set(positions_, &position, 0, sizeof(position)); + ggml_backend_tensor_set(cache_slot_, &slot, 0, sizeof(slot)); + modules::write_qwen_cached_step_mask(mask_, mask_values_, capacity_, cache_.valid_steps(), cache_.valid_steps()); + core::set_backend_threads(backend_, threads_); + const auto status = core::compute_backend_graph(backend_, graph_); + ggml_backend_synchronize(backend_); + if (status != GGML_STATUS_SUCCESS) throw std::runtime_error("OuteTTS cached-step graph compute failed"); + std::vector logits(static_cast(config_.vocab_size)); + ggml_backend_tensor_get(logits_, logits.data(), 0, logits.size() * sizeof(float)); + cache_.advance_after_direct_append(1); + return logits; + } + +private: + OuteTTSConfig config_; + const ModelWeights * weights_ = nullptr; + common::ConstantTensorCache * constants_ = nullptr; + ggml_backend_t backend_ = nullptr; + int threads_ = 1; + int64_t capacity_ = 0; + std::unique_ptr ctx_; + ggml_tensor * input_id_ = nullptr; + ggml_tensor * positions_ = nullptr; + ggml_tensor * cache_slot_ = nullptr; + ggml_tensor * mask_ = nullptr; + ggml_tensor * logits_ = nullptr; + runtime::TransformerKVCache cache_; + std::vector mask_values_; + ggml_cgraph * graph_ = nullptr; + ggml_backend_buffer_t buffer_ = nullptr; +}; + +} // namespace + +struct OuteTTSLlamaRuntime::Impl { + Impl( + std::shared_ptr assets_in, + core::BackendType backend_type, + int device, + int threads_in, + size_t weight_context_bytes, + size_t constant_context_bytes, + assets::TensorStorageType storage_type) + : assets(std::move(assets_in)), threads(std::max(1, threads_in)) { + if (assets == nullptr) { + throw std::runtime_error("OuteTTS Llama runtime requires assets"); + } + backend = core::init_backend({backend_type, device, threads}); + weights = load_weights(*assets, backend, backend_type, weight_context_bytes, storage_type); + constants = std::make_unique( + backend, threads, "outetts.llama.constants", constant_context_bytes); + } + + ~Impl() { + constants.reset(); + weights.store.reset(); + if (backend != nullptr) { + ggml_backend_free(backend); + } + } + + PrefillOutput prefill(const std::vector & ids) const { + const auto & c = assets->config; + const int64_t steps = static_cast(ids.size()); + // This correctness-first graph uses full-sequence prefill. A cached-step + // graph can replace it without changing weights, sampling, or package layout. + const size_t arena = std::max( + 1024ull * 1024ull * 1024ull, + static_cast(steps) * static_cast(steps) * 256ull + 512ull * 1024ull * 1024ull); + ggml_init_params params{arena, nullptr, true}; + std::unique_ptr ctx(ggml_init(params)); + if (!ctx) { + throw std::runtime_error("failed to create OuteTTS Llama graph context"); + } + core::ModuleBuildContext build{ctx.get(), "outetts.llama.prefill"}; + auto * ids_tensor = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_I32, steps, 1); + auto ids_value = core::wrap_tensor(ids_tensor, core::TensorShape::from_dims({1, steps}), GGML_TYPE_I32); + auto x = modules::EmbeddingModule({c.vocab_size, c.hidden_size}) + .build(build, ids_value, weights.embedding); + auto * positions = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_I32, steps); + auto position_value = core::wrap_tensor(positions, core::TensorShape::from_dims({steps}), GGML_TYPE_I32); + auto * mask = ggml_new_tensor_4d(ctx.get(), GGML_TYPE_F16, steps, steps, 1, 1); + auto mask_value = core::wrap_tensor( + mask, core::TensorShape::from_dims({1, 1, steps, steps}), GGML_TYPE_F16); + constants->begin_graph(); + auto output = modules::QwenCausalDecoderModule(decoder_config(c)).build( + build, x, position_value, graph_weights(weights, *constants), std::nullopt, mask_value); + std::vector keys; + std::vector values; + keys.reserve(output.state.layers.size()); + values.reserve(output.state.layers.size()); + for (const auto & layer : output.state.layers) { + if (!layer.key.has_value() || !layer.value.has_value()) { + throw std::runtime_error("OuteTTS prefill did not produce K/V state"); + } + auto * key = ggml_cpy(ctx.get(), layer.key->tensor, ggml_dup_tensor(ctx.get(), layer.key->tensor)); + auto * value = ggml_cpy(ctx.get(), layer.value->tensor, ggml_dup_tensor(ctx.get(), layer.value->tensor)); + ggml_set_output(key); + ggml_set_output(value); + keys.push_back(key); + values.push_back(value); + } + auto * logits = output.logits.tensor; + ggml_set_output(logits); + auto * graph = ggml_new_graph_custom(ctx.get(), 65536, false); + for (auto * key : keys) ggml_build_forward_expand(graph, key); + for (auto * value : values) ggml_build_forward_expand(graph, value); + ggml_build_forward_expand(graph, logits); + constants->finish_graph(); + constants->ensure_uploaded(); + ggml_gallocr_t allocator = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (allocator == nullptr || !ggml_gallocr_reserve(allocator, graph) || !ggml_gallocr_alloc_graph(allocator, graph)) { + if (allocator != nullptr) { + ggml_gallocr_free(allocator); + } + throw std::runtime_error("failed to allocate OuteTTS Llama graph"); + } + const auto position_values = modules::qwen_position_ids(steps); + const auto mask_values = modules::qwen_causal_prefill_mask_values(1, steps); + ggml_backend_tensor_set(ids_tensor, ids.data(), 0, ids.size() * sizeof(int32_t)); + ggml_backend_tensor_set(positions, position_values.data(), 0, position_values.size() * sizeof(int32_t)); + ggml_backend_tensor_set(mask, mask_values.data(), 0, mask_values.size() * sizeof(ggml_fp16_t)); + core::set_backend_threads(backend, threads); + const auto status = core::compute_backend_graph(backend, graph); + ggml_backend_synchronize(backend); + if (status != GGML_STATUS_SUCCESS) { + core::release_backend_graph_resources(backend, graph); + ggml_gallocr_free(allocator); + throw std::runtime_error("OuteTTS Llama graph compute failed"); + } + PrefillOutput result; + result.logits.resize(static_cast(c.vocab_size)); + ggml_backend_tensor_get(logits, result.logits.data(), 0, result.logits.size() * sizeof(float)); + result.state.current_end = steps; + result.state.layers.resize(keys.size()); + const size_t layer_elements = static_cast(steps * c.num_key_value_heads * c.head_dim); + for (size_t layer = 0; layer < keys.size(); ++layer) { + auto & state = result.state.layers[layer]; + state.valid_steps = steps; + state.key.resize(layer_elements); + state.value.resize(layer_elements); + ggml_backend_tensor_get(keys[layer], state.key.data(), 0, layer_elements * sizeof(float)); + ggml_backend_tensor_get(values[layer], state.value.data(), 0, layer_elements * sizeof(float)); + } + core::release_backend_graph_resources(backend, graph); + ggml_gallocr_free(allocator); + return result; + } + + std::shared_ptr assets; + ggml_backend_t backend = nullptr; + int threads = 1; + ModelWeights weights; + std::unique_ptr constants; +}; + +OuteTTSLlamaRuntime::OuteTTSLlamaRuntime( + std::shared_ptr assets, + core::BackendType backend_type, + int device, + int threads, + size_t weight_context_bytes, + size_t constant_context_bytes, + assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique( + std::move(assets), backend_type, device, threads, + weight_context_bytes, constant_context_bytes, weight_storage_type)) {} + +OuteTTSLlamaRuntime::~OuteTTSLlamaRuntime() = default; + +std::vector OuteTTSLlamaRuntime::generate( + const std::vector & prompt, + const OuteTTSGenerateOptions & options, + int32_t eos_id, + int32_t audio_end_id) const { + if (prompt.empty()) { + throw std::runtime_error("OuteTTS generation requires a prompt"); + } + if (options.max_new_tokens <= 0 || options.repetition_window < 0 || options.repetition_penalty <= 0.0F) { + throw std::runtime_error("invalid OuteTTS generation options"); + } + std::vector all = prompt; + std::vector generated; + generated.reserve(static_cast(options.max_new_tokens)); + std::mt19937 rng(options.seed); + OuteTTSGenerateOptions sampling_options = options; + auto prefill = impl_->prefill(prompt); + const int64_t capacity = std::min( + impl_->assets->generation.max_length, + static_cast(prompt.size()) + options.max_new_tokens); + CachedStepGraph step( + impl_->assets->config, + impl_->weights, + *impl_->constants, + impl_->backend, + impl_->threads, + capacity); + step.import_state(prefill.state); + std::vector logits = std::move(prefill.logits); + for (int64_t i = 0; i < options.max_new_tokens; ++i) { + if (static_cast(all.size()) >= impl_->assets->generation.max_length) { + break; + } + apply_repetition_penalty( + logits, all, options.repetition_window, options.repetition_penalty); + const int32_t token = sample_token( + std::move(logits), sampling_options, rng); + generated.push_back(token); + if (token == eos_id || token == audio_end_id) { + break; + } + all.push_back(token); + logits = step.run(token); + } + return generated; +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/loader.cpp b/src/models/outetts/loader.cpp new file mode 100644 index 000000000..b0f60ebd4 --- /dev/null +++ b/src/models/outetts/loader.cpp @@ -0,0 +1,135 @@ +#include "engine/models/outetts/loader.h" + +#include "engine/framework/assets/model_package.h" +#include "engine/models/outetts/session.h" + +namespace engine::models::outetts { +namespace { + +std::filesystem::path spec_path() { + return assets::default_model_package_spec_path("outetts"); +} + +runtime::CapabilitySet make_capabilities() { + runtime::CapabilitySet out; + out.supported_tasks = { + {runtime::VoiceTaskKind::Tts, {runtime::RunMode::Offline}}, + {runtime::VoiceTaskKind::VoiceCloning, {runtime::RunMode::Offline}}, + }; + out.supports_speaker_reference = true; + out.languages = { + "Auto", "Arabic", "Belarusian", "Bengali", "Chinese", + "Dutch", "English", "French", "Georgian", "German", + "Hungarian", "Italian", "Japanese", "Korean", "Latvian", + "Lithuanian", "Persian", "Polish", "Portuguese", "Russian", + "Spanish", "Swahili", "Tamil", "Ukrainian", + }; + return out; +} + +class Loader final : public runtime::IVoiceModelLoader { +public: + std::string family() const override { return "outetts"; } + bool can_load(const runtime::ModelLoadRequest &request) const override { + if (request.family_hint.has_value() && *request.family_hint != family()) + return false; + try { + (void)assets::load_resource_bundle_from_package_spec(request.model_path, + spec_path()); + return true; + } catch (...) { + return false; + } + } + runtime::ModelInspection + inspect(const runtime::ModelLoadRequest &request) const override { + const auto model_assets = load_outetts_assets(request.model_path); + runtime::ModelInspection out; + out.model_root = model_assets->resources.model_root(); + out.metadata.family = family(); + out.metadata.variant = "1.0-1B"; + out.metadata.description = "OuteTTS 1.0 1B with native IBM DAC speech " + "synthesis and voice cloning."; + out.capabilities = make_capabilities(); + out.cli.request_options = { + {"max_tokens", "n", "Maximum generated text/audio tokens."}, + {"temperature", "float", + "Sampling temperature; official cloning default 0.4."}, + {"top_k", "n", "Top-k sampling; official default 40."}, + {"top_p", "float", "Nucleus sampling; official default 0.9."}, + {"min_p", "float", + "Minimum probability relative to the best token; official default " + "0.05."}, + {"repetition_penalty", "float", + "Windowed repetition penalty; official default 1.1."}, + {"repetition_window", "n", + "Recent-token penalty window; official value 64."}, + {"seed", "n", + "Sampling seed; cloning defaults to 4099 for native weights and " + "42 for quantized weights."}, + {"reference_text", "text", + "Transcript matching the --voice-ref audio for voice cloning."}, + {"reference_language", "code", + "Language code used to align the reference transcript; default en."}, + }; + out.cli.session_options = { + {"outetts.weight_type", "native|f32|f16|bf16|q8_0", + "Language-model weight storage type. Quantized CUDA voice cloning " + "is expanded to F32 in memory for generation correctness."}, + {"outetts.llama_weight_context_mb", "n", + "Language-model weight context size in MiB."}, + {"outetts.constant_context_mb", "n", + "Language-model constant tensor context size in MiB."}, + {"outetts.dac_weight_context_mb", "n", + "DAC decoder weight context size in MiB."}, + {"outetts.dac_graph_context_mb", "n", + "DAC decoder graph context size in MiB."}, + {"outetts.aligner_model_path", "path", + "Optional Qwen3 Forced Aligner override. Cloning automatically uses " + "the aligner embedded in a standalone OuteTTS GGUF when present."}, + }; + out.discovered_configs = runtime::discover_named_assets_from_package_spec( + request.model_path, spec_path(), + assets::ModelPackageResourceKind::Files); + out.discovered_weights = runtime::discover_named_assets_from_package_spec( + request.model_path, spec_path(), + assets::ModelPackageResourceKind::Tensors); + return out; + } + std::unique_ptr + load(const runtime::ModelLoadRequest &request) const override { + return std::make_unique( + load_outetts_assets(request.model_path)); + } +}; + +} // namespace + +OuteTTSLoadedModel::OuteTTSLoadedModel( + std::shared_ptr assets) + : assets_(std::move(assets)), capabilities_(make_capabilities()) { + metadata_.family = "outetts"; + metadata_.variant = "1.0-1B"; + metadata_.description = + "OuteTTS 1.0 1B with native IBM DAC speech synthesis and voice cloning."; +} + +const runtime::ModelMetadata &OuteTTSLoadedModel::metadata() const noexcept { + return metadata_; +} +const runtime::CapabilitySet & +OuteTTSLoadedModel::capabilities() const noexcept { + return capabilities_; +} +std::unique_ptr +OuteTTSLoadedModel::create_task_session( + const runtime::TaskSpec &task, + const runtime::SessionOptions &options) const { + return std::make_unique(task, options, assets_); +} + +std::shared_ptr make_outetts_loader() { + return std::make_shared(); +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/session.cpp b/src/models/outetts/session.cpp new file mode 100644 index 000000000..c12326858 --- /dev/null +++ b/src/models/outetts/session.cpp @@ -0,0 +1,509 @@ +#include "engine/models/outetts/session.h" + +#include "engine/framework/audio/fft.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/debug/trace.h" +#include "engine/models/qwen3_asr/assets.h" +#include "engine/models/qwen3_forced_aligner/session.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::models::outetts { +namespace { + +assets::TensorStorageType requested_weight_type( + const runtime::SessionOptions &options) { + const auto it = options.options.find("outetts.weight_type"); + return it == options.options.end() + ? assets::TensorStorageType::Native + : assets::parse_tensor_storage_type(it->second); +} + +bool has_quantized_clone_weights(const runtime::SessionOptions &options, + const OuteTTSAssets &model_assets) { + constexpr std::string_view probe = + "model.layers.0.self_attn.q_proj.weight"; + const auto source_type = assets::tensor_storage_type_for_dtype( + model_assets.model_weights->require_metadata(probe).dtype); + const auto requested_type = assets::resolve_tensor_storage_type( + *model_assets.model_weights, probe, requested_weight_type(options)); + return ggml_is_quantized( + assets::ggml_type_for_tensor_storage(source_type)) || + ggml_is_quantized( + assets::ggml_type_for_tensor_storage(requested_type)); +} + +assets::TensorStorageType clone_weight_type( + const runtime::SessionOptions &options, + const OuteTTSAssets &model_assets) { + if (options.backend.type == core::BackendType::Cuda && + has_quantized_clone_weights(options, model_assets)) { + // CUDA execution with quantized OuteTTS weights diverges over the long + // reference-codec prompt used for cloning. The GGUF stays quantized on + // disk; only the in-memory language-model tensors are expanded to F32. + // F16 still produces phonetic but unintelligible speech from Q8 source + // tensors on this route, while F32 matches the coherent CPU decode. + debug::trace_log_scalar("outetts.cuda_clone_quantized_f32_fallback", true); + return assets::TensorStorageType::F32; + } + return requested_weight_type(options); +} + +OuteTTSGenerateOptions +generation_options(const runtime::TaskRequest &request, + const OuteTTSGenerationConfig &defaults, + bool voice_cloning, + bool quantized_cloning) { + OuteTTSGenerateOptions out; + out.temperature = voice_cloning ? 0.4F : defaults.temperature; + out.repetition_penalty = defaults.repetition_penalty; + out.repetition_window = defaults.repetition_window; + out.top_k = voice_cloning ? 40 : defaults.top_k; + out.top_p = voice_cloning ? 0.9F : defaults.top_p; + out.min_p = voice_cloning ? 0.05F : defaults.min_p; + if (const auto v = runtime::parse_i64_option(request.options, {"max_tokens"})) + out.max_new_tokens = *v; + if (const auto v = + runtime::parse_finite_float_option(request.options, {"temperature"})) + out.temperature = *v; + if (const auto v = runtime::parse_finite_float_option(request.options, + {"repetition_penalty"})) + out.repetition_penalty = *v; + if (const auto v = + runtime::parse_i64_option(request.options, {"repetition_window"})) + out.repetition_window = *v; + if (const auto v = runtime::parse_i64_option(request.options, {"top_k"})) + out.top_k = *v; + if (const auto v = + runtime::parse_finite_float_option(request.options, {"top_p"})) + out.top_p = *v; + if (const auto v = + runtime::parse_finite_float_option(request.options, {"min_p"})) + out.min_p = *v; + out.seed = runtime::parse_u32_option(request.options, {"seed"}) + .value_or(voice_cloning + ? (quantized_cloning ? 42u : 4099u) + : runtime::random_u32_seed()); + if (out.max_new_tokens <= 0 || out.repetition_window < 0 || out.top_k < 0 || + out.temperature < 0.0F || out.repetition_penalty <= 0.0F || + out.top_p <= 0.0F || out.top_p > 1.0F || out.min_p < 0.0F || + out.min_p > 1.0F) { + throw std::runtime_error("invalid OuteTTS generation options"); + } + return out; +} + +std::vector split_words(const std::string &text) { + std::istringstream input(text); + std::vector words; + std::string word; + while (input >> word) + words.push_back(word); + return words; +} + +size_t utf8_length(const std::string &text) { + return std::max( + 1, static_cast( + std::count_if(text.begin(), text.end(), [](unsigned char value) { + return (value & 0xc0) != 0x80; + }))); +} + +OuteTTSVoiceFeatures audio_features(const std::vector &samples, + size_t begin, size_t end) { + OuteTTSVoiceFeatures result; + if (begin >= end || begin >= samples.size()) + return result; + end = std::min(end, samples.size()); + double sum_sq = 0.0; + for (size_t i = begin; i < end; ++i) + sum_sq += static_cast(samples[i]) * samples[i]; + const double rms = std::sqrt(sum_sq / static_cast(end - begin)); + result.energy = + static_cast(std::clamp(std::lround(rms * 100.0), 0l, 100l)); + + const size_t count = end - begin; + std::vector> spectrum(count / 2u + 1u); + const auto fft = engine::audio::get_real_fft_plan(count); + fft->forward({count}, {static_cast(sizeof(float))}, + {static_cast(sizeof(std::complex))}, 0, + samples.data() + static_cast(begin), + spectrum.data()); + double magnitude_sum = 1.0e-10; + double weighted_frequency = 0.0; + for (size_t bin = 0; bin < spectrum.size(); ++bin) { + const double magnitude = std::abs(spectrum[bin]); + magnitude_sum += magnitude; + weighted_frequency += magnitude * static_cast(bin) * + 24000.0 / static_cast(count); + } + result.spectral_centroid = static_cast(std::clamp( + std::lround(weighted_frequency / magnitude_sum / 12000.0 * 100.0), + 0l, 100l)); + + if (count >= 400 && sum_sq >= 1.0e-8) { + constexpr int frame_length = 400; + constexpr int hop_length = 160; + constexpr int min_lag = 24000 / 600; + constexpr int max_lag_exclusive = 24000 / 75; + const size_t pad = + (frame_length - (count % hop_length)) % hop_length; + const size_t padded_count = count + pad; + const size_t frames = 1u + (padded_count - frame_length) / hop_length; + double pitch_sum = 0.0; + std::vector windowed(frame_length); + std::vector autocorrelation(frame_length); + for (size_t frame = 0; frame < frames; ++frame) { + const size_t offset = begin + frame * hop_length; + for (int i = 0; i < frame_length; ++i) { + const size_t source = offset + static_cast(i); + const double sample = source < end ? samples[source] : 0.0; + const double window = + 0.5 - 0.5 * std::cos(2.0 * 3.14159265358979323846 * i / + frame_length); + windowed[static_cast(i)] = sample * window; + } + for (int lag = 0; lag < frame_length; ++lag) { + double value = 0.0; + for (int i = 0; i + lag < frame_length; ++i) + value += windowed[static_cast(i)] * + windowed[static_cast(i + lag)]; + autocorrelation[static_cast(lag)] = value; + } + int best_lag = min_lag; + for (int lag = min_lag + 1; lag < max_lag_exclusive; ++lag) { + if (autocorrelation[static_cast(lag)] > + autocorrelation[static_cast(best_lag)]) + best_lag = lag; + } + double frequency = 75.0; + const double beta = autocorrelation[static_cast(best_lag)]; + if (autocorrelation[0] > 1.0e-10 && + beta / autocorrelation[0] > 0.3) { + const double alpha = + autocorrelation[static_cast(best_lag - 1)]; + const double gamma = + autocorrelation[static_cast(best_lag + 1)]; + const double delta = 0.5 * (alpha - gamma) / + (alpha - 2.0 * beta + gamma + 1.0e-8); + frequency = std::clamp(24000.0 / (best_lag + delta), 75.0, 600.0); + } + pitch_sum += frequency; + } + const double average_pitch = pitch_sum / static_cast(frames); + result.pitch = static_cast(std::clamp( + std::lround((average_pitch - 75.0) / 525.0 * 100.0), 0l, 100l)); + } + return result; +} + +struct ReferenceAlignment { + std::vector words; + int sample_rate = 16000; +}; + +OuteTTSVoiceProfile +make_voice_profile(OuteTTSDacDecoder::EncodedReference encoded, + std::string reference_text, + const ReferenceAlignment *alignment) { + auto words = split_words(reference_text); + if (words.empty()) + throw std::runtime_error("OuteTTS reference_text must not be empty"); + const size_t frame_count = + std::min(encoded.codebook1.size(), encoded.codebook2.size()); + if (frame_count == 0) + throw std::runtime_error( + "OuteTTS DAC encoder produced no reference codec frames"); + if (alignment != nullptr && alignment->words.size() != words.size()) { + words.clear(); + for (const auto &word : alignment->words) + words.push_back(word.word); + } + if (words.size() > frame_count) + words.resize(frame_count); + std::vector weights(words.size()); + size_t total_weight = 0; + for (size_t i = 0; i < words.size(); ++i) { + weights[i] = utf8_length(words[i]); + total_weight += weights[i]; + } + + OuteTTSVoiceProfile profile; + profile.text = reference_text; + profile.global_features = + audio_features(encoded.samples, 0, encoded.samples.size()); + debug::trace_log_scalar("outetts.reference.global.energy", + profile.global_features.energy); + debug::trace_log_scalar("outetts.reference.global.spectral_centroid", + profile.global_features.spectral_centroid); + debug::trace_log_scalar("outetts.reference.global.pitch", + profile.global_features.pitch); + size_t start = 0; + size_t cumulative_weight = 0; + for (size_t word_index = 0; word_index < words.size(); ++word_index) { + size_t feature_begin = 0; + size_t feature_end = 0; + size_t end = 0; + if (alignment != nullptr && word_index < alignment->words.size()) { + const auto &span = alignment->words[word_index].span; + const double begin_seconds = + static_cast(span.start_sample) / alignment->sample_rate; + const double end_seconds = + static_cast(span.end_sample) / alignment->sample_rate; + if (word_index == 0) { + const int64_t aligned_start = + static_cast(begin_seconds * 75.0) - 20; + start = static_cast(std::clamp( + aligned_start, 0, static_cast(frame_count - 1))); + } + int64_t aligned_end = static_cast(end_seconds * 75.0); + if (word_index + 1 == words.size()) + aligned_end += 20; + end = static_cast(std::clamp( + aligned_end, static_cast(start + 1), + static_cast(frame_count))); + feature_begin = static_cast(std::clamp( + static_cast(begin_seconds * 24000.0), 0, + static_cast(encoded.samples.size()))); + feature_end = static_cast(std::clamp( + static_cast(end_seconds * 24000.0), + static_cast(feature_begin), + static_cast(encoded.samples.size()))); + } else { + cumulative_weight += weights[word_index]; + end = word_index + 1 == words.size() + ? frame_count + : (frame_count * cumulative_weight + total_weight / 2) / + total_weight; + feature_begin = start * 320u; + feature_end = std::min(encoded.samples.size(), end * 320u); + } + end = std::max(end, std::min(frame_count, start + 1)); + OuteTTSVoiceWord word; + word.text = words[word_index]; + word.duration = + std::round(static_cast(end - start) / 75.0 * 100.0) / 100.0; + word.features = audio_features(encoded.samples, feature_begin, feature_end); + word.codebook1.assign( + encoded.codebook1.begin() + static_cast(start), + encoded.codebook1.begin() + static_cast(end)); + word.codebook2.assign( + encoded.codebook2.begin() + static_cast(start), + encoded.codebook2.begin() + static_cast(end)); + const std::string trace_prefix = + "outetts.reference.word." + std::to_string(word_index); + debug::trace_log_scalar(trace_prefix + ".text", word.text); + debug::trace_log_scalar(trace_prefix + ".duration", word.duration); + debug::trace_log_scalar(trace_prefix + ".energy", word.features.energy); + debug::trace_log_scalar(trace_prefix + ".spectral_centroid", + word.features.spectral_centroid); + debug::trace_log_scalar(trace_prefix + ".pitch", word.features.pitch); + profile.words.push_back(std::move(word)); + start = end; + } + return profile; +} + +std::optional align_reference( + const runtime::SessionOptions &options, + const OuteTTSAssets &model_assets, + const runtime::AudioBuffer &audio, + const std::string &text, + const std::string &language) { + const auto model_path = runtime::find_option( + options.options, + {"outetts.aligner_model_path", "outetts.forced_aligner_model_path"}); + runtime::SessionOptions aligner_options; + aligner_options.backend = options.backend; + for (const auto &[key, value] : options.options) { + if (key.rfind("qwen3_forced_aligner.", 0) == 0) + aligner_options.options.emplace(key, value); + } + std::shared_ptr + aligner_assets; + if (model_path.has_value()) { + aligner_assets = engine::models::qwen3_asr::load_qwen3_asr_assets( + std::filesystem::path(*model_path), "qwen3_forced_aligner"); + } else { + aligner_assets = model_assets.embedded_aligner; + } + if (aligner_assets == nullptr) { + throw std::runtime_error( + "OuteTTS voice cloning requires a GGUF with an embedded Qwen3 " + "Forced Aligner or --session-option " + "outetts.aligner_model_path="); + } + engine::models::qwen3_forced_aligner::Qwen3ForcedAlignerSession session( + {runtime::VoiceTaskKind::Alignment, runtime::RunMode::Offline}, + std::move(aligner_options), aligner_assets); + runtime::TaskRequest request; + request.audio_input = audio; + request.text_input = runtime::Transcript{text, language}; + request.options["audio_chunk_mode"] = "none"; + session.prepare(runtime::build_preparation_request(request)); + auto result = session.run(request); + if (result.word_timestamps.empty()) + throw std::runtime_error("OuteTTS reference aligner returned no words"); + return ReferenceAlignment{std::move(result.word_timestamps), + aligner_assets->config.sample_rate}; +} + +const runtime::AudioBuffer * +reference_audio(const runtime::TaskRequest &request) { + if (request.voice.has_value() && request.voice->speaker.has_value() && + request.voice->speaker->audio.has_value()) { + return &*request.voice->speaker->audio; + } + return request.audio_input.has_value() ? &*request.audio_input : nullptr; +} + +} // namespace + +OuteTTSSession::OuteTTSSession(runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets) + : RuntimeSessionBase(options), task_(task), assets_(std::move(assets)), + tokenizer_(assets_), + dac_(assets_, execution_context(), + runtime::parse_size_mb_option(options.options, + {"outetts.dac_weight_context_mb"}, + 1024ull * 1024ull * 1024ull), + runtime::parse_size_mb_option(options.options, + {"outetts.dac_graph_context_mb"}, + 1536ull * 1024ull * 1024ull), + assets::TensorStorageType::F32) { + if (assets_ == nullptr) + throw std::runtime_error("OuteTTS session requires assets"); + if ((task_.task != runtime::VoiceTaskKind::Tts && + task_.task != runtime::VoiceTaskKind::VoiceCloning) || + task_.mode != runtime::RunMode::Offline) { + throw std::runtime_error( + "OuteTTS supports offline TTS and voice cloning only"); + } +} + +OuteTTSLlamaRuntime &OuteTTSSession::llama(bool voice_cloning) { + auto &runtime_slot = voice_cloning ? clone_llama_ : llama_; + if (runtime_slot == nullptr) { + const auto storage_type = + voice_cloning ? clone_weight_type(options(), *assets_) + : requested_weight_type(options()); + runtime_slot = std::make_unique( + assets_, options().backend.type, options().backend.device, + std::max(1, options().backend.threads), + runtime::parse_size_mb_option(options().options, + {"outetts.llama_weight_context_mb"}, + 4096ull * 1024ull * 1024ull), + runtime::parse_size_mb_option(options().options, + {"outetts.constant_context_mb"}, + 256ull * 1024ull * 1024ull), + storage_type); + } + return *runtime_slot; +} + +std::string OuteTTSSession::family() const { return "outetts"; } +runtime::VoiceTaskKind OuteTTSSession::task_kind() const { return task_.task; } +runtime::RunMode OuteTTSSession::run_mode() const { return task_.mode; } +void OuteTTSSession::prepare( + const runtime::SessionPreparationRequest &request) { + voice_profile_.reset(); + if (request.voice.has_value() && request.voice->speaker.has_value() && + request.voice->speaker->audio.has_value()) { + const auto reference_text = + runtime::find_option(request.options, {"reference_text"}).value_or(""); + if (reference_text.empty()) { + throw std::runtime_error( + "OuteTTS voice cloning requires --reference-text"); + } + const auto &audio = *request.voice->speaker->audio; + const auto language = runtime::find_option( + request.options, {"reference_language"}) + .value_or(request.text.has_value() && + !request.text->language.empty() + ? request.text->language + : "en"); + const auto alignment = + align_reference(options(), *assets_, audio, reference_text, language); + voice_profile_ = make_voice_profile( + dac_.encode_reference(audio), reference_text, + alignment.has_value() ? &*alignment : nullptr); + } + mark_prepared(); +} + +runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { + require_prepared("OuteTTS run"); + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("OuteTTS requires text input"); + } + const auto *voice_audio = reference_audio(request); + std::optional request_profile; + if (voice_audio != nullptr && !voice_profile_.has_value()) { + const auto reference_text = + runtime::find_option(request.options, {"reference_text"}).value_or(""); + if (reference_text.empty()) + throw std::runtime_error( + "OuteTTS voice cloning requires --reference-text"); + const auto language = runtime::find_option( + request.options, {"reference_language"}) + .value_or(request.text_input.has_value() && + !request.text_input->language.empty() + ? request.text_input->language + : "en"); + const auto alignment = + align_reference(options(), *assets_, *voice_audio, reference_text, + language); + request_profile = make_voice_profile( + dac_.encode_reference(*voice_audio), reference_text, + alignment.has_value() ? &*alignment : nullptr); + } + const OuteTTSVoiceProfile *profile = + request_profile.has_value() + ? &*request_profile + : (voice_profile_.has_value() ? &*voice_profile_ : nullptr); + if (task_.task == runtime::VoiceTaskKind::VoiceCloning && + profile == nullptr) { + throw std::runtime_error( + "OuteTTS voice cloning requires --voice-ref and --reference-text"); + } + const auto prompt = + profile != nullptr + ? tokenizer_.build_clone_prompt(request.text_input->text, *profile) + : tokenizer_.build_prompt(request.text_input->text); + const bool quantized_cloning = + profile != nullptr && has_quantized_clone_weights(options(), *assets_); + auto generate_options = generation_options( + request, assets_->generation, profile != nullptr, quantized_cloning); + const auto generated = llama(profile != nullptr).generate( + prompt, generate_options, + tokenizer_.eos_id(), tokenizer_.audio_end_id()); + std::vector c1; + std::vector c2; + for (const int32_t token : generated) + tokenizer_.append_audio_code(token, c1, c2); + const size_t pairs = std::min(c1.size(), c2.size()); + c1.resize(pairs); + c2.resize(pairs); + if (pairs == 0) { + std::string detail; + for (size_t i = 0; i < std::min(generated.size(), 12); ++i) { + detail += (i == 0 ? "" : ",") + std::to_string(generated[i]); + } + throw std::runtime_error( + "OuteTTS generated no complete DAC code pairs (tokens=" + detail + ")"); + } + runtime::TaskResult result; + result.audio_output = dac_.decode(c1, c2); + return result; +} + +} // namespace engine::models::outetts diff --git a/src/models/outetts/tokenizer.cpp b/src/models/outetts/tokenizer.cpp new file mode 100644 index 000000000..6f8ec041d --- /dev/null +++ b/src/models/outetts/tokenizer.cpp @@ -0,0 +1,198 @@ +#include "engine/models/outetts/tokenizer.h" + +#include "engine/framework/tokenizers/llama_bpe.h" + +#include +#include +#include +#include +#include + +namespace engine::models::outetts { +namespace { + +int32_t require_id(const engine::tokenizers::LlamaBpeTokenizer &tokenizer, + const std::string &token) { + const auto id = tokenizer.find_token_id(token); + if (!id.has_value()) { + throw std::runtime_error("OuteTTS tokenizer is missing token: " + token); + } + return *id; +} + +bool is_punctuation(char value) { + return value == ',' || value == '.' || value == '?' || value == '!' || + value == ':' || value == ';'; +} + +std::string normalize_text(const std::string &input) { + std::string text; + text.reserve(input.size()); + bool pending_space = false; + for (const unsigned char value : input) { + if (value < 0x20 || value == 0x7f || value == '"') + continue; + if (std::isspace(value)) { + pending_space = !text.empty(); + continue; + } + const char character = static_cast(value); + if (is_punctuation(character)) { + while (!text.empty() && text.back() == ' ') + text.pop_back(); + if ((character == '?' || character == '!') && !text.empty() && + text.back() == character) { + pending_space = true; + continue; + } + if (character == '.' && text.size() >= 3 && + text[text.size() - 1] == '.' && text[text.size() - 2] == '.' && + text[text.size() - 3] == '.') { + pending_space = true; + continue; + } + text.push_back(character); + pending_space = true; + continue; + } + if (pending_space && !text.empty()) + text.push_back(' '); + text.push_back(character); + pending_space = false; + } + while (!text.empty() && text.back() == ' ') + text.pop_back(); + return text; +} + +std::string speaker_separator(const std::string &text) { + if (text.empty()) + return {}; + const char last = text.back(); + return last == '.' || last == '?' || last == '!' ? " " : ". "; +} + +std::string profile_codes(const OuteTTSVoiceProfile &profile, + const std::string &separator) { + std::ostringstream out; + for (size_t word_index = 0; word_index < profile.words.size(); ++word_index) { + const auto &word = profile.words[word_index]; + std::string word_text = normalize_text(word.text); + if (word_index + 1 == profile.words.size()) + word_text += normalize_text(separator); + out << "<|word_start|>" << word_text << "<|features|><|t_" << std::fixed + << std::setprecision(2) << word.duration << "|>" + << "<|energy_" << word.features.energy << "|>" + << "<|spectral_centroid_" << word.features.spectral_centroid << "|>" + << "<|pitch_" << word.features.pitch << "|>" + << "<|code|>"; + const size_t frames = + std::min(word.codebook1.size(), word.codebook2.size()); + for (size_t frame = 0; frame < frames; ++frame) { + out << "<|c1_" << word.codebook1[frame] << "|>" + << "<|c2_" << word.codebook2[frame] << "|>"; + } + out << "<|word_end|>"; + if (word_index + 1 != profile.words.size()) + out << '\n'; + } + return out.str(); +} + +} // namespace + +struct OuteTTSTokenizer::Impl { + explicit Impl(const OuteTTSAssets &assets) + : tokenizer({ + {}, + {}, + assets.resources.require_file("tokenizer_config"), + assets.resources.require_file("tokenizer"), + engine::tokenizers::LlamaBpePreTokenizer::Llama3, + }), + eos(require_id(tokenizer, "<|im_end|>")), + audio_end(require_id(tokenizer, "<|audio_end|>")), + word_end(require_id(tokenizer, "<|word_end|>")) { + for (int32_t code = 0; code <= 1024; ++code) { + c1.emplace(require_id(tokenizer, "<|c1_" + std::to_string(code) + "|>"), + code); + c2.emplace(require_id(tokenizer, "<|c2_" + std::to_string(code) + "|>"), + code); + } + } + + engine::tokenizers::LlamaBpeTokenizer tokenizer; + int32_t eos = 0; + int32_t audio_end = 0; + int32_t word_end = 0; + std::unordered_map c1; + std::unordered_map c2; +}; + +OuteTTSTokenizer::OuteTTSTokenizer( + std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("OuteTTS tokenizer requires assets"); + } + impl_ = std::make_shared(*assets); +} + +std::vector +OuteTTSTokenizer::build_prompt(const std::string &text) const { + if (text.empty()) { + throw std::runtime_error("OuteTTS requires non-empty text"); + } + const std::string prompt_text = normalize_text(text); + const std::string prompt = "<|im_start|>\n<|text_start|>" + prompt_text + + "<|text_end|>\n<|audio_start|>\n"; + return impl_->tokenizer.encode(prompt, true); +} + +std::vector +OuteTTSTokenizer::build_clone_prompt(const std::string &text, + const OuteTTSVoiceProfile &profile) const { + const std::string prompt_text = normalize_text(text); + const std::string reference_text = normalize_text(profile.text); + if (prompt_text.empty() || reference_text.empty() || profile.words.empty()) { + throw std::runtime_error("OuteTTS voice cloning requires text, " + "reference_text, and reference codec frames"); + } + const std::string separator = speaker_separator(reference_text); + const std::string merged = + normalize_text(reference_text + separator + prompt_text); + const std::string prompt = "<|im_start|>\n<|text_start|>" + merged + + "<|text_end|>\n<|audio_start|>\n" + + profile_codes(profile, separator) + + "\n<|word_start|>"; + return impl_->tokenizer.encode(prompt, true); +} + +bool OuteTTSTokenizer::is_stop_token(int32_t token) const noexcept { + return token == impl_->eos || token == impl_->audio_end; +} + +int32_t OuteTTSTokenizer::eos_id() const noexcept { return impl_->eos; } + +int32_t OuteTTSTokenizer::audio_end_id() const noexcept { + return impl_->audio_end; +} + +bool OuteTTSTokenizer::append_audio_code( + int32_t token, std::vector &codebook1, + std::vector &codebook2) const { + if (const auto it = impl_->c1.find(token); it != impl_->c1.end()) { + if (it->second < 1024) { + codebook1.push_back(it->second); + } + return true; + } + if (const auto it = impl_->c2.find(token); it != impl_->c2.end()) { + if (it->second < 1024) { + codebook2.push_back(it->second); + } + return true; + } + return false; +} + +} // namespace engine::models::outetts diff --git a/src/models/qwen3_asr/assets.cpp b/src/models/qwen3_asr/assets.cpp index ef576f5bb..3205dcfe5 100644 --- a/src/models/qwen3_asr/assets.cpp +++ b/src/models/qwen3_asr/assets.cpp @@ -165,6 +165,26 @@ assets::ResourceBundle make_resource_bundle( return resources; } +std::shared_ptr make_assets( + assets::ResourceBundle resources) { + if (!resources.has_file("preprocessor_config") && + !resources.has_file("processor_config")) { + throw std::runtime_error( + "Qwen3 ASR requires preprocessor_config.json or processor_config.json"); + } + const bool has_legacy_tokenizer = + resources.has_file("vocab") && resources.has_file("merges"); + if (!has_legacy_tokenizer && !resources.has_file("tokenizer_json")) { + throw std::runtime_error( + "Qwen3 ASR requires vocab.json plus merges.txt, or tokenizer.json"); + } + Qwen3ASRAssets assets; + assets.resources = std::move(resources); + assets.config = parse_config(assets.resources); + assets.model_weights = assets.resources.open_tensor_source("weights"); + return std::make_shared(std::move(assets)); +} + } // namespace std::shared_ptr load_qwen3_asr_assets(const std::filesystem::path & model_path) { @@ -174,12 +194,12 @@ std::shared_ptr load_qwen3_asr_assets(const std::filesyste std::shared_ptr load_qwen3_asr_assets( const std::filesystem::path & model_path, std::string_view package_family) { - auto resources = make_resource_bundle(model_path, package_family); - Qwen3ASRAssets assets; - assets.resources = std::move(resources); - assets.config = parse_config(assets.resources); - assets.model_weights = assets.resources.open_tensor_source("weights"); - return std::make_shared(std::move(assets)); + return make_assets(make_resource_bundle(model_path, package_family)); +} + +std::shared_ptr load_qwen3_asr_assets( + assets::ResourceBundle resources) { + return make_assets(std::move(resources)); } } // namespace engine::models::qwen3_asr diff --git a/tools/convert_outetts_dac.py b/tools/convert_outetts_dac.py new file mode 100644 index 000000000..097a47239 --- /dev/null +++ b/tools/convert_outetts_dac.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Convert OuteTTS 1.0's official IBM DAC checkpoint to safetensors. + +The official OuteTTS runtime loads a trusted PyTorch checkpoint. audio.cpp uses +the resulting plain tensor source for both native safetensors loading and GGUF +packing; no Python or pickle loader is needed at inference time. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import dac +from safetensors.torch import save_file + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("input", type=Path, help="weights_24khz_1.5kbps_v1.0.pth") + parser.add_argument("output", type=Path, help="output model.safetensors") + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + if args.output.exists() and not args.overwrite: + raise SystemExit(f"output exists (pass --overwrite): {args.output}") + + model = dac.DAC.load(str(args.input)).cpu().eval() + if model.sample_rate != 24000 or model.hop_length != 320: + raise RuntimeError( + f"expected OuteTTS DAC at 24 kHz with hop 320, got " + f"{model.sample_rate} Hz / hop {model.hop_length}" + ) + if model.n_codebooks != 2 or model.codebook_size != 1024: + raise RuntimeError( + f"expected two 1024-entry codebooks, got " + f"{model.n_codebooks} x {model.codebook_size}" + ) + + tensors = {name: tensor.detach().contiguous() for name, tensor in model.state_dict().items()} + args.output.parent.mkdir(parents=True, exist_ok=True) + save_file( + tensors, + str(args.output), + metadata={ + "format": "pt", + "source": "ibm-research/DAC.speech.v1.0", + "checkpoint": args.input.name, + }, + ) + print(f"wrote {args.output} ({len(tensors)} tensors)") + + +if __name__ == "__main__": + main() diff --git a/tools/model_manager.py b/tools/model_manager.py index e299bb0df..5c2a10b8d 100644 --- a/tools/model_manager.py +++ b/tools/model_manager.py @@ -934,6 +934,62 @@ def package_usage_examples(package: ModelPackage) -> list[str]: ), description="Installs Irodori-TTS VoiceDesign plus the sibling llm-jp tokenizer and DACVAE codec dependencies required by the framework runtime.", ), + ModelPackage( + id="outetts_1_0_1b", + display_name="OuteTTS 1.0 1B", + target_directory="Llama-OuteTTS-1.0-1B", + source=CompositeSnapshotSource( + placements=( + SnapshotPlacement( + source=SnapshotSource(repo_id="OuteAI/Llama-OuteTTS-1.0-1B"), + required_files=( + "config.json", + "generation_config.json", + "model.safetensors", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + ), + ), + SnapshotPlacement( + source=SnapshotSource(repo_id="ibm-research/DAC.speech.v1.0"), + target_subdir="../DAC.speech.v1.0", + required_files=("config.json", "weights_24khz_1.5kbps_v1.0.pth"), + ), + SnapshotPlacement( + source=SnapshotSource(repo_id="Qwen/Qwen3-ForcedAligner-0.6B"), + target_subdir="../Qwen3-ForcedAligner-0.6B", + required_files=( + "config.json", + "generation_config.json", + "model.safetensors", + "preprocessor_config.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt", + ), + ), + ), + ), + required_files=( + "config.json", + "generation_config.json", + "model.safetensors", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "../DAC.speech.v1.0/config.json", + "../DAC.speech.v1.0/model.safetensors", + "../Qwen3-ForcedAligner-0.6B/config.json", + "../Qwen3-ForcedAligner-0.6B/generation_config.json", + "../Qwen3-ForcedAligner-0.6B/model.safetensors", + "../Qwen3-ForcedAligner-0.6B/preprocessor_config.json", + "../Qwen3-ForcedAligner-0.6B/tokenizer_config.json", + "../Qwen3-ForcedAligner-0.6B/vocab.json", + "../Qwen3-ForcedAligner-0.6B/merges.txt", + ), + description="Installs OuteTTS, its IBM DAC 1.5 kbps codec, and Qwen3 Forced Aligner for reliable voice cloning.", + ), ModelPackage( id="stable_audio_3_small_music", display_name="Stable Audio 3 Small Music", @@ -1696,6 +1752,24 @@ def convert_irodori_dacvae_weights(root: Path) -> None: write_checked_safetensors(tensors, output_path, input_path, overwrite=True) +def convert_outetts_dac_weights(root: Path) -> None: + input_path = root / "weights_24khz_1.5kbps_v1.0.pth" + output_path = root / "model.safetensors" + payload = torch.load(input_path, map_location="cpu", weights_only=True) + state = checkpoint_state_dict(payload) + tensors = tensor_state_dict(state) + expected = { + "quantizer.quantizers.0.codebook.weight": (1024, 8), + "quantizer.quantizers.1.codebook.weight": (1024, 8), + "decoder.model.0.weight_v": (1536, 1024, 7), + "decoder.model.6.weight_v": (1, 96, 7), + } + for name, shape in expected.items(): + if name not in tensors or tuple(tensors[name].shape) != shape: + raise RuntimeError(f"unexpected OuteTTS DAC tensor {name}: {getattr(tensors.get(name), 'shape', None)}") + write_checked_safetensors(tensors, output_path, input_path, overwrite=True) + + def write_irodori_model_config(root: Path) -> None: input_path = root / "model.safetensors" output_path = root / "model_config.json" @@ -1792,6 +1866,10 @@ def install_composite_snapshot( dacvae_root = staged_package_root.parent / "Semantic-DACVAE-Japanese-32dim" if dacvae_root.exists(): convert_irodori_dacvae_weights(dacvae_root) + elif package.id == "outetts_1_0_1b": + dac_root = staged_package_root.parent / "DAC.speech.v1.0" + if dac_root.exists(): + convert_outetts_dac_weights(dac_root) elif package.id == "vibevoice_asr": copy_bundled_model_manager_assets( "vibevoice_1_5b", From 18d8383822f5990333836c5c3c077bb856374354 Mon Sep 17 00:00:00 2001 From: mirek190 Date: Wed, 15 Jul 2026 23:49:07 +0100 Subject: [PATCH 2/8] Address OuteTTS long-session review feedback --- CMakeLists.txt | 1 + docs/memory_saver.md | 3 + docs/outetts_validation.md | 137 +++++++++ docs/tts.md | 11 + include/engine/models/outetts/llama.h | 4 + include/engine/models/outetts/session.h | 37 ++- src/models/outetts/llama.cpp | 87 +++++- src/models/outetts/loader.cpp | 10 + src/models/outetts/session.cpp | 380 +++++++++++++++++++----- tests/outetts/outetts_warm_bench.cpp | 266 +++++++++++++++++ tests/outetts/warm_bench_requests.json | 47 +++ 11 files changed, 905 insertions(+), 78 deletions(-) create mode 100644 docs/outetts_validation.md create mode 100644 tests/outetts/outetts_warm_bench.cpp create mode 100644 tests/outetts/warm_bench_requests.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 3339855db..73d276061 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -722,6 +722,7 @@ if (ENGINE_BUILD_WARMBENCH) add_engine_warmbench(moss_tts_local_warm_bench tests/moss_tts_local/moss_tts_local_warm_bench.cpp) add_engine_warmbench(nemotron_asr_warm_bench tests/nemotron_asr/nemotron_asr_warm_bench.cpp) add_engine_warmbench(omnivoice_warm_bench tests/omnivoice/omnivoice_warm_bench.cpp) + add_engine_warmbench(outetts_warm_bench tests/outetts/outetts_warm_bench.cpp) add_engine_warmbench(pocket_tts_warm_bench tests/pocket_tts/pocket_tts_warm_bench.cpp) add_engine_warmbench(qwen3_asr_warm_bench tests/qwen3_asr/qwen3_asr_warm_bench.cpp) add_engine_warmbench(qwen3_forced_aligner_warm_bench tests/qwen3_forced_aligner/qwen3_forced_aligner_warm_bench.cpp) diff --git a/docs/memory_saver.md b/docs/memory_saver.md index 658c2a96a..037d0f821 100644 --- a/docs/memory_saver.md +++ b/docs/memory_saver.md @@ -31,6 +31,7 @@ Native/default weights were used for all rows. - VoxCPM2 used the OpenAI-compatible offline speech endpoint with a 2048-character voice-design request, `seed=1234`, `max_tokens=512`, `num_inference_steps=10`, and `guidance_scale=2.0`. The default and `mem_saver` WAV outputs were byte-identical. - IndexTTS2 used a five-request server sequence with the same seeds and references for default and `mem_saver`: normal text, longer text, longer emotion-text request with a different reference, shorter text, then longer text. - Irodori TTS 500M used a five-request server sequence with the same seeds and options for default and `mem_saver`: reference text, longer reference text, longer no-reference emoji/style text, shorter reference text, then longer reference text. +- OuteTTS 1.0 1B Q8 used a five-request long-lived-session sequence on an RTX 3090: repeated fixed-seed TTS, four-chunk long-form TTS, then repeated fixed-seed cloning with the same reference. The default and `mem_saver` WAV outputs were byte-identical. VRAM is total-device usage with no other CUDA workload; resident VRAM was sampled during a five-second post-sequence hold. | Model | Mode | Peak VRAM | Resident VRAM | Server wall | Audio | RTF | |---|---|---:|---:|---:|---:|---:| @@ -58,3 +59,5 @@ Native/default weights were used for all rows. | Irodori TTS 500M | mem_saver | 11222 MiB | 3570 MiB | 3276.203 ms | 95.4s | 0.0343418 | | Irodori TTS 500M 6000-char | default | 18693 MiB | 13367 MiB | 27185.7 ms | 777.92s | 0.0349466 | | Irodori TTS 500M 6000-char | mem_saver | 11588 MiB | 3609 MiB | 27828 ms | 777.92s | 0.0357724 | +| OuteTTS 1.0 1B Q8 | default | 17653 MiB | 294 MiB | 33342.24 ms | 11.424s | 2.918 | +| OuteTTS 1.0 1B Q8 | mem_saver | 5780 MiB | 294 MiB | 30990.93 ms | 11.424s | 2.712 | diff --git a/docs/outetts_validation.md b/docs/outetts_validation.md new file mode 100644 index 000000000..7e4d060d9 --- /dev/null +++ b/docs/outetts_validation.md @@ -0,0 +1,137 @@ +# OuteTTS validation + +This procedure exercises OuteTTS 1.0 1B in one long-lived audio.cpp session. +It covers normal TTS, framework long-form text chunking, voice cloning, repeated +reference-profile cache hits, cached-step graph reuse, stage timings, and +default-versus-`mem_saver` memory behavior. + +## Model setup + +Install the safetensors model, IBM DAC, and Qwen3 Forced Aligner resources: + +```bash +python tools/model_manager.py install outetts_1_0_1b --models-dir models +``` + +The standalone GGUF command is documented in [TTS](tts.md#outetts). The packed +file used below contains the OuteTTS language model, IBM DAC, Qwen aligner, +tokenizers, configuration, and package specification. + +## Python reference setup + +The maintainer can install the official OuteTTS reference without using any +audio.cpp conversion code: + +```bash +python -m venv .venv-outetts +. .venv-outetts/bin/activate +python -m pip install --upgrade pip outetts +``` + +On Windows PowerShell with the optional CUDA llama.cpp backend: + +```powershell +python -m venv .venv-outetts +.\.venv-outetts\Scripts\Activate.ps1 +$env:CMAKE_ARGS = "-DGGML_CUDA=on" +python -m pip install --upgrade pip outetts +``` + +Select `outetts.Backend.HF` for a Transformers comparison or +`outetts.Backend.LLAMACPP` for the official llama.cpp-backed route. Use +`outetts.Models.VERSION_1_0_SIZE_1B`, temperature `0.4`, repetition penalty +`1.1` over the latest 64 tokens, top-k `40`, top-p `0.9`, and min-p +`0.05`. + +## Build + +Windows CUDA: + +```powershell +$env:CUDA_PATH = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" +.\scripts\build_windows.ps1 -Preset windows-cuda-release -Target outetts_warm_bench -Jobs 4 +``` + +Linux CPU: + +```bash +cmake -S . -B build/cpu-release -DCMAKE_BUILD_TYPE=Release -DENGINE_BUILD_WARMBENCH=ON +cmake --build build/cpu-release --target outetts_warm_bench -j +``` + +## Long-lived session + +```powershell +build\windows-cuda-release\bin\outetts_warm_bench.exe ` + --model models\Llama-OuteTTS-1.0-1B-Q8_0\model.gguf ` + --backend cuda --threads 8 ` + --request-file tests\outetts\warm_bench_requests.json ` + --hold-seconds 5 ` + --audio-out-dir build\logs\warmbench\outetts-default ` + --log-file build\logs\warmbench\outetts-default.log +``` + +Expected trace evidence: + +- `outetts.text_chunk_count` is greater than one for `tts_longform`. +- `outetts.llama.step.graph_reused=1` appears after the first compatible + generation request or chunk. +- the second identical reference reports `outetts.reference_cache.hit=1`. +- `tts_cold` and `tts_repeat` are byte-identical, as are `clone_cold` + and `clone_repeat`; this verifies that warm graph/profile reuse does not + change deterministic output. +- `outetts.aligner.runtime_reused=1` is observable for uncached references + while the default session retains the aligner. +- only one active OuteTTS Llama runtime is retained; switching between native + TTS weights and the CUDA F32 cloning fallback replaces the previous runtime. + +Run the same request sequence in a fresh process with memory saver enabled: + +```powershell +build\windows-cuda-release\bin\outetts_warm_bench.exe ` + --model models\Llama-OuteTTS-1.0-1B-Q8_0\model.gguf ` + --backend cuda --threads 8 ` + --request-file tests\outetts\warm_bench_requests.json ` + --session-option outetts.mem_saver=true ` + --hold-seconds 5 ` + --audio-out-dir build\logs\warmbench\outetts-mem-saver ` + --log-file build\logs\warmbench\outetts-mem-saver.log +``` + +The memory-saver trace reports a positive +`outetts.llama.step.released_cache_capacity` after generation and +`outetts.aligner.runtime_released=1` after an uncached reference. + +Sample per-process VRAM once per second while each fresh benchmark runs: + +```powershell +nvidia-smi --query-compute-apps=pid,process_name,used_gpu_memory --format=csv -l 1 +``` + +Compare peak and final resident VRAM, request wall time, output duration, and RTF +between the two runs. Keep model, backend, device, seed, requests, and +quantization identical. + +## Measured validation + +The committed five-request sequence was measured with the packed Q8 GGUF on an +NVIDIA GeForce RTX 3090 (CUDA 12.4). VRAM was sampled every 250 ms from total +device usage with no other CUDA workload. Resident VRAM was sampled during the +five-second hold after all requests completed. + +| Mode | Sequence wall | Audio | RTF | Peak VRAM | Resident VRAM | +|---|---:|---:|---:|---:|---:| +| default | 33342.24 ms | 11.424s | 2.918 | 17653 MiB | 294 MiB | +| `outetts.mem_saver=true` | 30990.93 ms | 11.424s | 2.712 | 5780 MiB | 294 MiB | + +Both modes produced the same SHA-256 for the cold and repeated TTS pair and for +the cold and repeated clone pair. All ten generated WAV files passed an ffmpeg +decode check. The traces confirmed four framework chunks for the long-form +request, compatible step-graph reuse in default mode, explicit graph release in +memory-saver mode, and a reference-profile cache hit on the repeated clone. + +Generation is the dominant measured hot path. In the default CUDA run, the cold +reference path took about 890 ms (109 ms alignment, 139 ms DAC encoding, and +214 ms profile construction), while the repeated cached reference took 0.29 ms. +The memory-saver result is within normal run-to-run timing variation; its +purpose here is the lower peak, not a speedup guarantee. diff --git a/docs/tts.md b/docs/tts.md index 6037b4f98..7e4e07ebe 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -598,10 +598,21 @@ audiocpp_cli --task tts --family outetts \ | `--request-option min_p=` | float | `0.05` | Minimum probability relative to the most likely token. | | `--repetition-penalty` | float | `1.1` | Repetition penalty over the latest 64 tokens. | | `--request-option seed=` | integer | native clone: `4099`; quantized clone: `42` | Deterministic sampling seed. The defaults were separately verified for the native and Q8 cloning paths. | +| `--text-chunk-size` | characters | `2048` | Framework long-form text chunk size. Each chunk is generated and decoded in the same loaded session, then appended to the output WAV. | +| `--text-chunk-mode` | `default`, `tag_aware`, `japanese`, `endline` | `default` | Framework long-form text chunking mode. | | `--reference-text` | text | none | Exact transcript of `--voice-ref`; required for voice cloning. | | `--request-option reference_language=` | language code | `en` | Language used by the optional reference aligner. | | `--session-option outetts.weight_type=native|f32|f16|bf16|q8_0` | enum | `native` | Language-model weight storage type. For CUDA voice cloning, quantized weights remain compact in the GGUF but are expanded to F32 in VRAM to avoid generation divergence over long reference-codec prompts. Normal TTS and CPU cloning keep the selected type. | | `--session-option outetts.aligner_model_path=` | model path | embedded aligner | Optional external Qwen3 Forced Aligner override, required only for safetensors packages and older GGUFs without the embedded aligner. | +| `--session-option outetts.reference_cache_slots=` | integer | `1` | LRU slots for prepared reference profiles (alignment, DAC codes, and word features). Set `0` to disable reuse. | +| `--session-option outetts.mem_saver=true|false` | bool | `false` | Release the reusable Llama cached-step graph after each generated chunk and release the aligner runtime after preparing a reference. Model and DAC weights stay resident; later requests rebuild released state. | + +With logging enabled, OuteTTS reports framework chunk count, reference-profile +cache hits/evictions, Llama runtime and step-graph rebuild/reuse, released cache +capacity, and timings for reference alignment, DAC encode/decode, prompt +construction, generation, and the complete session request. See +[OuteTTS validation](outetts_validation.md) for the reproducible long-lived +session and memory test. ## Supertonic diff --git a/include/engine/models/outetts/llama.h b/include/engine/models/outetts/llama.h index 77503750f..697ffbd69 100644 --- a/include/engine/models/outetts/llama.h +++ b/include/engine/models/outetts/llama.h @@ -43,6 +43,10 @@ class OuteTTSLlamaRuntime final { int32_t eos_id, int32_t audio_end_id) const; + // Releases the reusable cached-step graph while keeping model weights + // resident. Returns the released KV-cache capacity in tokens. + int64_t release_cached_step_graph(); + private: struct Impl; std::unique_ptr impl_; diff --git a/include/engine/models/outetts/session.h b/include/engine/models/outetts/session.h index 33c7bc10b..af30dd476 100644 --- a/include/engine/models/outetts/session.h +++ b/include/engine/models/outetts/session.h @@ -1,13 +1,21 @@ #pragma once +#include "engine/framework/runtime/cache_slots.h" #include "engine/framework/runtime/session_base.h" #include "engine/models/outetts/assets.h" #include "engine/models/outetts/dac.h" #include "engine/models/outetts/llama.h" #include "engine/models/outetts/tokenizer.h" +#include +#include #include #include +#include + +namespace engine::models::qwen3_forced_aligner { +class Qwen3ForcedAlignerSession; +} namespace engine::models::outetts { @@ -16,6 +24,7 @@ class OuteTTSSession final : public runtime::RuntimeSessionBase, public: OuteTTSSession(runtime::TaskSpec task, runtime::SessionOptions options, std::shared_ptr assets); + ~OuteTTSSession() override; std::string family() const override; runtime::VoiceTaskKind task_kind() const override; runtime::RunMode run_mode() const override; @@ -23,14 +32,40 @@ class OuteTTSSession final : public runtime::RuntimeSessionBase, runtime::TaskResult run(const runtime::TaskRequest &request) override; private: + struct ReferenceProfileCacheKey { + uint64_t audio_hash = 0; + int sample_rate = 0; + int channels = 0; + size_t sample_count = 0; + std::string text; + std::string language; + }; + + struct ReferenceProfileCacheKeyEqual { + bool operator()(const ReferenceProfileCacheKey &lhs, + const ReferenceProfileCacheKey &rhs) const; + }; + OuteTTSLlamaRuntime &llama(bool voice_cloning); + OuteTTSVoiceProfile prepare_voice_profile( + const runtime::AudioBuffer &audio, const std::string &reference_text, + const std::string &language, bool &cache_hit); runtime::TaskSpec task_; std::shared_ptr assets_; OuteTTSTokenizer tokenizer_; std::unique_ptr llama_; - std::unique_ptr clone_llama_; + std::optional llama_storage_type_; + std::unique_ptr< + engine::models::qwen3_forced_aligner::Qwen3ForcedAlignerSession> + aligner_session_; + std::shared_ptr + aligner_assets_; OuteTTSDacDecoder dac_; + bool mem_saver_ = false; + runtime::CacheSlots + reference_profile_cache_; std::optional voice_profile_; }; diff --git a/src/models/outetts/llama.cpp b/src/models/outetts/llama.cpp index 541b12dc4..f9163a512 100644 --- a/src/models/outetts/llama.cpp +++ b/src/models/outetts/llama.cpp @@ -1,6 +1,8 @@ #include "engine/models/outetts/llama.h" #include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/debug/trace.h" #include "engine/framework/modules/attention/qwen_causal_decoder.h" #include "engine/framework/modules/lookup_modules.h" #include "engine/framework/modules/weight_binding.h" @@ -11,6 +13,7 @@ #include #include +#include #include #include #include @@ -280,6 +283,8 @@ class CachedStepGraph { if (buffer_ != nullptr) ggml_backend_buffer_free(buffer_); } + int64_t capacity() const noexcept { return capacity_; } + void import_state(const runtime::TransformerKVState & state) { cache_.import_state(state); } std::vector run(int32_t token) { @@ -349,6 +354,8 @@ struct OuteTTSLlamaRuntime::Impl { } PrefillOutput prefill(const std::vector & ids) const { + using Clock = std::chrono::steady_clock; + const auto total_start = Clock::now(); const auto & c = assets->config; const int64_t steps = static_cast(ids.size()); // This correctness-first graph uses full-sequence prefill. A cached-step @@ -404,6 +411,7 @@ struct OuteTTSLlamaRuntime::Impl { } throw std::runtime_error("failed to allocate OuteTTS Llama graph"); } + const auto build_end = Clock::now(); const auto position_values = modules::qwen_position_ids(steps); const auto mask_values = modules::qwen_causal_prefill_mask_values(1, steps); ggml_backend_tensor_set(ids_tensor, ids.data(), 0, ids.size() * sizeof(int32_t)); @@ -412,6 +420,7 @@ struct OuteTTSLlamaRuntime::Impl { core::set_backend_threads(backend, threads); const auto status = core::compute_backend_graph(backend, graph); ggml_backend_synchronize(backend); + const auto compute_end = Clock::now(); if (status != GGML_STATUS_SUCCESS) { core::release_backend_graph_resources(backend, graph); ggml_gallocr_free(allocator); @@ -431,16 +440,61 @@ struct OuteTTSLlamaRuntime::Impl { ggml_backend_tensor_get(keys[layer], state.key.data(), 0, layer_elements * sizeof(float)); ggml_backend_tensor_get(values[layer], state.value.data(), 0, layer_elements * sizeof(float)); } + const auto read_end = Clock::now(); core::release_backend_graph_resources(backend, graph); ggml_gallocr_free(allocator); + const auto release_end = Clock::now(); + debug::trace_log_scalar("outetts.llama.prefill.graph_rebuilt", true); + debug::trace_log_scalar("outetts.llama.prefill.graph_reused", false); + debug::trace_log_scalar("outetts.llama.prefill.tokens", steps); + debug::timing_log_scalar( + "outetts.llama.prefill.graph_build_ms", + debug::elapsed_ms(total_start, build_end)); + debug::timing_log_scalar( + "outetts.llama.prefill.compute_ms", + debug::elapsed_ms(build_end, compute_end)); + debug::timing_log_scalar( + "outetts.llama.prefill.read_ms", + debug::elapsed_ms(compute_end, read_end)); + debug::timing_log_scalar( + "outetts.llama.prefill.release_ms", + debug::elapsed_ms(read_end, release_end)); return result; } + CachedStepGraph & ensure_step_graph(int64_t capacity) { + const auto build_start = std::chrono::steady_clock::now(); + const bool rebuilt = + step_graph == nullptr || step_graph->capacity() < capacity; + if (rebuilt) { + step_graph = std::make_unique( + assets->config, weights, *constants, backend, threads, capacity); + } + debug::trace_log_scalar("outetts.llama.step.graph_rebuilt", rebuilt); + debug::trace_log_scalar("outetts.llama.step.graph_reused", !rebuilt); + debug::trace_log_scalar( + "outetts.llama.step.capacity", step_graph->capacity()); + debug::timing_log_scalar( + "outetts.llama.step.graph_build_ms", + rebuilt ? debug::elapsed_ms(build_start) : 0.0); + return *step_graph; + } + + int64_t release_cached_step_graph() { + if (step_graph == nullptr) { + return 0; + } + const int64_t capacity = step_graph->capacity(); + step_graph.reset(); + return capacity; + } + std::shared_ptr assets; ggml_backend_t backend = nullptr; int threads = 1; ModelWeights weights; std::unique_ptr constants; + std::unique_ptr step_graph; }; OuteTTSLlamaRuntime::OuteTTSLlamaRuntime( @@ -468,6 +522,7 @@ std::vector OuteTTSLlamaRuntime::generate( if (options.max_new_tokens <= 0 || options.repetition_window < 0 || options.repetition_penalty <= 0.0F) { throw std::runtime_error("invalid OuteTTS generation options"); } + const auto total_start = std::chrono::steady_clock::now(); std::vector all = prompt; std::vector generated; generated.reserve(static_cast(options.max_new_tokens)); @@ -477,31 +532,49 @@ std::vector OuteTTSLlamaRuntime::generate( const int64_t capacity = std::min( impl_->assets->generation.max_length, static_cast(prompt.size()) + options.max_new_tokens); - CachedStepGraph step( - impl_->assets->config, - impl_->weights, - *impl_->constants, - impl_->backend, - impl_->threads, - capacity); + constexpr int64_t kStepGraphCapacityQuantum = 256; + const int64_t reusable_capacity = std::min( + impl_->assets->generation.max_length, + ((capacity + kStepGraphCapacityQuantum - 1) / + kStepGraphCapacityQuantum) * + kStepGraphCapacityQuantum); + auto & step = impl_->ensure_step_graph(reusable_capacity); step.import_state(prefill.state); std::vector logits = std::move(prefill.logits); + double sample_ms = 0.0; + double cached_step_compute_ms = 0.0; for (int64_t i = 0; i < options.max_new_tokens; ++i) { if (static_cast(all.size()) >= impl_->assets->generation.max_length) { break; } + const auto sample_start = std::chrono::steady_clock::now(); apply_repetition_penalty( logits, all, options.repetition_window, options.repetition_penalty); const int32_t token = sample_token( std::move(logits), sampling_options, rng); + sample_ms += debug::elapsed_ms(sample_start); generated.push_back(token); if (token == eos_id || token == audio_end_id) { break; } all.push_back(token); + const auto step_start = std::chrono::steady_clock::now(); logits = step.run(token); + cached_step_compute_ms += debug::elapsed_ms(step_start); } + debug::trace_log_scalar( + "outetts.llama.generated_tokens", + static_cast(generated.size())); + debug::timing_log_scalar("outetts.llama.sample_ms", sample_ms); + debug::timing_log_scalar("outetts.llama.cached_step_compute_ms", + cached_step_compute_ms); + debug::timing_log_scalar("outetts.llama.generate_total_ms", + debug::elapsed_ms(total_start)); return generated; } +int64_t OuteTTSLlamaRuntime::release_cached_step_graph() { + return impl_->release_cached_step_graph(); +} + } // namespace engine::models::outetts diff --git a/src/models/outetts/loader.cpp b/src/models/outetts/loader.cpp index b0f60ebd4..5bb77f851 100644 --- a/src/models/outetts/loader.cpp +++ b/src/models/outetts/loader.cpp @@ -71,6 +71,10 @@ class Loader final : public runtime::IVoiceModelLoader { "Transcript matching the --voice-ref audio for voice cloning."}, {"reference_language", "code", "Language code used to align the reference transcript; default en."}, + {"text_chunk_size", "n", + "Framework long-form text chunk size; default 2048 characters."}, + {"text_chunk_mode", "default|tag_aware|japanese|endline", + "Framework long-form text chunking mode."}, }; out.cli.session_options = { {"outetts.weight_type", "native|f32|f16|bf16|q8_0", @@ -87,6 +91,12 @@ class Loader final : public runtime::IVoiceModelLoader { {"outetts.aligner_model_path", "path", "Optional Qwen3 Forced Aligner override. Cloning automatically uses " "the aligner embedded in a standalone OuteTTS GGUF when present."}, + {"outetts.reference_cache_slots", "n", + "Prepared reference-profile cache slots; default 1, set 0 to " + "disable."}, + {"outetts.mem_saver", "true|false", + "Release cached-step and aligner runtime state after use; default " + "false."}, }; out.discovered_configs = runtime::discover_named_assets_from_package_spec( request.model_path, spec_path(), diff --git a/src/models/outetts/session.cpp b/src/models/outetts/session.cpp index c12326858..a806fe959 100644 --- a/src/models/outetts/session.cpp +++ b/src/models/outetts/session.cpp @@ -3,14 +3,18 @@ #include "engine/framework/audio/fft.h" #include "engine/framework/runtime/options.h" #include "engine/framework/debug/trace.h" +#include "engine/framework/text/chunking.h" #include "engine/models/qwen3_asr/assets.h" #include "engine/models/qwen3_forced_aligner/session.h" #include +#include #include #include #include +#include #include +#include #include #include #include @@ -18,6 +22,55 @@ namespace engine::models::outetts { namespace { +using Clock = std::chrono::steady_clock; + +constexpr int64_t kDefaultTextChunkSize = 2048; +constexpr size_t kDefaultReferenceCacheSlots = 1; + +uint64_t mix_cache_key(uint64_t key, uint64_t value) { + key ^= value; + key *= 1099511628211ull; + return key; +} + +uint64_t reference_audio_hash(const runtime::AudioBuffer &audio) { + uint64_t key = 1469598103934665603ull; + key = mix_cache_key(key, static_cast(audio.sample_rate)); + key = mix_cache_key(key, static_cast(audio.channels)); + key = mix_cache_key(key, static_cast(audio.samples.size())); + for (const float sample : audio.samples) { + uint32_t bits = 0; + std::memcpy(&bits, &sample, sizeof(bits)); + key = mix_cache_key(key, static_cast(bits)); + } + return key; +} + +size_t reference_cache_slots(const runtime::SessionOptions &options) { + const int64_t slots = runtime::parse_i64_option( + options.options, + {"outetts.reference_cache_slots", + "reference_cache_slots"}) + .value_or( + static_cast( + kDefaultReferenceCacheSlots)); + if (slots < 0 || + static_cast(slots) > + static_cast(std::numeric_limits::max())) { + throw std::runtime_error( + "outetts.reference_cache_slots must be a non-negative size"); + } + return static_cast(slots); +} + +bool mem_saver_from_options(const runtime::SessionOptions &options) { + if (const auto value = runtime::find_option( + options.options, {"outetts.mem_saver", "mem_saver"})) { + return runtime::parse_bool_option(*value, "outetts.mem_saver"); + } + return false; +} + assets::TensorStorageType requested_weight_type( const runtime::SessionOptions &options) { const auto it = options.options.find("outetts.weight_type"); @@ -312,38 +365,36 @@ make_voice_profile(OuteTTSDacDecoder::EncodedReference encoded, return profile; } -std::optional align_reference( - const runtime::SessionOptions &options, - const OuteTTSAssets &model_assets, - const runtime::AudioBuffer &audio, - const std::string &text, - const std::string &language) { - const auto model_path = runtime::find_option( - options.options, - {"outetts.aligner_model_path", "outetts.forced_aligner_model_path"}); - runtime::SessionOptions aligner_options; - aligner_options.backend = options.backend; +runtime::SessionOptions aligner_session_options( + const runtime::SessionOptions &options) { + runtime::SessionOptions out; + out.backend = options.backend; for (const auto &[key, value] : options.options) { if (key.rfind("qwen3_forced_aligner.", 0) == 0) - aligner_options.options.emplace(key, value); + out.options.emplace(key, value); } - std::shared_ptr - aligner_assets; + return out; +} + +std::shared_ptr +resolve_aligner_assets(const runtime::SessionOptions &options, + const OuteTTSAssets &model_assets) { + const auto model_path = runtime::find_option( + options.options, + {"outetts.aligner_model_path", "outetts.forced_aligner_model_path"}); if (model_path.has_value()) { - aligner_assets = engine::models::qwen3_asr::load_qwen3_asr_assets( + return engine::models::qwen3_asr::load_qwen3_asr_assets( std::filesystem::path(*model_path), "qwen3_forced_aligner"); - } else { - aligner_assets = model_assets.embedded_aligner; - } - if (aligner_assets == nullptr) { - throw std::runtime_error( - "OuteTTS voice cloning requires a GGUF with an embedded Qwen3 " - "Forced Aligner or --session-option " - "outetts.aligner_model_path="); } - engine::models::qwen3_forced_aligner::Qwen3ForcedAlignerSession session( - {runtime::VoiceTaskKind::Alignment, runtime::RunMode::Offline}, - std::move(aligner_options), aligner_assets); + return model_assets.embedded_aligner; +} + +ReferenceAlignment align_reference( + engine::models::qwen3_forced_aligner::Qwen3ForcedAlignerSession &session, + const engine::models::qwen3_asr::Qwen3ASRAssets &aligner_assets, + const runtime::AudioBuffer &audio, + const std::string &text, + const std::string &language) { runtime::TaskRequest request; request.audio_input = audio; request.text_input = runtime::Transcript{text, language}; @@ -353,7 +404,7 @@ std::optional align_reference( if (result.word_timestamps.empty()) throw std::runtime_error("OuteTTS reference aligner returned no words"); return ReferenceAlignment{std::move(result.word_timestamps), - aligner_assets->config.sample_rate}; + aligner_assets.config.sample_rate}; } const runtime::AudioBuffer * @@ -379,7 +430,9 @@ OuteTTSSession::OuteTTSSession(runtime::TaskSpec task, runtime::parse_size_mb_option(options.options, {"outetts.dac_graph_context_mb"}, 1536ull * 1024ull * 1024ull), - assets::TensorStorageType::F32) { + assets::TensorStorageType::F32), + mem_saver_(mem_saver_from_options(options)), + reference_profile_cache_(reference_cache_slots(options)) { if (assets_ == nullptr) throw std::runtime_error("OuteTTS session requires assets"); if ((task_.task != runtime::VoiceTaskKind::Tts && @@ -390,13 +443,22 @@ OuteTTSSession::OuteTTSSession(runtime::TaskSpec task, } } +OuteTTSSession::~OuteTTSSession() = default; + OuteTTSLlamaRuntime &OuteTTSSession::llama(bool voice_cloning) { - auto &runtime_slot = voice_cloning ? clone_llama_ : llama_; - if (runtime_slot == nullptr) { - const auto storage_type = - voice_cloning ? clone_weight_type(options(), *assets_) - : requested_weight_type(options()); - runtime_slot = std::make_unique( + const auto ensure_start = Clock::now(); + const auto storage_type = + voice_cloning ? clone_weight_type(options(), *assets_) + : requested_weight_type(options()); + const bool rebuilt = + llama_ == nullptr || !llama_storage_type_.has_value() || + *llama_storage_type_ != storage_type; + if (rebuilt) { + // Keep only one language-model runtime resident. CUDA cloning may require + // an F32 runtime for quantized source weights, so switching routes replaces + // the previous runtime instead of retaining duplicate weights and graphs. + llama_.reset(); + llama_ = std::make_unique( assets_, options().backend.type, options().backend.device, std::max(1, options().backend.threads), runtime::parse_size_mb_option(options().options, @@ -406,8 +468,116 @@ OuteTTSLlamaRuntime &OuteTTSSession::llama(bool voice_cloning) { {"outetts.constant_context_mb"}, 256ull * 1024ull * 1024ull), storage_type); + llama_storage_type_ = storage_type; } - return *runtime_slot; + debug::trace_log_scalar("outetts.llama.runtime_rebuilt", rebuilt); + debug::trace_log_scalar("outetts.llama.runtime_reused", !rebuilt); + debug::trace_log_scalar("outetts.llama.clone_route", voice_cloning); + debug::timing_log_scalar( + "outetts.llama.ensure_runtime_ms", + rebuilt ? debug::elapsed_ms(ensure_start) : 0.0); + return *llama_; +} + +bool OuteTTSSession::ReferenceProfileCacheKeyEqual::operator()( + const ReferenceProfileCacheKey &lhs, + const ReferenceProfileCacheKey &rhs) const { + return lhs.audio_hash == rhs.audio_hash && + lhs.sample_rate == rhs.sample_rate && + lhs.channels == rhs.channels && + lhs.sample_count == rhs.sample_count && lhs.text == rhs.text && + lhs.language == rhs.language; +} + +OuteTTSVoiceProfile OuteTTSSession::prepare_voice_profile( + const runtime::AudioBuffer &audio, const std::string &reference_text, + const std::string &language, bool &cache_hit) { + const auto total_start = Clock::now(); + ReferenceProfileCacheKey key{ + reference_audio_hash(audio), audio.sample_rate, audio.channels, + audio.samples.size(), reference_text, language}; + if (const auto *cached = reference_profile_cache_.find(key)) { + cache_hit = true; + debug::trace_log_scalar("outetts.reference_cache.hit", true); + debug::trace_log_scalar( + "outetts.reference_cache.slots", + static_cast(reference_profile_cache_.capacity())); + debug::trace_log_scalar( + "outetts.reference_cache.entries", + static_cast(reference_profile_cache_.size())); + debug::trace_log_scalar("outetts.reference_cache.evicted", false); + debug::timing_log_scalar("outetts.reference.total_ms", + debug::elapsed_ms(total_start)); + return *cached; + } + + cache_hit = false; + const bool will_evict = + reference_profile_cache_.capacity() > 0 && + reference_profile_cache_.size() >= reference_profile_cache_.capacity(); + const auto ensure_aligner_start = Clock::now(); + const bool aligner_built = aligner_session_ == nullptr; + if (aligner_built) { + if (aligner_assets_ == nullptr) { + aligner_assets_ = resolve_aligner_assets(options(), *assets_); + } + if (aligner_assets_ == nullptr) { + throw std::runtime_error( + "OuteTTS voice cloning requires a GGUF with an embedded Qwen3 " + "Forced Aligner or --session-option " + "outetts.aligner_model_path="); + } + aligner_session_ = std::make_unique< + engine::models::qwen3_forced_aligner::Qwen3ForcedAlignerSession>( + runtime::TaskSpec{runtime::VoiceTaskKind::Alignment, + runtime::RunMode::Offline}, + aligner_session_options(options()), aligner_assets_); + } + debug::trace_log_scalar("outetts.aligner.runtime_rebuilt", aligner_built); + debug::trace_log_scalar("outetts.aligner.runtime_reused", !aligner_built); + debug::timing_log_scalar( + "outetts.aligner.ensure_runtime_ms", + aligner_built ? debug::elapsed_ms(ensure_aligner_start) : 0.0); + + const auto align_start = Clock::now(); + const auto alignment = align_reference(*aligner_session_, *aligner_assets_, + audio, reference_text, language); + debug::timing_log_scalar("outetts.reference.align_ms", + debug::elapsed_ms(align_start)); + + const auto encode_start = Clock::now(); + auto encoded = dac_.encode_reference(audio); + debug::timing_log_scalar("outetts.reference.dac_encode_ms", + debug::elapsed_ms(encode_start)); + + const auto profile_start = Clock::now(); + auto profile = make_voice_profile(std::move(encoded), reference_text, + &alignment); + debug::timing_log_scalar("outetts.reference.profile_ms", + debug::elapsed_ms(profile_start)); + reference_profile_cache_.put(std::move(key), profile); + + debug::trace_log_scalar("outetts.reference_cache.hit", false); + debug::trace_log_scalar( + "outetts.reference_cache.slots", + static_cast(reference_profile_cache_.capacity())); + debug::trace_log_scalar( + "outetts.reference_cache.entries", + static_cast(reference_profile_cache_.size())); + debug::trace_log_scalar("outetts.reference_cache.evicted", will_evict); + if (mem_saver_) { + const auto release_start = Clock::now(); + aligner_session_.reset(); + debug::trace_log_scalar("outetts.aligner.runtime_released", true); + debug::timing_log_scalar("outetts.aligner.release_ms", + debug::elapsed_ms(release_start)); + } else { + debug::trace_log_scalar("outetts.aligner.runtime_released", false); + debug::timing_log_scalar("outetts.aligner.release_ms", 0.0); + } + debug::timing_log_scalar("outetts.reference.total_ms", + debug::elapsed_ms(total_start)); + return profile; } std::string OuteTTSSession::family() const { return "outetts"; } @@ -431,23 +601,24 @@ void OuteTTSSession::prepare( !request.text->language.empty() ? request.text->language : "en"); - const auto alignment = - align_reference(options(), *assets_, audio, reference_text, language); - voice_profile_ = make_voice_profile( - dac_.encode_reference(audio), reference_text, - alignment.has_value() ? &*alignment : nullptr); + bool cache_hit = false; + voice_profile_ = + prepare_voice_profile(audio, reference_text, language, cache_hit); + debug::trace_log_scalar("outetts.prepare.reference_cache_hit", cache_hit); } mark_prepared(); } runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { + const auto wall_start = Clock::now(); require_prepared("OuteTTS run"); if (!request.text_input.has_value() || request.text_input->text.empty()) { throw std::runtime_error("OuteTTS requires text input"); } const auto *voice_audio = reference_audio(request); std::optional request_profile; - if (voice_audio != nullptr && !voice_profile_.has_value()) { + bool reference_cache_hit = false; + if (voice_audio != nullptr) { const auto reference_text = runtime::find_option(request.options, {"reference_text"}).value_or(""); if (reference_text.empty()) @@ -459,12 +630,8 @@ runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { !request.text_input->language.empty() ? request.text_input->language : "en"); - const auto alignment = - align_reference(options(), *assets_, *voice_audio, reference_text, - language); - request_profile = make_voice_profile( - dac_.encode_reference(*voice_audio), reference_text, - alignment.has_value() ? &*alignment : nullptr); + request_profile = prepare_voice_profile( + *voice_audio, reference_text, language, reference_cache_hit); } const OuteTTSVoiceProfile *profile = request_profile.has_value() @@ -475,34 +642,107 @@ runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { throw std::runtime_error( "OuteTTS voice cloning requires --voice-ref and --reference-text"); } - const auto prompt = - profile != nullptr - ? tokenizer_.build_clone_prompt(request.text_input->text, *profile) - : tokenizer_.build_prompt(request.text_input->text); + + const int64_t text_chunk_size = + engine::text::parse_text_chunk_size_override(request.options) + .value_or(kDefaultTextChunkSize); + const auto text_chunk_mode = + engine::text::parse_text_chunk_mode_override(request.options) + .value_or(engine::text::TextChunkMode::Default); + const auto chunk_requests = runtime::chunk_text_request( + request, text_chunk_size, text_chunk_mode); + if (chunk_requests.empty()) { + throw std::runtime_error("OuteTTS text chunking produced no requests"); + } + debug::trace_log_scalar("outetts.text_chunk_size", text_chunk_size); + debug::trace_log_scalar("outetts.text_chunk_mode", + engine::text::text_chunk_mode_name(text_chunk_mode)); + debug::trace_log_scalar( + "outetts.text_chunk_count", + static_cast(chunk_requests.size())); + debug::trace_log_scalar("outetts.reference.cache_hit", + reference_cache_hit); + const bool quantized_cloning = profile != nullptr && has_quantized_clone_weights(options(), *assets_); - auto generate_options = generation_options( - request, assets_->generation, profile != nullptr, quantized_cloning); - const auto generated = llama(profile != nullptr).generate( - prompt, generate_options, - tokenizer_.eos_id(), tokenizer_.audio_end_id()); - std::vector c1; - std::vector c2; - for (const int32_t token : generated) - tokenizer_.append_audio_code(token, c1, c2); - const size_t pairs = std::min(c1.size(), c2.size()); - c1.resize(pairs); - c2.resize(pairs); - if (pairs == 0) { - std::string detail; - for (size_t i = 0; i < std::min(generated.size(), 12); ++i) { - detail += (i == 0 ? "" : ",") + std::to_string(generated[i]); + + runtime::AudioBuffer merged_audio; + double prompt_ms = 0.0; + double generate_ms = 0.0; + double decode_ms = 0.0; + double release_ms = 0.0; + int64_t generated_tokens = 0; + int64_t released_cache_capacity = 0; + for (size_t chunk_index = 0; chunk_index < chunk_requests.size(); + ++chunk_index) { + const auto &chunk_request = chunk_requests[chunk_index]; + const auto prompt_start = Clock::now(); + const auto prompt = + profile != nullptr + ? tokenizer_.build_clone_prompt(chunk_request.text_input->text, + *profile) + : tokenizer_.build_prompt(chunk_request.text_input->text); + prompt_ms += debug::elapsed_ms(prompt_start); + + auto generate_options = + generation_options(chunk_request, assets_->generation, + profile != nullptr, quantized_cloning); + const auto generate_start = Clock::now(); + const auto generated = llama(profile != nullptr).generate( + prompt, generate_options, tokenizer_.eos_id(), + tokenizer_.audio_end_id()); + generate_ms += debug::elapsed_ms(generate_start); + generated_tokens += static_cast(generated.size()); + + std::vector c1; + std::vector c2; + for (const int32_t token : generated) + tokenizer_.append_audio_code(token, c1, c2); + const size_t pairs = std::min(c1.size(), c2.size()); + c1.resize(pairs); + c2.resize(pairs); + if (pairs == 0) { + std::string detail; + for (size_t i = 0; i < std::min(generated.size(), 12); ++i) { + detail += (i == 0 ? "" : ",") + std::to_string(generated[i]); + } + throw std::runtime_error( + "OuteTTS generated no complete DAC code pairs (tokens=" + detail + + ")"); + } + + const auto decode_start = Clock::now(); + runtime::append_audio_buffer(merged_audio, dac_.decode(c1, c2)); + decode_ms += debug::elapsed_ms(decode_start); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".prompt_tokens", + static_cast(prompt.size())); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".generated_tokens", + static_cast(generated.size())); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".codec_frames", + static_cast(pairs)); + + if (mem_saver_) { + const auto release_start = Clock::now(); + released_cache_capacity += llama_->release_cached_step_graph(); + release_ms += debug::elapsed_ms(release_start); } - throw std::runtime_error( - "OuteTTS generated no complete DAC code pairs (tokens=" + detail + ")"); } + runtime::TaskResult result; - result.audio_output = dac_.decode(c1, c2); + result.audio_output = std::move(merged_audio); + debug::trace_log_scalar("outetts.mem_saver", mem_saver_); + debug::trace_log_scalar("outetts.generated_tokens", generated_tokens); + debug::trace_log_scalar("outetts.llama.step.released_cache_capacity", + released_cache_capacity); + debug::timing_log_scalar("outetts.prompt_ms", prompt_ms); + debug::timing_log_scalar("outetts.generate_ms", generate_ms); + debug::timing_log_scalar("outetts.dac_decode_ms", decode_ms); + debug::timing_log_scalar("outetts.llama.step.release_ms", release_ms); + debug::timing_log_scalar("session.wall_ms", + debug::elapsed_ms(wall_start)); return result; } diff --git a/tests/outetts/outetts_warm_bench.cpp b/tests/outetts/outetts_warm_bench.cpp new file mode 100644 index 000000000..c1dbf662c --- /dev/null +++ b/tests/outetts/outetts_warm_bench.cpp @@ -0,0 +1,266 @@ +#include "engine/framework/audio/wav_reader.h" +#include "engine/framework/audio/wav_writer.h" +#include "engine/framework/debug/trace.h" +#include "engine/framework/io/json.h" +#include "engine/framework/runtime/registry.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct RequestCase { + std::string name; + std::string text; + std::string language; + std::filesystem::path voice_ref; + std::string reference_text; + std::unordered_map options; +}; + +std::string arg_value(int argc, char **argv, const std::string &name, + const std::string &fallback) { + for (int i = 1; i + 1 < argc; ++i) { + if (argv[i] == name) + return argv[i + 1]; + } + return fallback; +} + +std::vector arg_values(int argc, char **argv, + const std::string &name) { + std::vector out; + for (int i = 1; i + 1 < argc; ++i) { + if (argv[i] == name) + out.emplace_back(argv[i + 1]); + } + return out; +} + +int int_arg(int argc, char **argv, const std::string &name, int fallback) { + return std::stoi(arg_value(argc, argv, name, std::to_string(fallback))); +} + +engine::core::BackendType parse_backend(const std::string &value) { + if (value == "cpu") + return engine::core::BackendType::Cpu; + if (value == "cuda") + return engine::core::BackendType::Cuda; + if (value == "vulkan") + return engine::core::BackendType::Vulkan; + if (value == "best") + return engine::core::BackendType::BestAvailable; + throw std::runtime_error("unsupported backend: " + value); +} + +std::string scalar_option(const engine::io::json::Value &value) { + if (value.is_string()) + return value.as_string(); + if (value.is_bool()) + return value.as_bool() ? "true" : "false"; + if (value.is_number()) + return engine::io::json::stringify_number(value.as_number()); + throw std::runtime_error( + "OuteTTS warm-bench options must be strings, numbers, or booleans"); +} + +void copy_option_if_present( + std::unordered_map &options, + const engine::io::json::Value &item, const std::string &name) { + if (const auto *value = item.find(name); + value != nullptr && !value->is_null()) { + options[name] = scalar_option(*value); + } +} + +std::vector +load_requests(const std::filesystem::path &path, + const std::unordered_map &defaults) { + const auto root = engine::io::json::parse_file(path); + const auto &items = root.require("requests").as_array(); + if (items.empty()) + throw std::runtime_error("OuteTTS request file has no requests"); + std::vector out; + out.reserve(items.size()); + for (size_t index = 0; index < items.size(); ++index) { + const auto &item = items[index]; + RequestCase request; + request.name = engine::io::json::optional_string( + item, "name", "request_" + std::to_string(index)); + request.text = engine::io::json::require_string(item, "text"); + request.language = + engine::io::json::optional_string(item, "language", "en"); + request.voice_ref = + engine::io::json::optional_string(item, "voice_ref", ""); + request.reference_text = + engine::io::json::optional_string(item, "reference_text", ""); + request.options = defaults; + for (const char *name : + {"max_tokens", "seed", "temperature", "top_k", "top_p", "min_p", + "repetition_penalty", "repetition_window", "text_chunk_size", + "text_chunk_mode", "reference_language"}) { + copy_option_if_present(request.options, item, name); + } + if (!request.reference_text.empty()) + request.options["reference_text"] = request.reference_text; + out.push_back(std::move(request)); + } + return out; +} + +engine::runtime::AudioBuffer read_audio(const std::filesystem::path &path) { + const auto wav = engine::audio::read_wav_f32(path); + return {wav.sample_rate, wav.channels, wav.samples}; +} + +double audio_seconds(const engine::runtime::AudioBuffer &audio) { + if (audio.sample_rate <= 0 || audio.channels <= 0) + return 0.0; + return static_cast(audio.samples.size()) / + static_cast(audio.sample_rate * audio.channels); +} + +engine::runtime::TaskRequest make_request( + const RequestCase &request, + std::unordered_map + &audio_cache) { + engine::runtime::TaskRequest out; + out.text_input = + engine::runtime::Transcript{request.text, request.language}; + out.options = request.options; + if (!request.voice_ref.empty()) { + const std::string key = request.voice_ref.string(); + auto found = audio_cache.find(key); + if (found == audio_cache.end()) + found = audio_cache.emplace(key, read_audio(request.voice_ref)).first; + out.voice = engine::runtime::VoiceCondition{}; + out.voice->speaker = engine::runtime::VoiceReference{}; + out.voice->speaker->audio = found->second; + } + return out; +} + +} // namespace + +int main(int argc, char **argv) try { + const std::filesystem::path model_path = + arg_value(argc, argv, "--model", + "models/Llama-OuteTTS-1.0-1B-Q8_0/model.gguf"); + const std::filesystem::path request_file = + arg_value(argc, argv, "--request-file", ""); + if (request_file.empty()) + throw std::runtime_error("OuteTTS warm bench requires --request-file"); + const std::filesystem::path output_dir = + arg_value(argc, argv, "--audio-out-dir", + "build/logs/warmbench/outetts_audio"); + const std::filesystem::path log_file = + arg_value(argc, argv, "--log-file", + arg_value(argc, argv, "--trace-file", + "build/logs/warmbench/outetts.log")); + const std::string backend_name = + arg_value(argc, argv, "--backend", "cpu"); + const int device = int_arg(argc, argv, "--device", 0); + const int threads = int_arg(argc, argv, "--threads", 8); + const int iterations = int_arg(argc, argv, "--iterations", 1); + const int hold_seconds = int_arg(argc, argv, "--hold-seconds", 0); + if (iterations <= 0) + throw std::runtime_error("--iterations must be positive"); + if (hold_seconds < 0) + throw std::runtime_error("--hold-seconds must be non-negative"); + + std::unordered_map defaults; + for (const auto &option : arg_values(argc, argv, "--request-option")) { + const size_t equals = option.find('='); + if (equals == std::string::npos || equals == 0) + throw std::runtime_error("invalid --request-option: " + option); + defaults[option.substr(0, equals)] = option.substr(equals + 1); + } + const auto requests = load_requests(request_file, defaults); + + std::filesystem::create_directories(output_dir); + if (!log_file.parent_path().empty()) + std::filesystem::create_directories(log_file.parent_path()); + engine::debug::configure_logging( + engine::debug::LoggingConfig{true, log_file.string()}); + + auto registry = engine::runtime::make_default_registry(); + engine::runtime::ModelLoadRequest load_request; + load_request.model_path = model_path; + load_request.family_hint = "outetts"; + auto model = registry.load(load_request); + + engine::runtime::SessionOptions session_options; + session_options.backend.type = parse_backend(backend_name); + session_options.backend.device = device; + session_options.backend.threads = threads; + for (const auto &option : arg_values(argc, argv, "--session-option")) { + const size_t equals = option.find('='); + if (equals == std::string::npos || equals == 0) + throw std::runtime_error("invalid --session-option: " + option); + session_options.options[option.substr(0, equals)] = + option.substr(equals + 1); + } + + auto session_base = model->create_task_session( + {engine::runtime::VoiceTaskKind::Tts, + engine::runtime::RunMode::Offline}, + session_options); + auto *session = + dynamic_cast( + session_base.get()); + if (session == nullptr) + throw std::runtime_error( + "OuteTTS did not create an offline TTS session"); + session->prepare(engine::runtime::SessionPreparationRequest{}); + + std::unordered_map audio_cache; + for (size_t request_index = 0; request_index < requests.size(); + ++request_index) { + for (int iteration = 0; iteration < iterations; ++iteration) { + const auto started = std::chrono::steady_clock::now(); + auto result = session->run( + make_request(requests[request_index], audio_cache)); + const double wall_ms = + std::chrono::duration( + std::chrono::steady_clock::now() - started) + .count(); + if (!result.audio_output.has_value()) + throw std::runtime_error("OuteTTS produced no audio"); + const double seconds = audio_seconds(*result.audio_output); + const auto output_path = + output_dir / + (requests[request_index].name + "_" + + std::to_string(iteration + 1) + ".wav"); + engine::audio::write_pcm16_wav( + output_path, result.audio_output->sample_rate, + result.audio_output->channels, result.audio_output->samples); + std::cout << "request=" << requests[request_index].name << "\n"; + std::cout << "iteration=" << iteration + 1 << "\n"; + std::cout << "wall_ms=" << wall_ms << "\n"; + std::cout << "audio_seconds=" << seconds << "\n"; + std::cout << "rtf=" + << (seconds > 0.0 ? wall_ms / 1000.0 / seconds : 0.0) + << "\n"; + std::cout << "audio_out=" << output_path.string() << "\n"; + } + } + std::cout << "log_out=" << log_file.string() << "\n"; + if (hold_seconds > 0) { + std::cout << "holding_session_seconds=" << hold_seconds << "\n"; + std::cout.flush(); + std::this_thread::sleep_for(std::chrono::seconds(hold_seconds)); + } + return 0; +} catch (const std::exception &error) { + std::cerr << "outetts_warm_bench failed: " << error.what() << "\n"; + return 1; +} diff --git a/tests/outetts/warm_bench_requests.json b/tests/outetts/warm_bench_requests.json new file mode 100644 index 000000000..1237e13ec --- /dev/null +++ b/tests/outetts/warm_bench_requests.json @@ -0,0 +1,47 @@ +{ + "requests": [ + { + "name": "tts_cold", + "text": "This is the first OuteTTS request in a long-lived session.", + "language": "en", + "max_tokens": 256, + "seed": 1234 + }, + { + "name": "tts_repeat", + "text": "This is the first OuteTTS request in a long-lived session.", + "language": "en", + "max_tokens": 256, + "seed": 1234 + }, + { + "name": "tts_longform", + "text": "Long-form synthesis is split by the shared framework text chunker. Each sentence becomes a bounded request while model weights and the cached generation graph stay inside the same long-lived session. The generated audio chunks are appended in order to form one output waveform.", + "language": "en", + "max_tokens": 256, + "seed": 1234, + "text_chunk_size": 100, + "text_chunk_mode": "default" + }, + { + "name": "clone_cold", + "text": "This request builds and caches the reference voice profile.", + "language": "en", + "voice_ref": "assets/resources/b.wav", + "reference_text": "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you.", + "reference_language": "en", + "max_tokens": 256, + "seed": 42 + }, + { + "name": "clone_repeat", + "text": "This request builds and caches the reference voice profile.", + "language": "en", + "voice_ref": "assets/resources/b.wav", + "reference_text": "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you.", + "reference_language": "en", + "max_tokens": 256, + "seed": 42 + } + ] +} From 0e1d4036e0a791cfdeca5ca92a415fa1ee5f37da Mon Sep 17 00:00:00 2001 From: mirek190 Date: Thu, 16 Jul 2026 00:10:56 +0100 Subject: [PATCH 3/8] Document exact OuteTTS validation record --- docs/outetts_validation.md | 199 ++++++++++++++++++++++++++++++++----- 1 file changed, 173 insertions(+), 26 deletions(-) diff --git a/docs/outetts_validation.md b/docs/outetts_validation.md index 7e4d060d9..689dc069e 100644 --- a/docs/outetts_validation.md +++ b/docs/outetts_validation.md @@ -13,6 +13,13 @@ Install the safetensors model, IBM DAC, and Qwen3 Forced Aligner resources: python tools/model_manager.py install outetts_1_0_1b --models-dir models ``` +The model-manager package id is `outetts_1_0_1b`. It creates these model +roots: + +- `models/Llama-OuteTTS-1.0-1B` +- `models/DAC.speech.v1.0` +- `models/Qwen3-ForcedAligner-0.6B` + The standalone GGUF command is documented in [TTS](tts.md#outetts). The packed file used below contains the OuteTTS language model, IBM DAC, Qwen aligner, tokenizers, configuration, and package specification. @@ -43,32 +50,85 @@ Select `outetts.Backend.HF` for a Transformers comparison or `1.1` over the latest 64 tokens, top-k `40`, top-p `0.9`, and min-p `0.05`. -## Build +## Exact build commands + +The Windows validation used the project build script to establish the compiler +and backend configuration, then explicitly enabled the warm-benchmark targets. + +Windows CPU: + +```powershell +.\scripts\build_windows.ps1 -Preset windows-cpu-release -ConfigureOnly +cmake -S . -B build\windows-cpu-release -DENGINE_BUILD_WARMBENCH=ON +cmake --build build\windows-cpu-release --parallel 4 --target audiocpp_cli audiocpp_server outetts_warm_bench +``` -Windows CUDA: +Windows CUDA 12.4: ```powershell $env:CUDA_PATH = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" -.\scripts\build_windows.ps1 -Preset windows-cuda-release -Target outetts_warm_bench -Jobs 4 +.\scripts\build_windows.ps1 -Preset windows-cuda-release -ConfigureOnly +cmake -S . -B build\windows-cuda-release -DENGINE_BUILD_WARMBENCH=ON +cmake --build build\windows-cuda-release --parallel 4 --target audiocpp_cli audiocpp_server outetts_warm_bench ``` -Linux CPU: +## Exact standalone-GGUF path tests -```bash -cmake -S . -B build/cpu-release -DCMAKE_BUILD_TYPE=Release -DENGINE_BUILD_WARMBENCH=ON -cmake --build build/cpu-release --target outetts_warm_bench -j +The tested GGUF was deliberately outside the PR worktree and its directory +contained only one file: + +```text +..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf +``` + +Normal TTS: + +```powershell +build\windows-cuda-release\bin\audiocpp_cli.exe ` + --task tts --family outetts ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` + --backend cuda ` + --text "This is the standalone GGUF path test." ` + --max-tokens 256 --request-option seed=1234 ` + --out ..\outputs\outetts_path_test_tts.wav +``` + +Voice cloning using the aligner and DAC embedded in the same GGUF: + +```powershell +build\windows-cuda-release\bin\audiocpp_cli.exe ` + --task clon --family outetts ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` + --backend cuda ` + --voice-ref assets\resources\b.wav ` + --reference-text "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you." ` + --request-option reference_language=en ` + --text "This is the standalone GGUF cloning path test." ` + --max-tokens 256 --request-option seed=42 ` + --out ..\outputs\outetts_path_test_clone.wav ``` -## Long-lived session +Both commands loaded successfully without an external package spec, tokenizer, +DAC directory, aligner directory, or other sidecar. The generated 24 kHz mono +WAVs passed ffmpeg decoding: + +| Artifact | Duration | Bytes | SHA-256 | +|---|---:|---:|---| +| `..\outputs\outetts_path_test_tts.wav` | 1.399667s | 67228 | `92DA5D8438D4E6A79FBDDBD8EDA77D1AFD2A9B47F61085ECD3AD45C913102904` | +| `..\outputs\outetts_path_test_clone.wav` | 1.453000s | 69788 | `E5B46E62C2DB89A654F7EC21A7BE31F575D6F667F8578224386E7C552492B05E` | + +## Exact long-lived-session runs + +CUDA default: ```powershell build\windows-cuda-release\bin\outetts_warm_bench.exe ` - --model models\Llama-OuteTTS-1.0-1B-Q8_0\model.gguf ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` --backend cuda --threads 8 ` --request-file tests\outetts\warm_bench_requests.json ` --hold-seconds 5 ` - --audio-out-dir build\logs\warmbench\outetts-default ` - --log-file build\logs\warmbench\outetts-default.log + --audio-out-dir ..\outputs\outetts_review_cuda_default ` + --log-file build\logs\warmbench\outetts-cuda-default.log ``` Expected trace evidence: @@ -89,49 +149,136 @@ Run the same request sequence in a fresh process with memory saver enabled: ```powershell build\windows-cuda-release\bin\outetts_warm_bench.exe ` - --model models\Llama-OuteTTS-1.0-1B-Q8_0\model.gguf ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` --backend cuda --threads 8 ` --request-file tests\outetts\warm_bench_requests.json ` --session-option outetts.mem_saver=true ` --hold-seconds 5 ` - --audio-out-dir build\logs\warmbench\outetts-mem-saver ` - --log-file build\logs\warmbench\outetts-mem-saver.log + --audio-out-dir ..\outputs\outetts_review_cuda_mem_saver ` + --log-file build\logs\warmbench\outetts-cuda-mem_saver.log +``` + +CPU default: + +```powershell +build\windows-cpu-release\bin\outetts_warm_bench.exe ` + --model ..\models\Llama-OuteTTS-1.0-1B_Q8\Llama-OuteTTS-1.0-1B_Q8.gguf ` + --backend cpu --threads 8 ` + --request-file tests\outetts\warm_bench_requests.json ` + --hold-seconds 5 ` + --audio-out-dir ..\outputs\outetts_review_cpu_default ` + --log-file build\logs\warmbench\outetts-cpu-default-final.log ``` The memory-saver trace reports a positive `outetts.llama.step.released_cache_capacity` after generation and `outetts.aligner.runtime_released=1` after an uncached reference. -Sample per-process VRAM once per second while each fresh benchmark runs: +Sample total-device VRAM every 250 ms while each fresh benchmark runs. Ensure +that no unrelated CUDA workload is active: ```powershell -nvidia-smi --query-compute-apps=pid,process_name,used_gpu_memory --format=csv -l 1 +nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -lms 250 ``` Compare peak and final resident VRAM, request wall time, output duration, and RTF between the two runs. Keep model, backend, device, seed, requests, and quantization identical. +## Generated artifacts + +Each benchmark directory contains: + +```text +tts_cold_1.wav +tts_repeat_1.wav +tts_longform_1.wav +clone_cold_1.wav +clone_repeat_1.wav +``` + +The exact generated directories and logs were: + +- `..\outputs\outetts_review_cuda_default` +- `..\outputs\outetts_review_cuda_mem_saver` +- `..\outputs\outetts_review_cpu_default` +- `build\logs\warmbench\outetts-cuda-default.log` +- `build\logs\warmbench\outetts-cuda-mem_saver.log` +- `build\logs\warmbench\outetts-cpu-default-final.log` + +These validation artifacts are reproducible local outputs and are not committed +to the repository. + ## Measured validation The committed five-request sequence was measured with the packed Q8 GGUF on an NVIDIA GeForce RTX 3090 (CUDA 12.4). VRAM was sampled every 250 ms from total device usage with no other CUDA workload. Resident VRAM was sampled during the -five-second hold after all requests completed. +five-second hold after all requests completed. CPU process working set was +sampled every 250 ms over the same request sequence and hold. -| Mode | Sequence wall | Audio | RTF | Peak VRAM | Resident VRAM | -|---|---:|---:|---:|---:|---:| -| default | 33342.24 ms | 11.424s | 2.918 | 17653 MiB | 294 MiB | -| `outetts.mem_saver=true` | 30990.93 ms | 11.424s | 2.712 | 5780 MiB | 294 MiB | +| Backend | Mode | Sequence wall | Audio | RTF | Peak memory | Resident memory | +|---|---|---:|---:|---:|---:|---:| +| CUDA | default | 33342.24 ms | 11.424s | 2.918 | 17653 MiB VRAM | 294 MiB VRAM | +| CUDA | `outetts.mem_saver=true` | 30990.93 ms | 11.424s | 2.712 | 5780 MiB VRAM | 294 MiB VRAM | +| CPU | default | 103526.0 ms | 11.49067s | 9.009 | 14573.3 MiB RSS | 6458.0 MiB RSS | -Both modes produced the same SHA-256 for the cold and repeated TTS pair and for -the cold and repeated clone pair. All ten generated WAV files passed an ffmpeg -decode check. The traces confirmed four framework chunks for the long-form -request, compatible step-graph reuse in default mode, explicit graph release in -memory-saver mode, and a reference-profile cache hit on the repeated clone. +Both CUDA modes produced the same SHA-256 for the cold and repeated TTS pair and +for the cold and repeated clone pair. All ten CUDA and all five CPU generated +WAV files passed an ffmpeg decode check. The traces confirmed four framework +chunks for the long-form request, compatible step-graph reuse in default mode, +explicit graph release in memory-saver mode, and a reference-profile cache hit +on the repeated clone. Generation is the dominant measured hot path. In the default CUDA run, the cold reference path took about 890 ms (109 ms alignment, 139 ms DAC encoding, and 214 ms profile construction), while the repeated cached reference took 0.29 ms. The memory-saver result is within normal run-to-run timing variation; its purpose here is the lower peak, not a speedup guarantee. + +## Path and parity results + +- The standalone path test loaded the one-file GGUF from outside the worktree; + no adjacent sidecars or `model_specs` directory were present. +- Within CUDA, cold and repeated TTS were byte-identical with SHA-256 + `52A11609B165314E8F83919CD4F82AD899346B9945DBEA9D9676431F4E67C548`. +- Within CUDA, cold and repeated cloning were byte-identical with SHA-256 + `8816F29514944C7B8121D17E7620D095D06BA636247BFD4358793500FE8B6FA1`. +- The CUDA default and memory-saver outputs had the same hashes. +- Within CPU, cold and repeated TTS were byte-identical with SHA-256 + `A315C0AD425C99910597FCC4BAB9D80CB5F4C893176EBD9099BA6C6AA7B66BFD`. +- Within CPU, cold and repeated cloning were byte-identical with SHA-256 + `1A2009C0D512773A4B5082BF37EDEDD6A52A37CC6D556772C62EA26550A83B4D`. +- CPU and CUDA WAV bytes are not identical because backend floating-point + execution differs; deterministic reuse parity was therefore checked within + each backend rather than asserted across backends. +- The official Python reference has a clean setup path documented above. Per + maintainer request, Python output parity is intentionally left for maintainer + validation rather than claimed without their reference environment. + +## Backend coverage + +| Backend | Coverage | +|---|---| +| Windows CPU | CLI/server/warm-benchmark build; normal TTS, long-form TTS, cloning, cache reuse, graph reuse, WAV decode, timing, and RSS run | +| Windows CUDA 12.4 | CLI/server/warm-benchmark build; normal TTS, long-form TTS, cloning, standalone-path loading, cache/graph reuse, default/memory-saver A/B, WAV decode, timing, RTF, and VRAM run | +| Linux CPU | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | +| Linux Vulkan | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | +| macOS CPU | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | +| Metal | Not enabled or runtime-tested in this validation | + +## Known limitations + +- OuteTTS is offline-only in this implementation. +- Voice cloning requires an accurate transcript of the reference WAV and rejects + references longer than 20 seconds. +- Quantized CUDA cloning expands the OuteTTS language-model weights to F32 for + generation correctness. Default mode can therefore have a high transient + peak while the aligner is retained; `outetts.mem_saver=true` releases the + aligner and cached-step graph between phases. +- Long-form output concatenates independently generated chunks; `max_tokens` + applies to each chunk. +- CPU and CUDA outputs are deterministic within each tested backend but are not + expected to be byte-identical across backends. +- Python-reference output parity remains pending maintainer validation using the + exact setup and sampling parameters above. From ab926fd35469968e33e3013e472936df9a49369f Mon Sep 17 00:00:00 2001 From: mirek190 Date: Thu, 16 Jul 2026 00:45:03 +0100 Subject: [PATCH 4/8] Report per-request OuteTTS parity and performance --- docs/memory_saver.md | 6 +- docs/outetts_validation.md | 173 +++++++++++++++++++++++++++---------- 2 files changed, 131 insertions(+), 48 deletions(-) diff --git a/docs/memory_saver.md b/docs/memory_saver.md index 037d0f821..c9e9a5368 100644 --- a/docs/memory_saver.md +++ b/docs/memory_saver.md @@ -31,7 +31,7 @@ Native/default weights were used for all rows. - VoxCPM2 used the OpenAI-compatible offline speech endpoint with a 2048-character voice-design request, `seed=1234`, `max_tokens=512`, `num_inference_steps=10`, and `guidance_scale=2.0`. The default and `mem_saver` WAV outputs were byte-identical. - IndexTTS2 used a five-request server sequence with the same seeds and references for default and `mem_saver`: normal text, longer text, longer emotion-text request with a different reference, shorter text, then longer text. - Irodori TTS 500M used a five-request server sequence with the same seeds and options for default and `mem_saver`: reference text, longer reference text, longer no-reference emoji/style text, shorter reference text, then longer reference text. -- OuteTTS 1.0 1B Q8 used a five-request long-lived-session sequence on an RTX 3090: repeated fixed-seed TTS, four-chunk long-form TTS, then repeated fixed-seed cloning with the same reference. The default and `mem_saver` WAV outputs were byte-identical. VRAM is total-device usage with no other CUDA workload; resident VRAM was sampled during a five-second post-sequence hold. +- OuteTTS 1.0 1B Q8 used a five-request long-lived-session sequence on an RTX 3090: repeated fixed-seed TTS, four-chunk long-form TTS, then repeated fixed-seed cloning with the same reference. CUDA timing is the mean of three fresh processes per mode, alternated to reduce ordering bias; memory saver was 0.6-1.2% slower on every request and 0.99% slower over the mean sequence total. The default and `mem_saver` WAV outputs were byte-identical. VRAM is total-device usage with no other CUDA workload; resident VRAM was sampled during a five-second post-sequence hold. Per-request timing and RTF are recorded in `docs/outetts_validation.md`. | Model | Mode | Peak VRAM | Resident VRAM | Server wall | Audio | RTF | |---|---|---:|---:|---:|---:|---:| @@ -59,5 +59,5 @@ Native/default weights were used for all rows. | Irodori TTS 500M | mem_saver | 11222 MiB | 3570 MiB | 3276.203 ms | 95.4s | 0.0343418 | | Irodori TTS 500M 6000-char | default | 18693 MiB | 13367 MiB | 27185.7 ms | 777.92s | 0.0349466 | | Irodori TTS 500M 6000-char | mem_saver | 11588 MiB | 3609 MiB | 27828 ms | 777.92s | 0.0357724 | -| OuteTTS 1.0 1B Q8 | default | 17653 MiB | 294 MiB | 33342.24 ms | 11.424s | 2.918 | -| OuteTTS 1.0 1B Q8 | mem_saver | 5780 MiB | 294 MiB | 30990.93 ms | 11.424s | 2.712 | +| OuteTTS 1.0 1B Q8 | default | 17653 MiB | 294 MiB | 29896.19 ms | 11.424s | 2.617 | +| OuteTTS 1.0 1B Q8 | mem_saver | 5780 MiB | 294 MiB | 30193.41 ms | 11.424s | 2.643 | diff --git a/docs/outetts_validation.md b/docs/outetts_validation.md index 689dc069e..e0b4630ba 100644 --- a/docs/outetts_validation.md +++ b/docs/outetts_validation.md @@ -26,29 +26,43 @@ tokenizers, configuration, and package specification. ## Python reference setup -The maintainer can install the official OuteTTS reference without using any -audio.cpp conversion code: +The parity run used the official OuteTTS repository at commit +`f5eac6e70d792844c6a6959d900a47af2c061a5b` (`outetts` 0.4.4), Python +3.10, PyTorch 2.5.1+cu121, and Transformers 4.52.3: -```bash -python -m venv .venv-outetts -. .venv-outetts/bin/activate -python -m pip install --upgrade pip outetts +```powershell +git clone https://github.com/edwko/OuteTTS build\reference\OuteTTS +git -C build\reference\OuteTTS checkout f5eac6e70d792844c6a6959d900a47af2c061a5b +python -m venv --system-site-packages build\reference\venv +build\reference\venv\Scripts\python.exe -m pip install -e build\reference\OuteTTS --no-deps +build\reference\venv\Scripts\python.exe -m pip install transformers==4.52.3 llama-cpp-python==0.3.9 polars natsort mecab-python3 unidic-lite uroman openai-whisper ftfy pyloudnorm ``` -On Windows PowerShell with the optional CUDA llama.cpp backend: +The reference loaded `models/Llama-OuteTTS-1.0-1B` through +`outetts.Backend.HF` with BF16 CUDA weights and loaded the official IBM DAC +checkpoint from +`models/DAC.speech.v1.0/weights_24khz_1.5kbps_v1.0.pth`. Both implementations +used temperature `0.4`, repetition penalty `1.1` over the latest 64 tokens, +top-k `40`, top-p `0.9`, min-p `0.05`, the request-file seeds, and the same +maximum number of new tokens. + +For a controlled clone comparison, Qwen word timings were generated once and +passed to the official Python `AudioProcessor.create_speaker_from_dict` path: ```powershell -python -m venv .venv-outetts -.\.venv-outetts\Scripts\Activate.ps1 -$env:CMAKE_ARGS = "-DGGML_CUDA=on" -python -m pip install --upgrade pip outetts +build\windows-cuda-release\bin\audiocpp_cli.exe ` + --task align --family qwen3_forced_aligner ` + --model ..\models\Qwen3-ForcedAligner-0.6B_Q8\Qwen3-ForcedAligner-0.6B_Q8.gguf ` + --backend cuda --audio assets\resources\b.wav ` + --text "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you." ` + --language en --words-out build\reference\b_words.json ``` -Select `outetts.Backend.HF` for a Transformers comparison or -`outetts.Backend.LLAMACPP` for the official llama.cpp-backed route. Use -`outetts.Models.VERSION_1_0_SIZE_1B`, temperature `0.4`, repetition penalty -`1.1` over the latest 64 tokens, top-k `40`, top-p `0.9`, and min-p -`0.05`. +Python still performed the official DAC encoding and feature extraction. This +keeps different alignment algorithms from contaminating the TTS comparison. +The Python reference-profile build took 934.118 ms and is charged to +`clone_cold` below. The audio.cpp cold-clone timing also includes its Qwen +alignment, while the Python number starts from the supplied word timings. ## Exact build commands @@ -170,6 +184,19 @@ build\windows-cpu-release\bin\outetts_warm_bench.exe ` --log-file build\logs\warmbench\outetts-cpu-default-final.log ``` +Safetensors CUDA parity run (DAC is resolved by the package spec; the aligner +path is supplied because it is not embedded in the safetensors directory): + +```powershell +build\windows-cuda-release\bin\outetts_warm_bench.exe ` + --model ..\models\Llama-OuteTTS-1.0-1B ` + --backend cuda --threads 8 ` + --request-file tests\outetts\warm_bench_requests.json ` + --session-option outetts.aligner_model_path=..\models\Qwen3-ForcedAligner-0.6B ` + --audio-out-dir build\reference\cpp_safetensors ` + --log-file build\reference\cpp_safetensors.log +``` + The memory-saver trace reports a positive `outetts.llama.step.released_cache_capacity` after generation and `outetts.aligner.runtime_released=1` after an uncached reference. @@ -205,6 +232,12 @@ The exact generated directories and logs were: - `build\logs\warmbench\outetts-cuda-default.log` - `build\logs\warmbench\outetts-cuda-mem_saver.log` - `build\logs\warmbench\outetts-cpu-default-final.log` +- `build\logs\warmbench\outetts-ab\default-{1,2,3}-audio` +- `build\logs\warmbench\outetts-ab\mem_saver-{1,2,3}-audio` +- `build\reference\cpp_safetensors` +- `build\reference\python_outputs` (official 44.1 kHz wrapper output) +- `build\reference\python_outputs_24k` (metric input resampled to DAC-native + 24 kHz) These validation artifacts are reproducible local outputs and are not committed to the repository. @@ -212,29 +245,58 @@ to the repository. ## Measured validation The committed five-request sequence was measured with the packed Q8 GGUF on an -NVIDIA GeForce RTX 3090 (CUDA 12.4). VRAM was sampled every 250 ms from total -device usage with no other CUDA workload. Resident VRAM was sampled during the -five-second hold after all requests completed. CPU process working set was -sampled every 250 ms over the same request sequence and hold. - -| Backend | Mode | Sequence wall | Audio | RTF | Peak memory | Resident memory | -|---|---|---:|---:|---:|---:|---:| -| CUDA | default | 33342.24 ms | 11.424s | 2.918 | 17653 MiB VRAM | 294 MiB VRAM | -| CUDA | `outetts.mem_saver=true` | 30990.93 ms | 11.424s | 2.712 | 5780 MiB VRAM | 294 MiB VRAM | -| CPU | default | 103526.0 ms | 11.49067s | 9.009 | 14573.3 MiB RSS | 6458.0 MiB RSS | - -Both CUDA modes produced the same SHA-256 for the cold and repeated TTS pair and -for the cold and repeated clone pair. All ten CUDA and all five CPU generated -WAV files passed an ffmpeg decode check. The traces confirmed four framework -chunks for the long-form request, compatible step-graph reuse in default mode, -explicit graph release in memory-saver mode, and a reference-profile cache hit -on the repeated clone. - -Generation is the dominant measured hot path. In the default CUDA run, the cold -reference path took about 890 ms (109 ms alignment, 139 ms DAC encoding, and -214 ms profile construction), while the repeated cached reference took 0.29 ms. -The memory-saver result is within normal run-to-run timing variation; its -purpose here is the lower peak, not a speedup guarantee. +NVIDIA GeForce RTX 3090 (CUDA 12.4). Each CUDA mode was run in three fresh +processes in alternating order. Values below are mean +/- sample standard +deviation. The Python HF and audio.cpp safetensors columns are single loaded- +session runs; model load is excluded from every per-request measurement. + +| Request | Python HF wall / RTF | C++ safetensors wall / RTF | C++ Q8 wall / RTF | +|---|---:|---:|---:| +| `tts_cold` | 4144.65 ms / 3.141 | 3499.28 ms / 2.652 | 3207.55 +/- 20.46 ms / 2.431 | +| `tts_repeat` | 4068.42 ms / 3.083 | 3267.68 ms / 2.476 | 3026.10 +/- 6.42 ms / 2.293 | +| `tts_longform` | 16176.37 ms / 2.882 | 13169.50 ms / 2.347 | 12999.03 +/- 94.52 ms / 2.316 | +| `clone_cold` | 5078.82 ms / 3.342 | 5764.60 ms / 3.793 | 6429.54 +/- 37.51 ms / 4.053 | +| `clone_repeat` | 4149.47 ms / 2.730 | 3615.41 ms / 2.379 | 4233.98 +/- 14.79 ms / 2.669 | + +`clone_cold` includes the 934.118 ms official-Python reference-profile build as +described above. Generated durations are used independently for each RTF: +Python and C++ safetensors cloning produced 1.51968 seconds, while Q8 produced +1.58633 seconds. + +The memory-saver A/B is also reported per request rather than inferred from a +single aggregate run: + +| Request | Default CUDA Q8 | Memory saver CUDA Q8 | Wall delta | Default RTF | Memory-saver RTF | +|---|---:|---:|---:|---:|---:| +| `tts_cold` | 3207.55 +/- 20.46 ms | 3228.32 +/- 21.36 ms | +0.65% | 2.431 | 2.446 | +| `tts_repeat` | 3026.10 +/- 6.42 ms | 3047.81 +/- 25.96 ms | +0.72% | 2.293 | 2.310 | +| `tts_longform` | 12999.03 +/- 94.52 ms | 13138.60 +/- 113.97 ms | +1.07% | 2.316 | 2.341 | +| `clone_cold` | 6429.54 +/- 37.51 ms | 6506.54 +/- 51.57 ms | +1.20% | 4.053 | 4.102 | +| `clone_repeat` | 4233.98 +/- 14.79 ms | 4272.15 +/- 20.66 ms | +0.90% | 2.669 | 2.693 | + +Memory saver was slower on every request in this controlled repeat: about 0.6% +to 1.2%, and 0.99% over the mean five-request wall total. It remains useful +because the independently sampled peak VRAM fell from 17653 MiB to 5780 MiB +(67.3%) with the same 294 MiB post-session resident VRAM. It is therefore a +memory-versus-speed option, not a speed optimization. + +The Windows CPU Q8 run is likewise reported per request: + +| Request | Wall | Audio | RTF | +|---|---:|---:|---:| +| `tts_cold` | 10517.2 ms | 1.31967s | 7.970 | +| `tts_repeat` | 10349.9 ms | 1.31967s | 7.843 | +| `tts_longform` | 41604.0 ms | 5.67867s | 7.326 | +| `clone_cold` | 22849.3 ms | 1.58633s | 14.404 | +| `clone_repeat` | 18205.6 ms | 1.58633s | 11.477 | + +All generated WAV files passed an ffmpeg decode check. The traces confirmed +four framework chunks for the long-form request, compatible step-graph reuse in +default mode, explicit graph release in memory-saver mode, and a reference- +profile cache hit on the repeated clone. In the original instrumented default +CUDA run, cold reference preparation took about 890 ms (109 ms alignment, +139 ms DAC encoding, and 214 ms profile construction), while the repeated +cached reference took 0.29 ms. ## Path and parity results @@ -252,16 +314,36 @@ purpose here is the lower peak, not a speedup guarantee. - CPU and CUDA WAV bytes are not identical because backend floating-point execution differs; deterministic reuse parity was therefore checked within each backend rather than asserted across backends. -- The official Python reference has a clean setup path documented above. Per - maintainer request, Python output parity is intentionally left for maintainer - validation rather than claimed without their reference environment. + +Python parity uses the repository's +`tools/audiocpp_cli/compare_audiocpp_cli_path_results.py` implementation: PCM +WAV cosine plus an 80-band `log1p` mel-spectrogram cosine (`n_fft=1024`, +`hop=256`). The official Python `ModelOutput` upsamples native DAC audio to +44.1 kHz; it was resampled back to the DAC/audio.cpp native 24 kHz before the +metric was applied. Files were truncated to their common frame count, matching +the repository comparison helper. + +| Request | Safetensors WAV cosine | Safetensors log-mel cosine | Q8 WAV cosine | Q8 log-mel cosine | +|---|---:|---:|---:|---:| +| `tts_cold` | 0.037707 | 0.525263 | 0.038120 | 0.658969 | +| `tts_repeat` | 0.037707 | 0.525263 | 0.038120 | 0.658969 | +| `tts_longform` | 0.011433 | 0.589421 | 0.001456 | 0.688418 | +| `clone_cold` | -0.028749 | 0.766325 | 0.016037 | 0.367275 | +| `clone_repeat` | -0.028749 | 0.766325 | 0.016037 | 0.367275 | + +These are free-running sampled autoregressive outputs. The same seed does not +produce the same random stream across PyTorch and ggml, so the metric measures +rendered-audio similarity rather than sample identity. Raw WAV cosine is highly +phase-sensitive; log-mel cosine is the more informative parity value here. +Quantization can also change the sampled token path, so a higher value in one +row should not be interpreted as a general perceptual-quality ranking. ## Backend coverage | Backend | Coverage | |---|---| | Windows CPU | CLI/server/warm-benchmark build; normal TTS, long-form TTS, cloning, cache reuse, graph reuse, WAV decode, timing, and RSS run | -| Windows CUDA 12.4 | CLI/server/warm-benchmark build; normal TTS, long-form TTS, cloning, standalone-path loading, cache/graph reuse, default/memory-saver A/B, WAV decode, timing, RTF, and VRAM run | +| Windows CUDA 12.4 | CLI/server/warm-benchmark build; normal TTS, long-form TTS, cloning, standalone-path loading, cache/graph reuse, three-run default/memory-saver A/B, Python HF parity, WAV decode, timing, RTF, and VRAM run | | Linux CPU | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | | Linux Vulkan | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | | macOS CPU | GitHub Actions compile check passed for CLI, server, and GGUF converter; no model runtime measurement claimed | @@ -280,5 +362,6 @@ purpose here is the lower peak, not a speedup guarantee. applies to each chunk. - CPU and CUDA outputs are deterministic within each tested backend but are not expected to be byte-identical across backends. -- Python-reference output parity remains pending maintainer validation using the - exact setup and sampling parameters above. +- Python/C++ sampling uses different random-number implementations; fixed seeds + are deterministic within each runtime but do not imply identical token paths + between runtimes. From 795be97b8290a3149a512f792d371522234950f9 Mon Sep 17 00:00:00 2001 From: mirek190 Date: Thu, 16 Jul 2026 21:11:11 +0100 Subject: [PATCH 5/8] Match OuteTTS CUDA sampling with Python --- docs/memory_saver.md | 6 +- docs/outetts_validation.md | 164 +++++++++----- src/models/outetts/llama.cpp | 50 ++++- src/models/outetts/session.cpp | 2 +- src/models/outetts/tokenizer.cpp | 20 +- tools/audiocpp_cli/outetts_reference.py | 270 ++++++++++++++++++++++++ 6 files changed, 448 insertions(+), 64 deletions(-) create mode 100644 tools/audiocpp_cli/outetts_reference.py diff --git a/docs/memory_saver.md b/docs/memory_saver.md index c9e9a5368..044c250ea 100644 --- a/docs/memory_saver.md +++ b/docs/memory_saver.md @@ -31,7 +31,7 @@ Native/default weights were used for all rows. - VoxCPM2 used the OpenAI-compatible offline speech endpoint with a 2048-character voice-design request, `seed=1234`, `max_tokens=512`, `num_inference_steps=10`, and `guidance_scale=2.0`. The default and `mem_saver` WAV outputs were byte-identical. - IndexTTS2 used a five-request server sequence with the same seeds and references for default and `mem_saver`: normal text, longer text, longer emotion-text request with a different reference, shorter text, then longer text. - Irodori TTS 500M used a five-request server sequence with the same seeds and options for default and `mem_saver`: reference text, longer reference text, longer no-reference emoji/style text, shorter reference text, then longer reference text. -- OuteTTS 1.0 1B Q8 used a five-request long-lived-session sequence on an RTX 3090: repeated fixed-seed TTS, four-chunk long-form TTS, then repeated fixed-seed cloning with the same reference. CUDA timing is the mean of three fresh processes per mode, alternated to reduce ordering bias; memory saver was 0.6-1.2% slower on every request and 0.99% slower over the mean sequence total. The default and `mem_saver` WAV outputs were byte-identical. VRAM is total-device usage with no other CUDA workload; resident VRAM was sampled during a five-second post-sequence hold. Per-request timing and RTF are recorded in `docs/outetts_validation.md`. +- OuteTTS 1.0 1B Q8 used a five-request long-lived-session sequence on an RTX 3090: repeated fixed-seed TTS, four-chunk long-form TTS, then repeated fixed-seed cloning with the same reference. CUDA timing is the mean of three fresh processes per mode, alternated to reduce ordering bias; memory saver was 0.35-0.53% slower per request and 0.44% slower over the mean sequence total. This is close to run-to-run variance and is not evidence of a speed benefit. The default and `mem_saver` WAV outputs were byte-identical. VRAM is total-device usage with no other CUDA workload; resident VRAM was sampled during a five-second post-sequence hold. Per-request timing and RTF are recorded in `docs/outetts_validation.md`. | Model | Mode | Peak VRAM | Resident VRAM | Server wall | Audio | RTF | |---|---|---:|---:|---:|---:|---:| @@ -59,5 +59,5 @@ Native/default weights were used for all rows. | Irodori TTS 500M | mem_saver | 11222 MiB | 3570 MiB | 3276.203 ms | 95.4s | 0.0343418 | | Irodori TTS 500M 6000-char | default | 18693 MiB | 13367 MiB | 27185.7 ms | 777.92s | 0.0349466 | | Irodori TTS 500M 6000-char | mem_saver | 11588 MiB | 3609 MiB | 27828 ms | 777.92s | 0.0357724 | -| OuteTTS 1.0 1B Q8 | default | 17653 MiB | 294 MiB | 29896.19 ms | 11.424s | 2.617 | -| OuteTTS 1.0 1B Q8 | mem_saver | 5780 MiB | 294 MiB | 30193.41 ms | 11.424s | 2.643 | +| OuteTTS 1.0 1B Q8 | default | 17653 MiB | 294 MiB | 29911.38 ms | 11.237s | 2.662 | +| OuteTTS 1.0 1B Q8 | mem_saver | 5780 MiB | 294 MiB | 30043.25 ms | 11.237s | 2.674 | diff --git a/docs/outetts_validation.md b/docs/outetts_validation.md index e0b4630ba..48d92dfa0 100644 --- a/docs/outetts_validation.md +++ b/docs/outetts_validation.md @@ -39,13 +39,19 @@ build\reference\venv\Scripts\python.exe -m pip install transformers==4.52.3 llam ``` The reference loaded `models/Llama-OuteTTS-1.0-1B` through -`outetts.Backend.HF` with BF16 CUDA weights and loaded the official IBM DAC +`outetts.Backend.HF` with FP32 CUDA weights and loaded the official IBM DAC checkpoint from `models/DAC.speech.v1.0/weights_24khz_1.5kbps_v1.0.pth`. Both implementations used temperature `0.4`, repetition penalty `1.1` over the latest 64 tokens, top-k `40`, top-p `0.9`, min-p `0.05`, the request-file seeds, and the same maximum number of new tokens. +Temperature zero was deliberately not used. OuteTTS 1.0's official +`generation_config.json`, README example, and Python API all select sampled +generation with temperature `0.4`; changing to greedy generation would test a +different inference contract. OuteTTS has no diffusion or latent-noise input. +Its only request-time randomness is language-model token sampling. + For a controlled clone comparison, Qwen word timings were generated once and passed to the official Python `AudioProcessor.create_speaker_from_dict` path: @@ -60,9 +66,26 @@ build\windows-cuda-release\bin\audiocpp_cli.exe ` Python still performed the official DAC encoding and feature extraction. This keeps different alignment algorithms from contaminating the TTS comparison. -The Python reference-profile build took 934.118 ms and is charged to -`clone_cold` below. The audio.cpp cold-clone timing also includes its Qwen -alignment, while the Python number starts from the supplied word timings. +The audio.cpp cold-clone timing includes its Qwen alignment, while the Python +number starts from the supplied word timings. + +Run the committed official-reference driver after creating the alignment JSON: + +```powershell +build\reference\venv\Scripts\python.exe ` + tools\audiocpp_cli\outetts_reference.py ` + --model ..\models\Llama-OuteTTS-1.0-1B ` + --dac ..\models\DAC.speech.v1.0\weights_24khz_1.5kbps_v1.0.pth ` + --alignment-json build\reference\b_words.json ` + --request-file tests\outetts\warm_bench_requests.json ` + --device cuda --dtype fp32 ` + --out-dir build\reference\official_script_fp32 +``` + +The driver resets both `torch.manual_seed` and +`torch.cuda.manual_seed_all` immediately before every `model.generate` call, +after all prompt, alignment, and DAC-profile work. It writes native 24 kHz +WAVs plus `boundaries.json`, `memory.json`, `stdout.log`, and `command.json`. ## Exact build commands @@ -192,9 +215,10 @@ build\windows-cuda-release\bin\outetts_warm_bench.exe ` --model ..\models\Llama-OuteTTS-1.0-1B ` --backend cuda --threads 8 ` --request-file tests\outetts\warm_bench_requests.json ` + --session-option outetts.weight_type=f32 ` --session-option outetts.aligner_model_path=..\models\Qwen3-ForcedAligner-0.6B ` - --audio-out-dir build\reference\cpp_safetensors ` - --log-file build\reference\cpp_safetensors.log + --audio-out-dir build\reference\cpp_f32_final_explicit ` + --log-file build\reference\cpp_f32_final_explicit.log ``` The memory-saver trace reports a positive @@ -232,12 +256,12 @@ The exact generated directories and logs were: - `build\logs\warmbench\outetts-cuda-default.log` - `build\logs\warmbench\outetts-cuda-mem_saver.log` - `build\logs\warmbench\outetts-cpu-default-final.log` -- `build\logs\warmbench\outetts-ab\default-{1,2,3}-audio` -- `build\logs\warmbench\outetts-ab\mem_saver-{1,2,3}-audio` -- `build\reference\cpp_safetensors` -- `build\reference\python_outputs` (official 44.1 kHz wrapper output) -- `build\reference\python_outputs_24k` (metric input resampled to DAC-native - 24 kHz) +- `build\logs\warmbench\outetts-ab-torch-sampler\default-{1,2,3}-audio` +- `build\logs\warmbench\outetts-ab-torch-sampler\mem_saver-{1,2,3}-audio` +- `build\reference\cpp_f32_final_explicit` +- `build\reference\cpp_q8_torch_sampler` +- `build\reference\official_script_fp32\outputs` +- `build\reference\official_script_fp32\boundaries.json` These validation artifacts are reproducible local outputs and are not committed to the repository. @@ -250,32 +274,34 @@ processes in alternating order. Values below are mean +/- sample standard deviation. The Python HF and audio.cpp safetensors columns are single loaded- session runs; model load is excluded from every per-request measurement. -| Request | Python HF wall / RTF | C++ safetensors wall / RTF | C++ Q8 wall / RTF | +| Request | Python HF FP32 wall / RTF | C++ FP32 wall / RTF | C++ Q8 wall / RTF | |---|---:|---:|---:| -| `tts_cold` | 4144.65 ms / 3.141 | 3499.28 ms / 2.652 | 3207.55 +/- 20.46 ms / 2.431 | -| `tts_repeat` | 4068.42 ms / 3.083 | 3267.68 ms / 2.476 | 3026.10 +/- 6.42 ms / 2.293 | -| `tts_longform` | 16176.37 ms / 2.882 | 13169.50 ms / 2.347 | 12999.03 +/- 94.52 ms / 2.316 | -| `clone_cold` | 5078.82 ms / 3.342 | 5764.60 ms / 3.793 | 6429.54 +/- 37.51 ms / 4.053 | -| `clone_repeat` | 4149.47 ms / 2.730 | 3615.41 ms / 2.379 | 4233.98 +/- 14.79 ms / 2.669 | +| `tts_cold` | 4599.12 ms / 3.485 | 5366.24 ms / 4.066 | 3211.84 +/- 17.30 ms / 2.434 | +| `tts_repeat` | 3777.69 ms / 2.863 | 3864.98 ms / 2.929 | 3030.79 +/- 18.60 ms / 2.297 | +| `tts_longform` | 15129.91 ms / 2.690 | 15547.80 ms / 2.764 | 13015.83 +/- 171.29 ms / 2.342 | +| `clone_cold` | 5762.51 ms / 3.792 | 5988.01 ms / 3.940 | 6432.68 +/- 65.11 ms / 4.233 | +| `clone_repeat` | 5522.67 ms / 3.634 | 4231.19 ms / 2.784 | 4220.23 +/- 44.07 ms / 2.777 | -`clone_cold` includes the 934.118 ms official-Python reference-profile build as -described above. Generated durations are used independently for each RTF: -Python and C++ safetensors cloning produced 1.51968 seconds, while Q8 produced -1.58633 seconds. +These are per-request timings from a single loaded session; model loading is +excluded. Python and C++ FP32 generated 1.31967, 1.31967, 5.62533, 1.51967, +and 1.51967 seconds respectively. Q8 generated 5.55867 seconds for the +long-form request and matched the other four durations. The memory-saver A/B is also reported per request rather than inferred from a single aggregate run: | Request | Default CUDA Q8 | Memory saver CUDA Q8 | Wall delta | Default RTF | Memory-saver RTF | |---|---:|---:|---:|---:|---:| -| `tts_cold` | 3207.55 +/- 20.46 ms | 3228.32 +/- 21.36 ms | +0.65% | 2.431 | 2.446 | -| `tts_repeat` | 3026.10 +/- 6.42 ms | 3047.81 +/- 25.96 ms | +0.72% | 2.293 | 2.310 | -| `tts_longform` | 12999.03 +/- 94.52 ms | 13138.60 +/- 113.97 ms | +1.07% | 2.316 | 2.341 | -| `clone_cold` | 6429.54 +/- 37.51 ms | 6506.54 +/- 51.57 ms | +1.20% | 4.053 | 4.102 | -| `clone_repeat` | 4233.98 +/- 14.79 ms | 4272.15 +/- 20.66 ms | +0.90% | 2.669 | 2.693 | - -Memory saver was slower on every request in this controlled repeat: about 0.6% -to 1.2%, and 0.99% over the mean five-request wall total. It remains useful +| `tts_cold` | 3211.84 +/- 17.30 ms | 3223.24 +/- 12.89 ms | +0.35% | 2.434 | 2.443 | +| `tts_repeat` | 3030.79 +/- 18.60 ms | 3044.58 +/- 18.76 ms | +0.45% | 2.297 | 2.307 | +| `tts_longform` | 13015.83 +/- 171.29 ms | 13070.17 +/- 105.32 ms | +0.42% | 2.342 | 2.351 | +| `clone_cold` | 6432.68 +/- 65.11 ms | 6467.00 +/- 46.38 ms | +0.53% | 4.233 | 4.256 | +| `clone_repeat` | 4220.23 +/- 44.07 ms | 4238.27 +/- 17.07 ms | +0.43% | 2.777 | 2.789 | + +Memory saver was slightly slower on every request in this controlled repeat: +0.35% to 0.53%, and 0.44% over the mean five-request wall total. These small +differences are near normal run-to-run variance, so there is no evidence that +memory saver is a speed optimization. It remains useful because the independently sampled peak VRAM fell from 17653 MiB to 5780 MiB (67.3%) with the same 294 MiB post-session resident VRAM. It is therefore a memory-versus-speed option, not a speed optimization. @@ -303,9 +329,9 @@ cached reference took 0.29 ms. - The standalone path test loaded the one-file GGUF from outside the worktree; no adjacent sidecars or `model_specs` directory were present. - Within CUDA, cold and repeated TTS were byte-identical with SHA-256 - `52A11609B165314E8F83919CD4F82AD899346B9945DBEA9D9676431F4E67C548`. + `9D1AF0D25212129B3C26880F7C928CFFA813A4B9EF22F12F390A4C372D51D406`. - Within CUDA, cold and repeated cloning were byte-identical with SHA-256 - `8816F29514944C7B8121D17E7620D095D06BA636247BFD4358793500FE8B6FA1`. + `EB7E6B5A1330F845F45D3DA21C8AC28490572F95218A6F0D3DA2A83CB797EFE6`. - The CUDA default and memory-saver outputs had the same hashes. - Within CPU, cold and repeated TTS were byte-identical with SHA-256 `A315C0AD425C99910597FCC4BAB9D80CB5F4C893176EBD9099BA6C6AA7B66BFD`. @@ -318,25 +344,56 @@ cached reference took 0.29 ms. Python parity uses the repository's `tools/audiocpp_cli/compare_audiocpp_cli_path_results.py` implementation: PCM WAV cosine plus an 80-band `log1p` mel-spectrogram cosine (`n_fft=1024`, -`hop=256`). The official Python `ModelOutput` upsamples native DAC audio to -44.1 kHz; it was resampled back to the DAC/audio.cpp native 24 kHz before the -metric was applied. Files were truncated to their common frame count, matching -the repository comparison helper. +`hop=256`). Both sides write the DAC-native 24 kHz signal directly. Raw WAV +cosine is highly phase-sensitive; log-mel cosine is the more informative value +when the DAC encoders are numerically close but not bit-identical. + +The CUDA sampler now uses the framework's PyTorch-compatible exponential-race +random stream, equivalent to the `torch.multinomial` path used by Transformers. +Temporary trace-level instrumentation compared prompt strings, prompt token +IDs, generated token IDs, codec codes, and WAV frame counts at component +boundaries. It was removed after validation to avoid normal-log spam. The +permanent Python driver records the Python side in `boundaries.json`. -| Request | Safetensors WAV cosine | Safetensors log-mel cosine | Q8 WAV cosine | Q8 log-mel cosine | +Boundary results under the controlled FP32 run: + +| Request | Chunks | Python/C++ prompt IDs | Generated IDs per chunk | Python/C++ frames | |---|---:|---:|---:|---:| -| `tts_cold` | 0.037707 | 0.525263 | 0.038120 | 0.658969 | -| `tts_repeat` | 0.037707 | 0.525263 | 0.038120 | 0.658969 | -| `tts_longform` | 0.011433 | 0.589421 | 0.001456 | 0.688418 | -| `clone_cold` | -0.028749 | 0.766325 | 0.016037 | 0.367275 | -| `clone_repeat` | -0.028749 | 0.766325 | 0.016037 | 0.367275 | - -These are free-running sampled autoregressive outputs. The same seed does not -produce the same random stream across PyTorch and ggml, so the metric measures -rendered-audio similarity rather than sample identity. Raw WAV cosine is highly -phase-sensitive; log-mel cosine is the more informative parity value here. -Quantization can also change the sampled token path, so a higher value in one -row should not be interpreted as a general perceptual-quality ranking. +| `tts_cold` | 1 | 22 / 22 | 256 | 31672 / 31672 | +| `tts_repeat` | 1 | 22 / 22 | 256 | 31672 / 31672 | +| `tts_longform` | 4 | 20,22,14,21 / same | 256 each | 135008 / 135008 | +| `clone_cold` | 1 | 2367 / 2367 | 256 | 36472 / 36472 | +| `clone_repeat` | 1 | 2367 / 2367 | 256 | 36472 / 36472 | + +The sampled prompt/generated boundary values for all six ordinary-TTS chunks +had zero mismatches. Clone prompt construction also matches the official word +labels and token count. The C++ DAC reference encoder still differs slightly +from Python: among 1046 reference-profile frames, C1 differs on 48 and C2 on +118. That numerical boundary explains why cloning is not waveform-identical. + +Primary native/FP32 parity, measured only after the controlled random stream +and frame counts matched: + +| Request | WAV cosine | Log-mel cosine | C++ / Python frames | +|---|---:|---:|---:| +| `tts_cold` | 0.999812247 | 0.999997481 | 31672 / 31672 | +| `tts_repeat` | 0.999812238 | 0.999997483 | 31672 / 31672 | +| `tts_longform` | 0.997324530 | 0.999294328 | 135008 / 135008 | +| `clone_cold` | -0.007753064 | 0.889403845 | 36472 / 36472 | +| `clone_repeat` | -0.007753064 | 0.889403845 | 36472 / 36472 | + +Only after that path was clean was the packed Q8 model compared with the same +FP32 Python reference. Quantization is expected to perturb autoregressive +sampling, so this is a secondary characterization rather than the correctness +gate: + +| Request | WAV cosine | Log-mel cosine | C++ Q8 / Python frames | +|---|---:|---:|---:| +| `tts_cold` | -0.016326171 | 0.696795792 | 31672 / 31672 | +| `tts_repeat` | -0.016326591 | 0.696792751 | 31672 / 31672 | +| `tts_longform` | 0.577927575 | 0.920741352 | 133408 / 135008 | +| `clone_cold` | -0.033990775 | 0.882715855 | 36472 / 36472 | +| `clone_repeat` | -0.033990775 | 0.882715855 | 36472 / 36472 | ## Backend coverage @@ -362,6 +419,9 @@ row should not be interpreted as a general perceptual-quality ranking. applies to each chunk. - CPU and CUDA outputs are deterministic within each tested backend but are not expected to be byte-identical across backends. -- Python/C++ sampling uses different random-number implementations; fixed seeds - are deterministic within each runtime but do not imply identical token paths - between runtimes. +- The controlled CUDA sampling stream is PyTorch-compatible. CPU sampling is + deterministic for a fixed seed but still uses the C++ standard-library + distribution and is not claimed to reproduce PyTorch token-for-token. +- Quantized weights can change sampled tokens and output length even when the + seed and sampler implementation match, as shown by the Q8 long-form frame + drift above. diff --git a/src/models/outetts/llama.cpp b/src/models/outetts/llama.cpp index f9163a512..380348afd 100644 --- a/src/models/outetts/llama.cpp +++ b/src/models/outetts/llama.cpp @@ -6,6 +6,7 @@ #include "engine/framework/modules/attention/qwen_causal_decoder.h" #include "engine/framework/modules/lookup_modules.h" #include "engine/framework/modules/weight_binding.h" +#include "engine/framework/sampling/torch_random.h" #include "../common/constant_tensor_cache.h" #include @@ -201,7 +202,12 @@ void apply_repetition_penalty( } } -int32_t sample_token(std::vector logits, const OuteTTSGenerateOptions & o, std::mt19937 & rng) { +int32_t sample_token( + std::vector logits, + const OuteTTSGenerateOptions & o, + std::mt19937 & rng, + const sampling::TorchCudaSamplingPolicy & sampling_policy, + uint64_t call_index) { if (!(o.temperature > 0.0F) || !std::isfinite(o.temperature)) { return static_cast(std::max_element(logits.begin(), logits.end()) - logits.begin()); } @@ -236,6 +242,32 @@ int32_t sample_token(std::vector logits, const OuteTTSGenerateOptions & o } order.resize(std::max(1, kept)); probabilities.resize(order.size()); + if (sampling_policy.cuda_fast_path) { + double best_rank = -std::numeric_limits::infinity(); + int32_t best_token = -1; + for (size_t index = 0; index < order.size(); ++index) { + const float exponential = + sampling::torch_cuda_tensor_iterator_exponential_element( + o.seed, + static_cast(logits.size()), + static_cast(order[index]), + call_index, + sampling_policy.multiprocessor_count, + sampling_policy.max_threads_per_multiprocessor); + const double rank = + static_cast(probabilities[index]) / + static_cast(exponential); + if (rank > best_rank) { + best_rank = rank; + best_token = static_cast(order[index]); + } + } + if (best_token < 0) { + throw std::runtime_error( + "OuteTTS CUDA sampler failed to select a token"); + } + return best_token; + } std::discrete_distribution distribution(probabilities.begin(), probabilities.end()); return static_cast(order[static_cast(distribution(rng))]); } @@ -335,7 +367,17 @@ struct OuteTTSLlamaRuntime::Impl { size_t weight_context_bytes, size_t constant_context_bytes, assets::TensorStorageType storage_type) - : assets(std::move(assets_in)), threads(std::max(1, threads_in)) { + : assets(std::move(assets_in)), + threads(std::max(1, threads_in)), + sampling_policy( + backend_type == core::BackendType::Cuda + ? sampling::resolve_torch_cuda_sampling_policy( + backend_type, + device, + "outetts", + "OuteTTS", + sampling::TorchCudaSamplingPolicyFailureMode::StrictCuda) + : sampling::TorchCudaSamplingPolicy{}) { if (assets == nullptr) { throw std::runtime_error("OuteTTS Llama runtime requires assets"); } @@ -492,6 +534,7 @@ struct OuteTTSLlamaRuntime::Impl { std::shared_ptr assets; ggml_backend_t backend = nullptr; int threads = 1; + sampling::TorchCudaSamplingPolicy sampling_policy; ModelWeights weights; std::unique_ptr constants; std::unique_ptr step_graph; @@ -551,7 +594,8 @@ std::vector OuteTTSLlamaRuntime::generate( apply_repetition_penalty( logits, all, options.repetition_window, options.repetition_penalty); const int32_t token = sample_token( - std::move(logits), sampling_options, rng); + std::move(logits), sampling_options, rng, + impl_->sampling_policy, static_cast(i)); sample_ms += debug::elapsed_ms(sample_start); generated.push_back(token); if (token == eos_id || token == audio_end_id) { diff --git a/src/models/outetts/session.cpp b/src/models/outetts/session.cpp index a806fe959..31b799e68 100644 --- a/src/models/outetts/session.cpp +++ b/src/models/outetts/session.cpp @@ -275,7 +275,7 @@ make_voice_profile(OuteTTSDacDecoder::EncodedReference encoded, if (frame_count == 0) throw std::runtime_error( "OuteTTS DAC encoder produced no reference codec frames"); - if (alignment != nullptr && alignment->words.size() != words.size()) { + if (alignment != nullptr) { words.clear(); for (const auto &word : alignment->words) words.push_back(word.word); diff --git a/src/models/outetts/tokenizer.cpp b/src/models/outetts/tokenizer.cpp index 6f8ec041d..3b9ea4806 100644 --- a/src/models/outetts/tokenizer.cpp +++ b/src/models/outetts/tokenizer.cpp @@ -25,6 +25,17 @@ bool is_punctuation(char value) { value == ':' || value == ';'; } +std::string trim_text(const std::string &input) { + const auto first = std::find_if_not( + input.begin(), input.end(), + [](unsigned char value) { return std::isspace(value); }); + const auto last = std::find_if_not( + input.rbegin(), input.rend(), + [](unsigned char value) { return std::isspace(value); }) + .base(); + return first < last ? std::string(first, last) : std::string{}; +} + std::string normalize_text(const std::string &input) { std::string text; text.reserve(input.size()); @@ -77,9 +88,9 @@ std::string profile_codes(const OuteTTSVoiceProfile &profile, std::ostringstream out; for (size_t word_index = 0; word_index < profile.words.size(); ++word_index) { const auto &word = profile.words[word_index]; - std::string word_text = normalize_text(word.text); + std::string word_text = trim_text(word.text); if (word_index + 1 == profile.words.size()) - word_text += normalize_text(separator); + word_text += trim_text(separator); out << "<|word_start|>" << word_text << "<|features|><|t_" << std::fixed << std::setprecision(2) << word.duration << "|>" << "<|energy_" << word.features.energy << "|>" @@ -152,14 +163,13 @@ std::vector OuteTTSTokenizer::build_clone_prompt(const std::string &text, const OuteTTSVoiceProfile &profile) const { const std::string prompt_text = normalize_text(text); - const std::string reference_text = normalize_text(profile.text); + const std::string reference_text = trim_text(profile.text); if (prompt_text.empty() || reference_text.empty() || profile.words.empty()) { throw std::runtime_error("OuteTTS voice cloning requires text, " "reference_text, and reference codec frames"); } const std::string separator = speaker_separator(reference_text); - const std::string merged = - normalize_text(reference_text + separator + prompt_text); + const std::string merged = reference_text + separator + prompt_text; const std::string prompt = "<|im_start|>\n<|text_start|>" + merged + "<|text_end|>\n<|audio_start|>\n" + profile_codes(profile, separator) + diff --git a/tools/audiocpp_cli/outetts_reference.py b/tools/audiocpp_cli/outetts_reference.py new file mode 100644 index 000000000..41a3e1190 --- /dev/null +++ b/tools/audiocpp_cli/outetts_reference.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Run the official OuteTTS 1.0 HF reference for audio.cpp parity. + +The script intentionally keeps the official sampling configuration. It resets +the PyTorch CPU and CUDA generators immediately before each model.generate() +call, after prompt/profile construction, so no unrelated torch operation can +consume the request's sampling stream. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import re +import time +from pathlib import Path +from typing import Any + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_REQUESTS = REPO_ROOT / "tests" / "outetts" / "warm_bench_requests.json" + + +def framework_chunks(text: str, budget: int | None) -> list[str]: + text = text.strip() + if budget is None or len(text) <= budget: + return [text] + words = list(re.finditer(r"\S+", text)) + chunks: list[str] = [] + start = 0 + while start < len(words): + hard_end = start + while hard_end < len(words): + width = words[hard_end].end() - words[start].start() + if width > budget: + break + hard_end += 1 + if hard_end == start: + hard_end += 1 + end = hard_end + if hard_end < len(words) and hard_end > start + 1: + for index in range(hard_end, start + 1, -1): + if words[index - 1].group()[-1:] in ".!?": + end = index + break + if end == hard_end: + for index in range(hard_end, start + 1, -1): + if words[index - 1].group()[-1:] in ",;:": + end = index + break + chunks.append(text[words[start].start() : words[end - 1].end()].strip()) + start = end + return chunks + + +def resolve_path(value: str, base: Path) -> Path: + path = Path(value) + return path if path.is_absolute() else (base / path).resolve() + + +def reset_seed(torch_mod: Any, seed: int, device: str) -> None: + torch_mod.manual_seed(seed) + if device == "cuda": + torch_mod.cuda.manual_seed_all(seed) + + +def memory_snapshot(torch_mod: Any, device: str) -> dict[str, float]: + result: dict[str, float] = {} + try: + import psutil + + result["rss_mb"] = psutil.Process().memory_info().rss / (1024.0 * 1024.0) + except Exception: # psutil is optional for reference generation. + pass + if device == "cuda": + result["cuda_alloc_mb"] = torch_mod.cuda.memory_allocated() / (1024.0 * 1024.0) + result["cuda_peak_mb"] = torch_mod.cuda.max_memory_allocated() / (1024.0 * 1024.0) + result["cuda_reserved_mb"] = torch_mod.cuda.memory_reserved() / (1024.0 * 1024.0) + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--dac", type=Path, required=True) + parser.add_argument("--alignment-json", type=Path, required=True) + parser.add_argument("--request-file", type=Path, default=DEFAULT_REQUESTS) + parser.add_argument("--device", choices=("cpu", "cuda"), default="cuda") + parser.add_argument("--dtype", choices=("fp32", "bf16"), default="fp32") + parser.add_argument( + "--out-dir", type=Path, default=REPO_ROOT / "build" / "logs" / "outetts_python_fp32" + ) + args = parser.parse_args() + + import torch + import torchaudio + import outetts + + model_path = args.model.resolve() + dac_path = args.dac.resolve() + request_path = args.request_file.resolve() + request_base = request_path.parent.parent.parent if request_path.parent.name == "outetts" else REPO_ROOT + output_dir = args.out_dir.resolve() + wav_dir = output_dir / "outputs" + wav_dir.mkdir(parents=True, exist_ok=True) + + dtype = torch.float32 if args.dtype == "fp32" else torch.bfloat16 + config = outetts.ModelConfig( + model_path=str(model_path), + tokenizer_path=str(model_path), + interface_version=outetts.InterfaceVersion.V3, + backend=outetts.Backend.HF, + device=args.device, + dtype=dtype, + audio_codec_path=str(dac_path), + max_seq_length=8192, + ) + interface = outetts.Interface(config) + alignment = json.loads(args.alignment_json.read_text(encoding="utf-8")) + requests = json.loads(request_path.read_text(encoding="utf-8"))["requests"] + + speakers: dict[tuple[str, str], dict[str, Any]] = {} + boundaries: dict[str, list[dict[str, Any]]] = {} + memory: list[dict[str, Any]] = [] + stdout_lines = [ + "family=outetts", + "reference=official_hf", + f"dtype={args.dtype}", + f"device={args.device}", + ] + + for request in requests: + name = str(request["name"]) + seed = int(request["seed"]) + max_new_tokens = int(request["max_tokens"]) + profile_ms = 0.0 + speaker = None + + started = time.perf_counter() + voice_ref = request.get("voice_ref") + if voice_ref: + reference_text = str(request["reference_text"]) + reference_path = resolve_path(str(voice_ref), request_base) + key = (str(reference_path), reference_text) + if key not in speakers: + profile_started = time.perf_counter() + speakers[key] = interface.audio_processor.create_speaker_from_dict( + { + "audio": {"bytes": reference_path.read_bytes()}, + "text": reference_text, + "words": [ + { + "word": item["word"], + "start": item["start_sample"] / 16000.0, + "end": item["end_sample"] / 16000.0, + } + for item in alignment + ], + } + ) + speakers[key]["interface_version"] = outetts.InterfaceVersion.V3.value + profile_ms = (time.perf_counter() - profile_started) * 1000.0 + speaker = speakers[key] + + chunks = framework_chunks(request["text"], request.get("text_chunk_size")) + audio_chunks = [] + request_boundaries = [] + if args.device == "cuda": + torch.cuda.reset_peak_memory_stats() + + for chunk in chunks: + current_speaker = copy.deepcopy(speaker) + prompt_text = interface.prompt_processor.get_completion_prompt(chunk, current_speaker) + prompt = interface._prepare_prompt(prompt_text) + prompt_tokens = int(prompt.shape[-1]) + generation = outetts.GenerationConfig( + text=chunk, + speaker=current_speaker, + generation_type=outetts.GenerationType.REGULAR, + max_length=prompt_tokens + max_new_tokens, + sampler_config=outetts.SamplerConfig( + temperature=0.4, + repetition_penalty=1.1, + repetition_range=64, + top_k=40, + top_p=0.9, + min_p=0.05, + ), + ) + + # This reset is deliberately adjacent to generate(). Prompt/profile + # work above cannot consume the model-sampling RNG stream. + reset_seed(torch, seed, args.device) + full_tokens = interface.model.generate(prompt, generation) + generated = full_tokens[prompt_tokens:] + codebooks = interface.prompt_processor.extract_audio_from_tokens(generated) + audio = interface.get_audio(generated).detach().cpu().flatten() + audio_chunks.append(audio) + request_boundaries.append( + { + "prompt_text": prompt_text, + "prompt_ids": prompt.detach().cpu().flatten().tolist(), + "generated_ids": generated, + "c1": codebooks[0], + "c2": codebooks[1], + "audio_frames_24k": int(audio.numel()), + } + ) + + if args.device == "cuda": + torch.cuda.synchronize() + wall_ms = (time.perf_counter() - started) * 1000.0 + audio = torch.cat(audio_chunks) + wav_path = wav_dir / f"{name}_1.wav" + torchaudio.save(str(wav_path), audio.unsqueeze(0), 24000, encoding="PCM_S", bits_per_sample=16) + audio_seconds = audio.numel() / 24000.0 + rtf = wall_ms / 1000.0 / audio_seconds + + snapshot = memory_snapshot(torch, args.device) + snapshot.update( + { + "request_id": name, + "wall_ms": wall_ms, + "profile_ms": profile_ms, + "audio_seconds": audio_seconds, + "rtf": rtf, + } + ) + memory.append(snapshot) + boundaries[name] = request_boundaries + stdout_lines.extend( + [ + f"request_id={name}", + f"[TIMING] request.{name}.wall_ms {wall_ms}", + f"[TIMING] request.{name}.profile_ms {profile_ms}", + f"audio_seconds={audio_seconds}", + f"rtf={rtf}", + f"audio_out={wav_path}", + ] + ) + print(f"[REF] {name}: {wall_ms:.2f} ms, {audio_seconds:.5f}s, RTF={rtf:.3f}") + + (output_dir / "command.json").write_text( + json.dumps( + { + "reference": "official OuteTTS HF", + "model": str(model_path), + "dac": str(dac_path), + "requests": str(request_path), + "dtype": args.dtype, + "device": args.device, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + (output_dir / "stdout.log").write_text("\n".join(stdout_lines) + "\n", encoding="utf-8") + (output_dir / "memory.json").write_text( + json.dumps({"requests": memory}, indent=2) + "\n", encoding="utf-8" + ) + (output_dir / "boundaries.json").write_text( + json.dumps(boundaries, indent=2) + "\n", encoding="utf-8" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 19db84959a6a91bede804cf09541ba5b4460604c Mon Sep 17 00:00:00 2001 From: mirek190 Date: Thu, 16 Jul 2026 22:32:05 +0100 Subject: [PATCH 6/8] Refactor OuteTTS assets and model loading --- include/engine/models/outetts/loader.h | 26 ++-- src/models/outetts/assets.cpp | 36 +++-- src/models/outetts/loader.cpp | 188 +++++++++++++++---------- 3 files changed, 143 insertions(+), 107 deletions(-) diff --git a/include/engine/models/outetts/loader.h b/include/engine/models/outetts/loader.h index cef666f00..10430843b 100644 --- a/include/engine/models/outetts/loader.h +++ b/include/engine/models/outetts/loader.h @@ -3,25 +3,31 @@ #include "engine/framework/runtime/model.h" #include "engine/models/outetts/assets.h" +#include #include namespace engine::models::outetts { class OuteTTSLoadedModel final : public runtime::ILoadedVoiceModel { public: - explicit OuteTTSLoadedModel(std::shared_ptr assets); - const runtime::ModelMetadata & metadata() const noexcept override; - const runtime::CapabilitySet & capabilities() const noexcept override; - std::unique_ptr create_task_session( - const runtime::TaskSpec & task, - const runtime::SessionOptions & options) const override; + OuteTTSLoadedModel(runtime::ModelMetadata metadata, + runtime::CapabilitySet capabilities, + std::shared_ptr assets); + + const runtime::ModelMetadata &metadata() const noexcept override; + const runtime::CapabilitySet &capabilities() const noexcept override; + std::unique_ptr + create_task_session(const runtime::TaskSpec &task, + const runtime::SessionOptions &options) const override; private: - std::shared_ptr assets_; - runtime::ModelMetadata metadata_; - runtime::CapabilitySet capabilities_; + runtime::ModelMetadata metadata_; + runtime::CapabilitySet capabilities_; + std::shared_ptr assets_; }; +std::unique_ptr +load_outetts_model(const std::filesystem::path &model_path); std::shared_ptr make_outetts_loader(); -} // namespace engine::models::outetts +} // namespace engine::models::outetts diff --git a/src/models/outetts/assets.cpp b/src/models/outetts/assets.cpp index e7b2a3293..fb43d4a72 100644 --- a/src/models/outetts/assets.cpp +++ b/src/models/outetts/assets.cpp @@ -11,10 +11,6 @@ namespace engine::models::outetts { namespace json = engine::io::json; namespace { -std::filesystem::path spec_path() { - return assets::default_model_package_spec_path("outetts"); -} - OuteTTSConfig parse_config(const assets::ResourceBundle &resources) { const auto root = resources.parse_json("config"); if (json::require_string(root, "model_type") != "llama") { @@ -114,8 +110,8 @@ std::shared_ptr load_embedded_aligner(const assets::ResourceBundle &resources) { if (!resources.has_file("aligner_config")) return nullptr; - for (const char *id : {"aligner_generation_config", - "aligner_tokenizer_config"}) { + for (const char *id : + {"aligner_generation_config", "aligner_tokenizer_config"}) { if (!resources.has_file(id)) { throw std::runtime_error( std::string("OuteTTS embedded aligner is missing resource: ") + id); @@ -156,26 +152,26 @@ load_embedded_aligner(const assets::ResourceBundle &resources) { add_optional("vocab", "aligner_vocab"); add_optional("merges", "aligner_merges"); add_optional("tokenizer_json", "aligner_tokenizer_json"); - aligner.add_tensor_source("weights", resources.require_file("aligner_weights"), - "aligner_weights"); - return engine::models::qwen3_asr::load_qwen3_asr_assets( - std::move(aligner)); + aligner.add_tensor_source( + "weights", resources.require_file("aligner_weights"), "aligner_weights"); + return engine::models::qwen3_asr::load_qwen3_asr_assets(std::move(aligner)); } } // namespace std::shared_ptr load_outetts_assets(const std::filesystem::path &model_path) { - auto out = std::make_shared(); - out->resources = - assets::load_resource_bundle_from_package_spec(model_path, spec_path()); - out->config = parse_config(out->resources); - out->generation = parse_generation(out->resources); - out->model_weights = out->resources.open_tensor_source("model_weights"); - out->dac_weights = out->resources.open_tensor_source("dac_weights"); - out->embedded_aligner = load_embedded_aligner(out->resources); - validate_anchors(*out); - return out; + auto resources = assets::load_resource_bundle_from_package_spec( + model_path, assets::default_model_package_spec_path("outetts")); + OuteTTSAssets model_assets; + model_assets.config = parse_config(resources); + model_assets.generation = parse_generation(resources); + model_assets.model_weights = resources.open_tensor_source("model_weights"); + model_assets.dac_weights = resources.open_tensor_source("dac_weights"); + model_assets.embedded_aligner = load_embedded_aligner(resources); + model_assets.resources = std::move(resources); + validate_anchors(model_assets); + return std::make_shared(std::move(model_assets)); } } // namespace engine::models::outetts diff --git a/src/models/outetts/loader.cpp b/src/models/outetts/loader.cpp index 5bb77f851..0763ba788 100644 --- a/src/models/outetts/loader.cpp +++ b/src/models/outetts/loader.cpp @@ -3,14 +3,22 @@ #include "engine/framework/assets/model_package.h" #include "engine/models/outetts/session.h" +#include +#include + namespace engine::models::outetts { namespace { -std::filesystem::path spec_path() { - return assets::default_model_package_spec_path("outetts"); +runtime::ModelMetadata metadata(const OuteTTSAssets &) { + runtime::ModelMetadata out; + out.family = "outetts"; + out.variant = "1.0-1B"; + out.description = + "OuteTTS 1.0 1B with native IBM DAC speech synthesis and voice cloning."; + return out; } -runtime::CapabilitySet make_capabilities() { +runtime::CapabilitySet capabilities(const OuteTTSAssets &) { runtime::CapabilitySet out; out.supported_tasks = { {runtime::VoiceTaskKind::Tts, {runtime::RunMode::Offline}}, @@ -27,119 +35,145 @@ runtime::CapabilitySet make_capabilities() { return out; } -class Loader final : public runtime::IVoiceModelLoader { +runtime::ModelCliInterface cli(const OuteTTSAssets &) { + runtime::ModelCliInterface out; + out.request_options = { + {"max_tokens", "n", "Maximum generated text/audio tokens."}, + {"temperature", "float", + "Sampling temperature; official cloning default 0.4."}, + {"top_k", "n", "Top-k sampling; official default 40."}, + {"top_p", "float", "Nucleus sampling; official default 0.9."}, + {"min_p", "float", + "Minimum probability relative to the best token; official default " + "0.05."}, + {"repetition_penalty", "float", + "Windowed repetition penalty; official default 1.1."}, + {"repetition_window", "n", + "Recent-token penalty window; official value 64."}, + {"seed", "n", + "Sampling seed; cloning defaults to 4099 for native weights and " + "42 for quantized weights."}, + {"reference_text", "text", + "Transcript matching the --voice-ref audio for voice cloning."}, + {"reference_language", "code", + "Language code used to align the reference transcript; default en."}, + {"text_chunk_size", "n", + "Framework long-form text chunk size; default 2048 characters."}, + {"text_chunk_mode", "default|tag_aware|japanese|endline", + "Framework long-form text chunking mode."}, + }; + out.session_options = { + {"outetts.weight_type", "native|f32|f16|bf16|q8_0", + "Language-model weight storage type. Quantized CUDA voice cloning " + "is expanded to F32 in memory for generation correctness."}, + {"outetts.llama_weight_context_mb", "n", + "Language-model weight context size in MiB."}, + {"outetts.constant_context_mb", "n", + "Language-model constant tensor context size in MiB."}, + {"outetts.dac_weight_context_mb", "n", + "DAC decoder weight context size in MiB."}, + {"outetts.dac_graph_context_mb", "n", + "DAC decoder graph context size in MiB."}, + {"outetts.aligner_model_path", "path", + "Optional Qwen3 Forced Aligner override. Cloning automatically uses " + "the aligner embedded in a standalone OuteTTS GGUF when present."}, + {"outetts.reference_cache_slots", "n", + "Prepared reference-profile cache slots; default 1, set 0 to " + "disable."}, + {"outetts.mem_saver", "true|false", + "Release cached-step and aligner runtime state after use; default " + "false."}, + }; + return out; +} + +class OuteTTSLoader final : public runtime::IVoiceModelLoader { public: std::string family() const override { return "outetts"; } + bool can_load(const runtime::ModelLoadRequest &request) const override { - if (request.family_hint.has_value() && *request.family_hint != family()) - return false; try { + const auto package_spec = + assets::default_model_package_spec_path(family()); (void)assets::load_resource_bundle_from_package_spec(request.model_path, - spec_path()); - return true; + package_spec); + return !request.family_hint.has_value() || + *request.family_hint == family(); } catch (...) { return false; } } + runtime::ModelInspection inspect(const runtime::ModelLoadRequest &request) const override { const auto model_assets = load_outetts_assets(request.model_path); - runtime::ModelInspection out; - out.model_root = model_assets->resources.model_root(); - out.metadata.family = family(); - out.metadata.variant = "1.0-1B"; - out.metadata.description = "OuteTTS 1.0 1B with native IBM DAC speech " - "synthesis and voice cloning."; - out.capabilities = make_capabilities(); - out.cli.request_options = { - {"max_tokens", "n", "Maximum generated text/audio tokens."}, - {"temperature", "float", - "Sampling temperature; official cloning default 0.4."}, - {"top_k", "n", "Top-k sampling; official default 40."}, - {"top_p", "float", "Nucleus sampling; official default 0.9."}, - {"min_p", "float", - "Minimum probability relative to the best token; official default " - "0.05."}, - {"repetition_penalty", "float", - "Windowed repetition penalty; official default 1.1."}, - {"repetition_window", "n", - "Recent-token penalty window; official value 64."}, - {"seed", "n", - "Sampling seed; cloning defaults to 4099 for native weights and " - "42 for quantized weights."}, - {"reference_text", "text", - "Transcript matching the --voice-ref audio for voice cloning."}, - {"reference_language", "code", - "Language code used to align the reference transcript; default en."}, - {"text_chunk_size", "n", - "Framework long-form text chunk size; default 2048 characters."}, - {"text_chunk_mode", "default|tag_aware|japanese|endline", - "Framework long-form text chunking mode."}, - }; - out.cli.session_options = { - {"outetts.weight_type", "native|f32|f16|bf16|q8_0", - "Language-model weight storage type. Quantized CUDA voice cloning " - "is expanded to F32 in memory for generation correctness."}, - {"outetts.llama_weight_context_mb", "n", - "Language-model weight context size in MiB."}, - {"outetts.constant_context_mb", "n", - "Language-model constant tensor context size in MiB."}, - {"outetts.dac_weight_context_mb", "n", - "DAC decoder weight context size in MiB."}, - {"outetts.dac_graph_context_mb", "n", - "DAC decoder graph context size in MiB."}, - {"outetts.aligner_model_path", "path", - "Optional Qwen3 Forced Aligner override. Cloning automatically uses " - "the aligner embedded in a standalone OuteTTS GGUF when present."}, - {"outetts.reference_cache_slots", "n", - "Prepared reference-profile cache slots; default 1, set 0 to " - "disable."}, - {"outetts.mem_saver", "true|false", - "Release cached-step and aligner runtime state after use; default " - "false."}, - }; - out.discovered_configs = runtime::discover_named_assets_from_package_spec( - request.model_path, spec_path(), - assets::ModelPackageResourceKind::Files); - out.discovered_weights = runtime::discover_named_assets_from_package_spec( - request.model_path, spec_path(), - assets::ModelPackageResourceKind::Tensors); - return out; + runtime::ModelInspection inspection; + inspection.model_root = model_assets->resources.model_root(); + inspection.metadata = metadata(*model_assets); + inspection.capabilities = capabilities(*model_assets); + inspection.cli = cli(*model_assets); + const auto package_spec = assets::default_model_package_spec_path(family()); + inspection.discovered_configs = + runtime::discover_named_assets_from_package_spec( + request.model_path, package_spec, + assets::ModelPackageResourceKind::Files); + inspection.discovered_weights = + runtime::discover_named_assets_from_package_spec( + request.model_path, package_spec, + assets::ModelPackageResourceKind::Tensors); + return inspection; } + std::unique_ptr load(const runtime::ModelLoadRequest &request) const override { - return std::make_unique( - load_outetts_assets(request.model_path)); + return load_outetts_model(request.model_path); } }; } // namespace OuteTTSLoadedModel::OuteTTSLoadedModel( + runtime::ModelMetadata metadata, runtime::CapabilitySet capabilities, std::shared_ptr assets) - : assets_(std::move(assets)), capabilities_(make_capabilities()) { - metadata_.family = "outetts"; - metadata_.variant = "1.0-1B"; - metadata_.description = - "OuteTTS 1.0 1B with native IBM DAC speech synthesis and voice cloning."; + : metadata_(std::move(metadata)), capabilities_(std::move(capabilities)), + assets_(std::move(assets)) { + if (assets_ == nullptr) { + throw std::runtime_error("OuteTTS loaded model requires assets"); + } } const runtime::ModelMetadata &OuteTTSLoadedModel::metadata() const noexcept { return metadata_; } + const runtime::CapabilitySet & OuteTTSLoadedModel::capabilities() const noexcept { return capabilities_; } + std::unique_ptr OuteTTSLoadedModel::create_task_session( const runtime::TaskSpec &task, const runtime::SessionOptions &options) const { + if ((task.task != runtime::VoiceTaskKind::Tts && + task.task != runtime::VoiceTaskKind::VoiceCloning) || + task.mode != runtime::RunMode::Offline) { + throw std::runtime_error( + "OuteTTS supports offline TTS and voice cloning only"); + } return std::make_unique(task, options, assets_); } +std::unique_ptr +load_outetts_model(const std::filesystem::path &model_path) { + auto model_assets = load_outetts_assets(model_path); + return std::make_unique(metadata(*model_assets), + capabilities(*model_assets), + std::move(model_assets)); +} + std::shared_ptr make_outetts_loader() { - return std::make_shared(); + return std::make_shared(); } } // namespace engine::models::outetts From c8556caf136e2f59554d016a839d353214d4748f Mon Sep 17 00:00:00 2001 From: mirek190 Date: Sun, 19 Jul 2026 13:39:38 +0100 Subject: [PATCH 7/8] Fix OuteTTS teardown and long-form generation --- CMakeLists.txt | 8 + docs/outetts_validation.md | 7 +- docs/tts.md | 14 +- include/engine/models/outetts/llama.h | 19 +- include/engine/models/outetts/tokenizer.h | 13 ++ src/models/outetts/llama.cpp | 46 +++- src/models/outetts/loader.cpp | 7 +- src/models/outetts/session.cpp | 199 +++++++++++++++++- src/models/outetts/tokenizer.cpp | 29 +++ tests/outetts/warm_bench_requests.json | 9 +- .../test_outetts_generation_budget.cpp | 47 +++++ 11 files changed, 363 insertions(+), 35 deletions(-) create mode 100644 tests/unittests/test_outetts_generation_budget.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 73d276061..5742647d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -792,6 +792,14 @@ if (ENGINE_BUILD_TESTS) COMMAND audio_chunking_test ) + add_engine_unittest(outetts_generation_budget_test tests/unittests/test_outetts_generation_budget.cpp) + target_include_directories(outetts_generation_budget_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests) + + add_test( + NAME outetts_generation_budget_test + COMMAND outetts_generation_budget_test + ) + add_engine_unittest(subtitle_formatter_test tests/unittests/test_subtitle_formatter.cpp) add_test( diff --git a/docs/outetts_validation.md b/docs/outetts_validation.md index 48d92dfa0..8595aa4c9 100644 --- a/docs/outetts_validation.md +++ b/docs/outetts_validation.md @@ -415,8 +415,11 @@ gate: generation correctness. Default mode can therefore have a high transient peak while the aligner is retained; `outetts.mem_saver=true` releases the aligner and cached-step graph between phases. -- Long-form output concatenates independently generated chunks; `max_tokens` - applies to each chunk. +- Long-form output concatenates independently generated chunks. `max_tokens` + applies to each chunk; the runtime splits text further to fit an explicit cap + and retries a chunk that reaches the cap without EOS/audio-end as smaller + chunks rather than returning silently truncated speech. It reports an error + only when the remaining text cannot be split any further. - CPU and CUDA outputs are deterministic within each tested backend but are not expected to be byte-identical across backends. - The controlled CUDA sampling stream is PyTorch-compatible. CPU sampling is diff --git a/docs/tts.md b/docs/tts.md index 7e4e07ebe..ea7682156 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -591,14 +591,14 @@ audiocpp_cli --task tts --family outetts \ | Option | Values | Default | Meaning | |---|---|---:|---| -| `--max-tokens` | integer | `1024` | Maximum audio-token generation length. Too small a value truncates speech. | +| `--max-tokens` | integer | automatic | Maximum audio-token generation length per chunk. When omitted, OuteTTS estimates a safe budget from the chunk's word and character counts. An explicit smaller cap causes additional text splitting; a chunk that unexpectedly reaches the cap is retried as smaller chunks instead of silently truncating speech. | | `--temperature` | float | `0.4` for cloning; model default for TTS | Sampling temperature. Voice cloning follows the official OuteTTS default without changing temperature between words. | | `--top-k` | integer | `40` | Top-k sampling limit. | | `--top-p` | float | `0.9` | Nucleus sampling limit. | | `--request-option min_p=` | float | `0.05` | Minimum probability relative to the most likely token. | | `--repetition-penalty` | float | `1.1` | Repetition penalty over the latest 64 tokens. | | `--request-option seed=` | integer | native clone: `4099`; quantized clone: `42` | Deterministic sampling seed. The defaults were separately verified for the native and Q8 cloning paths. | -| `--text-chunk-size` | characters | `2048` | Framework long-form text chunk size. Each chunk is generated and decoded in the same loaded session, then appended to the output WAV. | +| `--text-chunk-size` | characters | `256` | Initial framework long-form text chunk size. Each chunk is split further when needed to fit `max_tokens`, generated and decoded in the same loaded session, then appended to the output WAV. | | `--text-chunk-mode` | `default`, `tag_aware`, `japanese`, `endline` | `default` | Framework long-form text chunking mode. | | `--reference-text` | text | none | Exact transcript of `--voice-ref`; required for voice cloning. | | `--request-option reference_language=` | language code | `en` | Language used by the optional reference aligner. | @@ -607,10 +607,12 @@ audiocpp_cli --task tts --family outetts \ | `--session-option outetts.reference_cache_slots=` | integer | `1` | LRU slots for prepared reference profiles (alignment, DAC codes, and word features). Set `0` to disable reuse. | | `--session-option outetts.mem_saver=true|false` | bool | `false` | Release the reusable Llama cached-step graph after each generated chunk and release the aligner runtime after preparing a reference. Model and DAC weights stay resident; later requests rebuild released state. | -With logging enabled, OuteTTS reports framework chunk count, reference-profile -cache hits/evictions, Llama runtime and step-graph rebuild/reuse, released cache -capacity, and timings for reference alignment, DAC encode/decode, prompt -construction, generation, and the complete session request. See +With logging enabled, OuteTTS reports framework chunk count and token budget, +per-chunk word/character counts, recommended and effective generation limits, +the natural stop reason, reference-profile cache hits/evictions, Llama runtime +and step-graph rebuild/reuse, released cache capacity, and timings for reference +alignment, DAC encode/decode, prompt construction, generation, and the complete +session request. See [OuteTTS validation](outetts_validation.md) for the reproducible long-lived session and memory test. diff --git a/include/engine/models/outetts/llama.h b/include/engine/models/outetts/llama.h index 697ffbd69..916e8473d 100644 --- a/include/engine/models/outetts/llama.h +++ b/include/engine/models/outetts/llama.h @@ -7,12 +7,13 @@ #include #include #include +#include #include namespace engine::models::outetts { struct OuteTTSGenerateOptions { - int64_t max_new_tokens = 2048; + int64_t max_new_tokens = 4096; float temperature = 0.4F; float repetition_penalty = 1.1F; int64_t repetition_window = 64; @@ -22,6 +23,20 @@ struct OuteTTSGenerateOptions { uint32_t seed = 0; }; +enum class OuteTTSStopReason { + Eos, + AudioEnd, + MaxTokens, + ContextLimit, +}; + +std::string_view outetts_stop_reason_name(OuteTTSStopReason reason) noexcept; + +struct OuteTTSGenerateResult { + std::vector tokens; + OuteTTSStopReason stop_reason = OuteTTSStopReason::MaxTokens; +}; + class OuteTTSLlamaRuntime final { public: OuteTTSLlamaRuntime( @@ -37,7 +52,7 @@ class OuteTTSLlamaRuntime final { OuteTTSLlamaRuntime(const OuteTTSLlamaRuntime &) = delete; OuteTTSLlamaRuntime & operator=(const OuteTTSLlamaRuntime &) = delete; - std::vector generate( + OuteTTSGenerateResult generate( const std::vector & prompt, const OuteTTSGenerateOptions & options, int32_t eos_id, diff --git a/include/engine/models/outetts/tokenizer.h b/include/engine/models/outetts/tokenizer.h index 3ea848e62..54acbdd1e 100644 --- a/include/engine/models/outetts/tokenizer.h +++ b/include/engine/models/outetts/tokenizer.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,18 @@ struct OuteTTSVoiceProfile { std::vector words; }; +struct OuteTTSTextGenerationBudget { + int64_t words = 0; + int64_t non_whitespace_codepoints = 0; + int64_t recommended_max_new_tokens = 0; +}; + +// Mirrors the sizing heuristic used by the upstream OuteTTS 1.0 runner. Audio +// code generation is much denser than text tokenization, so a character-sized +// chunk cannot safely use the same numeric value as its generated-token cap. +OuteTTSTextGenerationBudget +estimate_text_generation_budget(std::string_view text); + class OuteTTSTokenizer { public: explicit OuteTTSTokenizer(std::shared_ptr assets); diff --git a/src/models/outetts/llama.cpp b/src/models/outetts/llama.cpp index 380348afd..182617766 100644 --- a/src/models/outetts/llama.cpp +++ b/src/models/outetts/llama.cpp @@ -388,6 +388,11 @@ struct OuteTTSLlamaRuntime::Impl { } ~Impl() { + // CachedStepGraph owns backend buffers and asks the backend to release + // graph resources in its destructor. Destroy it before its constants, + // weights, and backend. The previous order left a live graph holding a + // freed backend and caused a Linux CUDA segfault during session teardown. + step_graph.reset(); constants.reset(); weights.store.reset(); if (backend != nullptr) { @@ -554,7 +559,21 @@ OuteTTSLlamaRuntime::OuteTTSLlamaRuntime( OuteTTSLlamaRuntime::~OuteTTSLlamaRuntime() = default; -std::vector OuteTTSLlamaRuntime::generate( +std::string_view outetts_stop_reason_name(OuteTTSStopReason reason) noexcept { + switch (reason) { + case OuteTTSStopReason::Eos: + return "eos"; + case OuteTTSStopReason::AudioEnd: + return "audio_end"; + case OuteTTSStopReason::MaxTokens: + return "max_tokens"; + case OuteTTSStopReason::ContextLimit: + return "context_limit"; + } + return "unknown"; +} + +OuteTTSGenerateResult OuteTTSLlamaRuntime::generate( const std::vector & prompt, const OuteTTSGenerateOptions & options, int32_t eos_id, @@ -567,8 +586,8 @@ std::vector OuteTTSLlamaRuntime::generate( } const auto total_start = std::chrono::steady_clock::now(); std::vector all = prompt; - std::vector generated; - generated.reserve(static_cast(options.max_new_tokens)); + OuteTTSGenerateResult result; + result.tokens.reserve(static_cast(options.max_new_tokens)); std::mt19937 rng(options.seed); OuteTTSGenerateOptions sampling_options = options; auto prefill = impl_->prefill(prompt); @@ -588,6 +607,7 @@ std::vector OuteTTSLlamaRuntime::generate( double cached_step_compute_ms = 0.0; for (int64_t i = 0; i < options.max_new_tokens; ++i) { if (static_cast(all.size()) >= impl_->assets->generation.max_length) { + result.stop_reason = OuteTTSStopReason::ContextLimit; break; } const auto sample_start = std::chrono::steady_clock::now(); @@ -597,8 +617,18 @@ std::vector OuteTTSLlamaRuntime::generate( std::move(logits), sampling_options, rng, impl_->sampling_policy, static_cast(i)); sample_ms += debug::elapsed_ms(sample_start); - generated.push_back(token); - if (token == eos_id || token == audio_end_id) { + result.tokens.push_back(token); + if (token == eos_id) { + result.stop_reason = OuteTTSStopReason::Eos; + break; + } + if (token == audio_end_id) { + result.stop_reason = OuteTTSStopReason::AudioEnd; + break; + } + // Do not execute one unused cached step after consuming the caller's + // final generation token. + if (i + 1 >= options.max_new_tokens) { break; } all.push_back(token); @@ -608,13 +638,15 @@ std::vector OuteTTSLlamaRuntime::generate( } debug::trace_log_scalar( "outetts.llama.generated_tokens", - static_cast(generated.size())); + static_cast(result.tokens.size())); + debug::trace_log_scalar("outetts.llama.stop_reason", + outetts_stop_reason_name(result.stop_reason)); debug::timing_log_scalar("outetts.llama.sample_ms", sample_ms); debug::timing_log_scalar("outetts.llama.cached_step_compute_ms", cached_step_compute_ms); debug::timing_log_scalar("outetts.llama.generate_total_ms", debug::elapsed_ms(total_start)); - return generated; + return result; } int64_t OuteTTSLlamaRuntime::release_cached_step_graph() { diff --git a/src/models/outetts/loader.cpp b/src/models/outetts/loader.cpp index 0763ba788..11ec6ea92 100644 --- a/src/models/outetts/loader.cpp +++ b/src/models/outetts/loader.cpp @@ -38,7 +38,9 @@ runtime::CapabilitySet capabilities(const OuteTTSAssets &) { runtime::ModelCliInterface cli(const OuteTTSAssets &) { runtime::ModelCliInterface out; out.request_options = { - {"max_tokens", "n", "Maximum generated text/audio tokens."}, + {"max_tokens", "n", + "Maximum generated audio tokens per chunk. When omitted, OuteTTS " + "estimates a safe value from each chunk."}, {"temperature", "float", "Sampling temperature; official cloning default 0.4."}, {"top_k", "n", "Top-k sampling; official default 40."}, @@ -58,7 +60,8 @@ runtime::ModelCliInterface cli(const OuteTTSAssets &) { {"reference_language", "code", "Language code used to align the reference transcript; default en."}, {"text_chunk_size", "n", - "Framework long-form text chunk size; default 2048 characters."}, + "Framework long-form text chunk size; default 256 characters. Chunks " + "are split further when required by max_tokens."}, {"text_chunk_mode", "default|tag_aware|japanese|endline", "Framework long-form text chunking mode."}, }; diff --git a/src/models/outetts/session.cpp b/src/models/outetts/session.cpp index 31b799e68..1afdb8bc4 100644 --- a/src/models/outetts/session.cpp +++ b/src/models/outetts/session.cpp @@ -4,6 +4,7 @@ #include "engine/framework/runtime/options.h" #include "engine/framework/debug/trace.h" #include "engine/framework/text/chunking.h" +#include "engine/framework/text/utf8.h" #include "engine/models/qwen3_asr/assets.h" #include "engine/models/qwen3_forced_aligner/session.h" @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -24,9 +26,16 @@ namespace { using Clock = std::chrono::steady_clock; -constexpr int64_t kDefaultTextChunkSize = 2048; +constexpr int64_t kDefaultTextChunkSize = 256; +constexpr int64_t kAutomaticChunkTokenBudget = 4096; constexpr size_t kDefaultReferenceCacheSlots = 1; +int64_t generation_budget_with_headroom(int64_t estimated_tokens) { + // The upstream heuristic is a useful lower bound, but sampled generation can + // occasionally need a little more room before emitting audio_end. + return estimated_tokens + std::max(128, estimated_tokens / 4); +} + uint64_t mix_cache_key(uint64_t key, uint64_t value) { key ^= value; key *= 1099511628211ull; @@ -113,8 +122,10 @@ OuteTTSGenerateOptions generation_options(const runtime::TaskRequest &request, const OuteTTSGenerationConfig &defaults, bool voice_cloning, - bool quantized_cloning) { + bool quantized_cloning, + int64_t automatic_max_new_tokens) { OuteTTSGenerateOptions out; + out.max_new_tokens = automatic_max_new_tokens; out.temperature = voice_cloning ? 0.4F : defaults.temperature; out.repetition_penalty = defaults.repetition_penalty; out.repetition_window = defaults.repetition_window; @@ -153,6 +164,52 @@ generation_options(const runtime::TaskRequest &request, return out; } +std::vector chunk_text_request_to_token_budget( + const runtime::TaskRequest &request, int64_t requested_chunk_size, + engine::text::TextChunkMode mode, int64_t token_budget) { + auto initial = runtime::chunk_text_request(request, requested_chunk_size, + mode); + std::deque pending(initial.begin(), initial.end()); + std::vector chunks; + while (!pending.empty()) { + auto candidate = std::move(pending.front()); + pending.pop_front(); + if (!candidate.text_input.has_value()) { + chunks.push_back(std::move(candidate)); + continue; + } + const auto budget = + estimate_text_generation_budget(candidate.text_input->text); + // The upstream estimate has an intentional 384-token floor. Preserve + // explicitly smaller limits for short requests instead of inventing empty + // or one-character chunks; a cap is still reported as an error after + // generation if the model cannot stop within that caller-provided limit. + if (budget.recommended_max_new_tokens <= token_budget || + token_budget < 384) { + chunks.push_back(std::move(candidate)); + continue; + } + + const int64_t codepoints = static_cast( + engine::text::utf8_codepoint_count(candidate.text_input->text, + "OuteTTS text chunk")); + if (codepoints <= 1) { + throw std::runtime_error( + "OuteTTS max_tokens is too small for the requested text chunk"); + } + const int64_t smaller_chunk_size = std::max(1, codepoints / 2); + auto smaller = runtime::chunk_text_request(candidate, smaller_chunk_size, + mode); + if (smaller.size() <= 1) { + throw std::runtime_error( + "OuteTTS could not split text to fit the max_tokens budget"); + } + for (auto it = smaller.rbegin(); it != smaller.rend(); ++it) + pending.push_front(std::move(*it)); + } + return chunks; +} + std::vector split_words(const std::string &text) { std::istringstream input(text); std::vector words; @@ -649,17 +706,36 @@ runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { const auto text_chunk_mode = engine::text::parse_text_chunk_mode_override(request.options) .value_or(engine::text::TextChunkMode::Default); - const auto chunk_requests = runtime::chunk_text_request( - request, text_chunk_size, text_chunk_mode); - if (chunk_requests.empty()) { + const auto explicit_max_tokens = + runtime::parse_i64_option(request.options, {"max_tokens"}); + if (explicit_max_tokens.has_value() && *explicit_max_tokens <= 0) { + throw std::runtime_error("OuteTTS max_tokens must be positive"); + } + const int64_t chunk_token_budget = std::min( + explicit_max_tokens.value_or(kAutomaticChunkTokenBudget), + kAutomaticChunkTokenBudget); + runtime::TaskRequest chunk_source = request; + // Reference conditioning is already represented by profile. Avoid copying + // the full reference waveform into every long-form chunk and retry. + if (profile != nullptr) { + chunk_source.voice.reset(); + chunk_source.audio_input.reset(); + } + const auto initial_chunk_requests = chunk_text_request_to_token_budget( + chunk_source, text_chunk_size, text_chunk_mode, chunk_token_budget); + if (initial_chunk_requests.empty()) { throw std::runtime_error("OuteTTS text chunking produced no requests"); } + std::deque pending_chunks( + initial_chunk_requests.begin(), initial_chunk_requests.end()); debug::trace_log_scalar("outetts.text_chunk_size", text_chunk_size); debug::trace_log_scalar("outetts.text_chunk_mode", engine::text::text_chunk_mode_name(text_chunk_mode)); + debug::trace_log_scalar("outetts.text_chunk_token_budget", + chunk_token_budget); debug::trace_log_scalar( "outetts.text_chunk_count", - static_cast(chunk_requests.size())); + static_cast(initial_chunk_requests.size())); debug::trace_log_scalar("outetts.reference.cache_hit", reference_cache_hit); @@ -672,10 +748,13 @@ runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { double decode_ms = 0.0; double release_ms = 0.0; int64_t generated_tokens = 0; + int64_t retry_generated_tokens = 0; int64_t released_cache_capacity = 0; - for (size_t chunk_index = 0; chunk_index < chunk_requests.size(); - ++chunk_index) { - const auto &chunk_request = chunk_requests[chunk_index]; + int64_t retry_count = 0; + size_t chunk_index = 0; + while (!pending_chunks.empty()) { + auto chunk_request = std::move(pending_chunks.front()); + pending_chunks.pop_front(); const auto prompt_start = Clock::now(); const auto prompt = profile != nullptr @@ -684,16 +763,108 @@ runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { : tokenizer_.build_prompt(chunk_request.text_input->text); prompt_ms += debug::elapsed_ms(prompt_start); + const auto text_budget = estimate_text_generation_budget( + chunk_request.text_input->text); + const int64_t automatic_max_new_tokens = generation_budget_with_headroom( + text_budget.recommended_max_new_tokens); + const int64_t expected_generation_tokens = std::min( + automatic_max_new_tokens, + explicit_max_tokens.value_or(automatic_max_new_tokens)); + const int64_t available_context = + assets_->generation.max_length - static_cast(prompt.size()); + if (available_context <= 0) { + throw std::runtime_error( + "OuteTTS chunk " + std::to_string(chunk_index + 1) + + " prompt exhausts the generation context; use a shorter cloning " + "reference"); + } + if (expected_generation_tokens > available_context) { + const int64_t codepoints = static_cast( + engine::text::utf8_codepoint_count(chunk_request.text_input->text, + "OuteTTS context retry")); + auto smaller = runtime::chunk_text_request( + chunk_request, std::max(1, codepoints / 2), + text_chunk_mode); + if (codepoints <= 1 || smaller.size() <= 1) { + throw std::runtime_error( + "OuteTTS chunk does not fit the generation context; reduce " + "text_chunk_size or use a shorter cloning reference"); + } + debug::trace_log_scalar( + "outetts.retry." + std::to_string(retry_count) + ".reason", + std::string_view("context_budget")); + debug::trace_log_scalar( + "outetts.retry." + std::to_string(retry_count) + ".split_count", + static_cast(smaller.size())); + ++retry_count; + for (auto it = smaller.rbegin(); it != smaller.rend(); ++it) + pending_chunks.push_front(std::move(*it)); + continue; + } + auto generate_options = generation_options(chunk_request, assets_->generation, - profile != nullptr, quantized_cloning); + profile != nullptr, quantized_cloning, + automatic_max_new_tokens); + generate_options.max_new_tokens = + std::min(generate_options.max_new_tokens, available_context); const auto generate_start = Clock::now(); - const auto generated = llama(profile != nullptr).generate( + const auto generation = llama(profile != nullptr).generate( prompt, generate_options, tokenizer_.eos_id(), tokenizer_.audio_end_id()); generate_ms += debug::elapsed_ms(generate_start); + const auto &generated = generation.tokens; + + if (generation.stop_reason == OuteTTSStopReason::MaxTokens || + generation.stop_reason == OuteTTSStopReason::ContextLimit) { + retry_generated_tokens += static_cast(generated.size()); + const int64_t codepoints = static_cast( + engine::text::utf8_codepoint_count(chunk_request.text_input->text, + "OuteTTS generation retry")); + auto smaller = runtime::chunk_text_request( + chunk_request, std::max(1, codepoints / 2), + text_chunk_mode); + if (codepoints <= 1 || smaller.size() <= 1) { + throw std::runtime_error( + "OuteTTS reached " + + std::string(outetts_stop_reason_name(generation.stop_reason)) + + " before an audio end token and cannot split the remaining text " + "further; increase max_tokens"); + } + const std::string retry_prefix = + "outetts.retry." + std::to_string(retry_count); + debug::trace_log_scalar(retry_prefix + ".reason", + outetts_stop_reason_name(generation.stop_reason)); + debug::trace_log_scalar(retry_prefix + ".generated_tokens", + static_cast(generated.size())); + debug::trace_log_scalar(retry_prefix + ".text_codepoints", + text_budget.non_whitespace_codepoints); + debug::trace_log_scalar(retry_prefix + ".split_count", + static_cast(smaller.size())); + ++retry_count; + for (auto it = smaller.rbegin(); it != smaller.rend(); ++it) + pending_chunks.push_front(std::move(*it)); + continue; + } generated_tokens += static_cast(generated.size()); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".text_codepoints", + text_budget.non_whitespace_codepoints); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".text_words", + text_budget.words); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + + ".recommended_max_new_tokens", + text_budget.recommended_max_new_tokens); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".max_new_tokens", + generate_options.max_new_tokens); + debug::trace_log_scalar( + "outetts.chunk." + std::to_string(chunk_index) + ".stop_reason", + outetts_stop_reason_name(generation.stop_reason)); + std::vector c1; std::vector c2; for (const int32_t token : generated) @@ -729,12 +900,18 @@ runtime::TaskResult OuteTTSSession::run(const runtime::TaskRequest &request) { released_cache_capacity += llama_->release_cached_step_graph(); release_ms += debug::elapsed_ms(release_start); } + ++chunk_index; } runtime::TaskResult result; result.audio_output = std::move(merged_audio); debug::trace_log_scalar("outetts.mem_saver", mem_saver_); debug::trace_log_scalar("outetts.generated_tokens", generated_tokens); + debug::trace_log_scalar("outetts.retry_generated_tokens", + retry_generated_tokens); + debug::trace_log_scalar("outetts.text_chunk_count_final", + static_cast(chunk_index)); + debug::trace_log_scalar("outetts.retry_count", retry_count); debug::trace_log_scalar("outetts.llama.step.released_cache_capacity", released_cache_capacity); debug::timing_log_scalar("outetts.prompt_ms", prompt_ms); diff --git a/src/models/outetts/tokenizer.cpp b/src/models/outetts/tokenizer.cpp index 3b9ea4806..74d4ccaad 100644 --- a/src/models/outetts/tokenizer.cpp +++ b/src/models/outetts/tokenizer.cpp @@ -1,5 +1,6 @@ #include "engine/models/outetts/tokenizer.h" +#include "engine/framework/text/utf8.h" #include "engine/framework/tokenizers/llama_bpe.h" #include @@ -112,6 +113,34 @@ std::string profile_codes(const OuteTTSVoiceProfile &profile, } // namespace +OuteTTSTextGenerationBudget +estimate_text_generation_budget(std::string_view text) { + OuteTTSTextGenerationBudget out; + size_t position = 0; + while (position < text.size()) { + while (position < text.size() && + std::isspace(static_cast(text[position])) != 0) { + ++position; + } + if (position >= text.size()) + break; + const size_t begin = position; + while (position < text.size() && + std::isspace(static_cast(text[position])) == 0) { + ++position; + } + ++out.words; + out.non_whitespace_codepoints += static_cast( + engine::text::utf8_codepoint_count(text.substr(begin, position - begin), + "OuteTTS text")); + } + const int64_t by_words = out.words * 72; + const int64_t by_characters = out.non_whitespace_codepoints * 12; + out.recommended_max_new_tokens = + std::max({256, by_words, by_characters}) + 128; + return out; +} + struct OuteTTSTokenizer::Impl { explicit Impl(const OuteTTSAssets &assets) : tokenizer({ diff --git a/tests/outetts/warm_bench_requests.json b/tests/outetts/warm_bench_requests.json index 1237e13ec..57d45a709 100644 --- a/tests/outetts/warm_bench_requests.json +++ b/tests/outetts/warm_bench_requests.json @@ -4,21 +4,20 @@ "name": "tts_cold", "text": "This is the first OuteTTS request in a long-lived session.", "language": "en", - "max_tokens": 256, + "max_tokens": 1024, "seed": 1234 }, { "name": "tts_repeat", "text": "This is the first OuteTTS request in a long-lived session.", "language": "en", - "max_tokens": 256, + "max_tokens": 1024, "seed": 1234 }, { "name": "tts_longform", "text": "Long-form synthesis is split by the shared framework text chunker. Each sentence becomes a bounded request while model weights and the cached generation graph stay inside the same long-lived session. The generated audio chunks are appended in order to form one output waveform.", "language": "en", - "max_tokens": 256, "seed": 1234, "text_chunk_size": 100, "text_chunk_mode": "default" @@ -30,7 +29,7 @@ "voice_ref": "assets/resources/b.wav", "reference_text": "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you.", "reference_language": "en", - "max_tokens": 256, + "max_tokens": 1024, "seed": 42 }, { @@ -40,7 +39,7 @@ "voice_ref": "assets/resources/b.wav", "reference_text": "Some call me nature. Others call me Mother Nature. I've been here for over 4.5 billion years. 22,500 times longer than you.", "reference_language": "en", - "max_tokens": 256, + "max_tokens": 1024, "seed": 42 } ] diff --git a/tests/unittests/test_outetts_generation_budget.cpp b/tests/unittests/test_outetts_generation_budget.cpp new file mode 100644 index 000000000..5b7b985a0 --- /dev/null +++ b/tests/unittests/test_outetts_generation_budget.cpp @@ -0,0 +1,47 @@ +#include "engine/models/outetts/tokenizer.h" + +#include "test_assert.h" + +#include + +int main() try { + using engine::models::outetts::estimate_text_generation_budget; + + const auto empty = estimate_text_generation_budget(""); + engine::test::require_eq(empty.words, int64_t{0}, "empty words"); + engine::test::require_eq(empty.non_whitespace_codepoints, int64_t{0}, + "empty codepoints"); + engine::test::require_eq(empty.recommended_max_new_tokens, int64_t{384}, + "empty minimum budget"); + + const auto words = + estimate_text_generation_budget("one two three four five six"); + engine::test::require_eq(words.words, int64_t{6}, "word count"); + engine::test::require_eq(words.non_whitespace_codepoints, int64_t{22}, + "non-whitespace codepoints"); + engine::test::require_eq(words.recommended_max_new_tokens, int64_t{560}, + "word-dominated budget"); + + const auto characters = estimate_text_generation_budget( + "supercalifragilisticexpialidocious"); + engine::test::require_eq(characters.words, int64_t{1}, + "long word count"); + engine::test::require_eq(characters.non_whitespace_codepoints, int64_t{34}, + "long word codepoints"); + engine::test::require_eq(characters.recommended_max_new_tokens, int64_t{536}, + "character-dominated budget"); + + const auto unicode = estimate_text_generation_budget(u8"Zażółć gęślą"); + engine::test::require_eq(unicode.words, int64_t{2}, "UTF-8 word count"); + engine::test::require_eq(unicode.non_whitespace_codepoints, int64_t{11}, + "UTF-8 codepoints"); + engine::test::require_eq(unicode.recommended_max_new_tokens, int64_t{384}, + "UTF-8 minimum budget"); + + std::cout << "outetts_generation_budget_test passed\n"; + return 0; +} catch (const std::exception &error) { + std::cerr << "outetts_generation_budget_test failed: " << error.what() + << "\n"; + return 1; +} From 347544a43dc031cc8260222b8a5ffcea4279637c Mon Sep 17 00:00:00 2001 From: 0xShug0 <231717474+0xShug0@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:50:57 -0400 Subject: [PATCH 8/8] Optimize OuteTTS sampling hot path --- src/models/outetts/llama.cpp | 68 ++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 19 deletions(-) diff --git a/src/models/outetts/llama.cpp b/src/models/outetts/llama.cpp index 182617766..cd9c23ab4 100644 --- a/src/models/outetts/llama.cpp +++ b/src/models/outetts/llama.cpp @@ -21,7 +21,6 @@ #include #include #include -#include namespace engine::models::outetts { namespace { @@ -157,7 +156,6 @@ modules::QwenCausalDecoderConfig decoder_config(const OuteTTSConfig & c) { out.stack.runtime.attention.prefill_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; out.stack.runtime.attention.static_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; out.stack.runtime.static_cache.update_mode = modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; - out.stack.runtime.static_cache.transpose_context = true; out.logits_size = c.vocab_size; return out; } @@ -183,17 +181,29 @@ modules::QwenCausalDecoderWeights graph_weights( return out; } +struct SamplingScratch { + std::vector repeated_ids; + std::vector order; + std::vector probabilities; +}; + void apply_repetition_penalty( std::vector & logits, const std::vector & ids, int64_t window, - float penalty) { + float penalty, + std::vector & repeated_ids) { if (penalty == 1.0F || window == 0) { return; } const size_t begin = ids.size() > static_cast(window) ? ids.size() - static_cast(window) : 0; - std::unordered_set seen(ids.begin() + static_cast(begin), ids.end()); - for (const int32_t id : seen) { + repeated_ids.clear(); + for (auto it = ids.begin() + static_cast(begin); it != ids.end(); ++it) { + if (std::find(repeated_ids.begin(), repeated_ids.end(), *it) == repeated_ids.end()) { + repeated_ids.push_back(*it); + } + } + for (const int32_t id : repeated_ids) { if (id < 0 || static_cast(id) >= logits.size()) { continue; } @@ -203,22 +213,38 @@ void apply_repetition_penalty( } int32_t sample_token( - std::vector logits, + const std::vector & logits, const OuteTTSGenerateOptions & o, std::mt19937 & rng, const sampling::TorchCudaSamplingPolicy & sampling_policy, - uint64_t call_index) { + uint64_t call_index, + SamplingScratch & scratch) { if (!(o.temperature > 0.0F) || !std::isfinite(o.temperature)) { return static_cast(std::max_element(logits.begin(), logits.end()) - logits.begin()); } - std::vector order(logits.size()); + auto & order = scratch.order; + order.resize(logits.size()); std::iota(order.begin(), order.end(), 0); - std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return logits[a] > logits[b]; }); - if (o.top_k > 0 && static_cast(o.top_k) < order.size()) { - order.resize(static_cast(o.top_k)); + const size_t kept_by_top_k = + o.top_k > 0 && static_cast(o.top_k) < order.size() + ? static_cast(o.top_k) + : order.size(); + const auto by_logit_desc = [&](size_t a, size_t b) { + return logits[a] > logits[b]; + }; + if (kept_by_top_k < order.size()) { + std::partial_sort( + order.begin(), + order.begin() + static_cast(kept_by_top_k), + order.end(), + by_logit_desc); + order.resize(kept_by_top_k); + } else { + std::sort(order.begin(), order.end(), by_logit_desc); } const float max_logit = logits[order.front()]; - std::vector probabilities(order.size(), 0.0F); + auto & probabilities = scratch.probabilities; + probabilities.assign(order.size(), 0.0F); double sum = 0.0; for (size_t i = 0; i < order.size(); ++i) { probabilities[i] = std::exp((logits[order[i]] - max_logit) / o.temperature); @@ -319,7 +345,7 @@ class CachedStepGraph { void import_state(const runtime::TransformerKVState & state) { cache_.import_state(state); } - std::vector run(int32_t token) { + void run(int32_t token, std::vector & logits) { if (cache_.valid_steps() >= capacity_) throw std::runtime_error("OuteTTS cached-step capacity exceeded"); ggml_backend_tensor_set(input_id_, &token, 0, sizeof(token)); const int32_t position = static_cast(cache_.current_end()); @@ -331,10 +357,11 @@ class CachedStepGraph { const auto status = core::compute_backend_graph(backend_, graph_); ggml_backend_synchronize(backend_); if (status != GGML_STATUS_SUCCESS) throw std::runtime_error("OuteTTS cached-step graph compute failed"); - std::vector logits(static_cast(config_.vocab_size)); + if (logits.size() != static_cast(config_.vocab_size)) { + logits.resize(static_cast(config_.vocab_size)); + } ggml_backend_tensor_get(logits_, logits.data(), 0, logits.size() * sizeof(float)); cache_.advance_after_direct_append(1); - return logits; } private: @@ -605,6 +632,7 @@ OuteTTSGenerateResult OuteTTSLlamaRuntime::generate( std::vector logits = std::move(prefill.logits); double sample_ms = 0.0; double cached_step_compute_ms = 0.0; + SamplingScratch sampling_scratch; for (int64_t i = 0; i < options.max_new_tokens; ++i) { if (static_cast(all.size()) >= impl_->assets->generation.max_length) { result.stop_reason = OuteTTSStopReason::ContextLimit; @@ -612,10 +640,12 @@ OuteTTSGenerateResult OuteTTSLlamaRuntime::generate( } const auto sample_start = std::chrono::steady_clock::now(); apply_repetition_penalty( - logits, all, options.repetition_window, options.repetition_penalty); + logits, all, options.repetition_window, options.repetition_penalty, + sampling_scratch.repeated_ids); const int32_t token = sample_token( - std::move(logits), sampling_options, rng, - impl_->sampling_policy, static_cast(i)); + logits, sampling_options, rng, + impl_->sampling_policy, static_cast(i), + sampling_scratch); sample_ms += debug::elapsed_ms(sample_start); result.tokens.push_back(token); if (token == eos_id) { @@ -633,7 +663,7 @@ OuteTTSGenerateResult OuteTTSLlamaRuntime::generate( } all.push_back(token); const auto step_start = std::chrono::steady_clock::now(); - logits = step.run(token); + step.run(token, logits); cached_step_compute_ms += debug::elapsed_ms(step_start); } debug::trace_log_scalar(