From 98b803fafa14104fc603c92a03a55725b59982d6 Mon Sep 17 00:00:00 2001 From: Tony Giorgio Date: Fri, 9 Jan 2026 12:08:13 -0600 Subject: [PATCH 01/22] feat: add iOS TTS support using ONNX Runtime - Add iOS-specific ort dependencies in Cargo.toml - Update build.rs to link ONNX Runtime xcframework for iOS - Enable TTS module for iOS in lib.rs (was desktop-only) - Add iOS-specific path handling in tts.rs for model storage - Update TTSContext.tsx to enable TTS on iOS - Add setup-ios-onnxruntime.sh script to download pre-built xcframework - Update GitHub Actions workflows to download ONNX Runtime before iOS builds - Add onnxruntime-ios/ to .gitignore Uses pre-built ONNX Runtime 1.20.1 xcframework from HuggingFace (csukuangfj/ios-onnxruntime) for static linking on iOS. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- .github/workflows/mobile-build.yml | 16 ++++ .github/workflows/testflight-on-comment.yml | 16 ++++ frontend/bun.lock | 1 + frontend/src-tauri/.gitignore | 3 + frontend/src-tauri/Cargo.toml | 15 ++++ frontend/src-tauri/build.rs | 30 +++++++ .../scripts/setup-ios-onnxruntime.sh | 78 +++++++++++++++++++ frontend/src-tauri/src/lib.rs | 41 +++++++++- frontend/src-tauri/src/tts.rs | 23 ++++-- frontend/src/services/tts/TTSContext.tsx | 6 +- 10 files changed, 219 insertions(+), 10 deletions(-) create mode 100755 frontend/src-tauri/scripts/setup-ios-onnxruntime.sh diff --git a/.github/workflows/mobile-build.yml b/.github/workflows/mobile-build.yml index 40e461ae4..2d732c013 100644 --- a/.github/workflows/mobile-build.yml +++ b/.github/workflows/mobile-build.yml @@ -53,6 +53,22 @@ jobs: working-directory: ./frontend run: bun install + - name: Cache ONNX Runtime iOS xcframework + uses: actions/cache@v4 + id: cache-onnxruntime + with: + path: frontend/src-tauri/onnxruntime-ios + key: onnxruntime-ios-1.20.1 + restore-keys: | + onnxruntime-ios- + + - name: Download ONNX Runtime for iOS + if: steps.cache-onnxruntime.outputs.cache-hit != 'true' + working-directory: ./frontend/src-tauri + run: | + chmod +x scripts/setup-ios-onnxruntime.sh + ./scripts/setup-ios-onnxruntime.sh + - name: Setup Xcode uses: maxim-lobanov/setup-xcode@v1 with: diff --git a/.github/workflows/testflight-on-comment.yml b/.github/workflows/testflight-on-comment.yml index f72da15a6..c4ce73e38 100644 --- a/.github/workflows/testflight-on-comment.yml +++ b/.github/workflows/testflight-on-comment.yml @@ -111,6 +111,22 @@ jobs: working-directory: ./frontend run: bun install + - name: Cache ONNX Runtime iOS xcframework + uses: actions/cache@v4 + id: cache-onnxruntime + with: + path: frontend/src-tauri/onnxruntime-ios + key: onnxruntime-ios-1.20.1 + restore-keys: | + onnxruntime-ios- + + - name: Download ONNX Runtime for iOS + if: steps.cache-onnxruntime.outputs.cache-hit != 'true' + working-directory: ./frontend/src-tauri + run: | + chmod +x scripts/setup-ios-onnxruntime.sh + ./scripts/setup-ios-onnxruntime.sh + - name: Setup Xcode uses: maxim-lobanov/setup-xcode@v1 with: diff --git a/frontend/bun.lock b/frontend/bun.lock index acf319d9d..847fabcdc 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "maple", diff --git a/frontend/src-tauri/.gitignore b/frontend/src-tauri/.gitignore index 502406b4e..c9f6eb3cb 100644 --- a/frontend/src-tauri/.gitignore +++ b/frontend/src-tauri/.gitignore @@ -2,3 +2,6 @@ # will have compiled files and executables /target/ /gen/schemas + +# ONNX Runtime iOS xcframework (downloaded at build time) +/onnxruntime-ios/ diff --git a/frontend/src-tauri/Cargo.toml b/frontend/src-tauri/Cargo.toml index 4bfb6eb83..fa100ba47 100644 --- a/frontend/src-tauri/Cargo.toml +++ b/frontend/src-tauri/Cargo.toml @@ -53,5 +53,20 @@ futures-util = "0.3" dirs = "5.0" sha2 = "0.10" +[target.'cfg(target_os = "ios")'.dependencies] +# TTS dependencies (Supertonic) - iOS +# Uses pre-built ONNX Runtime xcframework from HuggingFace +ort = { version = "2.0.0-rc.10", default-features = false } +ndarray = { version = "0.16" } +rand = "0.8" +rand_distr = "0.4" +hound = "3.5" +unicode-normalization = "0.1" +regex = "1.10" +reqwest = { version = "0.12", features = ["stream"] } +futures-util = "0.3" +dirs = "5.0" +sha2 = "0.10" + [target.'cfg(target_os = "android")'.dependencies] openssl = { version = "0.10", default-features = false, features = ["vendored"] } diff --git a/frontend/src-tauri/build.rs b/frontend/src-tauri/build.rs index d860e1e6a..ecad17e92 100644 --- a/frontend/src-tauri/build.rs +++ b/frontend/src-tauri/build.rs @@ -1,3 +1,33 @@ fn main() { + // iOS-specific build configuration for ONNX Runtime + #[cfg(target_os = "ios")] + { + // Get the path to the ONNX Runtime xcframework + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let ort_dir = format!("{}/onnxruntime-ios", manifest_dir); + + // Check if building for simulator or device + let target = std::env::var("TARGET").unwrap_or_default(); + let lib_path = if target.contains("sim") || target.contains("x86_64") { + format!("{}/onnxruntime.xcframework/ios-arm64_x86_64-simulator", ort_dir) + } else { + format!("{}/onnxruntime.xcframework/ios-arm64", ort_dir) + }; + + // Tell cargo where to find the ONNX Runtime static library + println!("cargo:rustc-link-search=native={}", lib_path); + println!("cargo:rustc-link-lib=static=onnxruntime"); + + // Link required iOS frameworks + println!("cargo:rustc-link-lib=framework=Foundation"); + println!("cargo:rustc-link-lib=framework=Accelerate"); + + // Set ORT_LIB_LOCATION for the ort crate + println!("cargo:rustc-env=ORT_LIB_LOCATION={}", ort_dir); + + // Rerun if the onnxruntime directory changes + println!("cargo:rerun-if-changed={}", ort_dir); + } + tauri_build::build() } diff --git a/frontend/src-tauri/scripts/setup-ios-onnxruntime.sh b/frontend/src-tauri/scripts/setup-ios-onnxruntime.sh new file mode 100755 index 000000000..143497d6c --- /dev/null +++ b/frontend/src-tauri/scripts/setup-ios-onnxruntime.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Setup ONNX Runtime for iOS builds +# This script downloads pre-built ONNX Runtime xcframework from HuggingFace +# or builds from source if needed. + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TAURI_DIR="$(dirname "$SCRIPT_DIR")" +ORT_VERSION="${ORT_VERSION:-1.20.1}" +ORT_DIR="$TAURI_DIR/onnxruntime-ios" +XCFRAMEWORK_DIR="$ORT_DIR/onnxruntime.xcframework" + +echo "Setting up ONNX Runtime $ORT_VERSION for iOS..." +echo "Target directory: $ORT_DIR" + +# Check if already downloaded +if [ -d "$XCFRAMEWORK_DIR" ]; then + echo "ONNX Runtime xcframework already exists at $XCFRAMEWORK_DIR" + echo "To re-download, remove the directory first: rm -rf $ORT_DIR" + exit 0 +fi + +# Create directory +mkdir -p "$ORT_DIR" + +# Download pre-built xcframework from HuggingFace +# Repository: https://huggingface.co/csukuangfj/ios-onnxruntime +HF_BASE_URL="https://huggingface.co/csukuangfj/ios-onnxruntime/resolve/main" + +echo "Downloading ONNX Runtime $ORT_VERSION xcframework from HuggingFace..." + +# Download the xcframework directory structure +# The structure is: +# onnxruntime.xcframework/ +# Info.plist +# Headers/ +# cpu_provider_factory.h +# onnxruntime_c_api.h +# onnxruntime_cxx_api.h +# onnxruntime_cxx_inline.h +# ios-arm64/ +# onnxruntime.a +# ios-arm64_x86_64-simulator/ +# onnxruntime.a + +mkdir -p "$XCFRAMEWORK_DIR/Headers" +mkdir -p "$XCFRAMEWORK_DIR/ios-arm64" +mkdir -p "$XCFRAMEWORK_DIR/ios-arm64_x86_64-simulator" + +echo "Downloading Info.plist..." +curl -L -o "$XCFRAMEWORK_DIR/Info.plist" \ + "$HF_BASE_URL/$ORT_VERSION/onnxruntime.xcframework/Info.plist" + +echo "Downloading headers..." +for header in cpu_provider_factory.h onnxruntime_c_api.h onnxruntime_cxx_api.h onnxruntime_cxx_inline.h; do + curl -L -o "$XCFRAMEWORK_DIR/Headers/$header" \ + "$HF_BASE_URL/$ORT_VERSION/onnxruntime.xcframework/Headers/$header" +done + +echo "Downloading iOS arm64 static library (this may take a while)..." +curl -L -o "$XCFRAMEWORK_DIR/ios-arm64/onnxruntime.a" \ + "$HF_BASE_URL/$ORT_VERSION/onnxruntime.xcframework/ios-arm64/onnxruntime.a" + +echo "Downloading iOS simulator static library (this may take a while)..." +curl -L -o "$XCFRAMEWORK_DIR/ios-arm64_x86_64-simulator/onnxruntime.a" \ + "$HF_BASE_URL/$ORT_VERSION/onnxruntime.xcframework/ios-arm64_x86_64-simulator/onnxruntime.a" + +echo "" +echo "ONNX Runtime xcframework downloaded successfully!" +echo "Location: $XCFRAMEWORK_DIR" +echo "" +echo "Contents:" +ls -la "$XCFRAMEWORK_DIR" +echo "" +echo "Static library sizes:" +ls -lh "$XCFRAMEWORK_DIR/ios-arm64/onnxruntime.a" +ls -lh "$XCFRAMEWORK_DIR/ios-arm64_x86_64-simulator/onnxruntime.a" diff --git a/frontend/src-tauri/src/lib.rs b/frontend/src-tauri/src/lib.rs index 2deadf0e7..a9cee65f2 100644 --- a/frontend/src-tauri/src/lib.rs +++ b/frontend/src-tauri/src/lib.rs @@ -3,7 +3,8 @@ use tauri_plugin_deep_link::DeepLinkExt; mod pdf_extractor; mod proxy; -#[cfg(desktop)] +// TTS is available on desktop and iOS (not Android) +#[cfg(any(desktop, target_os = "ios"))] mod tts; #[cfg(desktop)] @@ -240,6 +241,7 @@ pub fn run() { }) .plugin(tauri_plugin_updater::Builder::new().build()); + // Mobile (iOS and Android) configuration #[cfg(not(desktop))] let mut builder = tauri::Builder::default() .plugin( @@ -258,7 +260,14 @@ pub fn run() { builder = builder.plugin(tauri_plugin_sign_in_with_apple::init()); } - #[cfg(not(desktop))] + // Add TTS state management for iOS + #[cfg(all(not(desktop), target_os = "ios"))] + { + builder = builder.manage(tts::TTSState::new()); + } + + // Android-specific configuration (no TTS) + #[cfg(all(not(desktop), target_os = "android"))] let app = builder .invoke_handler(tauri::generate_handler![ pdf_extractor::extract_document_content, @@ -279,6 +288,34 @@ pub fn run() { }) .plugin(tauri_plugin_updater::Builder::new().build()); + // iOS-specific configuration (with TTS) + #[cfg(all(not(desktop), target_os = "ios"))] + let app = builder + .invoke_handler(tauri::generate_handler![ + pdf_extractor::extract_document_content, + tts::tts_get_status, + tts::tts_download_models, + tts::tts_load_models, + tts::tts_synthesize, + tts::tts_unload_models, + tts::tts_delete_models, + ]) + .setup(|app| { + // Set up the deep link handler for mobile + let app_handle = app.handle().clone(); + + // Register deep link handler - note that iOS does not support runtime registration + // but the handler for incoming URLs still works + app.deep_link().on_open_url(move |event| { + if let Some(url) = event.urls().first() { + handle_deep_link_event(url.as_ref(), &app_handle); + } + }); + + Ok(()) + }) + .plugin(tauri_plugin_updater::Builder::new().build()); + app.run(tauri::generate_context!()) .expect("error while running tauri application"); } diff --git a/frontend/src-tauri/src/tts.rs b/frontend/src-tauri/src/tts.rs index 5c5f25666..bfd5ab4a4 100644 --- a/frontend/src-tauri/src/tts.rs +++ b/frontend/src-tauri/src/tts.rs @@ -612,11 +612,24 @@ impl TextToSpeech { } fn get_tts_models_dir() -> Result { - let data_dir = dirs::data_local_dir() - .context("Failed to get local data directory")? - .join("cloud.opensecret.maple") - .join("tts_models"); - Ok(data_dir) + // On iOS, we need to use a different approach since dirs::data_local_dir() may not work + #[cfg(target_os = "ios")] + { + // On iOS, use the app's Documents directory which is accessible and persists + // NSHomeDirectory() + /Documents/tts_models + let home = std::env::var("HOME").context("Failed to get HOME directory on iOS")?; + let data_dir = PathBuf::from(home).join("Documents").join("tts_models"); + return Ok(data_dir); + } + + #[cfg(not(target_os = "ios"))] + { + let data_dir = dirs::data_local_dir() + .context("Failed to get local data directory")? + .join("cloud.opensecret.maple") + .join("tts_models"); + Ok(data_dir) + } } fn load_voice_style(models_dir: &Path) -> Result