From fdd67893d1580d0d9730e8b8210da09a37b17d04 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 2 Sep 2026 14:37:06 -0600 Subject: [PATCH 01/11] Fix cuda context lifecycle leak --- client-sdk-rust | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-sdk-rust b/client-sdk-rust index 1a477bc4..6bff3273 160000 --- a/client-sdk-rust +++ b/client-sdk-rust @@ -1 +1 @@ -Subproject commit 1a477bc422c6890537b3bcdb017f0ac094d49661 +Subproject commit 6bff3273ffe020ca4898f4f801f17d8a9c324eb7 From 8f4b9c45053ed44e8ae314e43b19742b67fd1ef1 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 2 Sep 2026 14:53:23 -0600 Subject: [PATCH 02/11] Maybe fix linux --- .github/workflows/builds.yml | 8 ++++++++ .github/workflows/tests.yml | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index f3fda0da..e572f06c 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -125,6 +125,14 @@ jobs: libcurl4-openssl-dev \ libwayland-dev libdecor-0-dev + # The libwebrtc artifact ships a hermetic libc++ built from LLVM trunk. + # Distro clang (14 on Ubuntu 22.04, 18 on 24.04) cannot compile its + # headers; the submodule script installs LLVM 21 and exports CC/CXX. + - name: Install clang for libwebrtc + if: runner.os == 'Linux' + shell: bash + run: client-sdk-rust/.github/scripts/install-clang.sh + - name: Install deps (macOS) if: runner.os == 'macOS' run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92f04bf5..47dac142 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -229,6 +229,14 @@ jobs: libwayland-dev libdecor-0-dev \ gdb jq + # The libwebrtc artifact ships a hermetic libc++ built from LLVM trunk. + # Distro clang (14 on Ubuntu 22.04, 18 on 24.04) cannot compile its + # headers; the submodule script installs LLVM 21 and exports CC/CXX. + - name: Install clang for libwebrtc + if: runner.os == 'Linux' + shell: bash + run: client-sdk-rust/.github/scripts/install-clang.sh + - name: Install deps (macOS) if: runner.os == 'macOS' run: | @@ -511,6 +519,12 @@ jobs: libwayland-dev libdecor-0-dev pip install --break-system-packages gcovr + # The coverage build compiles the Rust FFI too, so it needs the same + # compiler floor as the normal Linux build/test matrices. + - name: Install clang for libwebrtc + shell: bash + run: client-sdk-rust/.github/scripts/install-clang.sh + # ---------- Rust toolchain ---------- - name: Install Rust (stable) uses: dtolnay/rust-toolchain@6c977a6ca4077a0ceb28ffbe03f59d46e9ac8772 From dc214de059f172a21077920379fcfebf9082032d Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 3 Sep 2026 16:17:19 -0600 Subject: [PATCH 03/11] Add regression test for dispose Rust changes --- client-sdk-rust | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-sdk-rust b/client-sdk-rust index 6bff3273..e5fe5dab 160000 --- a/client-sdk-rust +++ b/client-sdk-rust @@ -1 +1 @@ -Subproject commit 6bff3273ffe020ca4898f4f801f17d8a9c324eb7 +Subproject commit e5fe5dab84e49320df58115d37a2a1a9478261d7 From 393044f48c0fd8e9207cffa32a355a5db866e549 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 3 Sep 2026 23:07:17 -0600 Subject: [PATCH 04/11] Track process memory/memory lifecycle tester --- AGENTS.md | 3 +- scripts/track_process_memory.py | 122 ++++++++++++++++++ src/tests/CMakeLists.txt | 46 +++++++ .../manual/memory_lifecycle_tester/README.md | 51 ++++++++ .../manual/memory_lifecycle_tester/main.cpp | 108 ++++++++++++++++ 5 files changed, 329 insertions(+), 1 deletion(-) create mode 100755 scripts/track_process_memory.py create mode 100644 src/tests/manual/memory_lifecycle_tester/README.md create mode 100644 src/tests/manual/memory_lifecycle_tester/main.cpp diff --git a/AGENTS.md b/AGENTS.md index 55da0430..b4c9d8ba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,8 @@ Be sure to update the directory layout in this file if the directory layout chan |------|-------------| | `include/livekit/` | Public API headers (what SDK consumers include) | | `src/` | Implementation files and internal-only headers (`ffi_client.h`, `lk_log.h`, etc.) | -| `src/tests/` | Google Test integration and stress tests | +| `src/tests/` | Google Test unit, integration, and stress tests | +| `src/tests/manual/` | Standalone testers built with the test targets but not registered with CTest | | `examples/` | In-tree example applications | | `client-sdk-rust/` | Git submodule holding the Rust core of the SDK| | `cpp-tools/` | Git submodule holding shared LiveKit C++ engineering guidance, clang-format / clang-tidy configs, scripts, docs, and CI workflow | diff --git a/scripts/track_process_memory.py b/scripts/track_process_memory.py new file mode 100755 index 00000000..0c4b760d --- /dev/null +++ b/scripts/track_process_memory.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# +# Copyright 2026 LiveKit +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run a command and report its resident memory usage.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import time + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run a command and report its initial, final, and peak RSS." + ) + parser.add_argument( + "--interval", + type=float, + default=0.1, + help="seconds between RSS samples (default: 0.1)", + ) + parser.add_argument( + "command", + nargs=argparse.REMAINDER, + help="command and its arguments; prefix it with -- when needed", + ) + args = parser.parse_args() + if args.interval <= 0: + parser.error("--interval must be greater than zero") + if not args.command: + parser.error("a command is required") + return args + + +def process_rss_kib(pid: int) -> int | None: + result = subprocess.run( + ["ps", "-o", "rss=", "-p", str(pid)], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None + + rss = result.stdout.strip() + if not rss: + return None + + rss_kib = int(rss) + # macOS reports 0 RSS for a child that has exited but has not yet been + # reaped. Do not overwrite the last live-process sample with that value. + return rss_kib if rss_kib > 0 else None + + +def format_rss(rss_kib: int) -> str: + return f"{rss_kib:,} KiB ({rss_kib / 1024:.2f} MiB)" + + +def main() -> int: + args = parse_args() + command = args.command + if command[0] == "--": + command = command[1:] + if not command: + print("error: a command is required after --", file=sys.stderr) + return 2 + + try: + process = subprocess.Popen(command) + except OSError as error: + print(f"error: could not start {command[0]!r}: {error}", file=sys.stderr) + return 127 + + started_at = time.monotonic() + initial_rss_kib: int | None = None + final_rss_kib: int | None = None + peak_rss_kib: int | None = None + + while process.poll() is None: + rss_kib = process_rss_kib(process.pid) + if rss_kib is not None: + if initial_rss_kib is None: + initial_rss_kib = rss_kib + final_rss_kib = rss_kib + peak_rss_kib = max(peak_rss_kib or rss_kib, rss_kib) + time.sleep(args.interval) + + elapsed_s = time.monotonic() - started_at + exit_code = process.wait() + print(f"command: {' '.join(command)}") + print(f"exit code: {exit_code}") + print(f"elapsed: {elapsed_s:.2f} s") + if initial_rss_kib is None: + print("RSS: no samples collected; the command exited before sampling began") + else: + assert final_rss_kib is not None + assert peak_rss_kib is not None + print(f"RSS initial: {format_rss(initial_rss_kib)}") + print(f"RSS final observed: {format_rss(final_rss_kib)}") + print(f"RSS peak: {format_rss(peak_rss_kib)}") + print(f"RSS change: {format_rss(final_rss_kib - initial_rss_kib)}") + + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 9583af73..47161f1f 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -348,3 +348,49 @@ endif() if(TARGET livekit_stress_tests) add_dependencies(run_all_tests livekit_stress_tests) endif() + +# ============================================================================ +# Manual testers (built with the test targets, not registered with CTest) +# ============================================================================ + +add_executable(livekit_memory_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/manual/memory_lifecycle_tester/main.cpp" +) + +target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) +target_include_directories(livekit_memory_lifecycle_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include) +target_compile_definitions(livekit_memory_lifecycle_tester PRIVATE + $<$:_USE_MATH_DEFINES> +) + +if(WIN32) + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/livekit_ffi.dll" + $ + COMMENT "Copying DLLs next to livekit_memory_lifecycle_tester" + ) +elseif(APPLE) + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.dylib" + $ + COMMENT "Copying dylibs next to livekit_memory_lifecycle_tester" + ) +else() + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.so" + $ + COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester" + ) +endif() diff --git a/src/tests/manual/memory_lifecycle_tester/README.md b/src/tests/manual/memory_lifecycle_tester/README.md new file mode 100644 index 00000000..dfe7c916 --- /dev/null +++ b/src/tests/manual/memory_lifecycle_tester/README.md @@ -0,0 +1,51 @@ +# Memory lifecycle tester + +This standalone application repeatedly exercises the public C++ SDK lifecycle +to expose retained lower-level Rust/WebRTC resources. + +Each iteration: + +- calls `livekit::initialize()`; +- creates an `AudioSource` and `LocalAudioTrack`; +- creates a 1280x720 `VideoSource` and `LocalVideoTrack`; +- creates a representative `DataTrackFrame`; +- destroys those objects before calling `livekit::shutdown()`. + +The video source intentionally receives no captured frame. This exercises +teardown of the Rust keepalive task that runs until the first raw video frame +arrives and previously retained roughly one 720p frame per lifecycle. + +`LocalDataTrack` itself cannot be created offline: its public factory publishes +through a connected `LocalParticipant`. The `DataTrackFrame` allocation covers +the offline data API surface but does not create a Rust data-track handle. + +The executable is built with the normal test targets but is not registered with +CTest, so it only runs when invoked manually. It does not connect to a server +and needs no LiveKit credentials. + +## Build + +```bash +./build.sh release-tests +``` + +## Run + +The default is 1,000 iterations. An alternate iteration count may be supplied +as the only argument: + +```bash +./build-release/bin/livekit_memory_lifecycle_tester +./build-release/bin/livekit_memory_lifecycle_tester 100 +``` + +To compare memory behavior before and after a lifecycle fix: + +```bash +python3 scripts/track_process_memory.py --interval 0.01 -- \ + ./build-release/bin/livekit_memory_lifecycle_tester +``` + +Use identical iteration counts and build configurations when comparing results. +Allocator caching means final RSS need not return to the initial value; the +useful regression signal is sustained or iteration-proportional growth. diff --git a/src/tests/manual/memory_lifecycle_tester/main.cpp b/src/tests/manual/memory_lifecycle_tester/main.cpp new file mode 100644 index 00000000..bf92b2dd --- /dev/null +++ b/src/tests/manual/memory_lifecycle_tester/main.cpp @@ -0,0 +1,108 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr int kDefaultIterations = 1'000; +constexpr int kAudioSampleRate = 48'000; +constexpr int kAudioChannels = 1; +constexpr int kAudioQueueSizeMs = 100; +constexpr int kVideoWidth = 1'280; +constexpr int kVideoHeight = 720; +constexpr std::size_t kDataPayloadSize = 1'024; + +int parseIterationCount(const char* value) { + try { + const int parsed = std::stoi(value); + if (parsed <= 0) { + throw std::runtime_error("iteration count must be greater than zero"); + } + return parsed; + } catch (const std::invalid_argument&) { + throw std::runtime_error("iteration count must be an integer"); + } catch (const std::out_of_range&) { + throw std::runtime_error("iteration count is out of range"); + } +} + +void exerciseCommonFeatures() { + auto audio_source = std::make_shared(kAudioSampleRate, kAudioChannels, kAudioQueueSizeMs); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("lifecycle-audio", audio_source); + if (!audio_track) { + throw std::runtime_error("failed to create local audio track"); + } + // Deliberately do not capture a frame. This covers teardown of the Rust + // keepalive task used before a raw video source receives its first frame. + auto video_source = std::make_shared(kVideoWidth, kVideoHeight); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("lifecycle-video", video_source); + if (!video_track) { + throw std::runtime_error("failed to create local video track"); + } + + // A LocalDataTrack requires a connected LocalParticipant. Constructing the + // public frame type still covers the common offline data allocation surface. + livekit::DataTrackFrame data_frame(std::vector(kDataPayloadSize, 0x5a)); + if (data_frame.payload.size() != kDataPayloadSize) { + throw std::runtime_error("failed to create data track frame"); + } +} + +} // namespace + +int main(int argc, char* argv[]) { + if (argc > 2) { + std::cerr << "usage: " << argv[0] << " [iteration-count]\n"; + return 2; + } + + try { + const int iteration_count = argc == 2 ? parseIterationCount(argv[1]) : kDefaultIterations; + std::cout << "Running " << iteration_count << " LiveKit initialize/shutdown cycles\n"; + + for (int iteration = 1; iteration <= iteration_count; ++iteration) { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("initialize failed at iteration " + std::to_string(iteration)); + } + + try { + exerciseCommonFeatures(); + } catch (...) { + livekit::shutdown(); + throw; + } + livekit::shutdown(); + + if (iteration % 100 == 0 || iteration == iteration_count) { + std::cout << "Completed " << iteration << "/" << iteration_count << " cycles\n"; + } + } + } catch (const std::exception& error) { + std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + return 1; + } + + return 0; +} From 006ef3e1ef641d95d9e54cda0c8f94a21439ac68 Mon Sep 17 00:00:00 2001 From: Alan George Date: Mon, 7 Sep 2026 10:08:40 -0600 Subject: [PATCH 05/11] Cuda tester --- CMakeLists.txt | 13 +- docs/building.md | 2 +- src/tests/CMakeLists.txt | 42 ++++ .../cuda_video_lifecycle_tester/README.md | 63 ++++++ .../cuda_video_lifecycle_tester/main.cpp | 207 ++++++++++++++++++ .../manual/memory_lifecycle_tester/README.md | 60 +++-- .../manual/memory_lifecycle_tester/main.cpp | 159 ++++++++++++-- 7 files changed, 497 insertions(+), 49 deletions(-) create mode 100644 src/tests/manual/cuda_video_lifecycle_tester/README.md create mode 100644 src/tests/manual/cuda_video_lifecycle_tester/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c7b244c..70b50d97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,13 +343,7 @@ else() ) endif() -file(GLOB_RECURSE RUST_SOURCES - "${RUST_ROOT}/livekit-ffi/src/*.rs" - "${RUST_ROOT}/livekit-ffi/Cargo.toml" -) - -add_custom_command( - OUTPUT "${RUST_LIB_DEBUG}" "${RUST_LIB_RELEASE}" +add_custom_target(build_rust_ffi COMMAND "${CMAKE_COMMAND}" -DCFG=$ -DRUST_ROOT=${RUST_ROOT} @@ -359,15 +353,10 @@ add_custom_command( -DGCC_LIB_DIR=${GCC_LIB_DIR} -P "${RUN_CARGO_SCRIPT}" WORKING_DIRECTORY "${RUST_ROOT}" - DEPENDS ${RUST_SOURCES} COMMENT "Building Rust FFI via cargo" VERBATIM ) -add_custom_target(build_rust_ffi - DEPENDS "${RUST_LIB_DEBUG}" "${RUST_LIB_RELEASE}" -) - # Note: protozero_plugin.o removal is no longer needed since we use dynamic libraries on Unix add_library(livekit SHARED diff --git a/docs/building.md b/docs/building.md index fd4cbb83..81f6e012 100644 --- a/docs/building.md +++ b/docs/building.md @@ -58,7 +58,7 @@ sudo apt update && sudo apt install -y \ build-essential cmake ninja-build pkg-config \ llvm-dev libclang-dev clang \ libprotobuf-dev protobuf-compiler libabsl-dev \ - libssl-dev + libssl-dev libcurl4-openssl-dev # Install Rust if you don't already have it curl https://sh.rustup.rs -sSf | sh diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 47161f1f..5e7695e8 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -394,3 +394,45 @@ else() COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester" ) endif() + +add_executable(livekit_cuda_video_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/manual/cuda_video_lifecycle_tester/main.cpp" +) + +target_link_libraries(livekit_cuda_video_lifecycle_tester PRIVATE livekit) +target_include_directories(livekit_cuda_video_lifecycle_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include) +target_compile_definitions(livekit_cuda_video_lifecycle_tester PRIVATE + $<$:_USE_MATH_DEFINES> +) + +if(WIN32) + add_custom_command(TARGET livekit_cuda_video_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/livekit_ffi.dll" + $ + COMMENT "Copying DLLs next to livekit_cuda_video_lifecycle_tester" + ) +elseif(APPLE) + add_custom_command(TARGET livekit_cuda_video_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.dylib" + $ + COMMENT "Copying dylibs next to livekit_cuda_video_lifecycle_tester" + ) +else() + add_custom_command(TARGET livekit_cuda_video_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.so" + $ + COMMENT "Copying shared libraries next to livekit_cuda_video_lifecycle_tester" + ) +endif() diff --git a/src/tests/manual/cuda_video_lifecycle_tester/README.md b/src/tests/manual/cuda_video_lifecycle_tester/README.md new file mode 100644 index 00000000..24464137 --- /dev/null +++ b/src/tests/manual/cuda_video_lifecycle_tester/README.md @@ -0,0 +1,63 @@ +# CUDA video lifecycle tester + +This standalone application repeatedly creates two C++ SDK rooms, publishes a +synthetic I420 video track from one, verifies that the other receives a frame, +and tears down both rooms and the SDK. It is intended to exercise the CUDA +video codec lifecycle on an NVIDIA system. + +It requires a CUDA-enabled SDK build, an NVIDIA driver with NVENC and NVDEC +support, a running LiveKit server, and two tokens for the same room. The +existing `memory_lifecycle_tester` remains the hardware-neutral lifecycle +test; this tester adds real video publication and reception. + +## Build + +```bash +./build.sh release-tests +``` + +The Rust build must not print the following message: + +```text +cuda.h not found; building without hardware accelerated video codec support for NVidia GPUs +``` + +## Run + +Start a local development server in one terminal: + +```bash +livekit-server --dev +``` + +In another terminal, load the test credentials and prefer the NVIDIA encoder: + +```bash +source scripts/set-test-tokens.sh +export LIVEKIT_PREFERRED_HW_ENCODER=nvenc + +./build-release/bin/livekit_cuda_video_lifecycle_tester --iterations 100 +``` + +The tester enables SDK Info logging. Confirm that the output includes messages +such as `Using NVIDIA HW encoder (NVENC) for H264` and `Using NVIDIA HW decoder +(NVDEC) for H264`. `LIVEKIT_PREFERRED_HW_ENCODER=nvenc` is a preference: it +does not turn an unavailable NVENC backend into a hard failure, so those log +messages are the current proof that the negotiated video path used NVIDIA +hardware. + +For a memory regression run: + +```bash +python3 scripts/track_process_memory.py --interval 0.01 -- \ + ./build-release/bin/livekit_cuda_video_lifecycle_tester --iterations 1000 +``` + +Monitor NVIDIA memory concurrently: + +```bash +watch -n 1 nvidia-smi +``` + +Allocator caching means final RSS need not return to the initial value. Look +for sustained or iteration-proportional host RSS or GPU-memory growth. diff --git a/src/tests/manual/cuda_video_lifecycle_tester/main.cpp b/src/tests/manual/cuda_video_lifecycle_tester/main.cpp new file mode 100644 index 00000000..40ab5cc0 --- /dev/null +++ b/src/tests/manual/cuda_video_lifecycle_tester/main.cpp @@ -0,0 +1,207 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace std::chrono_literals; + +constexpr int kDefaultIterations = 100; +constexpr int kVideoWidth = 640; +constexpr int kVideoHeight = 360; +constexpr int kFramesPerIteration = 500; +constexpr char kTrackName[] = "cuda-lifecycle-video"; + +struct Configuration { + std::string url; + std::string sender_token; + std::string receiver_token; +}; + +struct Options { + int iteration_count{kDefaultIterations}; +}; + +const char* requiredEnvironment(const char* name) { + const char* value = std::getenv(name); + if (value == nullptr || *value == '\0') { + throw std::runtime_error(std::string(name) + " must be set"); + } + return value; +} + +int parseIterationCount(const char* value) { + try { + const int parsed = std::stoi(value); + if (parsed <= 0) { + throw std::runtime_error("iteration count must be greater than zero"); + } + return parsed; + } catch (const std::invalid_argument&) { + throw std::runtime_error("iteration count must be an integer"); + } catch (const std::out_of_range&) { + throw std::runtime_error("iteration count is out of range"); + } +} + +Options parseOptions(int argc, char* argv[]) { + Options options; + if (argc == 1) { + return options; + } + if (argc == 3 && std::strcmp(argv[1], "--iterations") == 0) { + options.iteration_count = parseIterationCount(argv[2]); + return options; + } + if (argc == 2) { + options.iteration_count = parseIterationCount(argv[1]); + return options; + } + throw std::runtime_error("usage: cuda_video_lifecycle_tester [--iterations N]"); +} + +std::shared_ptr localParticipant(livekit::Room& room) { + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant is unavailable"); + } + return participant; +} + +void runIteration(const Configuration& config) { + livekit::Room receiver_room; + livekit::Room sender_room; + std::mutex mutex; + std::condition_variable frame_received; + bool received_frame = false; + + try { + if (!receiver_room.connect(config.url, config.receiver_token, {})) { + throw std::runtime_error("receiver failed to connect"); + } + if (!sender_room.connect(config.url, config.sender_token, {})) { + throw std::runtime_error("sender failed to connect"); + } + + const std::string sender_identity = localParticipant(sender_room)->identity(); + if (sender_identity.empty()) { + throw std::runtime_error("sender identity is empty"); + } + + receiver_room.setOnVideoFrameCallback( + sender_identity, kTrackName, + [&mutex, &frame_received, &received_frame](const livekit::VideoFrame&, std::int64_t) { + std::lock_guard lock(mutex); + received_frame = true; + frame_received.notify_all(); + }); + + auto source = std::make_shared(kVideoWidth, kVideoHeight); + auto track = livekit::LocalVideoTrack::createLocalVideoTrack(kTrackName, source); + if (!track) { + throw std::runtime_error("failed to create local video track"); + } + + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_CAMERA; + publish_options.simulcast = false; + localParticipant(sender_room)->publishTrack(track, publish_options); + + auto frame = livekit::VideoFrame::create(kVideoWidth, kVideoHeight, livekit::VideoBufferType::I420); + std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); + for (int frame_index = 0; frame_index < kFramesPerIteration; ++frame_index) { + source->captureFrame(frame); + + std::unique_lock lock(mutex); + if (frame_received.wait_for(lock, 20ms, [&received_frame] { return received_frame; })) { + break; + } + } + + { + std::lock_guard lock(mutex); + if (!received_frame) { + throw std::runtime_error("timed out waiting for the receiver to get a video frame"); + } + } + + if (const auto publication = track->publication()) { + localParticipant(sender_room)->unpublishTrack(publication->sid()); + } + receiver_room.clearOnVideoFrameCallback(sender_identity, kTrackName); + } catch (...) { + (void)sender_room.disconnect(); + (void)receiver_room.disconnect(); + throw; + } + + if (!sender_room.disconnect()) { + throw std::runtime_error("sender failed to disconnect"); + } + if (!receiver_room.disconnect()) { + throw std::runtime_error("receiver failed to disconnect"); + } +} + +} // namespace + +int main(int argc, char* argv[]) { + try { + const Options options = parseOptions(argc, argv); + const Configuration config{ + requiredEnvironment("LIVEKIT_URL"), + requiredEnvironment("LIVEKIT_TOKEN_A"), + requiredEnvironment("LIVEKIT_TOKEN_B"), + }; + const int progress_interval = options.iteration_count < 10 ? 1 : options.iteration_count / 10; + + std::cout << "Running " << options.iteration_count << " CUDA video lifecycle cycles\n"; + for (int iteration = 1; iteration <= options.iteration_count; ++iteration) { + if (!livekit::initialize(livekit::LogLevel::Info)) { + throw std::runtime_error("initialize failed at iteration " + std::to_string(iteration)); + } + + try { + runIteration(config); + } catch (...) { + livekit::shutdown(); + throw; + } + livekit::shutdown(); + + if (iteration % progress_interval == 0 || iteration == options.iteration_count) { + std::cout << "Completed " << iteration << "/" << options.iteration_count << " cycles\n"; + } + } + } catch (const std::exception& error) { + std::cerr << "CUDA video lifecycle tester failed: " << error.what() << '\n'; + return 1; + } + + return 0; +} diff --git a/src/tests/manual/memory_lifecycle_tester/README.md b/src/tests/manual/memory_lifecycle_tester/README.md index dfe7c916..e41b92ed 100644 --- a/src/tests/manual/memory_lifecycle_tester/README.md +++ b/src/tests/manual/memory_lifecycle_tester/README.md @@ -3,25 +3,23 @@ This standalone application repeatedly exercises the public C++ SDK lifecycle to expose retained lower-level Rust/WebRTC resources. -Each iteration: +Every iteration calls `livekit::initialize()`, runs the selected workload, and +then calls `livekit::shutdown()`. Workloads can be combined: -- calls `livekit::initialize()`; -- creates an `AudioSource` and `LocalAudioTrack`; -- creates a 1280x720 `VideoSource` and `LocalVideoTrack`; -- creates a representative `DataTrackFrame`; -- destroys those objects before calling `livekit::shutdown()`. +- `--sources`: creates an `AudioSource`/`LocalAudioTrack` and a 1280x720 + `VideoSource`/`LocalVideoTrack`; +- `--connect`: joins and leaves a LiveKit room; +- `--data-track`: publishes and unpublishes a `LocalDataTrack`; implies + `--connect`; +- `--data-frames`: immediately sends 10 1 KiB payloads; implies + `--data-track` and `--connect`. The video source intentionally receives no captured frame. This exercises teardown of the Rust keepalive task that runs until the first raw video frame arrives and previously retained roughly one 720p frame per lifecycle. -`LocalDataTrack` itself cannot be created offline: its public factory publishes -through a connected `LocalParticipant`. The `DataTrackFrame` allocation covers -the offline data API surface but does not create a Rust data-track handle. - The executable is built with the normal test targets but is not registered with -CTest, so it only runs when invoked manually. It does not connect to a server -and needs no LiveKit credentials. +CTest, so it only runs when invoked manually. ## Build @@ -31,21 +29,45 @@ and needs no LiveKit credentials. ## Run -The default is 1,000 iterations. An alternate iteration count may be supplied -as the only argument: +Start a local LiveKit server and load the test credentials: + +```bash +source scripts/set-test-tokens.sh +``` + +`--connect`, `--data-track`, and `--data-frames` require `LIVEKIT_URL` and +`LIVEKIT_TOKEN_A`. The default is 1,000 iterations. With no workload flags, +the tester runs every workload: ```bash ./build-release/bin/livekit_memory_lifecycle_tester -./build-release/bin/livekit_memory_lifecycle_tester 100 ``` -To compare memory behavior before and after a lifecycle fix: +Use `--iterations N` (or the legacy single numeric argument) to set the loop +count. Examples: + +```bash +# Offline audio/video-source lifecycle only. +./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --sources + +# Room setup and teardown only. +./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --connect + +# Incrementally add data-track allocation and data-frame delivery. +./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --data-track +./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --data-frames + +# Combine explicitly selected workloads. +./build-release/bin/livekit_memory_lifecycle_tester --iterations 100 --sources --data-frames +``` + +To compare memory behavior before and after a lifecycle fix, use identical +iteration counts and build configurations: ```bash python3 scripts/track_process_memory.py --interval 0.01 -- \ - ./build-release/bin/livekit_memory_lifecycle_tester + ./build-release/bin/livekit_memory_lifecycle_tester 100 ``` -Use identical iteration counts and build configurations when comparing results. -Allocator caching means final RSS need not return to the initial value; the +Allocator caching means final RSS need not return to the initial value. The useful regression signal is sustained or iteration-proportional growth. diff --git a/src/tests/manual/memory_lifecycle_tester/main.cpp b/src/tests/manual/memory_lifecycle_tester/main.cpp index bf92b2dd..e2f9a9f8 100644 --- a/src/tests/manual/memory_lifecycle_tester/main.cpp +++ b/src/tests/manual/memory_lifecycle_tester/main.cpp @@ -17,6 +17,8 @@ #include #include +#include +#include #include #include #include @@ -33,6 +35,30 @@ constexpr int kAudioQueueSizeMs = 100; constexpr int kVideoWidth = 1'280; constexpr int kVideoHeight = 720; constexpr std::size_t kDataPayloadSize = 1'024; +constexpr int kDataFrameCount = 10; +constexpr char kDataTrackName[] = "lifecycle-data"; + +struct Configuration { + std::string url; + std::string token; +}; + +struct Options { + int iteration_count{kDefaultIterations}; + bool sources{false}; + bool connect{false}; + bool data_track{false}; + bool data_frames{false}; + bool mode_selected{false}; +}; + +const char* requiredEnvironment(const char* name) { + const char* value = std::getenv(name); + if (value == nullptr || *value == '\0') { + throw std::runtime_error(std::string(name) + " must be set"); + } + return value; +} int parseIterationCount(const char* value) { try { @@ -48,7 +74,7 @@ int parseIterationCount(const char* value) { } } -void exerciseCommonFeatures() { +void runSources() { auto audio_source = std::make_shared(kAudioSampleRate, kAudioChannels, kAudioQueueSizeMs); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("lifecycle-audio", audio_source); if (!audio_track) { @@ -61,46 +87,145 @@ void exerciseCommonFeatures() { if (!video_track) { throw std::runtime_error("failed to create local video track"); } +} + +void runRoomIteration(const Configuration& config, const Options& options) { + livekit::Room room; + if (!room.connect(config.url, config.token, {})) { + throw std::runtime_error("failed to connect to the LiveKit room"); + } + + try { + if (!options.data_track) { + if (!room.disconnect()) { + throw std::runtime_error("failed to disconnect from the LiveKit room"); + } + return; + } + + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant is unavailable"); + } + + auto result = participant->publishDataTrack(kDataTrackName); + if (!result) { + throw std::runtime_error("failed to publish data track: " + result.error().message); + } + const auto& data_track = result.value(); + + if (options.data_frames) { + const livekit::DataTrackFrame data_frame(std::vector(kDataPayloadSize, 0x5a)); + for (int frame = 0; frame < kDataFrameCount; ++frame) { + auto push_result = data_track->tryPush(data_frame); + if (!push_result) { + throw std::runtime_error("failed to publish data: " + push_result.error().message); + } + } + } + + data_track->unpublishDataTrack(); + } catch (...) { + (void)room.disconnect(); + throw; + } - // A LocalDataTrack requires a connected LocalParticipant. Constructing the - // public frame type still covers the common offline data allocation surface. - livekit::DataTrackFrame data_frame(std::vector(kDataPayloadSize, 0x5a)); - if (data_frame.payload.size() != kDataPayloadSize) { - throw std::runtime_error("failed to create data track frame"); + if (!room.disconnect()) { + throw std::runtime_error("failed to disconnect from the LiveKit room"); } } -} // namespace +void printUsage(const char* executable) { + std::cerr << "usage: " << executable << " [--iterations N] [--sources] [--connect] [--data-track] [--data-frames]\n" + << " --sources Create local audio and video sources and tracks.\n" + << " --connect Connect to and leave a room.\n" + << " --data-track Publish and unpublish a data track (implies --connect).\n" + << " --data-frames Send data frames (implies --data-track and --connect).\n" + << " No mode flags runs all modes. A single numeric argument remains supported as the iteration count.\n"; +} -int main(int argc, char* argv[]) { - if (argc > 2) { - std::cerr << "usage: " << argv[0] << " [iteration-count]\n"; - return 2; +Options parseOptions(int argc, char* argv[]) { + Options options; + for (int argument = 1; argument < argc; ++argument) { + const char* value = argv[argument]; + if (std::strcmp(value, "--iterations") == 0) { + if (++argument == argc) { + throw std::runtime_error("--iterations requires a value"); + } + options.iteration_count = parseIterationCount(argv[argument]); + } else if (std::strcmp(value, "--sources") == 0) { + options.sources = true; + options.mode_selected = true; + } else if (std::strcmp(value, "--connect") == 0) { + options.connect = true; + options.mode_selected = true; + } else if (std::strcmp(value, "--data-track") == 0) { + options.data_track = true; + options.mode_selected = true; + } else if (std::strcmp(value, "--data-frames") == 0) { + options.data_frames = true; + options.mode_selected = true; + } else if (std::strcmp(value, "--help") == 0 || std::strcmp(value, "-h") == 0) { + printUsage(argv[0]); + std::exit(0); + } else if (argument == 1 && argc == 2) { + options.iteration_count = parseIterationCount(value); + } else { + throw std::runtime_error(std::string("unknown argument: ") + value); + } + } + + if (!options.mode_selected) { + options.sources = true; + options.connect = true; + options.data_track = true; + options.data_frames = true; + } else if (options.data_frames) { + options.data_track = true; + options.connect = true; + } else if (options.data_track) { + options.connect = true; } + return options; +} + +} // namespace +int main(int argc, char* argv[]) { try { - const int iteration_count = argc == 2 ? parseIterationCount(argv[1]) : kDefaultIterations; - std::cout << "Running " << iteration_count << " LiveKit initialize/shutdown cycles\n"; + const Options options = parseOptions(argc, argv); + const int progress_interval = options.iteration_count < 10 ? 1 : options.iteration_count / 10; + std::cout << "Running " << options.iteration_count << " LiveKit initialize/shutdown cycles\n"; - for (int iteration = 1; iteration <= iteration_count; ++iteration) { + for (int iteration = 1; iteration <= options.iteration_count; ++iteration) { if (!livekit::initialize(livekit::LogLevel::Warn)) { throw std::runtime_error("initialize failed at iteration " + std::to_string(iteration)); } try { - exerciseCommonFeatures(); + if (options.sources) { + runSources(); + } + if (options.connect) { + const Configuration config{ + requiredEnvironment("LIVEKIT_URL"), + requiredEnvironment("LIVEKIT_TOKEN_A"), + }; + runRoomIteration(config, options); + } } catch (...) { livekit::shutdown(); throw; } livekit::shutdown(); - if (iteration % 100 == 0 || iteration == iteration_count) { - std::cout << "Completed " << iteration << "/" << iteration_count << " cycles\n"; + if (iteration % progress_interval == 0 || iteration == options.iteration_count) { + std::cout << "Completed " << iteration << "/" << options.iteration_count << " cycles\n"; } } } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + printUsage(argv[0]); return 1; } From 1e0613116d5ed13db3c1baf3803378c5dadfa4d6 Mon Sep 17 00:00:00 2001 From: Alan George Date: Mon, 7 Sep 2026 13:31:10 -0600 Subject: [PATCH 06/11] Try encoder --- .../manual/cuda_video_lifecycle_tester/README.md | 16 ++++++++++------ .../manual/cuda_video_lifecycle_tester/main.cpp | 2 ++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/tests/manual/cuda_video_lifecycle_tester/README.md b/src/tests/manual/cuda_video_lifecycle_tester/README.md index 24464137..2d34f1c7 100644 --- a/src/tests/manual/cuda_video_lifecycle_tester/README.md +++ b/src/tests/manual/cuda_video_lifecycle_tester/README.md @@ -35,16 +35,20 @@ In another terminal, load the test credentials and prefer the NVIDIA encoder: ```bash source scripts/set-test-tokens.sh export LIVEKIT_PREFERRED_HW_ENCODER=nvenc +export RUST_LOG=libwebrtc=debug ./build-release/bin/livekit_cuda_video_lifecycle_tester --iterations 100 ``` -The tester enables SDK Info logging. Confirm that the output includes messages -such as `Using NVIDIA HW encoder (NVENC) for H264` and `Using NVIDIA HW decoder -(NVDEC) for H264`. `LIVEKIT_PREFERRED_HW_ENCODER=nvenc` is a preference: it -does not turn an unavailable NVENC backend into a hard failure, so those log -messages are the current proof that the negotiated video path used NVIDIA -hardware. +Confirm that stderr includes messages such as `Using NVIDIA HW encoder (NVENC) +for H264` and `Using NVIDIA HW decoder (NVDEC) for H264`. The native WebRTC +logs are forwarded to Rust with the `libwebrtc` target and Debug severity, so +`RUST_LOG=libwebrtc=debug` must be set before starting the tester. +The tester explicitly requests H264 because NVENC does not support the default +VP8 codec. `LIVEKIT_PREFERRED_HW_ENCODER=nvenc` selects the NVIDIA backend for +that codec, but it does not turn an unavailable NVENC backend into a hard +failure, so those log messages are the current proof that the negotiated video +path used NVIDIA hardware. For a memory regression run: diff --git a/src/tests/manual/cuda_video_lifecycle_tester/main.cpp b/src/tests/manual/cuda_video_lifecycle_tester/main.cpp index 40ab5cc0..595f9bd4 100644 --- a/src/tests/manual/cuda_video_lifecycle_tester/main.cpp +++ b/src/tests/manual/cuda_video_lifecycle_tester/main.cpp @@ -36,6 +36,7 @@ constexpr int kVideoWidth = 640; constexpr int kVideoHeight = 360; constexpr int kFramesPerIteration = 500; constexpr char kTrackName[] = "cuda-lifecycle-video"; +constexpr auto kH264VideoCodec = static_cast(1); struct Configuration { std::string url; @@ -130,6 +131,7 @@ void runIteration(const Configuration& config) { livekit::TrackPublishOptions publish_options; publish_options.source = livekit::TrackSource::SOURCE_CAMERA; publish_options.simulcast = false; + publish_options.video_codec = kH264VideoCodec; localParticipant(sender_room)->publishTrack(track, publish_options); auto frame = livekit::VideoFrame::create(kVideoWidth, kVideoHeight, livekit::VideoBufferType::I420); From a33214c2bbdfeb0e25ef59b54f899e98549a322a Mon Sep 17 00:00:00 2001 From: Alan George Date: Mon, 7 Sep 2026 15:00:43 -0600 Subject: [PATCH 07/11] Undo CMake changes --- CMakeLists.txt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 70b50d97..1c7b244c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,7 +343,13 @@ else() ) endif() -add_custom_target(build_rust_ffi +file(GLOB_RECURSE RUST_SOURCES + "${RUST_ROOT}/livekit-ffi/src/*.rs" + "${RUST_ROOT}/livekit-ffi/Cargo.toml" +) + +add_custom_command( + OUTPUT "${RUST_LIB_DEBUG}" "${RUST_LIB_RELEASE}" COMMAND "${CMAKE_COMMAND}" -DCFG=$ -DRUST_ROOT=${RUST_ROOT} @@ -353,10 +359,15 @@ add_custom_target(build_rust_ffi -DGCC_LIB_DIR=${GCC_LIB_DIR} -P "${RUN_CARGO_SCRIPT}" WORKING_DIRECTORY "${RUST_ROOT}" + DEPENDS ${RUST_SOURCES} COMMENT "Building Rust FFI via cargo" VERBATIM ) +add_custom_target(build_rust_ffi + DEPENDS "${RUST_LIB_DEBUG}" "${RUST_LIB_RELEASE}" +) + # Note: protozero_plugin.o removal is no longer needed since we use dynamic libraries on Unix add_library(livekit SHARED From ec627ba68e6ea484a7fd325c98edc07304c2975c Mon Sep 17 00:00:00 2001 From: Alan George Date: Mon, 7 Sep 2026 21:56:12 -0600 Subject: [PATCH 08/11] Use latest room fix --- client-sdk-rust | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-sdk-rust b/client-sdk-rust index e5fe5dab..9ed4c7a3 160000 --- a/client-sdk-rust +++ b/client-sdk-rust @@ -1 +1 @@ -Subproject commit e5fe5dab84e49320df58115d37a2a1a9478261d7 +Subproject commit 9ed4c7a30828c89881861e71dcb5f93b231a96b6 From 74028acf1e40752049b404e06b19ad5d39e6fe18 Mon Sep 17 00:00:00 2001 From: Alan George Date: Tue, 8 Sep 2026 09:39:19 -0600 Subject: [PATCH 09/11] Tester improvements --- .../manual/memory_lifecycle_tester/README.md | 31 +++- .../manual/memory_lifecycle_tester/main.cpp | 161 +++++++++++++----- 2 files changed, 145 insertions(+), 47 deletions(-) diff --git a/src/tests/manual/memory_lifecycle_tester/README.md b/src/tests/manual/memory_lifecycle_tester/README.md index e41b92ed..3178d90b 100644 --- a/src/tests/manual/memory_lifecycle_tester/README.md +++ b/src/tests/manual/memory_lifecycle_tester/README.md @@ -3,20 +3,29 @@ This standalone application repeatedly exercises the public C++ SDK lifecycle to expose retained lower-level Rust/WebRTC resources. -Every iteration calls `livekit::initialize()`, runs the selected workload, and -then calls `livekit::shutdown()`. Workloads can be combined: +By default, the application calls `livekit::initialize()` once, runs every +selected workload for the configured number of iterations, and calls +`livekit::shutdown()` once. This models a long-running process that repeatedly +joins and leaves rooms without relying on global shutdown to release each +room's resources. Pass `--ffi-cycles` to initialize and shut down the SDK on +every iteration instead. + +Workloads can be combined: - `--sources`: creates an `AudioSource`/`LocalAudioTrack` and a 1280x720 `VideoSource`/`LocalVideoTrack`; - `--connect`: joins and leaves a LiveKit room; +- `--media`: publishes audio and video tracks, captures three frames from each, + and unpublishes them; implies `--connect`; - `--data-track`: publishes and unpublishes a `LocalDataTrack`; implies `--connect`; - `--data-frames`: immediately sends 10 1 KiB payloads; implies `--data-track` and `--connect`. -The video source intentionally receives no captured frame. This exercises -teardown of the Rust keepalive task that runs until the first raw video frame -arrives and previously retained roughly one 720p frame per lifecycle. +The `--sources` video source intentionally receives no captured frame. This +exercises teardown of the Rust keepalive task that runs until the first raw +video frame arrives and previously retained roughly one 720p frame per +lifecycle. The separate `--media` source takes the normal first-capture path. The executable is built with the normal test targets but is not registered with CTest, so it only runs when invoked manually. @@ -35,9 +44,9 @@ Start a local LiveKit server and load the test credentials: source scripts/set-test-tokens.sh ``` -`--connect`, `--data-track`, and `--data-frames` require `LIVEKIT_URL` and -`LIVEKIT_TOKEN_A`. The default is 1,000 iterations. With no workload flags, -the tester runs every workload: +`--connect`, `--media`, `--data-track`, and `--data-frames` require +`LIVEKIT_URL` and `LIVEKIT_TOKEN_A`. The default is 1,000 iterations. With no +workload flags, the tester runs every workload in one SDK lifecycle: ```bash ./build-release/bin/livekit_memory_lifecycle_tester @@ -53,12 +62,18 @@ count. Examples: # Room setup and teardown only. ./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --connect +# Published audio/video capture and teardown. +./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --media + # Incrementally add data-track allocation and data-frame delivery. ./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --data-track ./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --data-frames # Combine explicitly selected workloads. ./build-release/bin/livekit_memory_lifecycle_tester --iterations 100 --sources --data-frames + +# Exercise full FFI initialization and disposal on every iteration. +./build-release/bin/livekit_memory_lifecycle_tester --iterations 100 --ffi-cycles ``` To compare memory behavior before and after a lifecycle fix, use identical diff --git a/src/tests/manual/memory_lifecycle_tester/main.cpp b/src/tests/manual/memory_lifecycle_tester/main.cpp index e2f9a9f8..44d9143e 100644 --- a/src/tests/manual/memory_lifecycle_tester/main.cpp +++ b/src/tests/manual/memory_lifecycle_tester/main.cpp @@ -16,6 +16,7 @@ #include +#include #include #include #include @@ -32,8 +33,10 @@ constexpr int kDefaultIterations = 1'000; constexpr int kAudioSampleRate = 48'000; constexpr int kAudioChannels = 1; constexpr int kAudioQueueSizeMs = 100; +constexpr int kAudioFrameDurationMs = 10; constexpr int kVideoWidth = 1'280; constexpr int kVideoHeight = 720; +constexpr int kMediaFrameCount = 3; constexpr std::size_t kDataPayloadSize = 1'024; constexpr int kDataFrameCount = 10; constexpr char kDataTrackName[] = "lifecycle-data"; @@ -47,8 +50,10 @@ struct Options { int iteration_count{kDefaultIterations}; bool sources{false}; bool connect{false}; + bool media{false}; bool data_track{false}; bool data_frames{false}; + bool ffi_cycles{false}; bool mode_selected{false}; }; @@ -74,7 +79,7 @@ int parseIterationCount(const char* value) { } } -void runSources() { +void runUnusedSources() { auto audio_source = std::make_shared(kAudioSampleRate, kAudioChannels, kAudioQueueSizeMs); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("lifecycle-audio", audio_source); if (!audio_track) { @@ -89,6 +94,51 @@ void runSources() { } } +template +void unpublishTrackIfPublished(const std::shared_ptr& participant, + const std::shared_ptr& track) { + const auto publication = track->publication(); + if (publication) { + participant->unpublishTrack(publication->sid()); + } +} + +void runMediaWork(const std::shared_ptr& participant) { + auto audio_source = std::make_shared(kAudioSampleRate, kAudioChannels, 0); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("lifecycle-audio-active", audio_source); + if (!audio_track) { + throw std::runtime_error("failed to create active local audio track"); + } + + auto video_source = std::make_shared(kVideoWidth, kVideoHeight); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("lifecycle-video-active", video_source); + if (!video_track) { + throw std::runtime_error("failed to create active local video track"); + } + + livekit::TrackPublishOptions audio_options; + audio_options.source = livekit::TrackSource::SOURCE_MICROPHONE; + livekit::TrackPublishOptions video_options; + video_options.source = livekit::TrackSource::SOURCE_CAMERA; + video_options.simulcast = false; + + participant->publishTrack(audio_track, audio_options); + participant->publishTrack(video_track, video_options); + + auto audio_frame = + livekit::AudioFrame::create(kAudioSampleRate, kAudioChannels, kAudioSampleRate * kAudioFrameDurationMs / 1'000); + auto video_frame = livekit::VideoFrame::create(kVideoWidth, kVideoHeight, livekit::VideoBufferType::RGBA); + std::fill(video_frame.data(), video_frame.data() + video_frame.dataSize(), 0x40); + + for (int frame = 0; frame < kMediaFrameCount; ++frame) { + audio_source->captureFrame(audio_frame, 1'000); + video_source->captureFrame(video_frame); + } + + unpublishTrackIfPublished(participant, video_track); + unpublishTrackIfPublished(participant, audio_track); +} + void runRoomIteration(const Configuration& config, const Options& options) { livekit::Room room; if (!room.connect(config.url, config.token, {})) { @@ -96,35 +146,34 @@ void runRoomIteration(const Configuration& config, const Options& options) { } try { - if (!options.data_track) { - if (!room.disconnect()) { - throw std::runtime_error("failed to disconnect from the LiveKit room"); - } - return; - } - auto participant = room.localParticipant().lock(); if (!participant) { throw std::runtime_error("local participant is unavailable"); } - auto result = participant->publishDataTrack(kDataTrackName); - if (!result) { - throw std::runtime_error("failed to publish data track: " + result.error().message); + if (options.media) { + runMediaWork(participant); } - const auto& data_track = result.value(); - - if (options.data_frames) { - const livekit::DataTrackFrame data_frame(std::vector(kDataPayloadSize, 0x5a)); - for (int frame = 0; frame < kDataFrameCount; ++frame) { - auto push_result = data_track->tryPush(data_frame); - if (!push_result) { - throw std::runtime_error("failed to publish data: " + push_result.error().message); + + if (options.data_track) { + auto result = participant->publishDataTrack(kDataTrackName); + if (!result) { + throw std::runtime_error("failed to publish data track: " + result.error().message); + } + const auto& data_track = result.value(); + + if (options.data_frames) { + const livekit::DataTrackFrame data_frame(std::vector(kDataPayloadSize, 0x5a)); + for (int frame = 0; frame < kDataFrameCount; ++frame) { + auto push_result = data_track->tryPush(data_frame); + if (!push_result) { + throw std::runtime_error("failed to publish data: " + push_result.error().message); + } } } - } - data_track->unpublishDataTrack(); + data_track->unpublishDataTrack(); + } } catch (...) { (void)room.disconnect(); throw; @@ -136,12 +185,16 @@ void runRoomIteration(const Configuration& config, const Options& options) { } void printUsage(const char* executable) { - std::cerr << "usage: " << executable << " [--iterations N] [--sources] [--connect] [--data-track] [--data-frames]\n" - << " --sources Create local audio and video sources and tracks.\n" + std::cerr << "usage: " << executable + << " [--iterations N] [--ffi-cycles] [--sources] [--connect] [--media] [--data-track] [--data-frames]\n" + << " --ffi-cycles Initialize and shut down the SDK on every iteration.\n" + << " --sources Create and drop unused local audio/video sources and tracks.\n" << " --connect Connect to and leave a room.\n" + << " --media Publish, capture, and unpublish audio/video tracks (implies --connect).\n" << " --data-track Publish and unpublish a data track (implies --connect).\n" << " --data-frames Send data frames (implies --data-track and --connect).\n" - << " No mode flags runs all modes. A single numeric argument remains supported as the iteration count.\n"; + << " By default the SDK is initialized once and all workloads run on every iteration.\n" + << " A single numeric argument remains supported as the iteration count.\n"; } Options parseOptions(int argc, char* argv[]) { @@ -159,12 +212,17 @@ Options parseOptions(int argc, char* argv[]) { } else if (std::strcmp(value, "--connect") == 0) { options.connect = true; options.mode_selected = true; + } else if (std::strcmp(value, "--media") == 0) { + options.media = true; + options.mode_selected = true; } else if (std::strcmp(value, "--data-track") == 0) { options.data_track = true; options.mode_selected = true; } else if (std::strcmp(value, "--data-frames") == 0) { options.data_frames = true; options.mode_selected = true; + } else if (std::strcmp(value, "--ffi-cycles") == 0) { + options.ffi_cycles = true; } else if (std::strcmp(value, "--help") == 0 || std::strcmp(value, "-h") == 0) { printUsage(argv[0]); std::exit(0); @@ -178,9 +236,13 @@ Options parseOptions(int argc, char* argv[]) { if (!options.mode_selected) { options.sources = true; options.connect = true; + options.media = true; options.data_track = true; options.data_frames = true; - } else if (options.data_frames) { + } else if (options.media) { + options.connect = true; + } + if (options.data_frames) { options.data_track = true; options.connect = true; } else if (options.data_track) { @@ -189,40 +251,61 @@ Options parseOptions(int argc, char* argv[]) { return options; } +void runIteration(const Configuration& config, const Options& options) { + if (options.sources) { + runUnusedSources(); + } + if (options.connect) { + runRoomIteration(config, options); + } +} + +void initializeSdk(int iteration) { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("initialize failed at iteration " + std::to_string(iteration)); + } +} + } // namespace int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); const int progress_interval = options.iteration_count < 10 ? 1 : options.iteration_count / 10; - std::cout << "Running " << options.iteration_count << " LiveKit initialize/shutdown cycles\n"; + const Configuration config = + options.connect ? Configuration{requiredEnvironment("LIVEKIT_URL"), requiredEnvironment("LIVEKIT_TOKEN_A")} + : Configuration{}; + + if (options.ffi_cycles) { + std::cout << "Running " << options.iteration_count << " LiveKit initialize/work/shutdown cycles\n"; + } else { + std::cout << "Running " << options.iteration_count << " workload cycles in one LiveKit SDK lifecycle\n"; + initializeSdk(1); + } for (int iteration = 1; iteration <= options.iteration_count; ++iteration) { - if (!livekit::initialize(livekit::LogLevel::Warn)) { - throw std::runtime_error("initialize failed at iteration " + std::to_string(iteration)); + if (options.ffi_cycles) { + initializeSdk(iteration); } try { - if (options.sources) { - runSources(); - } - if (options.connect) { - const Configuration config{ - requiredEnvironment("LIVEKIT_URL"), - requiredEnvironment("LIVEKIT_TOKEN_A"), - }; - runRoomIteration(config, options); - } + runIteration(config, options); } catch (...) { livekit::shutdown(); throw; } - livekit::shutdown(); + if (options.ffi_cycles) { + livekit::shutdown(); + } if (iteration % progress_interval == 0 || iteration == options.iteration_count) { std::cout << "Completed " << iteration << "/" << options.iteration_count << " cycles\n"; } } + + if (!options.ffi_cycles) { + livekit::shutdown(); + } } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; printUsage(argv[0]); From 6e90f47a14255941a5bef011176bbd01e39e233d Mon Sep 17 00:00:00 2001 From: Alan George Date: Tue, 8 Sep 2026 10:54:44 -0600 Subject: [PATCH 10/11] Use rust main to emulate release --- client-sdk-rust | 2 +- docs/building.md | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/client-sdk-rust b/client-sdk-rust index 9ed4c7a3..f76b7ac1 160000 --- a/client-sdk-rust +++ b/client-sdk-rust @@ -1 +1 @@ -Subproject commit 9ed4c7a30828c89881861e71dcb5f93b231a96b6 +Subproject commit f76b7ac1b97e419a42ea5907f01394b8831e5d4b diff --git a/docs/building.md b/docs/building.md index 81f6e012..4620eaa3 100644 --- a/docs/building.md +++ b/docs/building.md @@ -64,6 +64,25 @@ sudo apt update && sudo apt install -y \ curl https://sh.rustup.rs -sSf | sh ``` +The bundled `libwebrtc` artifact uses a hermetic libc++ built from LLVM trunk. +Ubuntu 24.04's Clang 18 is too old for it. The Linux CI uses LLVM 21.1.8; install +that toolchain and select it for the SDK build without replacing Ubuntu's +system compiler: + +```bash +export LLVM_VERSION=21.1.8 +export LLVM_ROOT="$HOME/.local/llvm-$LLVM_VERSION" +client-sdk-rust/.github/scripts/install-clang.sh +export CC="$LLVM_ROOT/bin/clang" +export CXX="$LLVM_ROOT/bin/clang++" + +# Confirm the selected compiler before configuring the SDK. +"$CXX" --version +``` + +Set `CC` and `CXX` before the first build, or remove the affected build +directory before reconfiguring so CMake does not retain the previous compiler. + If you plan to build the [example collection](https://github.com/livekit-examples/cpp-example-collection) (SDL-based renderer + camera/mic capture), also install: From f2fd42c2a1a6af101d2b8c3656ff9f435b59fe59 Mon Sep 17 00:00:00 2001 From: Alan George Date: Tue, 8 Sep 2026 12:08:39 -0600 Subject: [PATCH 11/11] Run memory checks in CI --- .github/workflows/nightly.yml | 3 + .github/workflows/tests.yml | 81 ++++- docs/testing.md | 26 ++ scripts/track_process_memory.py | 88 ++++++ src/tests/CMakeLists.txt | 3 + .../manual/memory_lifecycle_tester/README.md | 51 +++- .../manual/memory_lifecycle_tester/main.cpp | 278 +++++++++++++++++- 7 files changed, 511 insertions(+), 19 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index daf4bb45..9f0f34b3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -26,4 +26,7 @@ jobs: stress_timeout_minutes: 120 artifact_retention_days: 14 run_coverage: false + memory_ffi_iterations: 50 + memory_connected_iterations: 50 + memory_timeout_minutes: 20 secrets: inherit diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 398aa0f0..d3a8155d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,6 +54,21 @@ on: required: false type: boolean default: true + memory_ffi_iterations: + description: Memory lifecycle tester iterations for initialize/shutdown + unused sources. + required: false + type: number + default: 30 + memory_connected_iterations: + description: Memory lifecycle tester iterations for the connected software smoke. + required: false + type: number + default: 20 + memory_timeout_minutes: + description: Memory lifecycle smoke step timeout in minutes. + required: false + type: number + default: 10 workflow_dispatch: inputs: build_type: @@ -109,6 +124,21 @@ on: required: false type: boolean default: true + memory_ffi_iterations: + description: Memory lifecycle tester iterations for initialize/shutdown + unused sources. + required: false + type: number + default: 30 + memory_connected_iterations: + description: Memory lifecycle tester iterations for the connected software smoke. + required: false + type: number + default: 20 + memory_timeout_minutes: + description: Memory lifecycle smoke step timeout in minutes. + required: false + type: number + default: 10 permissions: contents: read @@ -357,7 +387,7 @@ jobs: --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml" - name: Start livekit-server - if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: matrix.e2e-testing id: livekit_server uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1 with: @@ -368,7 +398,7 @@ jobs: # Needed by token helper script - name: Install livekit-cli - if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: matrix.e2e-testing shell: bash env: # Windows installs lk via `gh api` / `gh release download`, which need this env var @@ -425,6 +455,51 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml + - name: Set memory lifecycle RSS cap + if: matrix.e2e-testing + shell: bash + run: | + set -euo pipefail + build="${{ inputs.build_type }}" + case "${RUNNER_OS}-${build}" in + Linux-release) cap=262144 ;; + Linux-debug) cap=524288 ;; + macOS-release) cap=262144 ;; + macOS-debug) cap=524288 ;; + Windows-release) cap=393216 ;; + Windows-debug) cap=786432 ;; + *) + echo "::error::No memory RSS cap for ${RUNNER_OS} ${build}" + exit 1 + ;; + esac + # Calibrated from a local macOS release run (~85 MiB connected, ~37 MiB + # ffi-cycles). Caps are ~3x that for release and ~2x again for debug, + # with extra Windows working-set headroom. Adjust from CI logs if needed. + echo "LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB=${cap}" >> "${GITHUB_ENV}" + echo "Memory RSS cap: ${cap} KiB (${RUNNER_OS} ${build})" + + - name: Run memory lifecycle smoke + if: matrix.e2e-testing + timeout-minutes: ${{ inputs.memory_timeout_minutes }} + shell: bash + run: | + set -euo pipefail + source scripts/set-test-tokens.sh + tester="${{ env.BUILD_DIR }}/bin/livekit_memory_lifecycle_tester" + bash scripts/run-with-backtrace.sh \ + "${tester}" \ + --iterations ${{ inputs.memory_ffi_iterations }} \ + --ffi-cycles \ + --sources + bash scripts/run-with-backtrace.sh \ + "${tester}" \ + --iterations ${{ inputs.memory_connected_iterations }} \ + --sources \ + --media \ + --data-frames \ + --receive + - name: Run stress tests if: matrix.e2e-testing && inputs.run_stress_tests timeout-minutes: ${{ inputs.stress_timeout_minutes }} @@ -441,7 +516,7 @@ jobs: --gtest_output=xml:${{ env.BUILD_DIR }}/stress-test-results.xml - name: Dump livekit-server log on failure - if: failure() && matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: failure() && matrix.e2e-testing shell: bash run: tail -n 500 "${{ steps.livekit_server.outputs.log-path }}" || true diff --git a/docs/testing.md b/docs/testing.md index 33919c0e..1a11d498 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -44,6 +44,7 @@ __Note:__ The tests require tokens and a running LiveKit server. See the section | `livekit_unit_tests` | Pure unit tests (no server required) | | `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality | | `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) | +| `livekit_memory_lifecycle_tester` | Manual/CI smoke for SDK connect/publish/subscribe teardown and RSS growth. Not registered with CTest. | ## Running a local LiveKit server for tests @@ -72,6 +73,9 @@ export LIVEKIT_TOKEN_B="" # Optional (for stress tests) export RPC_STRESS_DURATION_SECONDS=3600 # Test duration (default: 1 hour) export RPC_STRESS_CALLER_THREADS=4 # Concurrent caller threads (default: 4) + +# Optional (memory lifecycle tester / CI smoke) +export LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB=1048576 # fail if final RSS exceeds this many KiB ``` ### Generating tokens for the test suites @@ -105,6 +109,28 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **Audio frame**: frame creation, manipulation, edge cases. - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. +- **Memory lifecycle**: repeated source, room, media, data-track, and software + subscribe teardown. CI runs this as a short RSS-capped smoke test; longer + hardware runs stay manual. + +## Memory lifecycle smoke + +`livekit_memory_lifecycle_tester` is built with the test targets but is not +part of `ctest`. CI runs two short software-only invocations after the +integration suite: + +```bash +source scripts/set-test-tokens.sh +./build-release/bin/livekit_memory_lifecycle_tester --iterations 30 --ffi-cycles --sources +./build-release/bin/livekit_memory_lifecycle_tester --iterations 20 --sources --media --data-frames --receive +``` + +`--receive` needs `LIVEKIT_TOKEN_B` and a running LiveKit server. The tester +fails if `LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB` is set and final process RSS exceeds +that cap; leave the variable unset for local runs. CUDA and platform-audio +paths are not part of this smoke test; see +`src/tests/manual/cuda_video_lifecycle_tester` and +`src/tests/manual/memory_lifecycle_tester/README.md`. ## Memory checks (valgrind) diff --git a/scripts/track_process_memory.py b/scripts/track_process_memory.py index 0c4b760d..b97aa440 100755 --- a/scripts/track_process_memory.py +++ b/scripts/track_process_memory.py @@ -19,10 +19,13 @@ from __future__ import annotations import argparse +import os import subprocess import sys import time +MEMORY_LIMIT_ENV = "LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB" + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -34,6 +37,15 @@ def parse_args() -> argparse.Namespace: default=0.1, help="seconds between RSS samples (default: 0.1)", ) + parser.add_argument( + "--max-final-rss-kib", + type=int, + default=None, + help=( + "fail if the last observed RSS exceeds this many KiB; " + f"overrides {MEMORY_LIMIT_ENV} when set" + ), + ) parser.add_argument( "command", nargs=argparse.REMAINDER, @@ -42,12 +54,68 @@ def parse_args() -> argparse.Namespace: args = parser.parse_args() if args.interval <= 0: parser.error("--interval must be greater than zero") + if args.max_final_rss_kib is not None and args.max_final_rss_kib <= 0: + parser.error("--max-final-rss-kib must be greater than zero") if not args.command: parser.error("a command is required") return args +def resolve_max_final_rss_kib(explicit: int | None) -> int | None: + if explicit is not None: + return explicit + value = os.environ.get(MEMORY_LIMIT_ENV) + if value is None or value == "": + return None + try: + parsed = int(value) + except ValueError as error: + raise ValueError(f"{MEMORY_LIMIT_ENV} must be an integer") from error + if parsed <= 0: + raise ValueError(f"{MEMORY_LIMIT_ENV} must be greater than zero") + return parsed + + +def windows_rss_kib(pid: int) -> int | None: + import ctypes + from ctypes import wintypes + + class ProcessMemoryCounters(ctypes.Structure): + _fields_ = [ + ("cb", wintypes.DWORD), + ("PageFaultCount", wintypes.DWORD), + ("PeakWorkingSetSize", ctypes.c_size_t), + ("WorkingSetSize", ctypes.c_size_t), + ("QuotaPeakPagedPoolUsage", ctypes.c_size_t), + ("QuotaPagedPoolUsage", ctypes.c_size_t), + ("QuotaPeakNonPagedPoolUsage", ctypes.c_size_t), + ("QuotaNonPagedPoolUsage", ctypes.c_size_t), + ("PagefileUsage", ctypes.c_size_t), + ("PeakPagefileUsage", ctypes.c_size_t), + ] + + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + psapi = ctypes.WinDLL("psapi", use_last_error=True) + process_query_information = 0x0400 + process_vm_read = 0x0010 + handle = kernel32.OpenProcess(process_query_information | process_vm_read, False, pid) + if not handle: + return None + try: + counters = ProcessMemoryCounters() + counters.cb = ctypes.sizeof(counters) + if not psapi.GetProcessMemoryInfo(handle, ctypes.byref(counters), counters.cb): + return None + rss_kib = int(counters.WorkingSetSize) // 1024 + return rss_kib if rss_kib > 0 else None + finally: + kernel32.CloseHandle(handle) + + def process_rss_kib(pid: int) -> int | None: + if sys.platform == "win32": + return windows_rss_kib(pid) + result = subprocess.run( ["ps", "-o", "rss=", "-p", str(pid)], check=False, @@ -80,6 +148,12 @@ def main() -> int: print("error: a command is required after --", file=sys.stderr) return 2 + try: + max_final_rss_kib = resolve_max_final_rss_kib(args.max_final_rss_kib) + except ValueError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + try: process = subprocess.Popen(command) except OSError as error: @@ -107,6 +181,12 @@ def main() -> int: print(f"elapsed: {elapsed_s:.2f} s") if initial_rss_kib is None: print("RSS: no samples collected; the command exited before sampling began") + if max_final_rss_kib is not None and exit_code == 0: + print( + f"error: {MEMORY_LIMIT_ENV} is set but RSS could not be sampled", + file=sys.stderr, + ) + return 1 else: assert final_rss_kib is not None assert peak_rss_kib is not None @@ -114,6 +194,14 @@ def main() -> int: print(f"RSS final observed: {format_rss(final_rss_kib)}") print(f"RSS peak: {format_rss(peak_rss_kib)}") print(f"RSS change: {format_rss(final_rss_kib - initial_rss_kib)}") + if max_final_rss_kib is not None and final_rss_kib > max_final_rss_kib: + print( + "error: final RSS " + f"{format_rss(final_rss_kib)} exceeds limit {format_rss(max_final_rss_kib)}", + file=sys.stderr, + ) + if exit_code == 0: + return 1 return exit_code diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 5e7695e8..b15b3c8f 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -358,6 +358,9 @@ add_executable(livekit_memory_lifecycle_tester ) target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) +if(WIN32) + target_link_libraries(livekit_memory_lifecycle_tester PRIVATE psapi) +endif() target_include_directories(livekit_memory_lifecycle_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include) target_compile_definitions(livekit_memory_lifecycle_tester PRIVATE $<$:_USE_MATH_DEFINES> diff --git a/src/tests/manual/memory_lifecycle_tester/README.md b/src/tests/manual/memory_lifecycle_tester/README.md index 3178d90b..17767d52 100644 --- a/src/tests/manual/memory_lifecycle_tester/README.md +++ b/src/tests/manual/memory_lifecycle_tester/README.md @@ -20,15 +20,26 @@ Workloads can be combined: - `--data-track`: publishes and unpublishes a `LocalDataTrack`; implies `--connect`; - `--data-frames`: immediately sends 10 1 KiB payloads; implies - `--data-track` and `--connect`. + `--data-track` and `--connect`; +- `--receive`: connects a second participant, publishes modest synthetic + audio/video (640x360, default software codec), and waits until the subscriber + receives a few frames via `Room` frame callbacks. Implies `--connect` and + requires `LIVEKIT_TOKEN_B`. This is opt-in and is **not** part of the default + all-workloads run. The `--sources` video source intentionally receives no captured frame. This exercises teardown of the Rust keepalive task that runs until the first raw video frame arrives and previously retained roughly one 720p frame per lifecycle. The separate `--media` source takes the normal first-capture path. +`--receive` uses a smaller I420 source so CI can cover subscribe/decode and +`SubscriptionThreadDispatcher` teardown without NVIDIA hardware. + +This tester stays on the software media path. It does not set +`LIVEKIT_PREFERRED_HW_ENCODER` and does not exercise `PlatformAudio`. CUDA +encode/decode shakeouts belong in `src/tests/manual/cuda_video_lifecycle_tester`. The executable is built with the normal test targets but is not registered with -CTest, so it only runs when invoked manually. +CTest. CI invokes it as a short smoke test; longer runs are manual. ## Build @@ -45,8 +56,9 @@ source scripts/set-test-tokens.sh ``` `--connect`, `--media`, `--data-track`, and `--data-frames` require -`LIVEKIT_URL` and `LIVEKIT_TOKEN_A`. The default is 1,000 iterations. With no -workload flags, the tester runs every workload in one SDK lifecycle: +`LIVEKIT_URL` and `LIVEKIT_TOKEN_A`. `--receive` also requires +`LIVEKIT_TOKEN_B`. The default is 1,000 iterations. With no workload flags, the +tester runs every workload except `--receive` in one SDK lifecycle: ```bash ./build-release/bin/livekit_memory_lifecycle_tester @@ -69,6 +81,9 @@ count. Examples: ./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --data-track ./build-release/bin/livekit_memory_lifecycle_tester --iterations 1000 --data-frames +# Software subscribe/decode path with a second participant. +./build-release/bin/livekit_memory_lifecycle_tester --iterations 100 --receive + # Combine explicitly selected workloads. ./build-release/bin/livekit_memory_lifecycle_tester --iterations 100 --sources --data-frames @@ -76,6 +91,15 @@ count. Examples: ./build-release/bin/livekit_memory_lifecycle_tester --iterations 100 --ffi-cycles ``` +The tester prints RSS at progress intervals and `RSS final` at the end. Set +`LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB` to fail if that final sample exceeds the +limit. Leave it unset for local and hardware-lab runs; CI sets it per OS. + +```bash +LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB=1048576 \ + ./build-release/bin/livekit_memory_lifecycle_tester --iterations 20 --sources --media --data-frames --receive +``` + To compare memory behavior before and after a lifecycle fix, use identical iteration counts and build configurations: @@ -84,5 +108,22 @@ python3 scripts/track_process_memory.py --interval 0.01 -- \ ./build-release/bin/livekit_memory_lifecycle_tester 100 ``` +The helper accepts `--max-final-rss-kib N` and honors `LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB` +when the flag is omitted. + Allocator caching means final RSS need not return to the initial value. The -useful regression signal is sustained or iteration-proportional growth. +useful regression signal is sustained or iteration-proportional growth. The +optional RSS cap is a smoke ceiling, not a tight leak bound. + +## CI smoke + +PR CI runs two short invocations after the integration tests (release build, +software codecs only): + +```bash +./build-release/bin/livekit_memory_lifecycle_tester --iterations 30 --ffi-cycles --sources +./build-release/bin/livekit_memory_lifecycle_tester --iterations 20 --sources --media --data-frames --receive +``` + +Nightly uses the same commands with 50 iterations each on a debug build. Neither +job is a hardware shakeout. diff --git a/src/tests/manual/memory_lifecycle_tester/main.cpp b/src/tests/manual/memory_lifecycle_tester/main.cpp index 44d9143e..1d093a74 100644 --- a/src/tests/manual/memory_lifecycle_tester/main.cpp +++ b/src/tests/manual/memory_lifecycle_tester/main.cpp @@ -17,16 +17,39 @@ #include #include +#include +#include #include #include #include #include +#include #include #include +#include +#include +#include #include #include #include +#if defined(_WIN32) +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +// clang-format off +#include +#include +// clang-format on +#elif defined(__APPLE__) +#include +#else +#include +#endif + namespace { constexpr int kDefaultIterations = 1'000; @@ -40,10 +63,18 @@ constexpr int kMediaFrameCount = 3; constexpr std::size_t kDataPayloadSize = 1'024; constexpr int kDataFrameCount = 10; constexpr char kDataTrackName[] = "lifecycle-data"; +constexpr int kReceiveVideoWidth = 640; +constexpr int kReceiveVideoHeight = 360; +constexpr int kReceiveRequiredFrames = 3; +constexpr int kReceiveCaptureAttempts = 250; +constexpr char kReceiveAudioTrackName[] = "lifecycle-receive-audio"; +constexpr char kReceiveVideoTrackName[] = "lifecycle-receive-video"; +constexpr char kMemoryLimitEnv[] = "LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB"; struct Configuration { std::string url; std::string token; + std::string receiver_token; }; struct Options { @@ -53,6 +84,7 @@ struct Options { bool media{false}; bool data_track{false}; bool data_frames{false}; + bool receive{false}; bool ffi_cycles{false}; bool mode_selected{false}; }; @@ -79,6 +111,99 @@ int parseIterationCount(const char* value) { } } +std::optional parseMemoryLimitKib() { + const char* value = std::getenv(kMemoryLimitEnv); + if (value == nullptr || *value == '\0') { + return std::nullopt; + } + try { + const unsigned long long parsed = std::stoull(value); + if (parsed == 0) { + throw std::runtime_error(std::string(kMemoryLimitEnv) + " must be greater than zero"); + } + return static_cast(parsed); + } catch (const std::invalid_argument&) { + throw std::runtime_error(std::string(kMemoryLimitEnv) + " must be an integer"); + } catch (const std::out_of_range&) { + throw std::runtime_error(std::string(kMemoryLimitEnv) + " is out of range"); + } +} + +std::optional currentRssKib() { +#if defined(_WIN32) + PROCESS_MEMORY_COUNTERS counters{}; + counters.cb = sizeof(counters); + if (GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters)) == 0) { + return std::nullopt; + } + return static_cast(counters.WorkingSetSize) / 1024; +#elif defined(__APPLE__) + mach_task_basic_info_data_t info{}; + mach_msg_type_number_t count = MACH_TASK_BASIC_INFO_COUNT; + const kern_return_t result = + task_info(mach_task_self(), MACH_TASK_BASIC_INFO, reinterpret_cast(&info), &count); + if (result != KERN_SUCCESS) { + return std::nullopt; + } + return static_cast(info.resident_size) / 1024; +#else + std::ifstream status("/proc/self/status"); + if (!status) { + return std::nullopt; + } + std::string line; + while (std::getline(status, line)) { + if (line.compare(0, 6, "VmRSS:") != 0) { + continue; + } + std::istringstream fields(line.substr(6)); + std::uint64_t kib = 0; + fields >> kib; + if (!fields) { + return std::nullopt; + } + return kib; + } + return std::nullopt; +#endif +} + +std::string formatRssKib(std::uint64_t rss_kib) { + std::ostringstream stream; + stream << rss_kib << " KiB (" << std::fixed << std::setprecision(2) << static_cast(rss_kib) / 1024.0 + << " MiB)"; + return stream.str(); +} + +std::string formatRssSample() { + const auto rss_kib = currentRssKib(); + if (!rss_kib) { + return "unavailable"; + } + return formatRssKib(*rss_kib); +} + +void checkFinalRss() { + const auto rss_kib = currentRssKib(); + if (rss_kib) { + std::cout << "RSS final: " << formatRssKib(*rss_kib) << '\n'; + } else { + std::cout << "RSS final: unavailable\n"; + } + + const auto limit_kib = parseMemoryLimitKib(); + if (!limit_kib) { + return; + } + if (!rss_kib) { + throw std::runtime_error(std::string(kMemoryLimitEnv) + " is set but RSS could not be sampled"); + } + if (*rss_kib > *limit_kib) { + throw std::runtime_error("final RSS " + formatRssKib(*rss_kib) + " exceeds " + kMemoryLimitEnv + " " + + formatRssKib(*limit_kib)); + } +} + void runUnusedSources() { auto audio_source = std::make_shared(kAudioSampleRate, kAudioChannels, kAudioQueueSizeMs); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("lifecycle-audio", audio_source); @@ -103,6 +228,14 @@ void unpublishTrackIfPublished(const std::shared_ptr& } } +std::shared_ptr requireLocalParticipant(livekit::Room& room) { + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant is unavailable"); + } + return participant; +} + void runMediaWork(const std::shared_ptr& participant) { auto audio_source = std::make_shared(kAudioSampleRate, kAudioChannels, 0); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("lifecycle-audio-active", audio_source); @@ -139,6 +272,97 @@ void runMediaWork(const std::shared_ptr& participant) unpublishTrackIfPublished(participant, audio_track); } +void runReceiveWork(livekit::Room& sender_room, const Configuration& config) { + livekit::Room receiver_room; + std::mutex mutex; + std::condition_variable frames_received; + int audio_frames = 0; + int video_frames = 0; + + const auto received_enough = [&audio_frames, &video_frames] { + return audio_frames >= kReceiveRequiredFrames && video_frames >= kReceiveRequiredFrames; + }; + + try { + if (!receiver_room.connect(config.url, config.receiver_token, {})) { + throw std::runtime_error("receiver failed to connect to the LiveKit room"); + } + + const auto sender = requireLocalParticipant(sender_room); + const std::string sender_identity = sender->identity(); + if (sender_identity.empty()) { + throw std::runtime_error("sender identity is empty"); + } + + receiver_room.setOnAudioFrameCallback(sender_identity, kReceiveAudioTrackName, + [&mutex, &frames_received, &audio_frames](const livekit::AudioFrame&) { + std::lock_guard lock(mutex); + ++audio_frames; + frames_received.notify_all(); + }); + receiver_room.setOnVideoFrameCallback( + sender_identity, kReceiveVideoTrackName, + [&mutex, &frames_received, &video_frames](const livekit::VideoFrame&, std::int64_t) { + std::lock_guard lock(mutex); + ++video_frames; + frames_received.notify_all(); + }); + + auto audio_source = std::make_shared(kAudioSampleRate, kAudioChannels, 0); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack(kReceiveAudioTrackName, audio_source); + if (!audio_track) { + throw std::runtime_error("failed to create receive audio track"); + } + auto video_source = std::make_shared(kReceiveVideoWidth, kReceiveVideoHeight); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack(kReceiveVideoTrackName, video_source); + if (!video_track) { + throw std::runtime_error("failed to create receive video track"); + } + + livekit::TrackPublishOptions audio_options; + audio_options.source = livekit::TrackSource::SOURCE_MICROPHONE; + livekit::TrackPublishOptions video_options; + video_options.source = livekit::TrackSource::SOURCE_CAMERA; + video_options.simulcast = false; + sender->publishTrack(audio_track, audio_options); + sender->publishTrack(video_track, video_options); + + auto audio_frame = + livekit::AudioFrame::create(kAudioSampleRate, kAudioChannels, kAudioSampleRate * kAudioFrameDurationMs / 1'000); + auto video_frame = + livekit::VideoFrame::create(kReceiveVideoWidth, kReceiveVideoHeight, livekit::VideoBufferType::I420); + std::fill(video_frame.data(), video_frame.data() + video_frame.dataSize(), 0x7f); + + for (int attempt = 0; attempt < kReceiveCaptureAttempts; ++attempt) { + audio_source->captureFrame(audio_frame, 20); + video_source->captureFrame(video_frame); + std::unique_lock lock(mutex); + if (frames_received.wait_for(lock, std::chrono::milliseconds(20), received_enough)) { + break; + } + } + + { + std::lock_guard lock(mutex); + if (!received_enough()) { + throw std::runtime_error("timed out waiting for the receiver to get audio and video frames"); + } + } + + unpublishTrackIfPublished(sender, video_track); + unpublishTrackIfPublished(sender, audio_track); + receiver_room.clearOnVideoFrameCallback(sender_identity, kReceiveVideoTrackName); + receiver_room.clearOnAudioFrameCallback(sender_identity, kReceiveAudioTrackName); + } catch (...) { + (void)receiver_room.disconnect(); + throw; + } + + if (!receiver_room.disconnect()) { + throw std::runtime_error("failed to disconnect the receiver from the LiveKit room"); + } +} + void runRoomIteration(const Configuration& config, const Options& options) { livekit::Room room; if (!room.connect(config.url, config.token, {})) { @@ -146,10 +370,7 @@ void runRoomIteration(const Configuration& config, const Options& options) { } try { - auto participant = room.localParticipant().lock(); - if (!participant) { - throw std::runtime_error("local participant is unavailable"); - } + auto participant = requireLocalParticipant(room); if (options.media) { runMediaWork(participant); @@ -174,6 +395,10 @@ void runRoomIteration(const Configuration& config, const Options& options) { data_track->unpublishDataTrack(); } + + if (options.receive) { + runReceiveWork(room, config); + } } catch (...) { (void)room.disconnect(); throw; @@ -186,15 +411,20 @@ void runRoomIteration(const Configuration& config, const Options& options) { void printUsage(const char* executable) { std::cerr << "usage: " << executable - << " [--iterations N] [--ffi-cycles] [--sources] [--connect] [--media] [--data-track] [--data-frames]\n" + << " [--iterations N] [--ffi-cycles] [--sources] [--connect] [--media] [--data-track] " + "[--data-frames] [--receive]\n" << " --ffi-cycles Initialize and shut down the SDK on every iteration.\n" << " --sources Create and drop unused local audio/video sources and tracks.\n" << " --connect Connect to and leave a room.\n" << " --media Publish, capture, and unpublish audio/video tracks (implies --connect).\n" << " --data-track Publish and unpublish a data track (implies --connect).\n" << " --data-frames Send data frames (implies --data-track and --connect).\n" - << " By default the SDK is initialized once and all workloads run on every iteration.\n" - << " A single numeric argument remains supported as the iteration count.\n"; + << " --receive Subscribe to synthetic audio/video from a second participant " + "(implies --connect).\n" + << " By default the SDK is initialized once and all workloads except --receive run on every " + "iteration.\n" + << " A single numeric argument remains supported as the iteration count.\n" + << " Set " << kMemoryLimitEnv << " to fail if final RSS exceeds that many KiB.\n"; } Options parseOptions(int argc, char* argv[]) { @@ -221,6 +451,9 @@ Options parseOptions(int argc, char* argv[]) { } else if (std::strcmp(value, "--data-frames") == 0) { options.data_frames = true; options.mode_selected = true; + } else if (std::strcmp(value, "--receive") == 0) { + options.receive = true; + options.mode_selected = true; } else if (std::strcmp(value, "--ffi-cycles") == 0) { options.ffi_cycles = true; } else if (std::strcmp(value, "--help") == 0 || std::strcmp(value, "-h") == 0) { @@ -248,6 +481,9 @@ Options parseOptions(int argc, char* argv[]) { } else if (options.data_track) { options.connect = true; } + if (options.receive) { + options.connect = true; + } return options; } @@ -266,15 +502,26 @@ void initializeSdk(int iteration) { } } +Configuration loadConfiguration(const Options& options) { + Configuration config; + if (!options.connect) { + return config; + } + config.url = requiredEnvironment("LIVEKIT_URL"); + config.token = requiredEnvironment("LIVEKIT_TOKEN_A"); + if (options.receive) { + config.receiver_token = requiredEnvironment("LIVEKIT_TOKEN_B"); + } + return config; +} + } // namespace int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); const int progress_interval = options.iteration_count < 10 ? 1 : options.iteration_count / 10; - const Configuration config = - options.connect ? Configuration{requiredEnvironment("LIVEKIT_URL"), requiredEnvironment("LIVEKIT_TOKEN_A")} - : Configuration{}; + const Configuration config = loadConfiguration(options); if (options.ffi_cycles) { std::cout << "Running " << options.iteration_count << " LiveKit initialize/work/shutdown cycles\n"; @@ -299,12 +546,21 @@ int main(int argc, char* argv[]) { } if (iteration % progress_interval == 0 || iteration == options.iteration_count) { - std::cout << "Completed " << iteration << "/" << options.iteration_count << " cycles\n"; + std::cout << "Completed " << iteration << "/" << options.iteration_count << " cycles (RSS " << formatRssSample() + << ")\n"; } } if (!options.ffi_cycles) { + try { + checkFinalRss(); + } catch (...) { + livekit::shutdown(); + throw; + } livekit::shutdown(); + } else { + checkFinalRss(); } } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n';