Memory leak regression testing

.github/workflows/nightly.yml

@@ -26,4 +26,7 @@ jobs:
       stress_timeout_minutes: 120
       artifact_retention_days: 14
       run_coverage: false
+      memory_ffi_iterations: 50
+      memory_connected_iterations: 50
+      memory_timeout_minutes: 20
     secrets: inherit

.github/workflows/tests.yml

@@ -54,6 +54,21 @@ on:
         required: false
         type: boolean
         default: true
+      memory_ffi_iterations:
+        description: Memory lifecycle tester iterations for initialize/shutdown + unused sources.
+        required: false
+        type: number
+        default: 30
+      memory_connected_iterations:
+        description: Memory lifecycle tester iterations for the connected software smoke.
+        required: false
+        type: number
+        default: 20
+      memory_timeout_minutes:
+        description: Memory lifecycle smoke step timeout in minutes.
+        required: false
+        type: number
+        default: 10
   workflow_dispatch:
     inputs:
       build_type:
@@ -109,6 +124,21 @@ on:
         required: false
         type: boolean
         default: true
+      memory_ffi_iterations:
+        description: Memory lifecycle tester iterations for initialize/shutdown + unused sources.
+        required: false
+        type: number
+        default: 30
+      memory_connected_iterations:
+        description: Memory lifecycle tester iterations for the connected software smoke.
+        required: false
+        type: number
+        default: 20
+      memory_timeout_minutes:
+        description: Memory lifecycle smoke step timeout in minutes.
+        required: false
+        type: number
+        default: 10
 
 permissions:
   contents: read
@@ -357,7 +387,7 @@ jobs:
             --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml"
 
       - name: Start livekit-server
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: matrix.e2e-testing
         id: livekit_server
         uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1
         with:
@@ -368,7 +398,7 @@ jobs:
 
       # Needed by token helper script
       - name: Install livekit-cli
-        if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: matrix.e2e-testing
         shell: bash
         env:
           # Windows installs lk via `gh api` / `gh release download`, which need this env var
@@ -425,6 +455,51 @@ jobs:
             --gtest_recreate_environments_when_repeating=1 \
             --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml
 
+      - name: Set memory lifecycle RSS cap
+        if: matrix.e2e-testing
+        shell: bash
+        run: |
+          set -euo pipefail
+          build="${{ inputs.build_type }}"
+          case "${RUNNER_OS}-${build}" in
+            Linux-release) cap=262144 ;;
+            Linux-debug) cap=524288 ;;
+            macOS-release) cap=262144 ;;
+            macOS-debug) cap=524288 ;;
+            Windows-release) cap=393216 ;;
+            Windows-debug) cap=786432 ;;
+            *)
+              echo "::error::No memory RSS cap for ${RUNNER_OS} ${build}"
+              exit 1
+              ;;
+          esac
+          # Calibrated from a local macOS release run (~85 MiB connected, ~37 MiB
+          # ffi-cycles). Caps are ~3x that for release and ~2x again for debug,
+          # with extra Windows working-set headroom. Adjust from CI logs if needed.
+          echo "LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB=${cap}" >> "${GITHUB_ENV}"
+          echo "Memory RSS cap: ${cap} KiB (${RUNNER_OS} ${build})"
+
+      - name: Run memory lifecycle smoke
+        if: matrix.e2e-testing
+        timeout-minutes: ${{ inputs.memory_timeout_minutes }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          source scripts/set-test-tokens.sh
+          tester="${{ env.BUILD_DIR }}/bin/livekit_memory_lifecycle_tester"
+          bash scripts/run-with-backtrace.sh \
+            "${tester}" \
+            --iterations ${{ inputs.memory_ffi_iterations }} \
+            --ffi-cycles \
+            --sources
+          bash scripts/run-with-backtrace.sh \
+            "${tester}" \
+            --iterations ${{ inputs.memory_connected_iterations }} \
+            --sources \
+            --media \
+            --data-frames \
+            --receive
+
       - name: Run stress tests
         if: matrix.e2e-testing && inputs.run_stress_tests
         timeout-minutes: ${{ inputs.stress_timeout_minutes }}
@@ -441,7 +516,7 @@ jobs:
             --gtest_output=xml:${{ env.BUILD_DIR }}/stress-test-results.xml
 
       - name: Dump livekit-server log on failure
-        if: failure() && matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests)
+        if: failure() && matrix.e2e-testing
         shell: bash
         run: tail -n 500 "${{ steps.livekit_server.outputs.log-path }}" || true
 

AGENTS.md

@@ -85,7 +85,8 @@ Be sure to update the directory layout in this file if the directory layout chan
 |------|-------------|
 | `include/livekit/` | Public API headers (what SDK consumers include) |
 | `src/` | Implementation files and internal-only headers (`ffi_client.h`, `lk_log.h`, etc.) |
-| `src/tests/` | Google Test integration and stress tests |
+| `src/tests/` | Google Test unit, integration, and stress tests |
+| `src/tests/manual/` | Standalone testers built with the test targets but not registered with CTest |
 | `examples/` | In-tree example applications |
 | `client-sdk-rust/` | Git submodule holding the Rust core of the SDK|
 | `cpp-tools/` | Git submodule holding shared LiveKit C++ engineering guidance, clang-format / clang-tidy configs, scripts, docs, and CI workflow |

docs/building.md

@@ -58,12 +58,31 @@ sudo apt update && sudo apt install -y \
   build-essential cmake ninja-build pkg-config \
   llvm-dev libclang-dev clang \
   libprotobuf-dev protobuf-compiler libabsl-dev \
-  libssl-dev
+  libssl-dev libcurl4-openssl-dev
 
 # Install Rust if you don't already have it
 curl https://sh.rustup.rs -sSf | sh
 ```
 
+The bundled `libwebrtc` artifact uses a hermetic libc++ built from LLVM trunk.
+Ubuntu 24.04's Clang 18 is too old for it. The Linux CI uses LLVM 21.1.8; install
+that toolchain and select it for the SDK build without replacing Ubuntu's
+system compiler:
+
+```bash
+export LLVM_VERSION=21.1.8
+export LLVM_ROOT="$HOME/.local/llvm-$LLVM_VERSION"
+client-sdk-rust/.github/scripts/install-clang.sh
+export CC="$LLVM_ROOT/bin/clang"
+export CXX="$LLVM_ROOT/bin/clang++"
+
+# Confirm the selected compiler before configuring the SDK.
+"$CXX" --version
+```
+
+Set `CC` and `CXX` before the first build, or remove the affected build
+directory before reconfiguring so CMake does not retain the previous compiler.
+
 If you plan to build the [example collection](https://github.com/livekit-examples/cpp-example-collection)
 (SDL-based renderer + camera/mic capture), also install:
 

docs/testing.md

@@ -44,6 +44,7 @@ __Note:__ The tests require tokens and a running LiveKit server. See the section
 | `livekit_unit_tests` | Pure unit tests (no server required) |
 | `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality |
 | `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) |
+| `livekit_memory_lifecycle_tester` | Manual/CI smoke for SDK connect/publish/subscribe teardown and RSS growth. Not registered with CTest. |
 
 ## Running a local LiveKit server for tests
 
@@ -72,6 +73,9 @@ export LIVEKIT_TOKEN_B="<second participant token>"
 # Optional (for stress tests)
 export RPC_STRESS_DURATION_SECONDS=3600   # Test duration (default: 1 hour)
 export RPC_STRESS_CALLER_THREADS=4        # Concurrent caller threads (default: 4)
+
+# Optional (memory lifecycle tester / CI smoke)
+export LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB=1048576  # fail if final RSS exceeds this many KiB
 ```
 
 ### Generating tokens for the test suites
@@ -105,6 +109,28 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret -
 - **Audio frame**: frame creation, manipulation, edge cases.
 - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls.
 - **Stress**: high throughput, bidirectional RPC, memory pressure.
+- **Memory lifecycle**: repeated source, room, media, data-track, and software
+  subscribe teardown. CI runs this as a short RSS-capped smoke test; longer
+  hardware runs stay manual.
+
+## Memory lifecycle smoke
+
+`livekit_memory_lifecycle_tester` is built with the test targets but is not
+part of `ctest`. CI runs two short software-only invocations after the
+integration suite:
+
+```bash
+source scripts/set-test-tokens.sh
+./build-release/bin/livekit_memory_lifecycle_tester --iterations 30 --ffi-cycles --sources
+./build-release/bin/livekit_memory_lifecycle_tester --iterations 20 --sources --media --data-frames --receive
+```
+
+`--receive` needs `LIVEKIT_TOKEN_B` and a running LiveKit server. The tester
+fails if `LIVEKIT_MEMORY_MAX_FINAL_RSS_KIB` is set and final process RSS exceeds
+that cap; leave the variable unset for local runs. CUDA and platform-audio
+paths are not part of this smoke test; see
+`src/tests/manual/cuda_video_lifecycle_tester` and
+`src/tests/manual/memory_lifecycle_tester/README.md`.
 
 ## Memory checks (valgrind)