From d229144c92be3a22bb77c6e8974d1d89d26e28f9 Mon Sep 17 00:00:00 2001 From: kesaroid Date: Tue, 15 Sep 2026 11:34:31 -0400 Subject: [PATCH 1/2] fix(mv3dt): bump Kafka prerequisite version to 4.2.1 4.2.0 does not exist anymore. Keep setup and check scripts on the same Kafka download/install path. --- .../scripts/check_prerequisites.sh | 2 +- .../scripts/setup_prerequisites.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh index 25d963e3..731ce296 100755 --- a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/check_prerequisites.sh @@ -17,7 +17,7 @@ # Global variables - same as setup script BASE_DIR=${BASE_DIR:-$HOME} USE_INFERENCE_BUILDER=${USE_INFERENCE_BUILDER:-false} -KAFKA_VERSION="4.2.0" +KAFKA_VERSION="4.2.1" SCALA_VERSION="2.13" # Standardized paths diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh index 6c03240d..03c5e2bb 100755 --- a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/setup_prerequisites.sh @@ -53,7 +53,7 @@ export DEEPSTREAM_IMAGE="${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.1-trito REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BASE_DIR=${BASE_DIR:-$HOME} USE_INFERENCE_BUILDER=${USE_INFERENCE_BUILDER:-false} -KAFKA_VERSION="4.2.0" +KAFKA_VERSION="4.2.1" SCALA_VERSION="2.13" # Standardized paths From fd7d728a1c658b230bde103704214285f5a41301 Mon Sep 17 00:00:00 2001 From: kesaroid Date: Tue, 22 Sep 2026 15:24:23 -0400 Subject: [PATCH 2/2] fix(mv3dt): update Kafka version to 4.2.1 in manual setup and scripts; add PeopleNetTransformer model configuration --- src/.gitignore | 9 + .../README.md | 273 +++++++++--------- .../docs/manual-setup.md | 10 +- .../nvdsinfer_config.yaml | 21 ++ .../scripts/custom_4cam_ds.sh | 160 ++++++++++ 5 files changed, 334 insertions(+), 139 deletions(-) create mode 100644 src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/nvdsinfer_config.yaml create mode 100755 src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/custom_4cam_ds.sh diff --git a/src/.gitignore b/src/.gitignore index 8ead1456..23ef1445 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -4,3 +4,12 @@ *.pb.cc *.pb.h .build/ + + + +# Local run builds +local_build/ +apps/reference_apps/deepstream-tracker-3d-multi-view/datasets/* +apps/reference_apps/deepstream-tracker-3d-multi-view/experiments/* + +apps/reference_apps/deepstream-tracker-3d-multi-view/mv3dt_venv/ \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/README.md b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/README.md index d2b8b2f3..afd0f30f 100644 --- a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/README.md +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/README.md @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. --> -# Multi-View 3D Tracking in DeepStream +# Multi-View 3D Tracking in DeepStream MV3DT 12-camera live demo @@ -24,6 +24,7 @@ limitations under the License. This repository provides sample applications for Multi-View 3D Tracking (MV3DT) with DeepStream 9.1 SDK. MV3DT is a distributed, real-time multi-view multi-target 3D tracking framework built for large-scale, calibrated camera networks. It is designed to deliver robust object tracking and identity consistency across complex environments, leveraging camera calibration data as a prerequisite for accurate geometric reasoning. The sample applications support three detector models: [PeopleNet Transformer](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet_transformer_v2?version=deployable_v1.0), a general-purpose people detection transformer model, [PeopleNet v2.6.3](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplenet), a high-performance people detection model based on DetectNet_v2, and [RT-DETR 2D Warehouse](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/rtdetr_2d_warehouse?version=deployable_efficientvit_l2_v1.0), a real-time DETR model optimized for warehouse environments with multiple object classes. This repository aims to demonstrate MV3DT through live visualization of 3D tracking results, and is structured as follows: + - **[Prerequisites](#prerequisites)** - System requirements and setup instructions - **[Option 1: Sample applications using DeepStream Container](#option-1-running-mv3dt-using-deepstream-container)** - The DeepStream Container has the DeepStream SDK pre-installed. The samples automate MV3DT config generation and launch the DeepStream app inside the container. - **[Option 2: Sample applications using Inference Builder](#option-2-running-mv3dt-using-inference-builder)** - Inference Builder is an open-source tool that automates inference pipeline generation across AI frameworks and packages them as deployable containers. The samples in this repo are solely intended to demonstrate building and running MV3DT using Inference Builder. For additional capabilities, see the [Inference Builder README](https://github.com/NVIDIA-AI-IOT/inference_builder/tree/3f0c09f2e3da076cbbb75e17bdebd565b03d1a18). @@ -34,6 +35,7 @@ This repository aims to demonstrate MV3DT through live visualization of 3D track As shown in the repo structure, MV3DT can be run using either DeepStream Container or Inference Builder. You can choose either approach to run the sample applications. We recommend starting with DeepStream Container for quick start and optionally trying out Inference Builder for advanced use cases, for example, integrating it with other AI frameworks or microservices. ## Prerequisites + The sample applications in this repository require Ubuntu 24.04 and NVIDIA driver version 580.xx or higher; both x86 and Jetson platforms are supported. A graphical display server (e.g., X11) is required to view visualization results. If no physical display is available, a remote desktop solution such as VNC Viewer can be used as an alternative. #### Known Issues @@ -47,67 +49,67 @@ The sample applications in this repository require Ubuntu 24.04 and NVIDIA drive #### Setup 1. Please check [DeepStream Container Prerequisites](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_docker_containers.html#prerequisites) for DeepStream container setup. - 2. Git clone the current `deepstream_reference_apps` repository to the host machine and enter `deepstream-tracker-3d-multi-view` directory - ```bash - # Install Git LFS - sudo apt install git-lfs - git lfs install - git clone https://github.com/NVIDIA/DeepStream.git - cd DeepStream/src/apps/reference_apps/deepstream-tracker-3d-multi-view - git lfs pull # In case repo is already cloned before installing git-lfs - ``` + ```bash + # Install Git LFS + sudo apt install git-lfs + git lfs install + git clone https://github.com/NVIDIA/DeepStream.git + cd DeepStream/src/apps/reference_apps/deepstream-tracker-3d-multi-view + git lfs pull # In case repo is already cloned before installing git-lfs + ``` 3. Run the automated prerequisites setup script: - The setup script takes about 10-20 minutes to complete. And it may prompt you to enter your password for sudo access and github credentials. After the initial setup, if you close the session or restart your machine, you can still use this script to set up and check prerequisites, then the completion time will be much shorter. - - ```bash - # For DeepStream Container only (Option 1) - ./scripts/setup_prerequisites.sh + The setup script takes about 10-20 minutes to complete. And it may prompt you to enter your password for sudo access and github credentials. After the initial setup, if you close the session or restart your machine, you can still use this script to set up and check prerequisites, then the completion time will be much shorter. - # Additionally, if you want to try out Inference Builder (Option 2) - USE_INFERENCE_BUILDER=true ./scripts/setup_prerequisites.sh + ```bash + # For DeepStream Container only (Option 1) + ./scripts/setup_prerequisites.sh - # [Expected output]: For both options, you should see all items checked with "✓" under "PREREQUISITES CHECK SUMMARY", and in the last line you should see: - # [SUCCESS] Prerequisites check passed! You're ready to use MV3DT. - ``` + # Additionally, if you want to try out Inference Builder (Option 2) + USE_INFERENCE_BUILDER=true ./scripts/setup_prerequisites.sh - **Environment Variables:** - - `USE_INFERENCE_BUILDER` - Enable Inference Builder setup (default: false, DeepStream Container only) - - `BASE_DIR` - Base directory for Kafka and Inference Builder installations (default: `$HOME`) - - `DEEPSTREAM_IMAGE` - DeepStream Docker image (default: `nvcr.io/nvidia/deepstream:9.1-triton-multiarch` for x86 and Jetson platforms) - - * **Use case 1: If you want to use a different base directory** for Kafka and Inference Builder installations other than `$HOME`, you can set the `BASE_DIR` environment variable before running the script. - ```bash - export BASE_DIR=/path/to/your/preferred/base/directory - # If you want to use Inference Builder (Option 2), uncomment the lines below - # export USE_INFERENCE_BUILDER=true - # export INFERENCE_BUILDER_DIR="$BASE_DIR/inference_builder" - ./scripts/setup_prerequisites.sh - ``` - * **Use case 2: If you are on ARM SBSA platforms**, the DeepStream docker image will be different from the default one. Please set the `DEEPSTREAM_IMAGE` environment variable before running the script. - ```bash - export DEEPSTREAM_IMAGE=nvcr.io/nvidia/deepstream:9.1-triton-arm-sbsa - # If you want to use Inference Builder (Option 2), uncomment the line below - # export USE_INFERENCE_BUILDER=true - ./scripts/setup_prerequisites.sh - ``` - - - For manual setup, troubleshooting, or shutdown instructions for Kafka and Mqtt brokers, see: [Manual Setup Instructions](docs/manual-setup.md) + # [Expected output]: For both options, you should see all items checked with "✓" under "PREREQUISITES CHECK SUMMARY", and in the last line you should see: + # [SUCCESS] Prerequisites check passed! You're ready to use MV3DT. + ``` + **Environment Variables:** + + - `USE_INFERENCE_BUILDER` - Enable Inference Builder setup (default: false, DeepStream Container only) + - `BASE_DIR` - Base directory for Kafka and Inference Builder installations (default: `$HOME`) + - `DEEPSTREAM_IMAGE` - DeepStream Docker image (default: `nvcr.io/nvidia/deepstream:9.1-triton-multiarch` for x86 and Jetson platforms) + + * **Use case 1: If you want to use a different base directory** for Kafka and Inference Builder installations other than `$HOME`, you can set the `BASE_DIR` environment variable before running the script. + ```bash + export BASE_DIR=/path/to/your/preferred/base/directory + # If you want to use Inference Builder (Option 2), uncomment the lines below + # export USE_INFERENCE_BUILDER=true + # export INFERENCE_BUILDER_DIR="$BASE_DIR/inference_builder" + ./scripts/setup_prerequisites.sh + ``` + * **Use case 2: If you are on ARM SBSA platforms**, the DeepStream docker image will be different from the default one. Please set the `DEEPSTREAM_IMAGE` environment variable before running the script. + ```bash + export DEEPSTREAM_IMAGE=nvcr.io/nvidia/deepstream:9.1-triton-arm-sbsa + # If you want to use Inference Builder (Option 2), uncomment the line below + # export USE_INFERENCE_BUILDER=true + ./scripts/setup_prerequisites.sh + ``` + + For manual setup, troubleshooting, or shutdown instructions for Kafka and Mqtt brokers, see: [Manual Setup Instructions](docs/manual-setup.md) ## Option 1: Running MV3DT using DeepStream Container The following examples demonstrate running MV3DT using DeepStream container. Note that the configurations are auto-generated from `config_templates` using the [auto-configurator](utils/README.md#deepstream_auto_configuratorpy). ### Sample 1: 4-camera dataset + --- #### Quick Start -Run the provided script to quickly launch the 4-camera DeepStream pipeline. + +Run the provided script to quickly launch the 4-camera DeepStream pipeline. ```bash cd @@ -125,27 +127,28 @@ sudo xhost + # give container access to display, only need to run once per sessi # DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_4cam_ds.sh ``` -Two separate windows will be launched. One named **Bird-Eye View of Multi-View 3D Tracking**, and the other named **DeepStreamTest5App**. You may need to toggle, arrange, or resize the windows to see both views. If anything goes wrong or the windows are not showing, please follow the step-by-step instructions below; otherwise, the quick-start script covers the same processes. +Two separate windows will be launched. One named **Bird-Eye View of Multi-View 3D Tracking**, and the other named **DeepStreamTest5App**. You may need to toggle, arrange, or resize the windows to see both views. If anything goes wrong or the windows are not showing, please follow the step-by-step instructions below; otherwise, the quick-start script covers the same processes. **Note 1 (Important): When the script is launched for the first time, it tries to create model engine files use by MV3DT, which may take about 15 minutes, depending on HW platforms.** This process is only needed once for each dataset, and subsequent runs will use the generated engine files and launch immediately. **Note 2:** It's expected to see the following warnings. Those warnings will not affect the accuracy or performance of the pipeline. + - Load engine failed. Create engine again. - INT8 calibration file not specified. Trying FP16 mode. - GStreamer-WARNING **Window 1: Bird-Eye View of Multi-View 3D Tracking** + - This window shows the bird's-eye view of the multi-view 3D tracking results. It will show as blank map at the beginning. - **The BEV visualization window will not exit automatically. To close the window, select the window and press 'q'.** - **Window 2: DeepStreamTest5App** + - This window shows the DeepStream on-screen display (OSD) of 4 camera views in a grid. This window is directly from Deepstream pipeline, and will show as black window at the beginning. - To view a specific camera view in the DeepStream OSD window, left-click on the desired view. To return to the multi-camera grid view, simply right-click anywhere in the window. - **The DeepStreamTest5App will exit automatically. To quit the DeepStreamTest5App window early, select the window and press 'q'.** After you press 'q', the app will terminate within a few seconds, and finally you will see "App run successful" printed. - Note about window name: MV3DT is built upon DeepStreamTest5App with specific config files, which is why the window displays "DeepStreamTest5App" as its title. This base application provides essential IoT protocol support (Kafka and MQTT) required by MV3DT. Currently, the window name cannot be changed. For more details, see the [DeepStreamTest5App documentation](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_ref_app_test5.html). - **Expected output during engine generation period from window overview/spread:** Screenshot @@ -155,12 +158,15 @@ Two separate windows will be launched. One named **Bird-Eye View of Multi-View 3 Screenshot #### Step-by-step Instructions + For detailed step-by-step instructions, see [DeepStream Container: Step-by-step Instructions](docs/step-by-step-deepstream.md#sample-1-4-camera-dataset). ### Sample 2: 12-camera dataset + --- #### Quick Start + Run the provided script to quickly launch the 12-camera DeepStream pipeline. For detailed window explanations and important notes, see the [4-camera Quick Start section](#quick-start) above. ```bash @@ -175,7 +181,9 @@ Run the provided script to quickly launch the 12-camera DeepStream pipeline. For # To use PeopleNet v2.6.3 detector: # DETECTOR_MODEL=PeopleNet2.6.3 ./scripts/test_12cam_ds.sh ``` + #### Step-by-step Instructions + For detailed step-by-step instructions, see [DeepStream Container: Step-by-step Instructions](docs/step-by-step-deepstream.md#sample-2-12-camera-dataset). ## Option 2: Running MV3DT using Inference Builder @@ -185,11 +193,13 @@ Inference Builder is a tool that automatically generates inference pipelines and Compared to traditional DeepStream configuration files, the Inference Builder configuration (e.g., `config_templates/ds_mv3dt.yaml`) is significantly simpler and more concise, making it easier to define and modify. ### Sample 1: 4-camera dataset + --- #### Quick Start Run the provided script to quickly start the 4-camera DeepStream Inference Builder pipeline. + ```bash # sudo xhost + # give container access to display @@ -208,32 +218,33 @@ Run the provided script to quickly start the 4-camera DeepStream Inference Build For detailed window explanations and important notes, see the [4-camera DeepStream Quick Start section](#quick-start) above. Additional notes: - **Note 1:** By default, the application waits **1000 seconds** if there is no data being streamed before exiting gracefully. It is expected to see 0 FPS printed during the 1000 seconds wait time. This waiting time is controlled by the `inference_timeout` parameter in `config_templates/ds_mv3dt.yaml`. To avoid this delay: + * Option 1: Pre-generate the engine files and place them in the correct directories before starting the pipeline. Then, set `inference_timeout` to a lower value (e.g., 30 seconds). * Option 2: Use the default 1000-second timeout for the initial run to allow engine file creation. For later runs, reduce `inference_timeout` (e.g., to 30 seconds) so the container exits promptly after inference completes. **Note 2: To quit the app early**, run this command in a separate terminal to stop the container: -`docker ps -q --filter "ancestor=inference-builder-mv3dt" | xargs docker stop`. +`docker ps -q --filter "ancestor=inference-builder-mv3dt" | xargs docker stop`. If you are not able to see camera view or BEV view, please follow the step-by-step instructions; otherwise, the quick-start script covers the same processes. - **Expected output from window overview/spread:** Screenshot #### Step-by-step Instructions -For detailed step-by-step instructions, see [Inference Builder: Step-by-step Instructions](docs/step-by-step-inference-builder.md#sample-1-4-camera-dataset). +For detailed step-by-step instructions, see [Inference Builder: Step-by-step Instructions](docs/step-by-step-inference-builder.md#sample-1-4-camera-dataset). ### Sample 2: 12-camera dataset + --- #### Quick Start + Run the provided script to quickly start the 12-camera DeepStream Inference Builder pipeline. -For detailed window explanations and important notes, see the [4-camera DeepStream Quick Start section](#quick-start) above. +For detailed window explanations and important notes, see the [4-camera DeepStream Quick Start section](#quick-start) above. ```bash # sudo xhost + # give container access to display @@ -252,14 +263,17 @@ For detailed window explanations and important notes, see the [4-camera DeepStre ``` #### Step-by-step Instructions -For detailed step-by-step instructions, see [Inference Builder: Step-by-step Instructions](docs/step-by-step-inference-builder.md#sample-2-12-camera-dataset). +For detailed step-by-step instructions, see [Inference Builder: Step-by-step Instructions](docs/step-by-step-inference-builder.md#sample-2-12-camera-dataset). ## Output Visualization + Whether you use Option 1 (DeepStream Container) or Option 2 (Inference Builder), both approaches launch two windows with similar visualizations. This section explains and demonstrates the content of these visualization windows. ### DeepStream Direct Visualization + --- + When the pipeline is launched, DeepStream shows the output video like below while processing the input video. In the example frames below, you can see that objects detected across different cameras are assigned globally consistent IDs. And both 2D and 3D bounding boxes are visualized for each tracked object. In the Inference Builder OSD window, object IDs are visible directly in the grid view. In the DeepStream Container OSD window, object IDs are only visible when viewing a single camera. To enter single camera view with object IDs, left-click on the desired camera view. And to return to the multi-camera grid view, simply right-click anywhere in the window. @@ -280,37 +294,38 @@ In the Inference Builder OSD window, object IDs are visible directly in the grid If you don't need on-screen display, you can disable it: * **For DeepStream Container:** - * Remove the `--enable-osd` option from `deepstream_auto_configurator.py` command in the quick-start script and run again. + * Remove the `--enable-osd` option from `deepstream_auto_configurator.py` command in the quick-start script and run again. * **For Inference Builder:** - * Comment out the `render_config` section in `config_templates/ds_mv3dt.yaml` and run the quick-start script again. - + * Comment out the `render_config` section in `config_templates/ds_mv3dt.yaml` and run the quick-start script again. ### Real-time BEV visualization of 3D metadata from Kafka + --- -The `kafka_bev_visualizer.py` script provides real-time bird's-eye view (BEV) visualization of 3D tracking data streamed via Kafka. +The `kafka_bev_visualizer.py` script provides real-time bird's-eye view (BEV) visualization of 3D tracking data streamed via Kafka. Note that the BEV visualization script should be launched before launching the MV3DT app. * Command: - ```bash - python utils/kafka_bev_visualizer.py \ - --dataset-path=$DATASET_DIR \ - --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ - --show-ids \ - --average-multi-cam - ``` + + ```bash + python utils/kafka_bev_visualizer.py \ + --dataset-path=$DATASET_DIR \ + --msgconv-config=$EXPERIMENT_DIR/config_msgconv.txt \ + --show-ids \ + --average-multi-cam + ``` * Expected outputs (left: 4-camera, right: 12-camera) -
+ +
Sample 4-camera BEV tracking results Sample 12-camera BEV tracking results
- - * Note that the BEV visualization shows fused tracks, where trajectories of the same object from multiple cameras are averaged into one. If you want to see the individual trajectories from every camera, simply remove the `--average-multi-cam` option from the command. The output will then look like this: -
+ +
Sample 4-camera BEV tracking results Sample 12-camera BEV tracking results
@@ -333,19 +348,20 @@ python utils/kafka_client.py --broker localhost:9092 --topic mv3dt This section provides customization options for the MV3DT pipeline. If you are new to DeepStream and want to try MV3DT on your own dataset, see [Running MV3DT on Custom Datasets](#running-mv3dt-on-custom-datasets). If you already have a working 2D DeepStream tracking pipeline, see [Converting your Existing 2D DeepStream Tracking Pipeline to MV3DT](#converting-your-existing-2d-deepstream-tracking-pipeline-to-mv3dt) for simple transformation to multi-view 3D pipeline. - - ## Running MV3DT on Custom Datasets **Requirements:** + - Multi-view video streams must be synchronized. - All video streams must have the same resolution. - Camera calibration parameters (projection matrices) must be available. ### Steps + --- 1. **Organize your dataset** with the following structure: + ``` your_dataset/ ├── videos/ @@ -359,18 +375,15 @@ This section provides customization options for the MV3DT pipeline. If you are n ├── map.png (optional, for BEV visualization) └── transforms.yml (optional, for BEV visualization) ``` - 2. **Create camera calibration files** following the format of `datasets/mtmc_4cam/camInfo/Warehouse_Synthetic_Cam001.yml`. Replace the `projectionMatrix_3x4_w2p` values with your camera's projection matrix. For more details about these files, please refer to the [Single-View 3D Tracking](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#single-view-3d-tracking) and [The 3x4 Camera Projection Matrix](https://docs.nvidia.com/metropolis/deepstream/dev-guide/text/DS_plugin_gst-nvtracker.html#the-3x4-camera-projection-matrix) sections of the DeepStream documentation. - 3. **Optional: BEV visualization setup** - Prepare a BEV map image and create a `transforms.yml` file specifying the projection matrix that maps world coordinates (in meters) to BEV image coordinates, following the sample format in `datasets/mtmc_4cam/transforms.yml`. - 4. **Generate configurations** using the auto-configurator. Refer to `scripts/test_4cam_ds.sh` for the exact python command and environment variables needed. - 5. **Launch the MV3DT pipeline** using your generated configs. Refer to `scripts/test_4cam_ds.sh` for the exact Docker command and environment variables needed. - ### How It Works + --- + The auto-configurator generates a complete set of MV3DT config files tailored to your dataset: 1. **Detects dataset structure** - Scans your `videos/` and `camInfo/` directories to determine camera count and calibration files @@ -380,91 +393,90 @@ The auto-configurator generates a complete set of MV3DT config files tailored to For detailed usage and all available options, see the **[Auto-Configurator Documentation](utils/README.md)**. - ## Converting your Existing 2D DeepStream Tracking Pipeline to MV3DT + If you have an existing 2D detection and tracking pipeline using DeepStream, the auto-configurator can automatically enhance it to support multi-view 3D tracking, provided you have camera calibration files available. ### Steps + --- 1. **Organize your dataset** as in the previous [Running MV3DT on Custom Datasets](#running-mv3dt-on-custom-datasets) section. - 2. **Create camera calibration files** as in the previous section. +3. **Optional: BEV visualization setup** (required if you want to enable BEV visualization) -3. **Optional: BEV visualization setup** (required if you want to enable BEV visualization) - - Follow the instructions in the previous section. - + - Follow the instructions in the previous section. 4. **Optional: Set up your model configs** (required if using a different detector than PeopleNetTransformer) - - For example, if you are using PeopleNet as your detector, create a `PeopleNet` folder under `models` and modify `config_templates/config_pgie.txt` to point to your model files. - -5. **Optional: Set up your DeepStream pipeline config** (required if using a custom DeepStream pipeline config) - - Modify `config_templates/config_deepstream.txt` based on your use case. For example, if you have a custom `[pre-process]` section, copy that section to `config_templates/config_deepstream.txt`. Note that `[source%d]` and `[sink%d]` sections will be handled by the auto-configurator. - -6. **Place your tracker config file in the `config_templates` folder.** + - For example, if you are using PeopleNet as your detector, create a `PeopleNet` folder under `models` and modify `config_templates/config_pgie.txt` to point to your model files. +5. **Optional: Set up your DeepStream pipeline config** (required if using a custom DeepStream pipeline config) -7. **Generate MV3DT configs** - - Use the auto-configurator with `--tracker-config` argument, i.e. suppose your tracker config file is named `config_tracker_custom_2d.yml`, pass `--tracker-config=config_tracker_custom_2d.yml` to the auto-configurator. The auto-configurator assumes all template configs are located in the `config_templates` folder, so only the file name is needed, not the full path. - -8. **Launch the MV3DT pipeline** using your generated configs. + - Modify `config_templates/config_deepstream.txt` based on your use case. For example, if you have a custom `[pre-process]` section, copy that section to `config_templates/config_deepstream.txt`. Note that `[source%d]` and `[sink%d]` sections will be handled by the auto-configurator. +6. **Place your tracker config file in the `config_templates` folder.** +7. **Generate MV3DT configs** + - Use the auto-configurator with `--tracker-config` argument, i.e. suppose your tracker config file is named `config_tracker_custom_2d.yml`, pass `--tracker-config=config_tracker_custom_2d.yml` to the auto-configurator. The auto-configurator assumes all template configs are located in the `config_templates` folder, so only the file name is needed, not the full path. +8. **Launch the MV3DT pipeline** using your generated configs. ### A Step-by-step Example + --- 1. To simulate a custom dataset, let's create a 6-camera subset (randomly selected) from the 12-camera dataset. Run the following command to generate a new dataset in `datasets/mtmc_6cam`. - ```bash - ./scripts/create_6cam_subset.sh - ``` + ```bash + ./scripts/create_6cam_subset.sh + ``` 2. Assume you have an existing 2D tracking config file. In this example, we will use the `config_tracker_2d.yml` file in the `config_templates` folder. - 3. Set up output directories and run the auto-configurator to generate the MV3DT config files. - ```bash - export DATASET_DIR=$PWD/datasets/mtmc_6cam/ - export EXPERIMENT_DIR=$PWD/experiments/deepstream/6cam - mkdir -p $EXPERIMENT_DIR/infer-kitti-dump - mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + ```bash + export DATASET_DIR=$PWD/datasets/mtmc_6cam/ + export EXPERIMENT_DIR=$PWD/experiments/deepstream/6cam - python utils/deepstream_auto_configurator.py \ - --dataset-dir=$DATASET_DIR \ - --tracker-config=config_tracker_2d.yml \ - --enable-msg-broker \ - --enable-osd \ - --output-dir=$EXPERIMENT_DIR - ``` + mkdir -p $EXPERIMENT_DIR/infer-kitti-dump + mkdir -p $EXPERIMENT_DIR/tracker-kitti-dump + python utils/deepstream_auto_configurator.py \ + --dataset-dir=$DATASET_DIR \ + --tracker-config=config_tracker_2d.yml \ + --enable-msg-broker \ + --enable-osd \ + --output-dir=$EXPERIMENT_DIR + ``` 4. Launch the MV3DT pipeline: - ```bash - export MODEL_REPO=$PWD/models - - docker run -t --privileged --rm --net=host --runtime=nvidia \ - -v $MODEL_REPO:/workspace/models \ - -v $DATASET_DIR:/workspace/inputs \ - -v $EXPERIMENT_DIR:/workspace/experiments \ - -v /tmp/.X11-unix/:/tmp/.X11-unix \ - -e DISPLAY=$DISPLAY \ - -w /workspace/experiments \ - nvcr.io/nvidia/deepstream:9.1-triton-multiarch \ - deepstream-test5-app -c config_deepstream.txt - ``` + + ```bash + export MODEL_REPO=$PWD/models + + docker run -t --privileged --rm --net=host --runtime=nvidia \ + -v $MODEL_REPO:/workspace/models \ + -v $DATASET_DIR:/workspace/inputs \ + -v $EXPERIMENT_DIR:/workspace/experiments \ + -v /tmp/.X11-unix/:/tmp/.X11-unix \ + -e DISPLAY=$DISPLAY \ + -w /workspace/experiments \ + nvcr.io/nvidia/deepstream:9.1-triton-multiarch \ + deepstream-test5-app -c config_deepstream.txt + ``` * For convenience, the same process is automated in the following scripts: - ```bash - # Create the 6-camera dataset if not already done - # ./scripts/create_6cam_subset.sh + ```bash + # Create the 6-camera dataset if not already done + # ./scripts/create_6cam_subset.sh - # Option 1: using DeepStream Container - ./scripts/test_custom_2d_tracker_ds.sh + # Option 1: using DeepStream Container + ./scripts/test_custom_2d_tracker_ds.sh - # Option 2: using Inference Builder - ./scripts/test_custom_2d_tracker_ib.sh - ``` + # Option 2: using Inference Builder + ./scripts/test_custom_2d_tracker_ib.sh + ``` ### How It Works + --- + The auto-configurator generates a complete set of MV3DT config files based on your custom 2D tracker, pipeline, and model configs, as well as your dataset. 1. **Detects dataset structure** - Scans your `videos/` and `camInfo/` directories to determine camera count and calibration files @@ -475,14 +487,7 @@ The auto-configurator generates a complete set of MV3DT config files based on yo For detailed usage and all available options, see the **[Auto-Configurator Documentation](utils/README.md)**. - - - ## Python Util Scripts For more details on python utility scripts including auto-configuration generators and visualization tools, see 📁 **[Python Util Scripts Documentation](utils/README.md)** - - - - diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/manual-setup.md b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/manual-setup.md index 80edf9c1..f9f9238f 100644 --- a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/manual-setup.md +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/docs/manual-setup.md @@ -113,9 +113,9 @@ limitations under the License. sudo apt install openjdk-17-jdk # Get Kafka - wget https://dlcdn.apache.org/kafka/4.2.0/kafka_2.13-4.2.0.tgz - tar -xzf kafka_2.13-4.2.0.tgz - cd kafka_2.13-4.2.0 + wget https://dlcdn.apache.org/kafka/4.2.1/kafka_2.13-4.2.1.tgz + tar -xzf kafka_2.13-4.2.1.tgz + cd kafka_2.13-4.2.1 # Start the Kafka environment export KAFKA_CLUSTER_ID="$(bin/kafka-storage.sh random-uuid)" @@ -130,7 +130,7 @@ limitations under the License. * Create a `mv3dt` topic under broker server `localhost:9092`, and set the message retention to 30 seconds. ```bash - cd + cd ./bin/kafka-topics.sh --bootstrap-server localhost:9092 \ --create \ @@ -149,7 +149,7 @@ limitations under the License. * To stop a Kafka broker running in the background, you can use the following command: ```bash - cd + cd bin/kafka-server-stop.sh ``` diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/nvdsinfer_config.yaml b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/nvdsinfer_config.yaml new file mode 100644 index 00000000..49746009 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/models/PeopleNetTransformer/nvdsinfer_config.yaml @@ -0,0 +1,21 @@ +property: + gpu-id: 0 + offsets: 123.675;116.28;103.53 + net-scale-factor: 0.0173520735728 + labelfile-path: /workspace/models/PeopleNetTransformer/detector_labels.txt + onnx-file: /workspace/models/PeopleNetTransformer/peoplenet_transformer_model_op17.onnx + tlt-model-key: nvidia_tao + batch-size: 12 + network-mode: 2 + num-detected-classes: 4 + filter-out-class-ids: 0;2;3 + interval: 0 + gie-unique-id: 1 + output-blob-names: pred_boxes;pred_logits + infer-dims: 3;544;960 + cluster-mode: 4 + parse-bbox-func-name: NvDsInferParseCustomDDETRTAO + custom-lib-path: /workspace/models/PeopleNetTransformer/custom_parser/libnvds_infercustomparser_tao.so +class-attrs-all: + pre-cluster-threshold: 0.3 + topk: 20 \ No newline at end of file diff --git a/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/custom_4cam_ds.sh b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/custom_4cam_ds.sh new file mode 100755 index 00000000..86b72681 --- /dev/null +++ b/src/apps/reference_apps/deepstream-tracker-3d-multi-view/scripts/custom_4cam_ds.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Run MV3DT on a CUSTOM multi-camera dataset with off-the-shelf detectors. +# +# Usage (from the MV3DT app root): +# DATASET_DIR=/abs/path/to/my_dataset ./scripts/custom_4cam_ds.sh +# +# Expected dataset layout: +# my_dataset/ +# |-- videos/ cam_00.mp4, cam_01.mp4, ... (synchronized, same resolution) +# |-- camInfo/ cam_00.yml, cam_01.yml, ... (projectionMatrix_3x4_w2p + modelInfo) +# |-- map.png (optional, BEV only) +# `-- transforms.yml (optional, BEV only) +# +# Tunables (env vars): +# DATASET_DIR required, dataset root +# EXPERIMENT_DIR default experiments/deepstream/ +# MODEL_REPO default /models +# DETECTOR_MODEL PeopleNetTransformer (default) | RTDETR | PeopleNet2.6.3 +# TRACKER_CONFIG default config_tracker.yml (use config_tracker_2d.yml to convert a 2D pipeline) +# CONFIG_OVERRIDES default override_tracker_4cam.yml (empty string disables) +# RUN_MODE display (default when $DISPLAY is set) | headless +# ENABLE_BEV 1 (default) when map.png + transforms.yml exist +# DEEPSTREAM_IMAGE default nvcr.io/nvidia/deepstream:9.1-triton-multiarch + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +export DATASET_DIR="${DATASET_DIR:?Set DATASET_DIR to your custom dataset root (must contain videos/ and camInfo/)}" +DATASET_DIR="$(cd "$DATASET_DIR" && pwd)" +export EXPERIMENT_DIR="${EXPERIMENT_DIR:-$REPO_ROOT/experiments/deepstream/$(basename "$DATASET_DIR")}" +export MODEL_REPO="${MODEL_REPO:-$REPO_ROOT/models}" + +# ---------------------------------------------------------------- validation +[[ -d "$DATASET_DIR/videos" ]] || { echo "ERROR: $DATASET_DIR/videos missing" >&2; exit 1; } +[[ -d "$DATASET_DIR/camInfo" ]] || { echo "ERROR: $DATASET_DIR/camInfo missing" >&2; exit 1; } + +NUM_VIDEOS=$(find "$DATASET_DIR/videos" -maxdepth 1 -type f -name '*.mp4' | wc -l) +NUM_CALIB=$(find "$DATASET_DIR/camInfo" -maxdepth 1 -type f -name '*.yml' | wc -l) +NUM_CALIB_YAML=$(find "$DATASET_DIR/camInfo" -maxdepth 1 -type f -name '*.yaml' | wc -l) + +(( NUM_VIDEOS > 0 )) || { echo "ERROR: no .mp4 files in $DATASET_DIR/videos" >&2; exit 1; } + +# The auto-configurator only scans .yml; make non-destructive copies of .yaml exports (e.g. from AutoMagicCalib). +if (( NUM_CALIB == 0 )) && (( NUM_CALIB_YAML > 0 )); then + echo "Copying camInfo/*.yaml -> *.yml (existing .yml files are never overwritten)" + find "$DATASET_DIR/camInfo" -maxdepth 1 -type f -name '*.yaml' -print0 \ + | while IFS= read -r -d '' f; do cp -n "$f" "${f%.yaml}.yml"; done + NUM_CALIB=$(find "$DATASET_DIR/camInfo" -maxdepth 1 -type f -name '*.yml' | wc -l) +fi + +(( NUM_CALIB == NUM_VIDEOS )) || { + echo "ERROR: $NUM_VIDEOS videos but $NUM_CALIB camInfo/*.yml files - one calibration per camera is required" >&2 + exit 1 +} + +# Videos and calibrations are paired by basename; sorted order is the fallback and is easy to get wrong. +for v in "$DATASET_DIR"/videos/*.mp4; do + b="$(basename "$v" .mp4)" + [[ -f "$DATASET_DIR/camInfo/$b.yml" ]] || echo "WARN: no camInfo/$b.yml for videos/$b.mp4 - pairing falls back to sorted order" +done + +echo "Dataset: $DATASET_DIR ($NUM_VIDEOS cameras)" + +# ------------------------------------------------------------ detector/tracker +export DETECTOR_MODEL="${DETECTOR_MODEL:-PeopleNetTransformer}" +case "$DETECTOR_MODEL" in + PeopleNetTransformer) DETECTOR_CONFIG="config_pgie.txt" ;; + RTDETR) DETECTOR_CONFIG="config_pgie_rt_detr.txt" ;; + PeopleNet2.6.3) DETECTOR_CONFIG="config_pgie_peoplenet.txt" ;; + *) echo "ERROR: unsupported DETECTOR_MODEL=$DETECTOR_MODEL" >&2; exit 1 ;; +esac +TRACKER_CONFIG="${TRACKER_CONFIG:-config_tracker.yml}" +CONFIG_OVERRIDES="${CONFIG_OVERRIDES-override_tracker_4cam.yml}" + +echo "Using detector model: $DETECTOR_MODEL (detector=$DETECTOR_CONFIG, tracker=$TRACKER_CONFIG)" + +# ------------------------------------------------------------------- run mode +RUN_MODE="${RUN_MODE:-$([[ -n "${DISPLAY:-}" ]] && echo display || echo headless)}" +CONFIG_FLAGS=(--enable-msg-broker) +if [[ "$RUN_MODE" == "headless" ]]; then + CONFIG_FLAGS+=(--enable-file-output) +else + CONFIG_FLAGS+=(--enable-osd --enable-file-output) +fi + +ENABLE_BEV="${ENABLE_BEV:-1}" +if [[ "$ENABLE_BEV" == "1" && ( ! -f "$DATASET_DIR/map.png" || ! -f "$DATASET_DIR/transforms.yml" ) ]]; then + echo "WARN: map.png and/or transforms.yml missing - BEV visualization disabled" + ENABLE_BEV=0 +fi + +# ----------------------------------------------------------------- GPU runtime +if docker info 2>/dev/null | grep -q 'Runtimes.*nvidia'; then + GPU_FLAG="--runtime=nvidia" +elif docker run --help | grep -q -- "--gpus"; then + GPU_FLAG="--gpus all" +else + echo "No GPU support found in Docker." >&2 + exit 1 +fi + +mkdir -p "$EXPERIMENT_DIR"/{infer-kitti-dump,tracker-kitti-dump,outVideos,bev_outputs} + +# ------------------------------------------------------------ generate configs +source mv3dt_venv/bin/activate + +CONFIG_OVERRIDE_ARGS=() +[[ -n "$CONFIG_OVERRIDES" ]] && CONFIG_OVERRIDE_ARGS+=(--config-overrides="$CONFIG_OVERRIDES") + +python utils/deepstream_auto_configurator.py \ + --dataset-dir="$DATASET_DIR" \ + "${CONFIG_FLAGS[@]}" \ + --detector-config="$DETECTOR_CONFIG" \ + --tracker-config="$TRACKER_CONFIG" \ + "${CONFIG_OVERRIDE_ARGS[@]}" \ + --output-dir="$EXPERIMENT_DIR" + +# ---------------------------------------------------------------------- BEV +BEV_PID="" +if [[ "$ENABLE_BEV" == "1" && "$RUN_MODE" != "headless" ]]; then + python utils/kafka_bev_visualizer.py \ + --dataset-path="$DATASET_DIR" \ + --msgconv-config="$EXPERIMENT_DIR/config_msgconv.txt" \ + --average-multi-cam \ + --show-ids & + BEV_PID=$! +fi +trap '[[ -n "$BEV_PID" ]] && kill "$BEV_PID" 2>/dev/null || true' EXIT + +# ------------------------------------------------------------------- pipeline +DOCKER_DISPLAY_ARGS=() +[[ "$RUN_MODE" != "headless" ]] && DOCKER_DISPLAY_ARGS=(-v /tmp/.X11-unix/:/tmp/.X11-unix -e DISPLAY="$DISPLAY") + +docker run -t --privileged --rm --net=host $GPU_FLAG \ + -v "$MODEL_REPO:/workspace/models" \ + -v "$DATASET_DIR:/workspace/inputs" \ + -v "$EXPERIMENT_DIR:/workspace/experiments" \ + "${DOCKER_DISPLAY_ARGS[@]}" \ + -w /workspace/experiments \ + "${DEEPSTREAM_IMAGE:-nvcr.io/nvidia/deepstream:9.1-triton-multiarch}" \ + deepstream-test5-app -c config_deepstream.txt + +echo "Done. Artifacts under $EXPERIMENT_DIR (outVideos/tiled_display_raw.mp4, tracker-kitti-dump/)"